test_model_switch_custom_providers.py
1 """Regression tests for /model support of config.yaml custom_providers. 2 3 The terminal `hermes model` flow already exposes `custom_providers`, but the 4 shared slash-command pipeline (`/model` in CLI/gateway/Telegram) historically 5 only looked at `providers:`. 6 """ 7 8 import hermes_cli.providers as providers_mod 9 from hermes_cli.model_switch import list_authenticated_providers, switch_model 10 from hermes_cli.providers import resolve_provider_full 11 12 13 _MOCK_VALIDATION = { 14 "accepted": True, 15 "persist": True, 16 "recognized": True, 17 "message": None, 18 } 19 20 21 def test_list_authenticated_providers_includes_custom_providers(monkeypatch): 22 """No-args /model menus should include saved custom_providers entries.""" 23 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 24 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 25 26 providers = list_authenticated_providers( 27 current_provider="openai-codex", 28 user_providers={}, 29 custom_providers=[ 30 { 31 "name": "Local (127.0.0.1:4141)", 32 "base_url": "http://127.0.0.1:4141/v1", 33 "model": "rotator-openrouter-coding", 34 } 35 ], 36 max_models=50, 37 ) 38 39 assert any( 40 p["slug"] == "custom:local-(127.0.0.1:4141)" 41 and p["name"] == "Local (127.0.0.1:4141)" 42 and p["models"] == ["rotator-openrouter-coding"] 43 and p["api_url"] == "http://127.0.0.1:4141/v1" 44 for p in providers 45 ) 46 47 48 def test_resolve_provider_full_finds_named_custom_provider(): 49 """Explicit /model --provider should resolve saved custom_providers entries.""" 50 resolved = resolve_provider_full( 51 "custom:local-(127.0.0.1:4141)", 52 user_providers={}, 53 custom_providers=[ 54 { 55 "name": "Local (127.0.0.1:4141)", 56 "base_url": "http://127.0.0.1:4141/v1", 57 } 58 ], 59 ) 60 61 assert resolved is not None 62 assert resolved.id == "custom:local-(127.0.0.1:4141)" 63 assert resolved.name == "Local (127.0.0.1:4141)" 64 assert resolved.base_url == "http://127.0.0.1:4141/v1" 65 assert resolved.source == "user-config" 66 67 68 def test_switch_model_accepts_explicit_named_custom_provider(monkeypatch): 69 """Shared /model switch pipeline should accept --provider for custom_providers.""" 70 monkeypatch.setattr( 71 "hermes_cli.runtime_provider.resolve_runtime_provider", 72 lambda **kwargs: { 73 "api_key": "no-key-required", 74 "base_url": "http://127.0.0.1:4141/v1", 75 "api_mode": "chat_completions", 76 }, 77 ) 78 monkeypatch.setattr("hermes_cli.models.validate_requested_model", lambda *a, **k: _MOCK_VALIDATION) 79 monkeypatch.setattr("hermes_cli.model_switch.get_model_info", lambda *a, **k: None) 80 monkeypatch.setattr("hermes_cli.model_switch.get_model_capabilities", lambda *a, **k: None) 81 82 result = switch_model( 83 raw_input="rotator-openrouter-coding", 84 current_provider="openai-codex", 85 current_model="gpt-5.4", 86 current_base_url="https://chatgpt.com/backend-api/codex", 87 current_api_key="", 88 explicit_provider="custom:local-(127.0.0.1:4141)", 89 user_providers={}, 90 custom_providers=[ 91 { 92 "name": "Local (127.0.0.1:4141)", 93 "base_url": "http://127.0.0.1:4141/v1", 94 "model": "rotator-openrouter-coding", 95 } 96 ], 97 ) 98 99 assert result.success is True 100 assert result.target_provider == "custom:local-(127.0.0.1:4141)" 101 assert result.provider_label == "Local (127.0.0.1:4141)" 102 assert result.new_model == "rotator-openrouter-coding" 103 assert result.base_url == "http://127.0.0.1:4141/v1" 104 assert result.api_key == "no-key-required" 105 106 107 def test_list_groups_same_name_custom_providers_into_one_row(monkeypatch): 108 """Multiple custom_providers entries sharing a name should produce one row 109 with all models collected, not N duplicate rows.""" 110 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 111 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 112 113 providers = list_authenticated_providers( 114 current_provider="openrouter", 115 user_providers={}, 116 custom_providers=[ 117 {"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "qwen3-coder:480b-cloud"}, 118 {"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "glm-5.1:cloud"}, 119 {"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "kimi-k2.5"}, 120 {"name": "Ollama Cloud", "base_url": "https://ollama.com/v1", "model": "minimax-m2.7:cloud"}, 121 {"name": "Moonshot", "base_url": "https://api.moonshot.ai/v1", "model": "kimi-k2-thinking"}, 122 ], 123 max_models=50, 124 ) 125 126 ollama_rows = [p for p in providers if p["name"] == "Ollama Cloud"] 127 assert len(ollama_rows) == 1, f"Expected 1 Ollama Cloud row, got {len(ollama_rows)}" 128 assert ollama_rows[0]["models"] == [ 129 "qwen3-coder:480b-cloud", "glm-5.1:cloud", "kimi-k2.5", "minimax-m2.7:cloud" 130 ] 131 assert ollama_rows[0]["total_models"] == 4 132 133 moonshot_rows = [p for p in providers if p["name"] == "Moonshot"] 134 assert len(moonshot_rows) == 1 135 assert moonshot_rows[0]["models"] == ["kimi-k2-thinking"] 136 137 138 def test_list_deduplicates_same_model_in_group(monkeypatch): 139 """Duplicate model entries under the same provider name should not produce 140 duplicate entries in the models list.""" 141 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 142 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 143 144 providers = list_authenticated_providers( 145 current_provider="openrouter", 146 user_providers={}, 147 custom_providers=[ 148 {"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "llama3"}, 149 {"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "llama3"}, 150 {"name": "MyProvider", "base_url": "http://localhost:11434/v1", "model": "mistral"}, 151 ], 152 max_models=50, 153 ) 154 155 my_rows = [p for p in providers if p["name"] == "MyProvider"] 156 assert len(my_rows) == 1 157 assert my_rows[0]["models"] == ["llama3", "mistral"] 158 assert my_rows[0]["total_models"] == 2 159 160 161 def test_list_enumerates_dict_format_models_alongside_default(monkeypatch): 162 """custom_providers entry with dict-format ``models:`` plus singular 163 ``model:`` should surface the default and every dict key. 164 165 Regression: Hermes's own writer stores configured models as a dict 166 keyed by model id, but the /model picker previously only honored the 167 singular ``model:`` field, so multi-model custom providers appeared 168 to have only the active model. 169 """ 170 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 171 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 172 173 providers = list_authenticated_providers( 174 current_provider="openai-codex", 175 user_providers={}, 176 custom_providers=[ 177 { 178 "name": "DeepSeek", 179 "base_url": "https://api.deepseek.com", 180 "api_mode": "chat_completions", 181 "model": "deepseek-chat", 182 "models": { 183 "deepseek-chat": {"context_length": 128000}, 184 "deepseek-reasoner": {"context_length": 128000}, 185 }, 186 } 187 ], 188 max_models=50, 189 ) 190 191 ds_rows = [p for p in providers if p["name"] == "DeepSeek"] 192 assert len(ds_rows) == 1 193 assert ds_rows[0]["models"] == ["deepseek-chat", "deepseek-reasoner"] 194 assert ds_rows[0]["total_models"] == 2 195 196 197 def test_list_enumerates_dict_format_models_without_singular_model(monkeypatch): 198 """Dict-format ``models:`` with no singular ``model:`` should still 199 enumerate every dict key (previously the picker reported 0 models).""" 200 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 201 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 202 203 providers = list_authenticated_providers( 204 current_provider="openai-codex", 205 user_providers={}, 206 custom_providers=[ 207 { 208 "name": "Thor", 209 "base_url": "http://thor.lab:8337/v1", 210 "models": { 211 "gemma-4-26B-A4B-it-MXFP4_MOE": {"context_length": 262144}, 212 "Qwen3.5-35B-A3B-MXFP4_MOE": {"context_length": 262144}, 213 "gemma-4-31B-it-Q4_K_M": {"context_length": 262144}, 214 }, 215 } 216 ], 217 max_models=50, 218 ) 219 220 thor_rows = [p for p in providers if p["name"] == "Thor"] 221 assert len(thor_rows) == 1 222 assert set(thor_rows[0]["models"]) == { 223 "gemma-4-26B-A4B-it-MXFP4_MOE", 224 "Qwen3.5-35B-A3B-MXFP4_MOE", 225 "gemma-4-31B-it-Q4_K_M", 226 } 227 assert thor_rows[0]["total_models"] == 3 228 229 230 def test_list_dedupes_dict_model_matching_singular_default(monkeypatch): 231 """When the singular ``model:`` is also a key in the ``models:`` dict, 232 it must appear exactly once in the picker.""" 233 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 234 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 235 236 providers = list_authenticated_providers( 237 current_provider="openai-codex", 238 user_providers={}, 239 custom_providers=[ 240 { 241 "name": "DeepSeek", 242 "base_url": "https://api.deepseek.com", 243 "model": "deepseek-chat", 244 "models": { 245 "deepseek-chat": {"context_length": 128000}, 246 "deepseek-reasoner": {"context_length": 128000}, 247 }, 248 } 249 ], 250 max_models=50, 251 ) 252 253 ds_rows = [p for p in providers if p["name"] == "DeepSeek"] 254 assert ds_rows[0]["models"].count("deepseek-chat") == 1 255 assert ds_rows[0]["models"] == ["deepseek-chat", "deepseek-reasoner"] 256 257 258 259 # ───────────────────────────────────────────────────────────────────────────── 260 # #9210: group custom_providers by (base_url, api_key) in /model picker 261 # ───────────────────────────────────────────────────────────────────────────── 262 263 def test_list_authenticated_providers_groups_same_endpoint(monkeypatch): 264 """Multiple custom_providers entries sharing a base_url+api_key must be 265 returned as a single picker row with all their models merged.""" 266 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 267 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 268 269 providers = list_authenticated_providers( 270 current_provider="custom", 271 current_base_url="http://localhost:11434/v1", 272 user_providers={}, 273 custom_providers=[ 274 {"name": "Ollama — MiniMax M2.7", "base_url": "http://localhost:11434/v1", 275 "api_key": "ollama", "model": "minimax-m2.7"}, 276 {"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1", 277 "api_key": "ollama", "model": "glm-5.1"}, 278 {"name": "Ollama — Qwen3-coder", "base_url": "http://localhost:11434/v1", 279 "api_key": "ollama", "model": "qwen3-coder"}, 280 ], 281 max_models=50, 282 ) 283 284 custom_groups = [p for p in providers if p.get("is_user_defined")] 285 assert len(custom_groups) == 1, ( 286 "Expected 1 group for shared endpoint, got " 287 f"{[p['slug'] for p in custom_groups]}" 288 ) 289 group = custom_groups[0] 290 assert set(group["models"]) == {"minimax-m2.7", "glm-5.1", "qwen3-coder"} 291 assert group["total_models"] == 3 292 # Per-model suffix stripped from display name 293 assert group["name"] == "Ollama" 294 295 296 def test_list_authenticated_providers_current_endpoint_uses_current_slug(monkeypatch): 297 """When current_base_url matches the grouped endpoint, the slug must 298 equal current_provider so picker selection routes through the live 299 credential pipeline — provided current_provider is a real slug, not 300 the corrupt bare "custom" (see #17478).""" 301 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 302 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 303 304 providers = list_authenticated_providers( 305 current_provider="custom:ollama", 306 current_base_url="http://localhost:11434/v1", 307 user_providers={}, 308 custom_providers=[ 309 {"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1", 310 "api_key": "ollama", "model": "glm-5.1"}, 311 ], 312 max_models=50, 313 ) 314 315 matches = [p for p in providers if p.get("is_user_defined")] 316 assert len(matches) == 1 317 group = matches[0] 318 assert group["slug"] == "custom:ollama" 319 assert group["is_current"] is True 320 321 322 def test_list_authenticated_providers_bare_custom_slug_recovers(monkeypatch): 323 """Regression for #17478: when a prior failed switch left the bare 324 literal "custom" in model.provider, the picker must NOT propagate 325 that broken slug. It must fall back to the canonical 326 ``custom:<name>`` form so the picker stays usable.""" 327 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 328 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 329 330 providers = list_authenticated_providers( 331 current_provider="custom", 332 current_base_url="http://localhost:11434/v1", 333 user_providers={}, 334 custom_providers=[ 335 {"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1", 336 "api_key": "ollama", "model": "glm-5.1"}, 337 ], 338 max_models=50, 339 ) 340 341 matches = [p for p in providers if p.get("is_user_defined")] 342 assert len(matches) == 1 343 group = matches[0] 344 # Canonical slug, NOT the bare "custom" that caused #17478 345 assert group["slug"] == "custom:ollama" 346 347 348 def test_list_authenticated_providers_distinct_endpoints_stay_separate(monkeypatch): 349 """Entries with different base_urls must produce separate picker rows 350 even if some display names happen to be similar.""" 351 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 352 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 353 354 providers = list_authenticated_providers( 355 user_providers={}, 356 custom_providers=[ 357 {"name": "Ollama — GLM 5.1", "base_url": "http://localhost:11434/v1", 358 "api_key": "ollama", "model": "glm-5.1"}, 359 {"name": "Moonshot", "base_url": "https://api.moonshot.cn/v1", 360 "api_key": "sk-m", "model": "moonshot-v1"}, 361 {"name": "Ollama — Qwen3-coder", "base_url": "http://localhost:11434/v1", 362 "api_key": "ollama", "model": "qwen3-coder"}, 363 ], 364 max_models=50, 365 ) 366 367 custom_groups = [p for p in providers if p.get("is_user_defined")] 368 assert len(custom_groups) == 2 369 # Ollama endpoint collapses to one row with both models 370 ollama = next(p for p in custom_groups if p["name"] == "Ollama") 371 assert set(ollama["models"]) == {"glm-5.1", "qwen3-coder"} 372 moonshot = next(p for p in custom_groups if p["name"] == "Moonshot") 373 assert moonshot["models"] == ["moonshot-v1"] 374 375 376 def test_list_authenticated_providers_same_url_different_keys_disambiguated(monkeypatch): 377 """Two custom_providers entries with the same base_url but different 378 api_keys (and identical cleaned names) must both stay visible in the 379 picker — slug is suffixed to disambiguate.""" 380 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 381 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 382 383 providers = list_authenticated_providers( 384 user_providers={}, 385 custom_providers=[ 386 {"name": "OpenAI — key A", "base_url": "https://api.openai.com/v1", 387 "api_key": "sk-AAA", "model": "gpt-5.4"}, 388 {"name": "OpenAI — key B", "base_url": "https://api.openai.com/v1", 389 "api_key": "sk-BBB", "model": "gpt-4.6"}, 390 ], 391 max_models=50, 392 ) 393 394 custom_groups = [p for p in providers if p.get("is_user_defined")] 395 assert len(custom_groups) == 2 396 slugs = sorted(p["slug"] for p in custom_groups) 397 # First group keeps the base slug, second gets a numeric suffix 398 assert slugs == ["custom:openai", "custom:openai-2"] 399 # Each row has a distinct model 400 models = {p["slug"]: p["models"] for p in custom_groups} 401 assert models["custom:openai"] == ["gpt-5.4"] 402 assert models["custom:openai-2"] == ["gpt-4.6"] 403 404 405 def test_list_authenticated_providers_total_models_reflects_grouped_count(monkeypatch): 406 """After grouping six entries into one row, total_models must reflect 407 the full count, and every grouped model appears in the list.""" 408 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 409 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 410 411 entries = [ 412 {"name": f"Ollama \u2014 Model {i}", "base_url": "http://localhost:11434/v1", 413 "api_key": "ollama", "model": f"model-{i}"} 414 for i in range(6) 415 ] 416 providers = list_authenticated_providers( 417 user_providers={}, 418 custom_providers=entries, 419 max_models=4, 420 ) 421 422 groups = [p for p in providers if p.get("is_user_defined")] 423 assert len(groups) == 1 424 group = groups[0] 425 assert group["total_models"] == 6 426 # All six models are preserved in the grouped row. 427 assert sorted(group["models"]) == sorted(f"model-{i}" for i in range(6)) 428 429 430 def test_lmstudio_picker_probes_active_config_base_url(monkeypatch): 431 """When `provider: lmstudio` is saved with a remote base_url and no 432 LM_BASE_URL env var, the picker must probe the saved base_url — not 433 127.0.0.1. Regression: prior behavior always probed localhost, so users 434 with LM Studio on a lab box saw the wrong (or empty) model list. 435 """ 436 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 437 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 438 monkeypatch.delenv("LM_BASE_URL", raising=False) 439 monkeypatch.delenv("LM_API_KEY", raising=False) 440 441 captured: dict = {} 442 443 def _fake_fetch(api_key=None, base_url=None, timeout=5.0): 444 captured["base_url"] = base_url 445 captured["api_key"] = api_key 446 return ["qwen/qwen3-coder-30b"] 447 448 monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch) 449 450 list_authenticated_providers( 451 current_provider="lmstudio", 452 current_base_url="http://192.168.1.10:1234/v1", 453 current_model="qwen/qwen3-coder-30b", 454 ) 455 456 assert captured["base_url"] == "http://192.168.1.10:1234/v1" 457 458 459 def test_lmstudio_picker_lm_base_url_env_wins_over_active_config(monkeypatch): 460 """LM_BASE_URL env var must still take precedence over the saved 461 base_url so users can temporarily redirect the picker without editing 462 config.yaml. 463 """ 464 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 465 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 466 monkeypatch.setenv("LM_BASE_URL", "http://override.local:9999/v1") 467 monkeypatch.delenv("LM_API_KEY", raising=False) 468 469 captured: dict = {} 470 471 def _fake_fetch(api_key=None, base_url=None, timeout=5.0): 472 captured["base_url"] = base_url 473 return [] 474 475 monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch) 476 477 list_authenticated_providers( 478 current_provider="lmstudio", 479 current_base_url="http://192.168.1.10:1234/v1", 480 ) 481 482 assert captured["base_url"] == "http://override.local:9999/v1" 483 484 485 def test_lmstudio_picker_skips_probe_when_not_configured(monkeypatch): 486 """If the user has never configured LM Studio (no LM_API_KEY / LM_BASE_URL 487 and not on lmstudio), the picker must not pay the localhost probe cost 488 just to discover LM Studio is unavailable. 489 """ 490 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 491 monkeypatch.setattr(providers_mod, "HERMES_OVERLAYS", {}) 492 monkeypatch.delenv("LM_BASE_URL", raising=False) 493 monkeypatch.delenv("LM_API_KEY", raising=False) 494 495 captured: dict = {} 496 497 def _fake_fetch(api_key=None, base_url=None, timeout=5.0): 498 captured["base_url"] = base_url 499 return [] 500 501 monkeypatch.setattr("hermes_cli.models.fetch_lmstudio_models", _fake_fetch) 502 503 list_authenticated_providers( 504 current_provider="openrouter", 505 current_base_url="https://openrouter.ai/api/v1", 506 ) 507 508 assert "base_url" not in captured