test_user_providers_model_switch.py
1 """Tests for user-defined providers (providers: dict) in /model. 2 3 These tests ensure that providers defined in the config.yaml ``providers:`` section 4 are properly resolved for model switching and that their full ``models:`` lists 5 are exposed in the model picker. 6 """ 7 8 import pytest 9 from hermes_cli.model_switch import list_authenticated_providers, switch_model 10 from hermes_cli import runtime_provider as rp 11 12 13 # ============================================================================= 14 # Tests for list_authenticated_providers including full models list 15 # ============================================================================= 16 17 def test_list_authenticated_providers_includes_full_models_list_from_user_providers(monkeypatch): 18 """User-defined providers should expose both default_model and full models list. 19 20 Regression test: previously only default_model was shown in /model picker. 21 """ 22 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 23 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 24 25 user_providers = { 26 "local-ollama": { 27 "name": "Local Ollama", 28 "api": "http://localhost:11434/v1", 29 "default_model": "minimax-m2.7:cloud", 30 "models": [ 31 "minimax-m2.7:cloud", 32 "kimi-k2.5:cloud", 33 "glm-5.1:cloud", 34 "qwen3.5:cloud", 35 ], 36 } 37 } 38 39 providers = list_authenticated_providers( 40 current_provider="local-ollama", 41 user_providers=user_providers, 42 custom_providers=[], 43 max_models=50, 44 ) 45 46 # Find our user provider 47 user_prov = next( 48 (p for p in providers if p.get("is_user_defined") and p["slug"] == "local-ollama"), 49 None 50 ) 51 52 assert user_prov is not None, "User provider 'local-ollama' should be in results" 53 assert user_prov["total_models"] == 4, f"Expected 4 models, got {user_prov['total_models']}" 54 assert "minimax-m2.7:cloud" in user_prov["models"] 55 assert "kimi-k2.5:cloud" in user_prov["models"] 56 assert "glm-5.1:cloud" in user_prov["models"] 57 assert "qwen3.5:cloud" in user_prov["models"] 58 59 60 def test_list_authenticated_providers_dedupes_models_when_default_in_list(monkeypatch): 61 """When default_model is also in models list, don't duplicate.""" 62 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 63 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 64 65 user_providers = { 66 "my-provider": { 67 "api": "http://example.com/v1", 68 "default_model": "model-a", # Included in models list below 69 "models": ["model-a", "model-b", "model-c"], 70 } 71 } 72 73 providers = list_authenticated_providers( 74 current_provider="my-provider", 75 user_providers=user_providers, 76 custom_providers=[], 77 ) 78 79 user_prov = next( 80 (p for p in providers if p.get("is_user_defined")), 81 None 82 ) 83 84 assert user_prov is not None 85 assert user_prov["total_models"] == 3, "Should have 3 unique models, not 4" 86 assert user_prov["models"].count("model-a") == 1, "model-a should not be duplicated" 87 88 89 def test_list_authenticated_providers_enumerates_dict_format_models(monkeypatch): 90 """providers: dict entries with ``models:`` as a dict keyed by model id 91 (canonical Hermes write format) should surface every key in the picker. 92 93 Regression: the ``providers:`` dict path previously only accepted 94 list-format ``models:`` and silently dropped dict-format entries, 95 even though Hermes's own writer and downstream readers use dict format. 96 """ 97 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 98 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 99 100 user_providers = { 101 "local-ollama": { 102 "name": "Local Ollama", 103 "api": "http://localhost:11434/v1", 104 "default_model": "minimax-m2.7:cloud", 105 "models": { 106 "minimax-m2.7:cloud": {"context_length": 196608}, 107 "kimi-k2.5:cloud": {"context_length": 200000}, 108 "glm-5.1:cloud": {"context_length": 202752}, 109 }, 110 } 111 } 112 113 providers = list_authenticated_providers( 114 current_provider="local-ollama", 115 user_providers=user_providers, 116 custom_providers=[], 117 max_models=50, 118 ) 119 120 user_prov = next( 121 (p for p in providers if p.get("is_user_defined") and p["slug"] == "local-ollama"), 122 None, 123 ) 124 125 assert user_prov is not None 126 assert user_prov["total_models"] == 3 127 assert user_prov["models"] == [ 128 "minimax-m2.7:cloud", 129 "kimi-k2.5:cloud", 130 "glm-5.1:cloud", 131 ] 132 133 134 def test_list_authenticated_providers_uses_live_models_for_user_provider(monkeypatch): 135 """User-defined OpenAI-compatible providers should prefer live /models. 136 137 Regression: CRS-style providers with a stale config ``models:`` dict kept 138 showing only the configured subset in the /model picker, even though their 139 /v1/models endpoint exposed newly added models. 140 """ 141 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 142 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 143 monkeypatch.setenv("CRS_TEST_KEY", "sk-test") 144 145 calls = [] 146 147 def fake_fetch_api_models(api_key, base_url): 148 calls.append((api_key, base_url)) 149 return ["old-configured-model", "new-live-model"] 150 151 monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models) 152 153 user_providers = { 154 "crs-henkee": { 155 "name": "CRS Henkee", 156 "base_url": "http://127.0.0.1:3000/api/v1", 157 "key_env": "CRS_TEST_KEY", 158 "model": "old-configured-model", 159 "models": { 160 "old-configured-model": {"context_length": 200000}, 161 }, 162 } 163 } 164 165 providers = list_authenticated_providers( 166 current_provider="crs-henkee", 167 user_providers=user_providers, 168 custom_providers=[], 169 max_models=50, 170 ) 171 172 user_prov = next( 173 (p for p in providers if p.get("is_user_defined") and p["slug"] == "crs-henkee"), 174 None, 175 ) 176 177 assert user_prov is not None 178 assert calls == [("sk-test", "http://127.0.0.1:3000/api/v1")] 179 assert user_prov["models"] == ["old-configured-model", "new-live-model"] 180 assert user_prov["total_models"] == 2 181 182 183 def test_list_authenticated_providers_dict_models_without_default_model(monkeypatch): 184 """Dict-format ``models:`` without a ``default_model`` must still expose 185 every dict key, not collapse to an empty list.""" 186 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 187 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 188 189 user_providers = { 190 "multimodel": { 191 "api": "http://example.com/v1", 192 "models": { 193 "alpha": {"context_length": 8192}, 194 "beta": {"context_length": 16384}, 195 }, 196 } 197 } 198 199 providers = list_authenticated_providers( 200 current_provider="", 201 user_providers=user_providers, 202 custom_providers=[], 203 ) 204 205 user_prov = next( 206 (p for p in providers if p.get("is_user_defined") and p["slug"] == "multimodel"), 207 None, 208 ) 209 210 assert user_prov is not None 211 assert user_prov["total_models"] == 2 212 assert set(user_prov["models"]) == {"alpha", "beta"} 213 214 215 def test_list_authenticated_providers_dict_models_dedupe_with_default(monkeypatch): 216 """When ``default_model`` is also a key in the ``models:`` dict, it must 217 appear exactly once (list already had this for list-format models).""" 218 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 219 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 220 221 user_providers = { 222 "my-provider": { 223 "api": "http://example.com/v1", 224 "default_model": "model-a", 225 "models": { 226 "model-a": {"context_length": 8192}, 227 "model-b": {"context_length": 16384}, 228 "model-c": {"context_length": 32768}, 229 }, 230 } 231 } 232 233 providers = list_authenticated_providers( 234 current_provider="my-provider", 235 user_providers=user_providers, 236 custom_providers=[], 237 ) 238 239 user_prov = next( 240 (p for p in providers if p.get("is_user_defined")), 241 None, 242 ) 243 244 assert user_prov is not None 245 assert user_prov["total_models"] == 3 246 assert user_prov["models"].count("model-a") == 1 247 248 249 def test_openai_native_curated_catalog_is_non_empty(): 250 """Regression: built-in openai must have a static catalog for picker totals.""" 251 from hermes_cli.models import _PROVIDER_MODELS 252 253 assert _PROVIDER_MODELS.get("openai") 254 assert len(_PROVIDER_MODELS["openai"]) >= 4 255 256 257 def test_list_authenticated_providers_openai_built_in_nonzero_total(monkeypatch): 258 """Built-in openai row must not report total_models=0 when creds exist.""" 259 monkeypatch.setenv("OPENAI_API_KEY", "sk-test") 260 monkeypatch.setattr( 261 "agent.models_dev.fetch_models_dev", 262 lambda: {"openai": {"env": ["OPENAI_API_KEY"]}}, 263 ) 264 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 265 266 providers = list_authenticated_providers( 267 current_provider="", 268 current_base_url="", 269 user_providers={}, 270 custom_providers=[], 271 max_models=50, 272 ) 273 row = next((p for p in providers if p.get("slug") == "openai"), None) 274 assert row is not None 275 assert row["total_models"] > 0 276 277 278 def test_list_authenticated_providers_user_openai_official_url_fallback(monkeypatch): 279 """User providers: api.openai.com with no models list uses native curated fallback.""" 280 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 281 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 282 283 user_providers = { 284 "openai-direct": { 285 "name": "OpenAI Direct", 286 "api": "https://api.openai.com/v1", 287 } 288 } 289 providers = list_authenticated_providers( 290 current_provider="", 291 current_base_url="", 292 user_providers=user_providers, 293 custom_providers=[], 294 max_models=50, 295 ) 296 row = next((p for p in providers if p.get("slug") == "openai-direct"), None) 297 assert row is not None 298 assert row["total_models"] > 0 299 300 301 def test_list_authenticated_providers_fallback_to_default_only(monkeypatch): 302 """When no models array is provided, should fall back to default_model.""" 303 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 304 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 305 306 user_providers = { 307 "simple-provider": { 308 "name": "Simple Provider", 309 "api": "http://example.com/v1", 310 "default_model": "single-model", 311 # No 'models' key 312 } 313 } 314 315 providers = list_authenticated_providers( 316 current_provider="", 317 user_providers=user_providers, 318 custom_providers=[], 319 ) 320 321 user_prov = next( 322 (p for p in providers if p.get("is_user_defined")), 323 None 324 ) 325 326 assert user_prov is not None 327 assert user_prov["total_models"] == 1 328 assert user_prov["models"] == ["single-model"] 329 330 331 def test_list_authenticated_providers_accepts_base_url_and_singular_model(monkeypatch): 332 """providers: dict entries written in canonical Hermes shape 333 (``base_url`` + singular ``model``) should resolve the same as the 334 legacy ``api`` + ``default_model`` shape. 335 336 Regression: section 3 previously only read ``api``/``url`` and 337 ``default_model``, so new-shape entries written by Hermes's own writer 338 surfaced with empty ``api_url`` and no default. 339 """ 340 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 341 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 342 343 user_providers = { 344 "custom": { 345 "base_url": "http://example.com/v1", 346 "model": "gpt-5.4", 347 "models": { 348 "gpt-5.4": {}, 349 "grok-4.20-beta": {}, 350 "minimax-m2.7": {}, 351 }, 352 } 353 } 354 355 providers = list_authenticated_providers( 356 current_provider="custom", 357 user_providers=user_providers, 358 custom_providers=[], 359 max_models=50, 360 ) 361 362 custom = next((p for p in providers if p["slug"] == "custom"), None) 363 assert custom is not None 364 assert custom["api_url"] == "http://example.com/v1" 365 assert custom["models"] == ["gpt-5.4", "grok-4.20-beta", "minimax-m2.7"] 366 assert custom["total_models"] == 3 367 368 369 def test_list_authenticated_providers_dedupes_when_user_and_custom_overlap(monkeypatch): 370 """When the same slug appears in both ``providers:`` dict and 371 ``custom_providers:`` list, emit exactly one row (providers: dict wins 372 since it is processed first). 373 374 Regression: section 3 previously had no ``seen_slugs`` check, so 375 overlapping entries produced two picker rows for the same provider. 376 """ 377 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 378 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 379 380 providers = list_authenticated_providers( 381 current_provider="custom", 382 user_providers={ 383 "custom": { 384 "base_url": "http://example.com/v1", 385 "model": "gpt-5.4", 386 "models": { 387 "gpt-5.4": {}, 388 "grok-4.20-beta": {}, 389 }, 390 } 391 }, 392 custom_providers=[ 393 { 394 "name": "custom", 395 "base_url": "http://example.com/v1", 396 "model": "legacy-only-model", 397 } 398 ], 399 max_models=50, 400 ) 401 402 matches = [p for p in providers if p["slug"] == "custom"] 403 assert len(matches) == 1 404 # providers: dict wins — legacy-only-model is suppressed. 405 assert matches[0]["models"] == ["gpt-5.4", "grok-4.20-beta"] 406 407 408 def test_list_authenticated_providers_no_duplicate_labels_across_schemas(monkeypatch): 409 """Regression: same endpoint in both ``providers:`` dict AND ``custom_providers:`` 410 list (e.g. via ``get_compatible_custom_providers()``) must not emit two picker 411 rows with identical display names. 412 413 Before the fix, section 3 emitted bare-slug rows ("openrouter") and section 4 414 emitted ``custom:openrouter`` rows for the same endpoint — both labelled 415 identically, bypassing ``seen_slugs`` dedup because the slug shapes differ. 416 """ 417 monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {}) 418 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 419 420 shared_entries = [ 421 ("endpoint-a", "http://a.local/v1"), 422 ("endpoint-b", "http://b.local/v1"), 423 ("endpoint-c", "http://c.local/v1"), 424 ] 425 426 user_providers = { 427 name: {"name": name, "base_url": url, "model": "m1"} 428 for name, url in shared_entries 429 } 430 custom_providers = [ 431 {"name": name, "base_url": url, "model": "m1"} 432 for name, url in shared_entries 433 ] 434 435 providers = list_authenticated_providers( 436 current_provider="none", 437 user_providers=user_providers, 438 custom_providers=custom_providers, 439 max_models=50, 440 ) 441 442 user_rows = [p for p in providers if p.get("source") == "user-config"] 443 # Expect one row per shared entry — not two. 444 assert len(user_rows) == len(shared_entries), ( 445 f"Expected {len(shared_entries)} rows, got {len(user_rows)}: " 446 f"{[(p['slug'], p['name']) for p in user_rows]}" 447 ) 448 449 # And zero duplicate display labels. 450 labels = [p["name"].lower() for p in user_rows] 451 assert len(labels) == len(set(labels)), ( 452 f"Duplicate labels across picker rows: {labels}" 453 ) 454 455 456 def test_list_authenticated_providers_hides_custom_shadowing_builtin_endpoint(monkeypatch): 457 """#16970: a custom_providers entry whose ``base_url`` matches a built-in 458 provider's endpoint should be hidden. The built-in row already represents 459 that endpoint with its canonical slug, curated model list, and auth wiring. 460 461 Repro: user sets ``DASHSCOPE_API_KEY`` (triggers the built-in ``alibaba`` 462 row pointing at the static ``inference_base_url``) AND defines a 463 ``my-alibaba`` custom provider pointing at the same URL. Before the fix, 464 the picker showed both rows for one endpoint. 465 """ 466 monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-test") 467 monkeypatch.setattr( 468 "agent.models_dev.fetch_models_dev", 469 lambda: { 470 "alibaba": { 471 "name": "Alibaba Cloud (DashScope)", 472 "env": ["DASHSCOPE_API_KEY"], 473 } 474 }, 475 ) 476 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 477 478 custom_providers = [ 479 { 480 "name": "my-alibaba", 481 # Matches PROVIDER_REGISTRY['alibaba'].inference_base_url exactly. 482 "base_url": "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", 483 "api_key": "sk-sp-test", 484 "model": "qwen3.6-plus", 485 "models": {"qwen3.6-plus": {"context_length": 500000}}, 486 } 487 ] 488 489 providers = list_authenticated_providers( 490 current_provider="my-alibaba", 491 user_providers={}, 492 custom_providers=custom_providers, 493 max_models=50, 494 ) 495 496 slugs = [p["slug"] for p in providers] 497 # Built-in alibaba row should be present. 498 assert "alibaba" in slugs, ( 499 f"Expected built-in alibaba row, got slugs: {slugs}" 500 ) 501 # Custom shadow row should be hidden — its base_url matches the built-in's. 502 assert not any("my-alibaba" in s for s in slugs), ( 503 f"Custom my-alibaba should have been dedup'd against the built-in " 504 f"alibaba endpoint, got slugs: {slugs}" 505 ) 506 507 508 def test_list_authenticated_providers_keeps_custom_with_distinct_endpoint(monkeypatch): 509 """Dedup must only apply when the endpoint matches a built-in. A custom 510 provider on a genuinely distinct endpoint stays visible even if a 511 built-in is also authenticated.""" 512 monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-test") 513 monkeypatch.setattr( 514 "agent.models_dev.fetch_models_dev", 515 lambda: { 516 "alibaba": { 517 "name": "Alibaba Cloud (DashScope)", 518 "env": ["DASHSCOPE_API_KEY"], 519 } 520 }, 521 ) 522 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 523 524 custom_providers = [ 525 { 526 "name": "my-private-relay", 527 "base_url": "https://relay.example.internal/v1", 528 "api_key": "sk-relay-test", 529 "model": "qwen3.6-plus", 530 "models": {"qwen3.6-plus": {}}, 531 } 532 ] 533 534 providers = list_authenticated_providers( 535 current_provider="my-private-relay", 536 user_providers={}, 537 custom_providers=custom_providers, 538 max_models=50, 539 ) 540 541 slugs = [p["slug"] for p in providers] 542 assert any("my-private-relay" in s for s in slugs), ( 543 f"Custom provider on distinct endpoint must stay visible, got: {slugs}" 544 ) 545 546 547 def test_list_authenticated_providers_dedup_honors_base_url_env_override(monkeypatch): 548 """The dedup must track the EFFECTIVE endpoint — if DASHSCOPE_BASE_URL 549 overrides the static inference_base_url, a custom provider pointing at 550 the overridden URL (not the static one) should still be recognized as 551 a duplicate.""" 552 monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-test") 553 monkeypatch.setenv( 554 "DASHSCOPE_BASE_URL", 555 "https://custom-dashscope.example.com/v1", 556 ) 557 monkeypatch.setattr( 558 "agent.models_dev.fetch_models_dev", 559 lambda: { 560 "alibaba": { 561 "name": "Alibaba Cloud (DashScope)", 562 "env": ["DASHSCOPE_API_KEY"], 563 } 564 }, 565 ) 566 monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {}) 567 568 custom_providers = [ 569 { 570 "name": "my-dashscope-override", 571 # Same URL as DASHSCOPE_BASE_URL env override above. 572 "base_url": "https://custom-dashscope.example.com/v1", 573 "api_key": "sk-test", 574 "model": "qwen3.6-plus", 575 } 576 ] 577 578 providers = list_authenticated_providers( 579 current_provider="alibaba", 580 user_providers={}, 581 custom_providers=custom_providers, 582 max_models=50, 583 ) 584 585 slugs = [p["slug"] for p in providers] 586 assert not any("my-dashscope-override" in s for s in slugs), ( 587 f"Custom entry matching env-overridden built-in endpoint should be " 588 f"dedup'd, got: {slugs}" 589 ) 590 591 592 # ============================================================================= 593 # Tests for _get_named_custom_provider with providers: dict 594 # ============================================================================= 595 596 def test_get_named_custom_provider_finds_user_providers_by_key(monkeypatch, tmp_path): 597 """Should resolve providers from providers: dict (new-style), not just custom_providers.""" 598 config = { 599 "providers": { 600 "local-localhost:11434": { 601 "api": "http://localhost:11434/v1", 602 "name": "Local (localhost:11434)", 603 "default_model": "minimax-m2.7:cloud", 604 } 605 } 606 } 607 608 import yaml 609 config_file = tmp_path / "config.yaml" 610 config_file.write_text(yaml.dump(config)) 611 612 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 613 614 result = rp._get_named_custom_provider("local-localhost:11434") 615 616 assert result is not None 617 assert result["base_url"] == "http://localhost:11434/v1" 618 assert result["name"] == "Local (localhost:11434)" 619 620 621 def test_get_named_custom_provider_finds_by_display_name(monkeypatch, tmp_path): 622 """Should match providers by their 'name' field as well as key.""" 623 config = { 624 "providers": { 625 "my-ollama-xyz": { 626 "api": "http://ollama.example.com/v1", 627 "name": "My Production Ollama", 628 "default_model": "llama3", 629 } 630 } 631 } 632 633 import yaml 634 config_file = tmp_path / "config.yaml" 635 config_file.write_text(yaml.dump(config)) 636 637 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 638 639 # Should find by display name (normalized) 640 result = rp._get_named_custom_provider("my-production-ollama") 641 642 assert result is not None 643 assert result["base_url"] == "http://ollama.example.com/v1" 644 645 646 def test_get_named_custom_provider_falls_back_to_legacy_format(monkeypatch, tmp_path): 647 """Should still work with custom_providers: list format.""" 648 config = { 649 "providers": {}, 650 "custom_providers": [ 651 { 652 "name": "Custom Endpoint", 653 "base_url": "http://custom.example.com/v1", 654 } 655 ] 656 } 657 658 import yaml 659 config_file = tmp_path / "config.yaml" 660 config_file.write_text(yaml.dump(config)) 661 662 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 663 664 result = rp._get_named_custom_provider("custom-endpoint") 665 666 assert result is not None 667 668 669 def test_get_named_custom_provider_returns_none_for_unknown(monkeypatch, tmp_path): 670 """Should return None for providers that don't exist.""" 671 config = { 672 "providers": { 673 "known-provider": { 674 "api": "http://known.example.com/v1", 675 } 676 } 677 } 678 679 import yaml 680 config_file = tmp_path / "config.yaml" 681 config_file.write_text(yaml.dump(config)) 682 683 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 684 685 result = rp._get_named_custom_provider("other-provider") 686 687 # "unknown-provider" partial-matches "known-provider" because "unknown" doesn't match 688 # but our matching is loose (substring). Let's verify a truly non-matching provider 689 result = rp._get_named_custom_provider("completely-different-name") 690 assert result is None 691 692 693 def test_get_named_custom_provider_skips_empty_base_url(monkeypatch, tmp_path): 694 """Should skip providers without a base_url.""" 695 config = { 696 "providers": { 697 "incomplete-provider": { 698 "name": "Incomplete", 699 # No api/base_url field 700 } 701 } 702 } 703 704 import yaml 705 config_file = tmp_path / "config.yaml" 706 config_file.write_text(yaml.dump(config)) 707 708 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 709 710 result = rp._get_named_custom_provider("incomplete-provider") 711 712 assert result is None 713 714 715 # ============================================================================= 716 # Integration test for switch_model with user providers 717 # ============================================================================= 718 719 def test_switch_model_resolves_user_provider_credentials(monkeypatch, tmp_path): 720 """/model switch should resolve credentials for providers: dict providers.""" 721 import yaml 722 723 config = { 724 "providers": { 725 "local-ollama": { 726 "api": "http://localhost:11434/v1", 727 "name": "Local Ollama", 728 "default_model": "minimax-m2.7:cloud", 729 } 730 } 731 } 732 733 config_file = tmp_path / "config.yaml" 734 config_file.write_text(yaml.dump(config)) 735 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 736 737 # Mock validation to pass 738 monkeypatch.setattr( 739 "hermes_cli.models.validate_requested_model", 740 lambda *a, **k: {"accepted": True, "persist": True, "recognized": True, "message": None} 741 ) 742 743 result = switch_model( 744 raw_input="kimi-k2.5:cloud", 745 current_provider="local-ollama", 746 current_model="minimax-m2.7:cloud", 747 current_base_url="http://localhost:11434/v1", 748 is_global=False, 749 user_providers=config["providers"], 750 ) 751 752 assert result.success is True 753 assert result.error_message == "" 754 755 756 # ============================================================================= 757 # Regression: providers: dict ``transport`` field must be honored 758 # ============================================================================= 759 760 761 def test_get_named_custom_provider_reads_transport_field(monkeypatch): 762 """v12+ ``providers:`` dict stores api mode under ``transport:`` (not the 763 legacy ``api_mode:``). ``_get_named_custom_provider`` must accept both 764 field names. 765 766 Bug: this function read only ``entry.get("api_mode")`` for v12+ entries. 767 After ``migrate_config()`` writes ``transport`` on every entry, the 768 lookup returns None and ``_resolve_named_custom_runtime`` falls back 769 through ``_detect_api_mode_for_url(base_url) or "chat_completions"`` 770 — silently downgrading every codex_responses / anthropic_messages 771 provider to chat_completions. 772 """ 773 config = { 774 "_config_version": 12, 775 "providers": { 776 "my-codex-provider": { 777 "name": "my-codex-provider", 778 "api": "http://127.0.0.1:4000/v1", 779 "api_key": "test-key", 780 "default_model": "gpt-5", 781 "transport": "codex_responses", 782 }, 783 }, 784 } 785 786 monkeypatch.setattr(rp, "load_config", lambda: config) 787 788 result = rp._get_named_custom_provider("my-codex-provider") 789 assert result is not None 790 assert result["api_mode"] == "codex_responses" 791 assert result["base_url"] == "http://127.0.0.1:4000/v1" 792 assert result["model"] == "gpt-5" 793 794 795 def test_get_named_custom_provider_legacy_api_mode_field_still_works(monkeypatch): 796 """Hand-edited configs that used ``api_mode:`` (legacy spelling) inside 797 the v12+ providers: dict shape must keep working — the migration writer 798 produces ``transport:`` but human-edited configs may carry the older 799 spelling forward.""" 800 config = { 801 "_config_version": 12, 802 "providers": { 803 "anthropic-proxy": { 804 "name": "anthropic-proxy", 805 "api": "http://127.0.0.1:8082", 806 "api_key": "test-key", 807 "default_model": "claude-opus-4-7", 808 "api_mode": "anthropic_messages", # legacy spelling 809 }, 810 }, 811 } 812 813 monkeypatch.setattr(rp, "load_config", lambda: config) 814 815 result = rp._get_named_custom_provider("anthropic-proxy") 816 assert result is not None 817 assert result["api_mode"] == "anthropic_messages" 818 819 820 def test_get_named_custom_provider_transport_resolves_via_display_name(monkeypatch): 821 """When the requested name matches the entry's ``name:`` field rather 822 than its dict key, the same transport-vs-api_mode logic must apply 823 (second branch in ``_get_named_custom_provider``).""" 824 config = { 825 "_config_version": 12, 826 "providers": { 827 "slug-different-from-name": { 828 "name": "Codex Provider", # display name 829 "api": "http://127.0.0.1:4000/v1", 830 "api_key": "test-key", 831 "default_model": "gpt-5", 832 "transport": "codex_responses", 833 }, 834 }, 835 } 836 837 monkeypatch.setattr(rp, "load_config", lambda: config) 838 839 result = rp._get_named_custom_provider("Codex Provider") 840 assert result is not None 841 assert result["api_mode"] == "codex_responses" 842 843 844 # ============================================================================= 845 # Regression: user_providers override for private models not listed by /v1/models 846 # ============================================================================= 847 848 _REJECTED_VALIDATION = { 849 "accepted": False, 850 "persist": False, 851 "recognized": False, 852 "message": "not found", 853 } 854 855 856 def _run_user_provider_override_case( 857 *, 858 slug, 859 name, 860 base_url, 861 models, 862 raw_input, 863 ): 864 """Run ``switch_model`` with a private user provider and a rejected API check. 865 866 The bug in PR #17964 was that ``user_providers`` was treated like a list, 867 so private models listed in ``models:`` never triggered the override path. 868 These tests keep the validation failure in place and prove the config list 869 still wins for both dict- and list-shaped ``models`` entries. 870 """ 871 from unittest.mock import patch 872 873 user_providers = { 874 slug: { 875 "name": name, 876 "api": base_url, 877 "discover_models": False, 878 "models": models, 879 } 880 } 881 882 with patch("hermes_cli.model_switch.resolve_alias", return_value=None), \ 883 patch("hermes_cli.model_switch.list_provider_models", return_value=[]), \ 884 patch("hermes_cli.model_switch.normalize_model_for_provider", side_effect=lambda model, provider: model), \ 885 patch("hermes_cli.models.validate_requested_model", return_value=_REJECTED_VALIDATION), \ 886 patch("hermes_cli.models.detect_provider_for_model", return_value=None), \ 887 patch("hermes_cli.model_switch.get_model_info", return_value=None), \ 888 patch("hermes_cli.model_switch.get_model_capabilities", return_value=None), \ 889 patch("hermes_cli.runtime_provider.resolve_runtime_provider", return_value={"api_key": "***", "base_url": base_url, "api_mode": "anthropic_messages"}): 890 return switch_model( 891 raw_input=raw_input, 892 current_provider=slug, 893 current_model="old-model", 894 current_base_url=base_url, 895 user_providers=user_providers, 896 custom_providers=[], 897 ) 898 899 900 @pytest.mark.parametrize( 901 ("slug", "name", "base_url", "models", "raw_input", "expected_model"), 902 [ 903 ( 904 "kimi-coding", 905 "Kimi Coding Plan", 906 "https://api.kimi.com/coding", 907 {"kimi-k2.6": {}}, 908 "kimi-k2.6", 909 "kimi-k2.6", 910 ), 911 ( 912 "kimi-dedicated", 913 "Kimi Dedicated", 914 "https://api.kimi.com/v1", 915 [{"name": "moonshotai/Kimi-K2.6-ACED"}], 916 "moonshotai/Kimi-K2.6-ACED", 917 "moonshotai/Kimi-K2.6-ACED", 918 ), 919 ], 920 ids=["kimi-coding-plan-dict", "kimi-k2-6-aced-list"], 921 ) 922 def test_user_provider_override_accepts_listed_private_models( 923 slug, 924 name, 925 base_url, 926 models, 927 raw_input, 928 expected_model, 929 ): 930 """Private models listed in providers: config should override /v1/models misses. 931 932 Covers both config shapes the fix now accepts: 933 - dict models for the Kimi Coding Plan K2p6 case 934 - list-of-dicts models for the Kimi-K2.6-ACED dedicated case 935 """ 936 result = _run_user_provider_override_case( 937 slug=slug, 938 name=name, 939 base_url=base_url, 940 models=models, 941 raw_input=raw_input, 942 ) 943 944 assert result.success is True 945 assert result.new_model == expected_model 946 assert result.error_message == "" 947 948 949 @pytest.mark.parametrize( 950 ("slug", "name", "base_url", "models", "raw_input"), 951 [ 952 ( 953 "kimi-coding", 954 "Kimi Coding Plan", 955 "https://api.kimi.com/coding", 956 {"kimi-k2.6": {}}, 957 "kimi-k2.6-mangled", 958 ), 959 ( 960 "kimi-dedicated", 961 "Kimi Dedicated", 962 "https://api.kimi.com/v1", 963 [{"name": "moonshotai/Kimi-K2.6-ACED"}], 964 "moonshotai/Kimi-K2.6-ACED!!!", 965 ), 966 ], 967 ids=["kimi-coding-plan-dict-mangled", "kimi-k2-6-aced-list-mangled"], 968 ) 969 def test_user_provider_override_rejects_mangled_private_models( 970 slug, 971 name, 972 base_url, 973 models, 974 raw_input, 975 ): 976 """Malformed model names should fail cleanly, not crash or auto-accept.""" 977 result = _run_user_provider_override_case( 978 slug=slug, 979 name=name, 980 base_url=base_url, 981 models=models, 982 raw_input=raw_input, 983 ) 984 985 assert result.success is False 986 assert result.error_message == "not found"