test_setup_model_provider.py
1 """Regression tests for interactive setup provider/model persistence. 2 3 Since setup_model_provider delegates to select_provider_and_model() 4 from hermes_cli.main, these tests mock the delegation point and verify 5 that the setup wizard correctly syncs config from disk after the call. 6 """ 7 8 from __future__ import annotations 9 10 from hermes_cli.config import load_config, save_config, save_env_value 11 from hermes_cli.nous_subscription import NousFeatureState, NousSubscriptionFeatures 12 from hermes_cli.setup import _print_setup_summary, setup_model_provider 13 14 15 def _maybe_keep_current_tts(question, choices): 16 if question != "Select TTS provider:": 17 return None 18 assert choices[-1].startswith("Keep current (") 19 return len(choices) - 1 20 21 22 def _clear_provider_env(monkeypatch): 23 for key in ( 24 "HERMES_INFERENCE_PROVIDER", 25 "OPENAI_BASE_URL", 26 "OPENAI_API_KEY", 27 "OPENROUTER_API_KEY", 28 "GITHUB_TOKEN", 29 "GH_TOKEN", 30 "GLM_API_KEY", 31 "KIMI_API_KEY", 32 "MINIMAX_API_KEY", 33 "MINIMAX_CN_API_KEY", 34 "ANTHROPIC_TOKEN", 35 "ANTHROPIC_API_KEY", 36 ): 37 monkeypatch.delenv(key, raising=False) 38 39 40 def _stub_tts(monkeypatch): 41 monkeypatch.setattr("hermes_cli.setup.prompt_choice", lambda q, c, d=0: ( 42 _maybe_keep_current_tts(q, c) if _maybe_keep_current_tts(q, c) is not None 43 else d 44 )) 45 monkeypatch.setattr("hermes_cli.setup.prompt_yes_no", lambda *a, **kw: False) 46 47 48 def _write_model_config(provider, base_url="", model_name="test-model"): 49 """Simulate what a _model_flow_* function writes to disk.""" 50 cfg = load_config() 51 m = cfg.get("model") 52 if not isinstance(m, dict): 53 m = {"default": m} if m else {} 54 cfg["model"] = m 55 m["provider"] = provider 56 if base_url: 57 m["base_url"] = base_url 58 else: 59 m.pop("base_url", None) 60 if model_name: 61 m["default"] = model_name 62 m.pop("api_mode", None) 63 save_config(cfg) 64 65 66 def test_setup_keep_current_custom_from_config_does_not_fall_through(tmp_path, monkeypatch): 67 """Keep-current custom should not fall through to the generic model menu.""" 68 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 69 _clear_provider_env(monkeypatch) 70 _stub_tts(monkeypatch) 71 72 # Pre-set custom provider 73 _write_model_config("custom", "http://localhost:8080/v1", "local-model") 74 75 config = load_config() 76 assert config["model"]["provider"] == "custom" 77 78 def fake_select(): 79 pass # user chose "cancel" or "keep current" 80 81 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 82 83 setup_model_provider(config) 84 save_config(config) 85 86 reloaded = load_config() 87 assert isinstance(reloaded["model"], dict) 88 assert reloaded["model"]["provider"] == "custom" 89 assert reloaded["model"]["base_url"] == "http://localhost:8080/v1" 90 91 92 def test_setup_keep_current_config_provider_uses_provider_specific_model_menu( 93 tmp_path, monkeypatch 94 ): 95 """Keeping current provider preserves the config on disk.""" 96 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 97 _clear_provider_env(monkeypatch) 98 _stub_tts(monkeypatch) 99 100 _write_model_config("zai", "https://open.bigmodel.cn/api/paas/v4", "glm-5") 101 102 config = load_config() 103 104 def fake_select(): 105 pass # keep current 106 107 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 108 109 setup_model_provider(config) 110 save_config(config) 111 112 reloaded = load_config() 113 assert isinstance(reloaded["model"], dict) 114 assert reloaded["model"]["provider"] == "zai" 115 116 117 def test_setup_same_provider_rotation_strategy_saved_for_multi_credential_pool(tmp_path, monkeypatch): 118 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 119 _clear_provider_env(monkeypatch) 120 save_env_value("OPENROUTER_API_KEY", "or-key") 121 122 # Pre-write config so the pool step sees provider="openrouter" 123 _write_model_config("openrouter", "", "anthropic/claude-opus-4.6") 124 125 config = load_config() 126 127 class _Entry: 128 def __init__(self, label): 129 self.label = label 130 131 class _Pool: 132 def entries(self): 133 return [_Entry("primary"), _Entry("secondary")] 134 135 def fake_select(): 136 pass # no-op — config already has provider set 137 138 def fake_prompt_choice(question, choices, default=0): 139 if "rotation strategy" in question: 140 return 1 # round robin 141 tts_idx = _maybe_keep_current_tts(question, choices) 142 if tts_idx is not None: 143 return tts_idx 144 return default 145 146 def fake_prompt_yes_no(question, default=True): 147 return False 148 149 # Patch directly on the module objects to ensure local imports pick them up. 150 import hermes_cli.main as _main_mod 151 import hermes_cli.setup as _setup_mod 152 import agent.credential_pool as _pool_mod 153 import agent.auxiliary_client as _aux_mod 154 155 monkeypatch.setattr(_main_mod, "select_provider_and_model", fake_select) 156 # NOTE: _stub_tts overwrites prompt_choice, so set our mock AFTER it. 157 _stub_tts(monkeypatch) 158 monkeypatch.setattr(_setup_mod, "prompt_choice", fake_prompt_choice) 159 monkeypatch.setattr(_setup_mod, "prompt_yes_no", fake_prompt_yes_no) 160 monkeypatch.setattr(_setup_mod, "prompt", lambda *args, **kwargs: "") 161 monkeypatch.setattr(_pool_mod, "load_pool", lambda provider: _Pool()) 162 monkeypatch.setattr(_aux_mod, "get_available_vision_backends", lambda: []) 163 164 setup_model_provider(config) 165 166 # The pool has 2 entries, so the strategy prompt should fire 167 strategy = config.get("credential_pool_strategies", {}).get("openrouter") 168 assert strategy == "round_robin", f"Expected round_robin but got {strategy}" 169 170 171 def test_setup_same_provider_fallback_can_add_another_credential(tmp_path, monkeypatch): 172 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 173 _clear_provider_env(monkeypatch) 174 save_env_value("OPENROUTER_API_KEY", "or-key") 175 176 # Pre-write config so the pool step sees provider="openrouter" 177 _write_model_config("openrouter", "", "anthropic/claude-opus-4.6") 178 179 config = load_config() 180 pool_sizes = iter([1, 2]) 181 add_calls = [] 182 183 class _Entry: 184 def __init__(self, label): 185 self.label = label 186 187 class _Pool: 188 def __init__(self, size): 189 self._size = size 190 191 def entries(self): 192 return [_Entry(f"cred-{idx}") for idx in range(self._size)] 193 194 def fake_load_pool(provider): 195 return _Pool(next(pool_sizes)) 196 197 def fake_auth_add_command(args): 198 add_calls.append(args.provider) 199 200 def fake_select(): 201 pass # no-op — config already has provider set 202 203 def fake_prompt_choice(question, choices, default=0): 204 if question == "Select same-provider rotation strategy:": 205 return 0 206 tts_idx = _maybe_keep_current_tts(question, choices) 207 if tts_idx is not None: 208 return tts_idx 209 return default 210 211 yes_no_answers = iter([True, False]) 212 213 def fake_prompt_yes_no(question, default=True): 214 if question == "Add another credential for same-provider fallback?": 215 return next(yes_no_answers) 216 return False 217 218 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 219 _stub_tts(monkeypatch) 220 monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) 221 monkeypatch.setattr("hermes_cli.setup.prompt_yes_no", fake_prompt_yes_no) 222 monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") 223 monkeypatch.setattr("agent.credential_pool.load_pool", fake_load_pool) 224 monkeypatch.setattr("hermes_cli.auth_commands.auth_add_command", fake_auth_add_command) 225 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: []) 226 227 setup_model_provider(config) 228 229 assert add_calls == ["openrouter"] 230 assert config.get("credential_pool_strategies", {}).get("openrouter") == "fill_first" 231 232 233 def test_setup_same_provider_single_credential_keeps_existing_rotation_strategy(tmp_path, monkeypatch): 234 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 235 _clear_provider_env(monkeypatch) 236 save_env_value("OPENROUTER_API_KEY", "or-key") 237 238 _write_model_config("openrouter", "", "anthropic/claude-opus-4.6") 239 240 config = load_config() 241 config["credential_pool_strategies"] = {"openrouter": "round_robin"} 242 save_config(config) 243 244 class _Entry: 245 def __init__(self, label): 246 self.label = label 247 248 class _Pool: 249 def entries(self): 250 return [_Entry("primary")] 251 252 def fake_select(): 253 pass 254 255 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 256 _stub_tts(monkeypatch) 257 monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") 258 monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: _Pool()) 259 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: []) 260 261 setup_model_provider(config) 262 263 assert config.get("credential_pool_strategies", {}).get("openrouter") == "round_robin" 264 265 266 def test_setup_pool_step_shows_manual_vs_auto_detected_counts(tmp_path, monkeypatch, capsys): 267 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 268 _clear_provider_env(monkeypatch) 269 save_env_value("OPENROUTER_API_KEY", "or-key") 270 271 # Pre-write config so the pool step sees provider="openrouter" 272 _write_model_config("openrouter", "", "anthropic/claude-opus-4.6") 273 274 config = load_config() 275 276 class _Entry: 277 def __init__(self, label, source): 278 self.label = label 279 self.source = source 280 281 class _Pool: 282 def entries(self): 283 return [ 284 _Entry("primary", "manual"), 285 _Entry("secondary", "manual"), 286 _Entry("OPENROUTER_API_KEY", "env:OPENROUTER_API_KEY"), 287 ] 288 289 def fake_select(): 290 pass # no-op — config already has provider set 291 292 def fake_prompt_choice(question, choices, default=0): 293 if "rotation strategy" in question: 294 return 0 295 tts_idx = _maybe_keep_current_tts(question, choices) 296 if tts_idx is not None: 297 return tts_idx 298 return default 299 300 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 301 _stub_tts(monkeypatch) 302 monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) 303 monkeypatch.setattr("hermes_cli.setup.prompt_yes_no", lambda *args, **kwargs: False) 304 monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") 305 monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: _Pool()) 306 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: []) 307 308 setup_model_provider(config) 309 310 out = capsys.readouterr().out 311 assert "Current pooled credentials for openrouter: 3 (2 manual, 1 auto-detected from env/shared auth)" in out 312 313 314 def test_setup_copilot_acp_skips_same_provider_pool_step(tmp_path, monkeypatch): 315 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 316 _clear_provider_env(monkeypatch) 317 318 config = load_config() 319 320 def fake_prompt_choice(question, choices, default=0): 321 if question == "Select your inference provider:": 322 return 15 # GitHub Copilot ACP 323 if question == "Select default model:": 324 return 0 325 if question == "Configure vision:": 326 return len(choices) - 1 327 tts_idx = _maybe_keep_current_tts(question, choices) 328 if tts_idx is not None: 329 return tts_idx 330 raise AssertionError(f"Unexpected prompt_choice call: {question}") 331 332 def fake_prompt_yes_no(question, default=True): 333 if question == "Add another credential for same-provider fallback?": 334 raise AssertionError("same-provider pool prompt should not appear for copilot-acp") 335 return False 336 337 monkeypatch.setattr("hermes_cli.setup.prompt_choice", fake_prompt_choice) 338 monkeypatch.setattr("hermes_cli.setup.prompt_yes_no", fake_prompt_yes_no) 339 monkeypatch.setattr("hermes_cli.setup.prompt", lambda *args, **kwargs: "") 340 monkeypatch.setattr("hermes_cli.auth.get_active_provider", lambda: None) 341 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: []) 342 343 setup_model_provider(config) 344 345 assert config.get("credential_pool_strategies", {}) == {} 346 347 348 def test_setup_copilot_uses_gh_auth_and_saves_provider(tmp_path, monkeypatch): 349 """Copilot provider saves correctly through delegation.""" 350 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 351 _clear_provider_env(monkeypatch) 352 _stub_tts(monkeypatch) 353 354 config = load_config() 355 356 def fake_select(): 357 _write_model_config("copilot", "https://models.github.ai/inference/v1", "gpt-4o") 358 359 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 360 361 setup_model_provider(config) 362 save_config(config) 363 364 reloaded = load_config() 365 assert isinstance(reloaded["model"], dict) 366 assert reloaded["model"]["provider"] == "copilot" 367 368 369 def test_setup_copilot_acp_uses_model_picker_and_saves_provider(tmp_path, monkeypatch): 370 """Copilot ACP provider saves correctly through delegation.""" 371 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 372 _clear_provider_env(monkeypatch) 373 _stub_tts(monkeypatch) 374 375 config = load_config() 376 377 def fake_select(): 378 _write_model_config("copilot-acp", "", "claude-sonnet-4") 379 380 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 381 382 setup_model_provider(config) 383 save_config(config) 384 385 reloaded = load_config() 386 assert isinstance(reloaded["model"], dict) 387 assert reloaded["model"]["provider"] == "copilot-acp" 388 389 390 def test_setup_switch_custom_to_codex_clears_custom_endpoint_and_updates_config( 391 tmp_path, monkeypatch 392 ): 393 """Switching from custom to codex updates config correctly.""" 394 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 395 _clear_provider_env(monkeypatch) 396 _stub_tts(monkeypatch) 397 398 # Start with custom 399 _write_model_config("custom", "http://localhost:11434/v1", "qwen3.5:32b") 400 401 config = load_config() 402 assert config["model"]["provider"] == "custom" 403 404 def fake_select(): 405 _write_model_config("openai-codex", "https://api.openai.com/v1", "gpt-4o") 406 407 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 408 409 setup_model_provider(config) 410 save_config(config) 411 412 reloaded = load_config() 413 assert isinstance(reloaded["model"], dict) 414 assert reloaded["model"]["provider"] == "openai-codex" 415 assert reloaded["model"]["default"] == "gpt-4o" 416 417 418 def test_setup_switch_preserves_non_model_config(tmp_path, monkeypatch): 419 """Provider switch preserves other config sections (terminal, display, etc.).""" 420 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 421 _clear_provider_env(monkeypatch) 422 _stub_tts(monkeypatch) 423 424 config = load_config() 425 config["terminal"]["timeout"] = 999 426 save_config(config) 427 428 config = load_config() 429 430 def fake_select(): 431 _write_model_config("openrouter", model_name="gpt-4o") 432 433 monkeypatch.setattr("hermes_cli.main.select_provider_and_model", fake_select) 434 435 setup_model_provider(config) 436 save_config(config) 437 438 reloaded = load_config() 439 assert reloaded["terminal"]["timeout"] == 999 440 assert reloaded["model"]["provider"] == "openrouter" 441 442 443 def test_setup_summary_marks_anthropic_auth_as_vision_available(tmp_path, monkeypatch, capsys): 444 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 445 _clear_provider_env(monkeypatch) 446 monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-api03-key") 447 monkeypatch.setattr("shutil.which", lambda _name: None) 448 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: ["anthropic"]) 449 450 _print_setup_summary(load_config(), tmp_path) 451 output = capsys.readouterr().out 452 453 assert "Vision (image analysis)" in output 454 assert "missing run 'hermes setup' to configure" not in output 455 456 457 def test_setup_summary_shows_camofox_when_browser_feature_is_camofox(tmp_path, monkeypatch, capsys): 458 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 459 _clear_provider_env(monkeypatch) 460 monkeypatch.setattr( 461 "hermes_cli.setup.get_nous_subscription_features", 462 lambda config: NousSubscriptionFeatures( 463 subscribed=False, 464 nous_auth_present=False, 465 provider_is_nous=False, 466 features={ 467 "web": NousFeatureState("web", "Web tools", True, False, False, False, False, True, ""), 468 "image_gen": NousFeatureState("image_gen", "Image generation", True, False, False, False, False, True, ""), 469 "tts": NousFeatureState("tts", "OpenAI TTS", True, False, False, False, False, True, ""), 470 "browser": NousFeatureState("browser", "Browser automation", True, True, True, False, True, True, "Camofox"), 471 "modal": NousFeatureState("modal", "Modal execution", False, False, False, False, False, True, "local"), 472 }, 473 ), 474 ) 475 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: []) 476 477 _print_setup_summary(load_config(), tmp_path) 478 output = capsys.readouterr().out 479 480 assert "Browser Automation (Camofox)" in output 481 482 483 def test_setup_summary_does_not_mark_incomplete_browserbase_as_available(tmp_path, monkeypatch, capsys): 484 monkeypatch.setenv("HERMES_HOME", str(tmp_path)) 485 _clear_provider_env(monkeypatch) 486 monkeypatch.setenv("BROWSERBASE_API_KEY", "bb-key") 487 monkeypatch.setattr( 488 "hermes_cli.setup.get_nous_subscription_features", 489 lambda config: NousSubscriptionFeatures( 490 subscribed=False, 491 nous_auth_present=False, 492 provider_is_nous=False, 493 features={ 494 "web": NousFeatureState("web", "Web tools", True, False, False, False, False, True, ""), 495 "image_gen": NousFeatureState("image_gen", "Image generation", True, False, False, False, False, True, ""), 496 "tts": NousFeatureState("tts", "OpenAI TTS", True, False, False, False, False, True, ""), 497 "browser": NousFeatureState("browser", "Browser automation", True, False, False, False, False, True, "Browserbase"), 498 "modal": NousFeatureState("modal", "Modal execution", False, False, False, False, False, True, "local"), 499 }, 500 ), 501 ) 502 monkeypatch.setattr("agent.auxiliary_client.get_available_vision_backends", lambda: []) 503 504 _print_setup_summary(load_config(), tmp_path) 505 output = capsys.readouterr().out 506 507 assert "Browser Automation (Browserbase)" not in output 508 assert "Browser Automation" in output 509 assert "BROWSERBASE_API_KEY/BROWSERBASE_PROJECT_ID" in output