test_fallback_cmd.py
1 """Tests for `hermes fallback` — chain reading, add/remove/clear, legacy migration.""" 2 from __future__ import annotations 3 4 import io 5 import types 6 from pathlib import Path 7 from unittest.mock import patch 8 9 import pytest 10 import yaml 11 12 13 # --------------------------------------------------------------------------- 14 # Shared fixture — isolate HERMES_HOME so save_config writes to tmp_path 15 # --------------------------------------------------------------------------- 16 17 @pytest.fixture() 18 def isolated_home(tmp_path, monkeypatch): 19 monkeypatch.setattr(Path, "home", lambda: tmp_path) 20 home = tmp_path / ".hermes" 21 home.mkdir(exist_ok=True) 22 monkeypatch.setenv("HERMES_HOME", str(home)) 23 return tmp_path 24 25 26 def _write_config(home: Path, data: dict) -> None: 27 config_path = home / ".hermes" / "config.yaml" 28 config_path.write_text(yaml.safe_dump(data), encoding="utf-8") 29 30 31 def _read_config(home: Path) -> dict: 32 config_path = home / ".hermes" / "config.yaml" 33 return yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} 34 35 36 # --------------------------------------------------------------------------- 37 # _read_chain / _write_chain 38 # --------------------------------------------------------------------------- 39 40 class TestReadChain: 41 def test_returns_empty_list_when_unset(self): 42 from hermes_cli.fallback_cmd import _read_chain 43 assert _read_chain({}) == [] 44 45 def test_reads_new_list_format(self): 46 from hermes_cli.fallback_cmd import _read_chain 47 cfg = { 48 "fallback_providers": [ 49 {"provider": "openrouter", "model": "anthropic/claude-sonnet-4.6"}, 50 {"provider": "nous", "model": "Hermes-4-Llama-3.1-405B"}, 51 ] 52 } 53 assert _read_chain(cfg) == [ 54 {"provider": "openrouter", "model": "anthropic/claude-sonnet-4.6"}, 55 {"provider": "nous", "model": "Hermes-4-Llama-3.1-405B"}, 56 ] 57 58 def test_migrates_legacy_single_dict(self): 59 from hermes_cli.fallback_cmd import _read_chain 60 cfg = {"fallback_model": {"provider": "openrouter", "model": "gpt-5.4"}} 61 assert _read_chain(cfg) == [{"provider": "openrouter", "model": "gpt-5.4"}] 62 63 def test_skips_incomplete_entries(self): 64 from hermes_cli.fallback_cmd import _read_chain 65 cfg = { 66 "fallback_providers": [ 67 {"provider": "openrouter"}, # missing model 68 {"model": "gpt-5.4"}, # missing provider 69 {"provider": "nous", "model": "foo"}, # valid 70 "not-a-dict", # noise 71 ] 72 } 73 assert _read_chain(cfg) == [{"provider": "nous", "model": "foo"}] 74 75 def test_returns_copies_not_aliases(self): 76 from hermes_cli.fallback_cmd import _read_chain 77 cfg = {"fallback_providers": [{"provider": "nous", "model": "foo"}]} 78 result = _read_chain(cfg) 79 result[0]["provider"] = "mutated" 80 assert cfg["fallback_providers"][0]["provider"] == "nous" 81 82 83 # --------------------------------------------------------------------------- 84 # _extract_fallback_from_model_cfg 85 # --------------------------------------------------------------------------- 86 87 class TestExtractFallback: 88 def test_extracts_from_default_field(self): 89 from hermes_cli.fallback_cmd import _extract_fallback_from_model_cfg 90 model_cfg = {"provider": "openrouter", "default": "anthropic/claude-sonnet-4.6"} 91 assert _extract_fallback_from_model_cfg(model_cfg) == { 92 "provider": "openrouter", 93 "model": "anthropic/claude-sonnet-4.6", 94 } 95 96 def test_extracts_optional_base_url_and_api_mode(self): 97 from hermes_cli.fallback_cmd import _extract_fallback_from_model_cfg 98 model_cfg = { 99 "provider": "custom", 100 "default": "local-model", 101 "base_url": "http://localhost:11434/v1", 102 "api_mode": "chat_completions", 103 } 104 assert _extract_fallback_from_model_cfg(model_cfg) == { 105 "provider": "custom", 106 "model": "local-model", 107 "base_url": "http://localhost:11434/v1", 108 "api_mode": "chat_completions", 109 } 110 111 def test_returns_none_without_provider(self): 112 from hermes_cli.fallback_cmd import _extract_fallback_from_model_cfg 113 assert _extract_fallback_from_model_cfg({"default": "foo"}) is None 114 115 def test_returns_none_without_model(self): 116 from hermes_cli.fallback_cmd import _extract_fallback_from_model_cfg 117 assert _extract_fallback_from_model_cfg({"provider": "openrouter"}) is None 118 119 def test_returns_none_for_non_dict(self): 120 from hermes_cli.fallback_cmd import _extract_fallback_from_model_cfg 121 assert _extract_fallback_from_model_cfg("plain-string") is None 122 assert _extract_fallback_from_model_cfg(None) is None 123 124 125 # --------------------------------------------------------------------------- 126 # cmd_fallback_list 127 # --------------------------------------------------------------------------- 128 129 class TestListCommand: 130 def test_list_empty(self, isolated_home, capsys): 131 _write_config(isolated_home, {}) 132 from hermes_cli.fallback_cmd import cmd_fallback_list 133 cmd_fallback_list(types.SimpleNamespace()) 134 out = capsys.readouterr().out 135 assert "No fallback providers configured" in out 136 assert "hermes fallback add" in out 137 138 def test_list_with_entries(self, isolated_home, capsys): 139 _write_config(isolated_home, { 140 "model": {"provider": "anthropic", "default": "claude-sonnet-4-6"}, 141 "fallback_providers": [ 142 {"provider": "openrouter", "model": "anthropic/claude-sonnet-4.6"}, 143 {"provider": "nous", "model": "Hermes-4"}, 144 ], 145 }) 146 from hermes_cli.fallback_cmd import cmd_fallback_list 147 cmd_fallback_list(types.SimpleNamespace()) 148 out = capsys.readouterr().out 149 assert "Fallback chain (2 entries)" in out 150 assert "anthropic/claude-sonnet-4.6" in out 151 assert "Hermes-4" in out 152 # Primary should be shown too 153 assert "claude-sonnet-4-6" in out 154 155 def test_list_migrates_legacy_for_display(self, isolated_home, capsys): 156 _write_config(isolated_home, { 157 "fallback_model": {"provider": "openrouter", "model": "gpt-5.4"}, 158 }) 159 from hermes_cli.fallback_cmd import cmd_fallback_list 160 cmd_fallback_list(types.SimpleNamespace()) 161 out = capsys.readouterr().out 162 assert "1 entry" in out 163 assert "gpt-5.4" in out 164 165 166 # --------------------------------------------------------------------------- 167 # cmd_fallback_add — mock select_provider_and_model 168 # --------------------------------------------------------------------------- 169 170 class TestAddCommand: 171 def test_add_appends_new_entry(self, isolated_home, capsys): 172 _write_config(isolated_home, { 173 "model": {"provider": "anthropic", "default": "claude-sonnet-4-6"}, 174 }) 175 176 def fake_picker(args=None): 177 # Simulate what the real picker does: writes the selection to config["model"] 178 from hermes_cli.config import load_config, save_config 179 cfg = load_config() 180 cfg["model"] = { 181 "provider": "openrouter", 182 "default": "anthropic/claude-sonnet-4.6", 183 "base_url": "https://openrouter.ai/api/v1", 184 "api_mode": "chat_completions", 185 } 186 save_config(cfg) 187 188 with patch("hermes_cli.main.select_provider_and_model", side_effect=fake_picker), \ 189 patch("hermes_cli.main._require_tty"): 190 from hermes_cli.fallback_cmd import cmd_fallback_add 191 cmd_fallback_add(types.SimpleNamespace()) 192 193 cfg = _read_config(isolated_home) 194 # Primary is preserved 195 assert cfg["model"]["provider"] == "anthropic" 196 assert cfg["model"]["default"] == "claude-sonnet-4-6" 197 # Fallback was appended 198 assert cfg["fallback_providers"] == [ 199 { 200 "provider": "openrouter", 201 "model": "anthropic/claude-sonnet-4.6", 202 "base_url": "https://openrouter.ai/api/v1", 203 "api_mode": "chat_completions", 204 } 205 ] 206 out = capsys.readouterr().out 207 assert "Added fallback" in out 208 209 def test_add_rejects_duplicate(self, isolated_home, capsys): 210 _write_config(isolated_home, { 211 "model": {"provider": "anthropic", "default": "claude-sonnet-4-6"}, 212 "fallback_providers": [ 213 {"provider": "openrouter", "model": "gpt-5.4"}, 214 ], 215 }) 216 217 def fake_picker(args=None): 218 from hermes_cli.config import load_config, save_config 219 cfg = load_config() 220 cfg["model"] = {"provider": "openrouter", "default": "gpt-5.4"} 221 save_config(cfg) 222 223 with patch("hermes_cli.main.select_provider_and_model", side_effect=fake_picker), \ 224 patch("hermes_cli.main._require_tty"): 225 from hermes_cli.fallback_cmd import cmd_fallback_add 226 cmd_fallback_add(types.SimpleNamespace()) 227 228 cfg = _read_config(isolated_home) 229 # Should still have exactly one entry 230 assert len(cfg["fallback_providers"]) == 1 231 out = capsys.readouterr().out 232 assert "already in the fallback chain" in out 233 234 def test_add_rejects_same_as_primary(self, isolated_home, capsys): 235 _write_config(isolated_home, { 236 "model": {"provider": "openrouter", "default": "gpt-5.4"}, 237 }) 238 239 def fake_picker(args=None): 240 # User picks the same thing that's already the primary 241 from hermes_cli.config import load_config, save_config 242 cfg = load_config() 243 cfg["model"] = {"provider": "openrouter", "default": "gpt-5.4"} 244 save_config(cfg) 245 246 with patch("hermes_cli.main.select_provider_and_model", side_effect=fake_picker), \ 247 patch("hermes_cli.main._require_tty"): 248 from hermes_cli.fallback_cmd import cmd_fallback_add 249 cmd_fallback_add(types.SimpleNamespace()) 250 251 cfg = _read_config(isolated_home) 252 assert "fallback_providers" not in cfg or cfg["fallback_providers"] == [] 253 out = capsys.readouterr().out 254 assert "matches the current primary" in out 255 256 def test_add_preserves_primary_when_picker_changes_it(self, isolated_home): 257 """The picker mutates config["model"]; fallback_add must restore the primary.""" 258 _write_config(isolated_home, { 259 "model": { 260 "provider": "anthropic", 261 "default": "claude-sonnet-4-6", 262 "base_url": "https://api.anthropic.com", 263 "api_mode": "anthropic_messages", 264 }, 265 }) 266 267 def fake_picker(args=None): 268 from hermes_cli.config import load_config, save_config 269 cfg = load_config() 270 cfg["model"] = { 271 "provider": "openrouter", 272 "default": "anthropic/claude-sonnet-4.6", 273 "base_url": "https://openrouter.ai/api/v1", 274 "api_mode": "chat_completions", 275 } 276 save_config(cfg) 277 278 with patch("hermes_cli.main.select_provider_and_model", side_effect=fake_picker), \ 279 patch("hermes_cli.main._require_tty"): 280 from hermes_cli.fallback_cmd import cmd_fallback_add 281 cmd_fallback_add(types.SimpleNamespace()) 282 283 cfg = _read_config(isolated_home) 284 # Primary exactly as it was 285 assert cfg["model"]["provider"] == "anthropic" 286 assert cfg["model"]["default"] == "claude-sonnet-4-6" 287 assert cfg["model"]["base_url"] == "https://api.anthropic.com" 288 assert cfg["model"]["api_mode"] == "anthropic_messages" 289 # Fallback added 290 assert len(cfg["fallback_providers"]) == 1 291 assert cfg["fallback_providers"][0]["provider"] == "openrouter" 292 293 def test_add_noop_when_picker_cancelled(self, isolated_home, capsys): 294 _write_config(isolated_home, { 295 "model": {"provider": "anthropic", "default": "claude-sonnet-4-6"}, 296 }) 297 298 def fake_picker(args=None): 299 # User cancelled — no change to config 300 pass 301 302 with patch("hermes_cli.main.select_provider_and_model", side_effect=fake_picker), \ 303 patch("hermes_cli.main._require_tty"): 304 from hermes_cli.fallback_cmd import cmd_fallback_add 305 cmd_fallback_add(types.SimpleNamespace()) 306 307 cfg = _read_config(isolated_home) 308 assert "fallback_providers" not in cfg or cfg["fallback_providers"] == [] 309 out = capsys.readouterr().out 310 # Either "No fallback added" (picker fully cancelled) or "matches the current primary" 311 # (picker left config untouched) — both indicate a non-add outcome. 312 assert ("No fallback added" in out) or ("matches the current primary" in out) 313 314 def test_add_noop_when_picker_clears_model(self, isolated_home, capsys): 315 """Simulate picker explicitly clearing model.default (unusual but possible).""" 316 _write_config(isolated_home, { 317 "model": {"provider": "anthropic", "default": "claude-sonnet-4-6"}, 318 }) 319 320 def fake_picker(args=None): 321 from hermes_cli.config import load_config, save_config 322 cfg = load_config() 323 cfg["model"] = {"provider": "", "default": ""} 324 save_config(cfg) 325 326 with patch("hermes_cli.main.select_provider_and_model", side_effect=fake_picker), \ 327 patch("hermes_cli.main._require_tty"): 328 from hermes_cli.fallback_cmd import cmd_fallback_add 329 cmd_fallback_add(types.SimpleNamespace()) 330 331 out = capsys.readouterr().out 332 assert "No fallback added" in out 333 334 335 # --------------------------------------------------------------------------- 336 # cmd_fallback_remove 337 # --------------------------------------------------------------------------- 338 339 class TestRemoveCommand: 340 def test_remove_empty_chain(self, isolated_home, capsys): 341 _write_config(isolated_home, {}) 342 from hermes_cli.fallback_cmd import cmd_fallback_remove 343 cmd_fallback_remove(types.SimpleNamespace()) 344 out = capsys.readouterr().out 345 assert "nothing to remove" in out 346 347 def test_remove_selected_entry(self, isolated_home, capsys): 348 _write_config(isolated_home, { 349 "fallback_providers": [ 350 {"provider": "openrouter", "model": "gpt-5.4"}, 351 {"provider": "nous", "model": "Hermes-4"}, 352 {"provider": "anthropic", "model": "claude-sonnet-4-6"}, 353 ], 354 }) 355 356 # Picker returns index 1 (the middle entry, "nous / Hermes-4") 357 with patch("hermes_cli.setup._curses_prompt_choice", return_value=1): 358 from hermes_cli.fallback_cmd import cmd_fallback_remove 359 cmd_fallback_remove(types.SimpleNamespace()) 360 361 cfg = _read_config(isolated_home) 362 assert cfg["fallback_providers"] == [ 363 {"provider": "openrouter", "model": "gpt-5.4"}, 364 {"provider": "anthropic", "model": "claude-sonnet-4-6"}, 365 ] 366 out = capsys.readouterr().out 367 assert "Removed fallback" in out 368 assert "Hermes-4" in out 369 370 def test_remove_cancel_keeps_chain(self, isolated_home): 371 _write_config(isolated_home, { 372 "fallback_providers": [ 373 {"provider": "openrouter", "model": "gpt-5.4"}, 374 ], 375 }) 376 377 # Cancel = last item (index == len(chain) == 1 in our menu) 378 with patch("hermes_cli.setup._curses_prompt_choice", return_value=1): 379 from hermes_cli.fallback_cmd import cmd_fallback_remove 380 cmd_fallback_remove(types.SimpleNamespace()) 381 382 cfg = _read_config(isolated_home) 383 assert len(cfg["fallback_providers"]) == 1 384 385 386 # --------------------------------------------------------------------------- 387 # cmd_fallback_clear 388 # --------------------------------------------------------------------------- 389 390 class TestClearCommand: 391 def test_clear_empty_chain(self, isolated_home, capsys): 392 _write_config(isolated_home, {}) 393 from hermes_cli.fallback_cmd import cmd_fallback_clear 394 cmd_fallback_clear(types.SimpleNamespace()) 395 out = capsys.readouterr().out 396 assert "nothing to clear" in out 397 398 def test_clear_with_confirmation(self, isolated_home, capsys, monkeypatch): 399 _write_config(isolated_home, { 400 "fallback_providers": [ 401 {"provider": "openrouter", "model": "gpt-5.4"}, 402 {"provider": "nous", "model": "Hermes-4"}, 403 ], 404 }) 405 monkeypatch.setattr("builtins.input", lambda *a, **kw: "y") 406 from hermes_cli.fallback_cmd import cmd_fallback_clear 407 cmd_fallback_clear(types.SimpleNamespace()) 408 409 cfg = _read_config(isolated_home) 410 assert cfg.get("fallback_providers") == [] 411 out = capsys.readouterr().out 412 assert "Fallback chain cleared" in out 413 414 def test_clear_cancelled(self, isolated_home, monkeypatch): 415 _write_config(isolated_home, { 416 "fallback_providers": [{"provider": "openrouter", "model": "gpt-5.4"}], 417 }) 418 monkeypatch.setattr("builtins.input", lambda *a, **kw: "n") 419 from hermes_cli.fallback_cmd import cmd_fallback_clear 420 cmd_fallback_clear(types.SimpleNamespace()) 421 422 cfg = _read_config(isolated_home) 423 assert len(cfg["fallback_providers"]) == 1 424 425 426 # --------------------------------------------------------------------------- 427 # cmd_fallback dispatcher 428 # --------------------------------------------------------------------------- 429 430 class TestDispatcher: 431 def test_no_subcommand_lists(self, isolated_home, capsys): 432 _write_config(isolated_home, {}) 433 from hermes_cli.fallback_cmd import cmd_fallback 434 cmd_fallback(types.SimpleNamespace(fallback_command=None)) 435 out = capsys.readouterr().out 436 assert "No fallback providers configured" in out 437 438 def test_list_alias(self, isolated_home, capsys): 439 _write_config(isolated_home, {}) 440 from hermes_cli.fallback_cmd import cmd_fallback 441 cmd_fallback(types.SimpleNamespace(fallback_command="ls")) 442 out = capsys.readouterr().out 443 assert "No fallback providers configured" in out 444 445 def test_remove_alias(self, isolated_home, capsys): 446 _write_config(isolated_home, {}) 447 from hermes_cli.fallback_cmd import cmd_fallback 448 cmd_fallback(types.SimpleNamespace(fallback_command="rm")) 449 out = capsys.readouterr().out 450 assert "nothing to remove" in out 451 452 def test_unknown_subcommand_exits(self, isolated_home): 453 _write_config(isolated_home, {}) 454 from hermes_cli.fallback_cmd import cmd_fallback 455 with pytest.raises(SystemExit): 456 cmd_fallback(types.SimpleNamespace(fallback_command="nope")) 457 458 459 # --------------------------------------------------------------------------- 460 # argparse wiring — verify the subparser is registered 461 # --------------------------------------------------------------------------- 462 463 class TestArgparseWiring: 464 """Verify `hermes fallback` is wired into main.py's argparse tree. 465 466 main() builds the parser inline, so we invoke main([...]) via subprocess 467 with --help to introspect registered subcommands without side effects. 468 """ 469 470 def test_fallback_help_lists_subcommands(self): 471 import subprocess 472 import sys 473 result = subprocess.run( 474 [sys.executable, "-m", "hermes_cli.main", "fallback", "--help"], 475 capture_output=True, 476 text=True, 477 timeout=30, 478 ) 479 # --help exits 0 480 assert result.returncode == 0, f"stderr: {result.stderr}" 481 out = result.stdout + result.stderr 482 # All four subcommands should appear in help 483 assert "list" in out 484 assert "add" in out 485 assert "remove" in out 486 assert "clear" in out