test_fast_command.py
1 """Tests for the /fast CLI command and service-tier config handling.""" 2 3 import unittest 4 from types import SimpleNamespace 5 from unittest.mock import MagicMock, patch 6 7 8 def _import_cli(): 9 import hermes_cli.config as config_mod 10 11 if not hasattr(config_mod, "save_env_value_secure"): 12 config_mod.save_env_value_secure = lambda key, value: { 13 "success": True, 14 "stored_as": key, 15 "validated": False, 16 } 17 18 import cli as cli_mod 19 20 return cli_mod 21 22 23 class TestParseServiceTierConfig(unittest.TestCase): 24 def _parse(self, raw): 25 cli_mod = _import_cli() 26 return cli_mod._parse_service_tier_config(raw) 27 28 def test_fast_maps_to_priority(self): 29 self.assertEqual(self._parse("fast"), "priority") 30 self.assertEqual(self._parse("priority"), "priority") 31 32 def test_normal_disables_service_tier(self): 33 self.assertIsNone(self._parse("normal")) 34 self.assertIsNone(self._parse("off")) 35 self.assertIsNone(self._parse("")) 36 37 38 class TestHandleFastCommand(unittest.TestCase): 39 def _make_cli(self, service_tier=None): 40 return SimpleNamespace( 41 service_tier=service_tier, 42 provider="openai-codex", 43 requested_provider="openai-codex", 44 model="gpt-5.4", 45 _fast_command_available=lambda: True, 46 agent=MagicMock(), 47 ) 48 49 def test_no_args_shows_status(self): 50 cli_mod = _import_cli() 51 stub = self._make_cli(service_tier=None) 52 with ( 53 patch.object(cli_mod, "_cprint") as mock_cprint, 54 patch.object(cli_mod, "save_config_value") as mock_save, 55 ): 56 cli_mod.HermesCLI._handle_fast_command(stub, "/fast") 57 58 # Bare /fast shows status, does not change config 59 mock_save.assert_not_called() 60 # Should have printed the status line 61 printed = " ".join(str(c) for c in mock_cprint.call_args_list) 62 self.assertIn("normal", printed) 63 64 def test_no_args_shows_fast_when_enabled(self): 65 cli_mod = _import_cli() 66 stub = self._make_cli(service_tier="priority") 67 with ( 68 patch.object(cli_mod, "_cprint") as mock_cprint, 69 patch.object(cli_mod, "save_config_value") as mock_save, 70 ): 71 cli_mod.HermesCLI._handle_fast_command(stub, "/fast") 72 73 mock_save.assert_not_called() 74 printed = " ".join(str(c) for c in mock_cprint.call_args_list) 75 self.assertIn("fast", printed) 76 77 def test_normal_argument_clears_service_tier(self): 78 cli_mod = _import_cli() 79 stub = self._make_cli(service_tier="priority") 80 with ( 81 patch.object(cli_mod, "_cprint"), 82 patch.object(cli_mod, "save_config_value", return_value=True) as mock_save, 83 ): 84 cli_mod.HermesCLI._handle_fast_command(stub, "/fast normal") 85 86 mock_save.assert_called_once_with("agent.service_tier", "normal") 87 self.assertIsNone(stub.service_tier) 88 self.assertIsNone(stub.agent) 89 90 def test_unsupported_model_does_not_expose_fast(self): 91 cli_mod = _import_cli() 92 stub = SimpleNamespace( 93 service_tier=None, 94 provider="openai-codex", 95 requested_provider="openai-codex", 96 model="gpt-5.3-codex", 97 _fast_command_available=lambda: False, 98 agent=MagicMock(), 99 ) 100 101 with ( 102 patch.object(cli_mod, "_cprint") as mock_cprint, 103 patch.object(cli_mod, "save_config_value") as mock_save, 104 ): 105 cli_mod.HermesCLI._handle_fast_command(stub, "/fast") 106 107 mock_save.assert_not_called() 108 self.assertTrue(mock_cprint.called) 109 110 111 class TestPriorityProcessingModels(unittest.TestCase): 112 """Verify the expanded Priority Processing model registry.""" 113 114 def test_all_documented_models_supported(self): 115 from hermes_cli.models import model_supports_fast_mode 116 117 # All OpenAI flagship models support Priority Processing — including 118 # future releases (gpt-5.5, 5.6...) via pattern matching. 119 supported = [ 120 "gpt-5.5", "gpt-5.5-mini", 121 "gpt-5.4", "gpt-5.4-mini", "gpt-5.2", 122 "gpt-5.1", "gpt-5", "gpt-5-mini", 123 "gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano", 124 "gpt-4o", "gpt-4o-mini", 125 "o1", "o1-mini", "o3", "o3-mini", "o4-mini", 126 ] 127 for model in supported: 128 assert model_supports_fast_mode(model), f"{model} should support fast mode" 129 130 def test_all_anthropic_models_supported(self): 131 """Per Anthropic docs, fast mode is currently Opus 4.6 only. 132 133 Sending speed=fast to Opus 4.7, Sonnet, or Haiku returns HTTP 400. 134 Pre-fix this test asserted all Claude variants supported fast mode, 135 which mirrored the bug rather than the API contract. 136 """ 137 from hermes_cli.models import model_supports_fast_mode 138 139 # Supported: Opus 4.6 in any form 140 supported = [ 141 "claude-opus-4-6", "claude-opus-4.6", 142 "anthropic/claude-opus-4-6", "anthropic/claude-opus-4.6", 143 ] 144 for model in supported: 145 assert model_supports_fast_mode(model), f"{model} should support fast mode" 146 147 # Unsupported per Anthropic API: Opus 4.7, Sonnet, Haiku 148 unsupported = [ 149 "claude-opus-4-7", 150 "claude-sonnet-4-6", "claude-sonnet-4.6", "claude-sonnet-4", 151 "claude-haiku-4-5", "claude-3-5-haiku", 152 ] 153 for model in unsupported: 154 assert not model_supports_fast_mode(model), ( 155 f"{model} should NOT support fast mode — Anthropic restricts " 156 f"speed=fast to Opus 4.6" 157 ) 158 159 def test_codex_models_excluded(self): 160 """Codex models route through Responses API and don't accept service_tier.""" 161 from hermes_cli.models import model_supports_fast_mode 162 163 for model in ["gpt-5-codex", "gpt-5.2-codex", "gpt-5.3-codex", "gpt-5.1-codex-max"]: 164 assert not model_supports_fast_mode(model), f"{model} is codex — should not expose /fast" 165 166 def test_vendor_prefix_stripped(self): 167 from hermes_cli.models import model_supports_fast_mode 168 169 assert model_supports_fast_mode("openai/gpt-5.4") is True 170 assert model_supports_fast_mode("openai/gpt-4.1") is True 171 assert model_supports_fast_mode("openai/o3") is True 172 173 def test_non_priority_models_rejected(self): 174 from hermes_cli.models import model_supports_fast_mode 175 176 # Codex-series models route through the Codex Responses API and 177 # don't accept service_tier, so they're excluded. 178 assert model_supports_fast_mode("gpt-5.3-codex") is False 179 assert model_supports_fast_mode("gpt-5.2-codex") is False 180 assert model_supports_fast_mode("gpt-5-codex") is False 181 # Non-OpenAI, non-Anthropic models 182 assert model_supports_fast_mode("gemini-3-pro-preview") is False 183 assert model_supports_fast_mode("kimi-k2-thinking") is False 184 assert model_supports_fast_mode("deepseek-chat") is False 185 assert model_supports_fast_mode("") is False 186 assert model_supports_fast_mode(None) is False 187 188 def test_resolve_overrides_returns_service_tier(self): 189 from hermes_cli.models import resolve_fast_mode_overrides 190 191 result = resolve_fast_mode_overrides("gpt-5.4") 192 assert result == {"service_tier": "priority"} 193 194 result = resolve_fast_mode_overrides("gpt-4.1") 195 assert result == {"service_tier": "priority"} 196 197 def test_resolve_overrides_none_for_unsupported(self): 198 from hermes_cli.models import resolve_fast_mode_overrides 199 200 assert resolve_fast_mode_overrides("gpt-5.3-codex") is None 201 assert resolve_fast_mode_overrides("gemini-3-pro-preview") is None 202 assert resolve_fast_mode_overrides("kimi-k2-thinking") is None 203 204 205 class TestFastModeRouting(unittest.TestCase): 206 def test_fast_command_exposed_for_model_even_when_provider_is_auto(self): 207 cli_mod = _import_cli() 208 stub = SimpleNamespace(provider="auto", requested_provider="auto", model="gpt-5.4", agent=None) 209 210 assert cli_mod.HermesCLI._fast_command_available(stub) is True 211 212 def test_fast_command_exposed_for_non_codex_models(self): 213 cli_mod = _import_cli() 214 stub = SimpleNamespace(provider="openai", requested_provider="openai", model="gpt-4.1", agent=None) 215 assert cli_mod.HermesCLI._fast_command_available(stub) is True 216 217 stub = SimpleNamespace(provider="openrouter", requested_provider="openrouter", model="o3", agent=None) 218 assert cli_mod.HermesCLI._fast_command_available(stub) is True 219 220 def test_turn_route_injects_overrides_without_provider_switch(self): 221 """Fast mode should add request_overrides but NOT change the provider/runtime.""" 222 cli_mod = _import_cli() 223 stub = SimpleNamespace( 224 model="gpt-5.4", 225 api_key="primary-key", 226 base_url="https://openrouter.ai/api/v1", 227 provider="openrouter", 228 api_mode="chat_completions", 229 acp_command=None, 230 acp_args=[], 231 _credential_pool=None, 232 service_tier="priority", 233 ) 234 235 route = cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi") 236 237 # Provider should NOT have changed 238 assert route["runtime"]["provider"] == "openrouter" 239 assert route["runtime"]["api_mode"] == "chat_completions" 240 # But request_overrides should be set 241 assert route["request_overrides"] == {"service_tier": "priority"} 242 243 def test_turn_route_keeps_primary_runtime_when_model_has_no_fast_backend(self): 244 cli_mod = _import_cli() 245 stub = SimpleNamespace( 246 model="gpt-5.3-codex", 247 api_key="primary-key", 248 base_url="https://openrouter.ai/api/v1", 249 provider="openrouter", 250 api_mode="chat_completions", 251 acp_command=None, 252 acp_args=[], 253 _credential_pool=None, 254 service_tier="priority", 255 ) 256 257 route = cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi") 258 259 assert route["runtime"]["provider"] == "openrouter" 260 assert route.get("request_overrides") is None 261 262 263 class TestAnthropicFastMode(unittest.TestCase): 264 """Verify Anthropic Fast Mode model support and override resolution.""" 265 266 def test_anthropic_opus_supported(self): 267 from hermes_cli.models import model_supports_fast_mode 268 269 # Native Anthropic format (hyphens) 270 assert model_supports_fast_mode("claude-opus-4-6") is True 271 # OpenRouter format (dots) 272 assert model_supports_fast_mode("claude-opus-4.6") is True 273 # With vendor prefix 274 assert model_supports_fast_mode("anthropic/claude-opus-4-6") is True 275 assert model_supports_fast_mode("anthropic/claude-opus-4.6") is True 276 277 def test_anthropic_non_opus46_models_excluded(self): 278 """Anthropic restricts fast mode to Opus 4.6 — others must be excluded. 279 280 Per https://platform.claude.com/docs/en/build-with-claude/fast-mode, 281 sending speed=fast to Opus 4.7, Sonnet, or Haiku returns HTTP 400. 282 """ 283 from hermes_cli.models import model_supports_fast_mode 284 285 assert model_supports_fast_mode("claude-sonnet-4-6") is False 286 assert model_supports_fast_mode("claude-sonnet-4.6") is False 287 assert model_supports_fast_mode("claude-haiku-4-5") is False 288 assert model_supports_fast_mode("claude-opus-4-7") is False 289 assert model_supports_fast_mode("anthropic/claude-sonnet-4.6") is False 290 assert model_supports_fast_mode("anthropic/claude-opus-4-7") is False 291 292 def test_non_claude_models_not_anthropic_fast(self): 293 """Non-Claude models should not be treated as Anthropic fast-mode.""" 294 from hermes_cli.models import _is_anthropic_fast_model 295 296 assert _is_anthropic_fast_model("gpt-5.4") is False 297 assert _is_anthropic_fast_model("gemini-3-pro") is False 298 assert _is_anthropic_fast_model("kimi-k2-thinking") is False 299 300 def test_anthropic_variant_tags_stripped(self): 301 from hermes_cli.models import model_supports_fast_mode 302 303 # OpenRouter variant tags after colon should be stripped 304 assert model_supports_fast_mode("claude-opus-4.6:fast") is True 305 assert model_supports_fast_mode("claude-opus-4.6:beta") is True 306 307 def test_resolve_overrides_returns_speed_for_anthropic(self): 308 from hermes_cli.models import resolve_fast_mode_overrides 309 310 result = resolve_fast_mode_overrides("claude-opus-4-6") 311 assert result == {"speed": "fast"} 312 313 result = resolve_fast_mode_overrides("anthropic/claude-opus-4.6") 314 assert result == {"speed": "fast"} 315 316 def test_resolve_overrides_returns_none_for_unsupported_claude(self): 317 """Opus 4.7 and other Claude models don't support fast mode (API 400s). 318 319 Per Anthropic docs, fast mode is currently Opus 4.6 only. 320 """ 321 from hermes_cli.models import resolve_fast_mode_overrides 322 323 assert resolve_fast_mode_overrides("claude-opus-4-7") is None 324 assert resolve_fast_mode_overrides("claude-sonnet-4-6") is None 325 assert resolve_fast_mode_overrides("claude-haiku-4-5") is None 326 327 def test_resolve_overrides_returns_service_tier_for_openai(self): 328 """OpenAI models should still get service_tier, not speed.""" 329 from hermes_cli.models import resolve_fast_mode_overrides 330 331 result = resolve_fast_mode_overrides("gpt-5.4") 332 assert result == {"service_tier": "priority"} 333 334 def test_is_anthropic_fast_model(self): 335 """Fast mode is currently Opus 4.6 only — other Claude variants must be excluded.""" 336 from hermes_cli.models import _is_anthropic_fast_model 337 338 # Supported: Opus 4.6 in any form 339 assert _is_anthropic_fast_model("claude-opus-4-6") is True 340 assert _is_anthropic_fast_model("claude-opus-4.6") is True 341 assert _is_anthropic_fast_model("anthropic/claude-opus-4-6") is True 342 assert _is_anthropic_fast_model("claude-opus-4.6:fast") is True 343 344 # Unsupported per Anthropic API contract — would 400 if we sent speed=fast 345 assert _is_anthropic_fast_model("claude-opus-4-7") is False 346 assert _is_anthropic_fast_model("claude-sonnet-4-6") is False 347 assert _is_anthropic_fast_model("claude-haiku-4-5") is False 348 349 # Non-Claude 350 assert _is_anthropic_fast_model("gpt-5.4") is False 351 assert _is_anthropic_fast_model("") is False 352 353 def test_fast_command_exposed_for_anthropic_model(self): 354 cli_mod = _import_cli() 355 stub = SimpleNamespace( 356 provider="anthropic", requested_provider="anthropic", 357 model="claude-opus-4-6", agent=None, 358 ) 359 assert cli_mod.HermesCLI._fast_command_available(stub) is True 360 361 def test_fast_command_hidden_for_anthropic_sonnet(self): 362 """Sonnet doesn't support fast mode (Opus 4.6 only) — /fast must be hidden.""" 363 cli_mod = _import_cli() 364 stub = SimpleNamespace( 365 provider="anthropic", requested_provider="anthropic", 366 model="claude-sonnet-4-6", agent=None, 367 ) 368 assert cli_mod.HermesCLI._fast_command_available(stub) is False 369 370 def test_fast_command_hidden_for_anthropic_opus_47(self): 371 """Opus 4.7 doesn't support fast mode — /fast must be hidden.""" 372 cli_mod = _import_cli() 373 stub = SimpleNamespace( 374 provider="anthropic", requested_provider="anthropic", 375 model="claude-opus-4-7", agent=None, 376 ) 377 assert cli_mod.HermesCLI._fast_command_available(stub) is False 378 379 def test_fast_command_hidden_for_non_claude_non_openai(self): 380 """Non-Claude, non-OpenAI models should not expose /fast.""" 381 cli_mod = _import_cli() 382 stub = SimpleNamespace( 383 provider="gemini", requested_provider="gemini", 384 model="gemini-3-pro-preview", agent=None, 385 ) 386 assert cli_mod.HermesCLI._fast_command_available(stub) is False 387 388 def test_turn_route_injects_speed_for_anthropic(self): 389 """Anthropic models should get speed:'fast' override, not service_tier.""" 390 cli_mod = _import_cli() 391 stub = SimpleNamespace( 392 model="claude-opus-4-6", 393 api_key="sk-ant-test", 394 base_url="https://api.anthropic.com", 395 provider="anthropic", 396 api_mode="anthropic_messages", 397 acp_command=None, 398 acp_args=[], 399 _credential_pool=None, 400 service_tier="priority", 401 ) 402 403 route = cli_mod.HermesCLI._resolve_turn_agent_config(stub, "hi") 404 405 assert route["runtime"]["provider"] == "anthropic" 406 assert route["request_overrides"] == {"speed": "fast"} 407 408 409 class TestAnthropicFastModeAdapter(unittest.TestCase): 410 """Verify build_anthropic_kwargs handles fast_mode parameter.""" 411 412 def test_fast_mode_adds_speed_and_beta(self): 413 from agent.anthropic_adapter import build_anthropic_kwargs, _FAST_MODE_BETA 414 415 kwargs = build_anthropic_kwargs( 416 model="claude-opus-4-6", 417 messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], 418 tools=None, 419 max_tokens=None, 420 reasoning_config=None, 421 fast_mode=True, 422 ) 423 assert kwargs.get("extra_body", {}).get("speed") == "fast" 424 assert "speed" not in kwargs 425 assert "extra_headers" in kwargs 426 assert _FAST_MODE_BETA in kwargs["extra_headers"].get("anthropic-beta", "") 427 428 def test_fast_mode_off_no_speed(self): 429 from agent.anthropic_adapter import build_anthropic_kwargs 430 431 kwargs = build_anthropic_kwargs( 432 model="claude-opus-4-6", 433 messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], 434 tools=None, 435 max_tokens=None, 436 reasoning_config=None, 437 fast_mode=False, 438 ) 439 assert kwargs.get("extra_body", {}).get("speed") is None 440 assert "speed" not in kwargs 441 assert "extra_headers" not in kwargs 442 443 def test_fast_mode_skipped_for_third_party_endpoint(self): 444 from agent.anthropic_adapter import build_anthropic_kwargs 445 446 kwargs = build_anthropic_kwargs( 447 model="claude-opus-4-6", 448 messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], 449 tools=None, 450 max_tokens=None, 451 reasoning_config=None, 452 fast_mode=True, 453 base_url="https://api.minimax.io/anthropic/v1", 454 ) 455 # Third-party endpoints should NOT get speed or fast-mode beta 456 assert kwargs.get("extra_body", {}).get("speed") is None 457 assert "speed" not in kwargs 458 assert "extra_headers" not in kwargs 459 460 def test_fast_mode_kwargs_are_safe_for_sdk_unpacking(self): 461 from agent.anthropic_adapter import build_anthropic_kwargs 462 463 kwargs = build_anthropic_kwargs( 464 model="claude-opus-4-6", 465 messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], 466 tools=None, 467 max_tokens=None, 468 reasoning_config=None, 469 fast_mode=True, 470 ) 471 assert "speed" not in kwargs 472 assert kwargs.get("extra_body", {}).get("speed") == "fast" 473 474 475 class TestConfigDefault(unittest.TestCase): 476 def test_default_config_has_service_tier(self): 477 from hermes_cli.config import DEFAULT_CONFIG 478 479 agent = DEFAULT_CONFIG.get("agent", {}) 480 self.assertIn("service_tier", agent) 481 self.assertEqual(agent["service_tier"], "")