test_auxiliary_main_first.py
1 """Regression tests for the ``auto`` → main-model-first policy. 2 3 Prior to this change, aggregator users (OpenRouter / Nous Portal) had aux 4 tasks routed through a cheap provider-side default (Gemini Flash) while 5 non-aggregator users got their main model. This made behavior inconsistent 6 and surprising — users picked Claude but got Gemini Flash summaries. 7 8 The current policy: ``auto`` means "use my main chat model" for every user, 9 regardless of provider type. Explicit per-task overrides in ``config.yaml`` 10 (``auxiliary.<task>.provider``) still win. The cheap fallback chain only 11 runs when the main provider has no working client. 12 """ 13 14 from __future__ import annotations 15 16 from unittest.mock import MagicMock, patch 17 18 import pytest 19 20 21 # ── Text aux tasks — _resolve_auto ────────────────────────────────────────── 22 23 24 class TestResolveAutoMainFirst: 25 """_resolve_auto() must prefer main provider + main model for every user.""" 26 27 def test_openrouter_main_uses_main_model_for_aux(self, monkeypatch): 28 """OpenRouter main user → aux uses their picked OR model, not Gemini Flash.""" 29 monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key") 30 31 with patch( 32 "agent.auxiliary_client._read_main_provider", 33 return_value="openrouter", 34 ), patch( 35 "agent.auxiliary_client._read_main_model", 36 return_value="anthropic/claude-sonnet-4.6", 37 ), patch( 38 "agent.auxiliary_client.resolve_provider_client" 39 ) as mock_resolve: 40 mock_client = MagicMock() 41 mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6") 42 43 from agent.auxiliary_client import _resolve_auto 44 45 client, model = _resolve_auto() 46 47 assert client is mock_client 48 assert model == "anthropic/claude-sonnet-4.6" 49 # Verify it asked resolve_provider_client for the MAIN provider+model, 50 # not a fallback-chain provider 51 mock_resolve.assert_called_once() 52 assert mock_resolve.call_args.args[0] == "openrouter" 53 assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6" 54 55 def test_nous_main_uses_main_model_for_aux(self, monkeypatch): 56 """Nous Portal main user → aux uses their picked Nous model, not free-tier MiMo.""" 57 # No OPENROUTER_API_KEY → ensures if main failed we'd fall to chain 58 with patch( 59 "agent.auxiliary_client._read_main_provider", return_value="nous", 60 ), patch( 61 "agent.auxiliary_client._read_main_model", 62 return_value="anthropic/claude-opus-4.6", 63 ), patch( 64 "agent.auxiliary_client.resolve_provider_client" 65 ) as mock_resolve: 66 mock_client = MagicMock() 67 mock_resolve.return_value = (mock_client, "anthropic/claude-opus-4.6") 68 69 from agent.auxiliary_client import _resolve_auto 70 71 client, model = _resolve_auto() 72 73 assert client is mock_client 74 assert model == "anthropic/claude-opus-4.6" 75 assert mock_resolve.call_args.args[0] == "nous" 76 77 def test_non_aggregator_main_still_uses_main(self, monkeypatch): 78 """Non-aggregator main (DeepSeek) → unchanged behavior, main model used.""" 79 monkeypatch.setenv("DEEPSEEK_API_KEY", "ds-test") 80 81 with patch( 82 "agent.auxiliary_client._read_main_provider", return_value="deepseek", 83 ), patch( 84 "agent.auxiliary_client._read_main_model", return_value="deepseek-chat", 85 ), patch( 86 "agent.auxiliary_client.resolve_provider_client" 87 ) as mock_resolve: 88 mock_client = MagicMock() 89 mock_resolve.return_value = (mock_client, "deepseek-chat") 90 91 from agent.auxiliary_client import _resolve_auto 92 93 client, model = _resolve_auto() 94 95 assert client is mock_client 96 assert model == "deepseek-chat" 97 assert mock_resolve.call_args.args[0] == "deepseek" 98 99 def test_main_unavailable_falls_through_to_chain(self, monkeypatch): 100 """Main provider with no working client → fall back to aux chain.""" 101 monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") 102 103 chain_client = MagicMock() 104 with patch( 105 "agent.auxiliary_client._read_main_provider", return_value="anthropic", 106 ), patch( 107 "agent.auxiliary_client._read_main_model", return_value="claude-opus", 108 ), patch( 109 "agent.auxiliary_client.resolve_provider_client", 110 return_value=(None, None), # main provider has no client 111 ), patch( 112 "agent.auxiliary_client._try_openrouter", 113 return_value=(chain_client, "google/gemini-3-flash-preview"), 114 ): 115 from agent.auxiliary_client import _resolve_auto 116 117 client, model = _resolve_auto() 118 119 assert client is chain_client 120 assert model == "google/gemini-3-flash-preview" 121 122 def test_no_main_config_uses_chain_directly(self): 123 """No main provider configured → skip step 1, use chain (no regression).""" 124 chain_client = MagicMock() 125 with patch( 126 "agent.auxiliary_client._read_main_provider", return_value="", 127 ), patch( 128 "agent.auxiliary_client._read_main_model", return_value="", 129 ), patch( 130 "agent.auxiliary_client._try_openrouter", 131 return_value=(chain_client, "google/gemini-3-flash-preview"), 132 ): 133 from agent.auxiliary_client import _resolve_auto 134 135 client, model = _resolve_auto() 136 137 assert client is chain_client 138 139 def test_runtime_override_wins_over_config(self, monkeypatch): 140 """main_runtime kwarg overrides config-read main provider/model.""" 141 with patch( 142 "agent.auxiliary_client._read_main_provider", 143 return_value="openrouter", 144 ), patch( 145 "agent.auxiliary_client._read_main_model", return_value="config-model", 146 ), patch( 147 "agent.auxiliary_client.resolve_provider_client" 148 ) as mock_resolve: 149 mock_resolve.return_value = (MagicMock(), "runtime-model") 150 151 from agent.auxiliary_client import _resolve_auto 152 153 _resolve_auto(main_runtime={ 154 "provider": "anthropic", 155 "model": "runtime-model", 156 "base_url": "", 157 "api_key": "", 158 "api_mode": "", 159 }) 160 161 # Runtime override wins 162 assert mock_resolve.call_args.args[0] == "anthropic" 163 assert mock_resolve.call_args.args[1] == "runtime-model" 164 165 166 # ── Vision — resolve_vision_provider_client ───────────────────────────────── 167 168 169 class TestResolveVisionMainFirst: 170 """Vision auto-detection prefers the main provider first.""" 171 172 def test_openrouter_main_vision_uses_main_model(self, monkeypatch): 173 """OpenRouter main with vision-capable model → aux vision uses main model.""" 174 monkeypatch.setenv("OPENROUTER_API_KEY", "or-key") 175 176 with patch( 177 "agent.auxiliary_client._read_main_provider", return_value="openrouter", 178 ), patch( 179 "agent.auxiliary_client._read_main_model", 180 return_value="anthropic/claude-sonnet-4.6", 181 ), patch( 182 "agent.auxiliary_client.resolve_provider_client" 183 ) as mock_resolve, patch( 184 "agent.auxiliary_client._resolve_task_provider_model", 185 return_value=("auto", None, None, None, None), 186 ): 187 mock_client = MagicMock() 188 mock_resolve.return_value = (mock_client, "anthropic/claude-sonnet-4.6") 189 190 from agent.auxiliary_client import resolve_vision_provider_client 191 192 provider, client, model = resolve_vision_provider_client() 193 194 assert provider == "openrouter" 195 assert client is mock_client 196 assert model == "anthropic/claude-sonnet-4.6" 197 # Verify it did NOT call the strict vision backend for OpenRouter 198 # (which would have used a cheap gemini-flash-preview default) 199 mock_resolve.assert_called_once() 200 assert mock_resolve.call_args.args[0] == "openrouter" 201 assert mock_resolve.call_args.args[1] == "anthropic/claude-sonnet-4.6" 202 assert mock_resolve.call_args.kwargs.get("is_vision") is True 203 204 def test_nous_main_vision_uses_paid_nous_vision_backend(self): 205 """Paid Nous main → aux vision uses the dedicated Nous vision backend.""" 206 with patch( 207 "agent.auxiliary_client._read_main_provider", return_value="nous", 208 ), patch( 209 "agent.auxiliary_client._read_main_model", 210 return_value="openai/gpt-5", 211 ), patch( 212 "agent.auxiliary_client._resolve_task_provider_model", 213 return_value=("auto", None, None, None, None), 214 ), patch( 215 "agent.auxiliary_client._resolve_strict_vision_backend", 216 return_value=(MagicMock(), "google/gemini-3-flash-preview"), 217 ): 218 from agent.auxiliary_client import resolve_vision_provider_client 219 220 provider, client, model = resolve_vision_provider_client() 221 222 assert provider == "nous" 223 assert client is not None 224 assert model == "google/gemini-3-flash-preview" 225 226 def test_nous_main_vision_uses_free_tier_nous_vision_backend(self): 227 """Free-tier Nous main → aux vision uses MiMo omni, not the text main model.""" 228 with patch( 229 "agent.auxiliary_client._read_main_provider", return_value="nous", 230 ), patch( 231 "agent.auxiliary_client._read_main_model", 232 return_value="xiaomi/mimo-v2-pro", 233 ), patch( 234 "agent.auxiliary_client._resolve_task_provider_model", 235 return_value=("auto", None, None, None, None), 236 ), patch( 237 "agent.auxiliary_client._resolve_strict_vision_backend", 238 return_value=(MagicMock(), "xiaomi/mimo-v2-omni"), 239 ): 240 from agent.auxiliary_client import resolve_vision_provider_client 241 242 provider, client, model = resolve_vision_provider_client() 243 244 assert provider == "nous" 245 assert client is not None 246 assert model == "xiaomi/mimo-v2-omni" 247 248 def test_exotic_provider_with_vision_override_preserved(self): 249 """xiaomi → mimo-v2.5 override still wins over main_model.""" 250 with patch( 251 "agent.auxiliary_client._read_main_provider", return_value="xiaomi", 252 ), patch( 253 "agent.auxiliary_client._read_main_model", 254 return_value="mimo-v2-pro", # text model 255 ), patch( 256 "agent.auxiliary_client.resolve_provider_client" 257 ) as mock_resolve, patch( 258 "agent.auxiliary_client._resolve_task_provider_model", 259 return_value=("auto", None, None, None, None), 260 ): 261 mock_resolve.return_value = (MagicMock(), "mimo-v2.5") 262 263 from agent.auxiliary_client import resolve_vision_provider_client 264 265 provider, client, model = resolve_vision_provider_client() 266 267 assert provider == "xiaomi" 268 # Should use mimo-v2.5 (vision override), not mimo-v2-pro (text main) 269 assert mock_resolve.call_args.args[1] == "mimo-v2.5" 270 assert mock_resolve.call_args.kwargs.get("is_vision") is True 271 272 def test_copilot_vision_sets_vision_header(self, monkeypatch): 273 """Copilot vision requests include the header required for vision routing.""" 274 monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "ghu_test-token") 275 276 captured = {} 277 278 def fake_headers(*, is_agent_turn=False, is_vision=False): 279 captured["is_agent_turn"] = is_agent_turn 280 captured["is_vision"] = is_vision 281 return {"Copilot-Vision-Request": "true"} if is_vision else {} 282 283 with patch( 284 "agent.auxiliary_client._read_main_provider", return_value="copilot", 285 ), patch( 286 "agent.auxiliary_client._read_main_model", return_value="configured-copilot-model", 287 ), patch( 288 "agent.auxiliary_client._resolve_task_provider_model", 289 return_value=("auto", None, None, None, None), 290 ), patch( 291 "agent.auxiliary_client.OpenAI", 292 ) as mock_openai, patch( 293 "hermes_cli.auth.resolve_api_key_provider_credentials", 294 return_value={ 295 "provider": "copilot", 296 "api_key": "copilot-api-token", 297 "base_url": "https://api.githubcopilot.com", 298 }, 299 ), patch( 300 "hermes_cli.copilot_auth.copilot_request_headers", 301 side_effect=fake_headers, 302 ): 303 mock_client = MagicMock() 304 mock_openai.return_value = mock_client 305 306 from agent.auxiliary_client import resolve_vision_provider_client 307 308 provider, client, model = resolve_vision_provider_client() 309 310 assert provider == "copilot" 311 assert client is mock_client 312 assert model == "configured-copilot-model" 313 assert captured == {"is_agent_turn": True, "is_vision": True} 314 assert mock_openai.call_args.kwargs["default_headers"]["Copilot-Vision-Request"] == "true" 315 316 def test_text_copilot_does_not_set_vision_header(self, monkeypatch): 317 """Text Copilot requests keep the vision-only header off.""" 318 monkeypatch.setenv("COPILOT_GITHUB_TOKEN", "ghu_test-token") 319 320 captured = {} 321 322 def fake_headers(*, is_agent_turn=False, is_vision=False): 323 captured["is_agent_turn"] = is_agent_turn 324 captured["is_vision"] = is_vision 325 return {"Copilot-Vision-Request": "true"} if is_vision else {} 326 327 with patch( 328 "agent.auxiliary_client.OpenAI", 329 ) as mock_openai, patch( 330 "hermes_cli.auth.resolve_api_key_provider_credentials", 331 return_value={ 332 "provider": "copilot", 333 "api_key": "copilot-api-token", 334 "base_url": "https://api.githubcopilot.com", 335 }, 336 ), patch( 337 "hermes_cli.copilot_auth.copilot_request_headers", 338 side_effect=fake_headers, 339 ): 340 mock_client = MagicMock() 341 mock_openai.return_value = mock_client 342 343 from agent.auxiliary_client import resolve_provider_client 344 345 client, model = resolve_provider_client("copilot", "gpt-5-mini") 346 347 assert client is mock_client 348 assert model == "gpt-5-mini" 349 assert captured == {"is_agent_turn": True, "is_vision": False} 350 assert "default_headers" not in mock_openai.call_args.kwargs 351 352 def test_main_unavailable_vision_falls_through_to_aggregators(self): 353 """Main provider fails → fall back to OpenRouter/Nous strict backends.""" 354 fallback_client = MagicMock() 355 with patch( 356 "agent.auxiliary_client._read_main_provider", return_value="deepseek", 357 ), patch( 358 "agent.auxiliary_client._read_main_model", return_value="deepseek-chat", 359 ), patch( 360 "agent.auxiliary_client.resolve_provider_client", 361 return_value=(None, None), 362 ), patch( 363 "agent.auxiliary_client._resolve_strict_vision_backend", 364 return_value=(fallback_client, "google/gemini-3-flash-preview"), 365 ), patch( 366 "agent.auxiliary_client._resolve_task_provider_model", 367 return_value=("auto", None, None, None, None), 368 ): 369 from agent.auxiliary_client import resolve_vision_provider_client 370 371 provider, client, model = resolve_vision_provider_client() 372 373 assert client is fallback_client 374 assert provider in ("openrouter", "nous") 375 376 def test_explicit_provider_override_still_wins(self): 377 """Explicit config override bypasses main-first policy.""" 378 with patch( 379 "agent.auxiliary_client._read_main_provider", return_value="openrouter", 380 ), patch( 381 "agent.auxiliary_client._read_main_model", 382 return_value="anthropic/claude-opus-4.6", 383 ), patch( 384 "agent.auxiliary_client._resolve_task_provider_model", 385 return_value=("nous", None, None, None, None), # explicit override 386 ), patch( 387 "agent.auxiliary_client._resolve_strict_vision_backend" 388 ) as mock_strict: 389 mock_strict.return_value = (MagicMock(), "nous-default-model") 390 391 from agent.auxiliary_client import resolve_vision_provider_client 392 393 provider, client, model = resolve_vision_provider_client() 394 395 # Explicit "nous" override → uses strict backend, NOT main model path 396 assert provider == "nous" 397 mock_strict.assert_called_once_with("nous", None) 398 399 400 # ── Constant cleanup ──────────────────────────────────────────────────────── 401 402 403 def test_aggregator_providers_constant_removed(): 404 """The dead _AGGREGATOR_PROVIDERS constant should no longer live in the module. 405 406 Removed when the main-first policy made the aggregator-skip guard obsolete. 407 """ 408 import agent.auxiliary_client as aux_mod 409 410 assert not hasattr(aux_mod, "_AGGREGATOR_PROVIDERS"), ( 411 "_AGGREGATOR_PROVIDERS was removed when _resolve_auto stopped " 412 "treating aggregators specially. If you re-added it, the main-first " 413 "policy may have regressed." 414 )