test_auxiliary_config_bridge.py
1 """Tests for auxiliary model config bridging — verifies that config.yaml values 2 are properly mapped to environment variables by both CLI and gateway loaders. 3 4 Also tests the vision_tools and browser_tool model override env vars. 5 """ 6 7 import json 8 import os 9 import sys 10 from pathlib import Path 11 from unittest.mock import patch, MagicMock 12 13 import pytest 14 import yaml 15 16 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) 17 18 19 def _run_auxiliary_bridge(config_dict, monkeypatch): 20 """Simulate the auxiliary config → env var bridging logic shared by CLI and gateway. 21 22 This mirrors the code in cli.py load_cli_config() and gateway/run.py. 23 Both use the same pattern; we test it once here. 24 """ 25 # Clear env vars 26 for key in ( 27 "AUXILIARY_VISION_PROVIDER", "AUXILIARY_VISION_MODEL", 28 "AUXILIARY_VISION_BASE_URL", "AUXILIARY_VISION_API_KEY", 29 "AUXILIARY_WEB_EXTRACT_PROVIDER", "AUXILIARY_WEB_EXTRACT_MODEL", 30 "AUXILIARY_WEB_EXTRACT_BASE_URL", "AUXILIARY_WEB_EXTRACT_API_KEY", 31 ): 32 monkeypatch.delenv(key, raising=False) 33 34 # Compression config is read directly from config.yaml — no env var bridging. 35 36 # Auxiliary bridge 37 auxiliary_cfg = config_dict.get("auxiliary", {}) 38 if auxiliary_cfg and isinstance(auxiliary_cfg, dict): 39 aux_task_env = { 40 "vision": { 41 "provider": "AUXILIARY_VISION_PROVIDER", 42 "model": "AUXILIARY_VISION_MODEL", 43 "base_url": "AUXILIARY_VISION_BASE_URL", 44 "api_key": "AUXILIARY_VISION_API_KEY", 45 }, 46 "web_extract": { 47 "provider": "AUXILIARY_WEB_EXTRACT_PROVIDER", 48 "model": "AUXILIARY_WEB_EXTRACT_MODEL", 49 "base_url": "AUXILIARY_WEB_EXTRACT_BASE_URL", 50 "api_key": "AUXILIARY_WEB_EXTRACT_API_KEY", 51 }, 52 } 53 for task_key, env_map in aux_task_env.items(): 54 task_cfg = auxiliary_cfg.get(task_key, {}) 55 if not isinstance(task_cfg, dict): 56 continue 57 prov = str(task_cfg.get("provider", "")).strip() 58 model = str(task_cfg.get("model", "")).strip() 59 base_url = str(task_cfg.get("base_url", "")).strip() 60 api_key = str(task_cfg.get("api_key", "")).strip() 61 if prov and prov != "auto": 62 os.environ[env_map["provider"]] = prov 63 if model: 64 os.environ[env_map["model"]] = model 65 if base_url: 66 os.environ[env_map["base_url"]] = base_url 67 if api_key: 68 os.environ[env_map["api_key"]] = api_key 69 70 71 # ── Config bridging tests ──────────────────────────────────────────────────── 72 73 74 class TestAuxiliaryConfigBridge: 75 """Verify the config.yaml → env var bridging logic used by CLI and gateway.""" 76 77 def test_vision_provider_bridged(self, monkeypatch): 78 config = { 79 "auxiliary": { 80 "vision": {"provider": "openrouter", "model": ""}, 81 "web_extract": {"provider": "auto", "model": ""}, 82 } 83 } 84 _run_auxiliary_bridge(config, monkeypatch) 85 assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter" 86 # auto should not be set 87 assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None 88 89 def test_vision_model_bridged(self, monkeypatch): 90 config = { 91 "auxiliary": { 92 "vision": {"provider": "auto", "model": "openai/gpt-4o"}, 93 } 94 } 95 _run_auxiliary_bridge(config, monkeypatch) 96 assert os.environ.get("AUXILIARY_VISION_MODEL") == "openai/gpt-4o" 97 # auto provider should not be set 98 assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None 99 100 def test_web_extract_bridged(self, monkeypatch): 101 config = { 102 "auxiliary": { 103 "web_extract": {"provider": "nous", "model": "gemini-2.5-flash"}, 104 } 105 } 106 _run_auxiliary_bridge(config, monkeypatch) 107 assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") == "nous" 108 assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "gemini-2.5-flash" 109 110 def test_direct_endpoint_bridged(self, monkeypatch): 111 config = { 112 "auxiliary": { 113 "vision": { 114 "base_url": "http://localhost:1234/v1", 115 "api_key": "local-key", 116 "model": "qwen2.5-vl", 117 } 118 } 119 } 120 _run_auxiliary_bridge(config, monkeypatch) 121 assert os.environ.get("AUXILIARY_VISION_BASE_URL") == "http://localhost:1234/v1" 122 assert os.environ.get("AUXILIARY_VISION_API_KEY") == "local-key" 123 assert os.environ.get("AUXILIARY_VISION_MODEL") == "qwen2.5-vl" 124 125 def test_empty_values_not_bridged(self, monkeypatch): 126 config = { 127 "auxiliary": { 128 "vision": {"provider": "auto", "model": ""}, 129 } 130 } 131 _run_auxiliary_bridge(config, monkeypatch) 132 assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None 133 assert os.environ.get("AUXILIARY_VISION_MODEL") is None 134 135 def test_missing_auxiliary_section_safe(self, monkeypatch): 136 """Config without auxiliary section should not crash.""" 137 config = {"model": {"default": "test-model"}} 138 _run_auxiliary_bridge(config, monkeypatch) 139 assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None 140 141 def test_non_dict_task_config_ignored(self, monkeypatch): 142 """Malformed task config (e.g. string instead of dict) is safely ignored.""" 143 config = { 144 "auxiliary": { 145 "vision": "openrouter", # should be a dict 146 } 147 } 148 _run_auxiliary_bridge(config, monkeypatch) 149 assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None 150 151 def test_mixed_tasks(self, monkeypatch): 152 config = { 153 "auxiliary": { 154 "vision": {"provider": "openrouter", "model": ""}, 155 "web_extract": {"provider": "auto", "model": "custom-llm"}, 156 } 157 } 158 _run_auxiliary_bridge(config, monkeypatch) 159 assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter" 160 assert os.environ.get("AUXILIARY_VISION_MODEL") is None 161 assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None 162 assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "custom-llm" 163 164 def test_all_tasks_with_overrides(self, monkeypatch): 165 config = { 166 "auxiliary": { 167 "vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"}, 168 "web_extract": {"provider": "nous", "model": "gemini-3-flash"}, 169 } 170 } 171 _run_auxiliary_bridge(config, monkeypatch) 172 assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter" 173 assert os.environ.get("AUXILIARY_VISION_MODEL") == "google/gemini-2.5-flash" 174 assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") == "nous" 175 assert os.environ.get("AUXILIARY_WEB_EXTRACT_MODEL") == "gemini-3-flash" 176 177 def test_whitespace_in_values_stripped(self, monkeypatch): 178 config = { 179 "auxiliary": { 180 "vision": {"provider": " openrouter ", "model": " my-model "}, 181 } 182 } 183 _run_auxiliary_bridge(config, monkeypatch) 184 assert os.environ.get("AUXILIARY_VISION_PROVIDER") == "openrouter" 185 assert os.environ.get("AUXILIARY_VISION_MODEL") == "my-model" 186 187 def test_empty_auxiliary_dict_safe(self, monkeypatch): 188 config = {"auxiliary": {}} 189 _run_auxiliary_bridge(config, monkeypatch) 190 assert os.environ.get("AUXILIARY_VISION_PROVIDER") is None 191 assert os.environ.get("AUXILIARY_WEB_EXTRACT_PROVIDER") is None 192 193 194 # ── Gateway bridge parity test ─────────────────────────────────────────────── 195 196 197 class TestGatewayBridgeCodeParity: 198 """Verify the gateway/run.py config bridge contains the auxiliary section.""" 199 200 def test_gateway_has_auxiliary_bridge(self): 201 """The gateway config bridge must include auxiliary.* bridging.""" 202 gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py" 203 content = gateway_path.read_text() 204 # Check for key patterns that indicate the bridge is present 205 assert "AUXILIARY_VISION_PROVIDER" in content 206 assert "AUXILIARY_VISION_MODEL" in content 207 assert "AUXILIARY_VISION_BASE_URL" in content 208 assert "AUXILIARY_VISION_API_KEY" in content 209 assert "AUXILIARY_WEB_EXTRACT_PROVIDER" in content 210 assert "AUXILIARY_WEB_EXTRACT_MODEL" in content 211 assert "AUXILIARY_WEB_EXTRACT_BASE_URL" in content 212 assert "AUXILIARY_WEB_EXTRACT_API_KEY" in content 213 214 def test_gateway_no_compression_env_bridge(self): 215 """Gateway should NOT bridge compression config to env vars (config-only).""" 216 gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py" 217 content = gateway_path.read_text() 218 assert "CONTEXT_COMPRESSION_PROVIDER" not in content 219 assert "CONTEXT_COMPRESSION_MODEL" not in content 220 221 222 # ── Vision model override tests ────────────────────────────────────────────── 223 224 225 class TestVisionModelOverride: 226 """Test that AUXILIARY_VISION_MODEL env var overrides the default model in the handler.""" 227 228 def test_env_var_overrides_default(self, monkeypatch): 229 monkeypatch.setenv("AUXILIARY_VISION_MODEL", "openai/gpt-4o") 230 from tools.vision_tools import _handle_vision_analyze 231 with patch("tools.vision_tools.vision_analyze_tool", new_callable=MagicMock) as mock_tool: 232 mock_tool.return_value = '{"success": true}' 233 _handle_vision_analyze({"image_url": "http://test.jpg", "question": "test"}) 234 call_args = mock_tool.call_args 235 # 3rd positional arg = model 236 assert call_args[0][2] == "openai/gpt-4o" 237 238 def test_default_model_when_no_override(self, monkeypatch): 239 monkeypatch.delenv("AUXILIARY_VISION_MODEL", raising=False) 240 from tools.vision_tools import _handle_vision_analyze 241 with patch("tools.vision_tools.vision_analyze_tool", new_callable=MagicMock) as mock_tool: 242 mock_tool.return_value = '{"success": true}' 243 _handle_vision_analyze({"image_url": "http://test.jpg", "question": "test"}) 244 call_args = mock_tool.call_args 245 # With no AUXILIARY_VISION_MODEL env var, model should be None 246 # (the centralized call_llm router picks the provider default) 247 assert call_args[0][2] is None 248 249 250 # ── DEFAULT_CONFIG shape tests ─────────────────────────────────────────────── 251 252 253 class TestDefaultConfigShape: 254 """Verify the DEFAULT_CONFIG in hermes_cli/config.py has correct auxiliary structure.""" 255 256 def test_auxiliary_section_exists(self): 257 from hermes_cli.config import DEFAULT_CONFIG 258 assert "auxiliary" in DEFAULT_CONFIG 259 260 def test_vision_task_structure(self): 261 from hermes_cli.config import DEFAULT_CONFIG 262 vision = DEFAULT_CONFIG["auxiliary"]["vision"] 263 assert "provider" in vision 264 assert "model" in vision 265 assert vision["provider"] == "auto" 266 assert vision["model"] == "" 267 268 def test_web_extract_task_structure(self): 269 from hermes_cli.config import DEFAULT_CONFIG 270 web = DEFAULT_CONFIG["auxiliary"]["web_extract"] 271 assert "provider" in web 272 assert "model" in web 273 assert web["provider"] == "auto" 274 assert web["model"] == "" 275 276 277 # ── CLI defaults parity ───────────────────────────────────────────────────── 278 279 280 class TestCLIDefaultsHaveAuxiliaryKeys: 281 """Verify cli.py load_cli_config() defaults dict does NOT include auxiliary 282 (it comes from config.yaml deep merge, not hardcoded defaults).""" 283 284 def test_cli_defaults_can_merge_auxiliary(self): 285 """The load_cli_config deep merge logic handles keys not in defaults. 286 Verify auxiliary would be picked up from config.yaml.""" 287 # This is a structural assertion: cli.py's second-pass loop 288 # carries over keys from file_config that aren't in defaults. 289 # So auxiliary config from config.yaml gets merged even though 290 # cli.py's defaults dict doesn't define it. 291 import cli as _cli_mod 292 source = Path(_cli_mod.__file__).read_text() 293 assert "auxiliary_config = defaults.get(\"auxiliary\"" in source 294 assert "AUXILIARY_VISION_PROVIDER" in source 295 assert "AUXILIARY_VISION_MODEL" in source