test_gemini_provider.py
1 """Tests for Google AI Studio (Gemini) provider integration.""" 2 3 import os 4 import pytest 5 from unittest.mock import patch, MagicMock 6 7 from hermes_cli.auth import PROVIDER_REGISTRY, resolve_provider, resolve_api_key_provider_credentials 8 from hermes_cli.models import _PROVIDER_MODELS, _PROVIDER_LABELS, _PROVIDER_ALIASES, normalize_provider 9 from hermes_cli.model_normalize import normalize_model_for_provider, detect_vendor 10 from agent.model_metadata import get_model_context_length 11 from agent.models_dev import PROVIDER_TO_MODELS_DEV, list_agentic_models, _NOISE_PATTERNS 12 13 14 # ── Provider Registry ── 15 16 class TestGeminiProviderRegistry: 17 def test_gemini_in_registry(self): 18 assert "gemini" in PROVIDER_REGISTRY 19 20 def test_gemini_config(self): 21 pconfig = PROVIDER_REGISTRY["gemini"] 22 assert pconfig.id == "gemini" 23 assert pconfig.name == "Google AI Studio" 24 assert pconfig.auth_type == "api_key" 25 assert pconfig.inference_base_url == "https://generativelanguage.googleapis.com/v1beta" 26 27 def test_gemini_env_vars(self): 28 pconfig = PROVIDER_REGISTRY["gemini"] 29 assert pconfig.api_key_env_vars == ("GOOGLE_API_KEY", "GEMINI_API_KEY") 30 assert pconfig.base_url_env_var == "GEMINI_BASE_URL" 31 32 def test_gemini_base_url(self): 33 assert "generativelanguage.googleapis.com" in PROVIDER_REGISTRY["gemini"].inference_base_url 34 35 36 # ── Provider Aliases ── 37 38 PROVIDER_ENV_VARS = ( 39 "OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", 40 "GOOGLE_API_KEY", "GEMINI_API_KEY", "GEMINI_BASE_URL", 41 "GLM_API_KEY", "ZAI_API_KEY", "KIMI_API_KEY", 42 "MINIMAX_API_KEY", "DEEPSEEK_API_KEY", 43 ) 44 45 @pytest.fixture(autouse=True) 46 def _clean_provider_env(monkeypatch): 47 for var in PROVIDER_ENV_VARS: 48 monkeypatch.delenv(var, raising=False) 49 50 51 class TestGeminiAliases: 52 def test_explicit_gemini(self): 53 assert resolve_provider("gemini") == "gemini" 54 55 def test_alias_google(self): 56 assert resolve_provider("google") == "gemini" 57 58 def test_alias_google_gemini(self): 59 assert resolve_provider("google-gemini") == "gemini" 60 61 def test_alias_google_ai_studio(self): 62 assert resolve_provider("google-ai-studio") == "gemini" 63 64 def test_models_py_aliases(self): 65 assert _PROVIDER_ALIASES.get("google") == "gemini" 66 assert _PROVIDER_ALIASES.get("google-gemini") == "gemini" 67 assert _PROVIDER_ALIASES.get("google-ai-studio") == "gemini" 68 69 def test_normalize_provider(self): 70 assert normalize_provider("google") == "gemini" 71 assert normalize_provider("gemini") == "gemini" 72 assert normalize_provider("google-ai-studio") == "gemini" 73 74 75 # ── Auto-detection ── 76 77 class TestGeminiAutoDetection: 78 def test_auto_detects_google_api_key(self, monkeypatch): 79 monkeypatch.setenv("GOOGLE_API_KEY", "test-google-key") 80 assert resolve_provider("auto") == "gemini" 81 82 def test_auto_detects_gemini_api_key(self, monkeypatch): 83 monkeypatch.setenv("GEMINI_API_KEY", "test-gemini-key") 84 assert resolve_provider("auto") == "gemini" 85 86 def test_google_api_key_priority_over_gemini(self, monkeypatch): 87 monkeypatch.setenv("GOOGLE_API_KEY", "primary-key") 88 monkeypatch.setenv("GEMINI_API_KEY", "alias-key") 89 creds = resolve_api_key_provider_credentials("gemini") 90 assert creds["api_key"] == "primary-key" 91 assert creds["source"] == "GOOGLE_API_KEY" 92 93 94 # ── Credential Resolution ── 95 96 class TestGeminiCredentials: 97 def test_resolve_with_google_api_key(self, monkeypatch): 98 monkeypatch.setenv("GOOGLE_API_KEY", "google-secret") 99 creds = resolve_api_key_provider_credentials("gemini") 100 assert creds["provider"] == "gemini" 101 assert creds["api_key"] == "google-secret" 102 assert creds["base_url"] == "https://generativelanguage.googleapis.com/v1beta" 103 104 def test_resolve_with_gemini_api_key(self, monkeypatch): 105 monkeypatch.setenv("GEMINI_API_KEY", "gemini-secret") 106 creds = resolve_api_key_provider_credentials("gemini") 107 assert creds["api_key"] == "gemini-secret" 108 109 def test_resolve_with_custom_base_url(self, monkeypatch): 110 monkeypatch.setenv("GOOGLE_API_KEY", "key") 111 monkeypatch.setenv("GEMINI_BASE_URL", "https://custom.endpoint/v1") 112 creds = resolve_api_key_provider_credentials("gemini") 113 assert creds["base_url"] == "https://custom.endpoint/v1" 114 115 def test_runtime_gemini(self, monkeypatch): 116 monkeypatch.setenv("GOOGLE_API_KEY", "google-key") 117 from hermes_cli.runtime_provider import resolve_runtime_provider 118 result = resolve_runtime_provider(requested="gemini") 119 assert result["provider"] == "gemini" 120 assert result["api_mode"] == "chat_completions" 121 assert result["api_key"] == "google-key" 122 assert result["base_url"] == "https://generativelanguage.googleapis.com/v1beta" 123 124 125 # ── Model Catalog ── 126 127 class TestGeminiModelCatalog: 128 def test_provider_entry_exists(self): 129 """Gemini provider has a model catalog entry. Specific model names 130 are data that changes with Google releases and don't belong in tests. 131 """ 132 assert "gemini" in _PROVIDER_MODELS 133 assert len(_PROVIDER_MODELS["gemini"]) >= 1 134 135 def test_provider_label(self): 136 assert "gemini" in _PROVIDER_LABELS 137 assert _PROVIDER_LABELS["gemini"] == "Google AI Studio" 138 139 140 # ── Model Normalization ── 141 142 class TestGeminiModelNormalization: 143 def test_passthrough_bare_name(self): 144 assert normalize_model_for_provider("gemini-2.5-flash", "gemini") == "gemini-2.5-flash" 145 146 def test_strip_vendor_prefix(self): 147 assert normalize_model_for_provider("google/gemini-2.5-flash", "gemini") == "google/gemini-2.5-flash" 148 149 def test_gemma_vendor_detection(self): 150 assert detect_vendor("gemma-4-31b-it") == "google" 151 152 def test_gemini_vendor_detection(self): 153 assert detect_vendor("gemini-2.5-flash") == "google" 154 155 def test_aggregator_prepends_vendor(self): 156 result = normalize_model_for_provider("gemini-2.5-flash", "openrouter") 157 assert result == "google/gemini-2.5-flash" 158 159 def test_gemma_aggregator_prepends_vendor(self): 160 result = normalize_model_for_provider("gemma-4-31b-it", "openrouter") 161 assert result == "google/gemma-4-31b-it" 162 163 164 # ── Context Length ── 165 166 class TestGeminiContextLength: 167 def test_gemma_4_31b_context(self): 168 # Mock external API lookups to test against hardcoded defaults 169 # (models.dev and OpenRouter may return different values like 262144). 170 with patch("agent.models_dev.lookup_models_dev_context", return_value=None), \ 171 patch("agent.model_metadata.fetch_model_metadata", return_value={}): 172 ctx = get_model_context_length("gemma-4-31b-it", provider="gemini") 173 assert ctx == 256000 174 175 def test_gemini_3_context(self): 176 ctx = get_model_context_length("gemini-3.1-pro-preview", provider="gemini") 177 assert ctx == 1048576 178 179 180 # ── Agent Init (no SyntaxError) ── 181 182 class TestGeminiAgentInit: 183 def test_agent_imports_without_error(self): 184 """Verify run_agent.py has no SyntaxError (the critical bug).""" 185 import importlib 186 import run_agent 187 importlib.reload(run_agent) 188 189 def test_gemini_agent_uses_chat_completions(self, monkeypatch): 190 """Gemini still reports chat_completions even though the transport is native.""" 191 monkeypatch.setenv("GOOGLE_API_KEY", "test-key") 192 with patch("agent.gemini_native_adapter.GeminiNativeClient") as mock_client: 193 mock_client.return_value = MagicMock() 194 from run_agent import AIAgent 195 agent = AIAgent( 196 model="gemini-2.5-flash", 197 provider="gemini", 198 api_key="test-key", 199 base_url="https://generativelanguage.googleapis.com/v1beta", 200 ) 201 assert agent.api_mode == "chat_completions" 202 assert agent.provider == "gemini" 203 204 def test_gemini_agent_uses_native_client(self, monkeypatch): 205 monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_REAL_KEY") 206 with patch("agent.gemini_native_adapter.GeminiNativeClient") as mock_client, \ 207 patch("run_agent.OpenAI") as mock_openai, \ 208 patch("run_agent.ContextCompressor") as mock_compressor: 209 mock_client.return_value = MagicMock() 210 mock_compressor.return_value = MagicMock(context_length=1048576, threshold_tokens=524288) 211 from run_agent import AIAgent 212 AIAgent( 213 model="gemini-2.5-flash", 214 provider="gemini", 215 api_key="AIzaSy_REAL_KEY", 216 base_url="https://generativelanguage.googleapis.com/v1beta", 217 ) 218 assert mock_client.called 219 mock_openai.assert_not_called() 220 221 def test_gemini_custom_base_url_keeps_openai_client(self, monkeypatch): 222 monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_REAL_KEY") 223 with patch("agent.gemini_native_adapter.GeminiNativeClient") as mock_client, \ 224 patch("run_agent.OpenAI") as mock_openai, \ 225 patch("run_agent.ContextCompressor") as mock_compressor: 226 mock_openai.return_value = MagicMock() 227 mock_compressor.return_value = MagicMock(context_length=128000, threshold_tokens=64000) 228 from run_agent import AIAgent 229 AIAgent( 230 model="gemini-2.5-flash", 231 provider="gemini", 232 api_key="AIzaSy_REAL_KEY", 233 base_url="https://proxy.example.com/v1", 234 ) 235 mock_openai.assert_called_once() 236 237 def test_gemini_openai_compat_base_url_keeps_openai_client(self, monkeypatch): 238 monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_REAL_KEY") 239 with patch("agent.gemini_native_adapter.GeminiNativeClient") as mock_client, \ 240 patch("run_agent.OpenAI") as mock_openai, \ 241 patch("run_agent.ContextCompressor") as mock_compressor: 242 mock_openai.return_value = MagicMock() 243 mock_compressor.return_value = MagicMock(context_length=1048576, threshold_tokens=524288) 244 from run_agent import AIAgent 245 AIAgent( 246 model="gemini-2.5-flash", 247 provider="gemini", 248 api_key="AIzaSy_REAL_KEY", 249 base_url="https://generativelanguage.googleapis.com/v1beta/openai", 250 ) 251 mock_openai.assert_called_once() 252 253 def test_gemini_resolve_provider_client_uses_native_client(self, monkeypatch): 254 """resolve_provider_client('gemini') should build GeminiNativeClient.""" 255 monkeypatch.setenv("GEMINI_API_KEY", "AIzaSy_TEST_KEY") 256 with patch("agent.gemini_native_adapter.GeminiNativeClient") as mock_client, \ 257 patch("agent.auxiliary_client.OpenAI") as mock_openai: 258 mock_client.return_value = MagicMock() 259 from agent.auxiliary_client import resolve_provider_client 260 resolve_provider_client("gemini") 261 assert mock_client.called 262 mock_openai.assert_not_called() 263 264 def test_gemini_resolve_provider_client_keeps_openai_for_non_native_base_url(self, monkeypatch): 265 monkeypatch.setenv("GOOGLE_API_KEY", "AIzaSy_TEST_KEY") 266 monkeypatch.setenv("GEMINI_BASE_URL", "https://proxy.example.com/v1") 267 with patch("agent.gemini_native_adapter.GeminiNativeClient") as mock_client, \ 268 patch("agent.auxiliary_client.OpenAI") as mock_openai: 269 mock_openai.return_value = MagicMock() 270 from agent.auxiliary_client import resolve_provider_client 271 resolve_provider_client("gemini") 272 mock_openai.assert_called_once() 273 274 275 # ── models.dev Integration ── 276 277 class TestGeminiModelsDev: 278 def test_gemini_mapped_to_google(self): 279 assert PROVIDER_TO_MODELS_DEV.get("gemini") == "google" 280 281 def test_noise_filter_excludes_tts(self): 282 assert _NOISE_PATTERNS.search("gemini-2.5-pro-preview-tts") 283 284 def test_noise_filter_excludes_dated_preview(self): 285 assert _NOISE_PATTERNS.search("gemini-2.5-flash-preview-04-17") 286 287 def test_noise_filter_excludes_embedding(self): 288 assert _NOISE_PATTERNS.search("gemini-embedding-001") 289 290 def test_noise_filter_excludes_live(self): 291 assert _NOISE_PATTERNS.search("gemini-live-2.5-flash") 292 293 def test_noise_filter_excludes_image(self): 294 assert _NOISE_PATTERNS.search("gemini-2.5-flash-image") 295 296 def test_noise_filter_excludes_customtools(self): 297 assert _NOISE_PATTERNS.search("gemini-3.1-pro-preview-customtools") 298 299 def test_noise_filter_passes_stable(self): 300 assert not _NOISE_PATTERNS.search("gemini-2.5-flash") 301 302 def test_noise_filter_passes_preview(self): 303 # Non-dated preview (e.g. gemini-3-flash-preview) should pass 304 assert not _NOISE_PATTERNS.search("gemini-3-flash-preview") 305 306 def test_noise_filter_passes_gemma(self): 307 assert not _NOISE_PATTERNS.search("gemma-4-31b-it") 308 309 def test_list_agentic_models_with_mock_data(self): 310 """list_agentic_models filters correctly from mock models.dev data.""" 311 mock_data = { 312 "google": { 313 "models": { 314 "gemini-3-flash-preview": {"tool_call": True}, 315 "gemini-2.5-pro": {"tool_call": True}, 316 "gemini-embedding-001": {"tool_call": False}, 317 "gemini-2.5-flash-preview-tts": {"tool_call": False}, 318 "gemini-live-2.5-flash": {"tool_call": True}, 319 "gemini-2.5-flash-preview-04-17": {"tool_call": True}, 320 "gemma-4-31b-it": {"tool_call": True}, 321 } 322 } 323 } 324 with patch("agent.models_dev.fetch_models_dev", return_value=mock_data): 325 result = list_agentic_models("gemini") 326 assert "gemini-3-flash-preview" in result 327 assert "gemini-2.5-pro" in result 328 assert "gemma-4-31b-it" not in result 329 # Filtered out: 330 assert "gemini-embedding-001" not in result # no tool_call 331 assert "gemini-2.5-flash-preview-tts" not in result # no tool_call 332 assert "gemini-live-2.5-flash" not in result # noise: live- 333 assert "gemini-2.5-flash-preview-04-17" not in result # noise: dated preview 334 335 def test_list_provider_models_hides_low_tpm_google_gemmas(self): 336 mock_data = { 337 "google": { 338 "models": { 339 "gemini-2.5-pro": {}, 340 "gemma-4-31b-it": {}, 341 "gemma-3-27b-it": {}, 342 "gemini-1.5-pro": {}, 343 "gemini-2.0-flash": {}, 344 } 345 } 346 } 347 with patch("agent.models_dev.fetch_models_dev", return_value=mock_data): 348 from agent.models_dev import list_provider_models 349 350 result = list_provider_models("gemini") 351 352 assert "gemini-2.5-pro" in result 353 assert "gemma-4-31b-it" not in result 354 assert "gemma-3-27b-it" not in result 355 assert "gemini-1.5-pro" not in result 356 assert "gemini-2.0-flash" not in result