test_config_validation.py
1 """Tests for config.yaml structure validation (validate_config_structure).""" 2 3 import pytest 4 5 from hermes_cli.config import validate_config_structure, ConfigIssue 6 7 8 class TestCustomProvidersValidation: 9 """custom_providers must be a YAML list, not a dict.""" 10 11 def test_dict_instead_of_list(self): 12 """The exact Discord user scenario — custom_providers as flat dict.""" 13 issues = validate_config_structure({ 14 "custom_providers": { 15 "name": "Generativelanguage.googleapis.com", 16 "base_url": "https://generativelanguage.googleapis.com/v1beta", 17 "api_key": "xxx", 18 "model": "models/gemini-2.5-flash", 19 "rate_limit_delay": 2.0, 20 "fallback_model": { 21 "provider": "openrouter", 22 "model": "qwen/qwen3.6-plus:free", 23 }, 24 }, 25 "fallback_providers": [], 26 }) 27 errors = [i for i in issues if i.severity == "error"] 28 assert any("dict" in i.message and "list" in i.message for i in errors), ( 29 "Should detect custom_providers as dict instead of list" 30 ) 31 32 def test_dict_detects_misplaced_fields(self): 33 """When custom_providers is a dict, detect fields that look misplaced.""" 34 issues = validate_config_structure({ 35 "custom_providers": { 36 "name": "test", 37 "base_url": "https://example.com", 38 "api_key": "xxx", 39 }, 40 }) 41 warnings = [i for i in issues if i.severity == "warning"] 42 # Should flag base_url, api_key as looking like custom_providers entry fields 43 misplaced = [i for i in warnings if "custom_providers entry fields" in i.message] 44 assert len(misplaced) == 1 45 46 def test_dict_detects_nested_fallback(self): 47 """When fallback_model gets swallowed into custom_providers dict.""" 48 issues = validate_config_structure({ 49 "custom_providers": { 50 "name": "test", 51 "fallback_model": {"provider": "openrouter", "model": "test"}, 52 }, 53 }) 54 errors = [i for i in issues if i.severity == "error"] 55 assert any("fallback_model" in i.message and "inside" in i.message for i in errors) 56 57 def test_valid_list_no_issues(self): 58 """Properly formatted custom_providers should produce no issues.""" 59 issues = validate_config_structure({ 60 "custom_providers": [ 61 {"name": "gemini", "base_url": "https://example.com/v1"}, 62 ], 63 "model": {"provider": "custom", "default": "test"}, 64 }) 65 assert len(issues) == 0 66 67 def test_list_entry_missing_name(self): 68 """List entry without name should warn.""" 69 issues = validate_config_structure({ 70 "custom_providers": [{"base_url": "https://example.com/v1"}], 71 "model": {"provider": "custom"}, 72 }) 73 assert any("missing 'name'" in i.message for i in issues) 74 75 def test_list_entry_missing_base_url(self): 76 """List entry without base_url should warn.""" 77 issues = validate_config_structure({ 78 "custom_providers": [{"name": "test"}], 79 "model": {"provider": "custom"}, 80 }) 81 assert any("missing 'base_url'" in i.message for i in issues) 82 83 def test_list_entry_not_dict(self): 84 """Non-dict list entries should warn.""" 85 issues = validate_config_structure({ 86 "custom_providers": ["not-a-dict"], 87 "model": {"provider": "custom"}, 88 }) 89 assert any("not a dict" in i.message for i in issues) 90 91 def test_none_custom_providers_no_issues(self): 92 """No custom_providers at all should be fine.""" 93 issues = validate_config_structure({ 94 "model": {"provider": "openrouter"}, 95 }) 96 assert len(issues) == 0 97 98 99 class TestFallbackModelValidation: 100 """fallback_model should be a top-level dict with provider + model.""" 101 102 def test_missing_provider(self): 103 issues = validate_config_structure({ 104 "fallback_model": {"model": "anthropic/claude-sonnet-4"}, 105 }) 106 assert any("missing 'provider'" in i.message for i in issues) 107 108 def test_missing_model(self): 109 issues = validate_config_structure({ 110 "fallback_model": {"provider": "openrouter"}, 111 }) 112 assert any("missing 'model'" in i.message for i in issues) 113 114 def test_valid_fallback(self): 115 issues = validate_config_structure({ 116 "fallback_model": { 117 "provider": "openrouter", 118 "model": "anthropic/claude-sonnet-4", 119 }, 120 }) 121 # Only fallback-related issues should be absent 122 fb_issues = [i for i in issues if "fallback" in i.message.lower()] 123 assert len(fb_issues) == 0 124 125 def test_non_dict_fallback(self): 126 issues = validate_config_structure({ 127 "fallback_model": "openrouter:anthropic/claude-sonnet-4", 128 }) 129 assert any("should be a dict" in i.message for i in issues) 130 131 def test_empty_fallback_dict_no_issues(self): 132 """Empty fallback_model dict means disabled — no warnings needed.""" 133 issues = validate_config_structure({ 134 "fallback_model": {}, 135 }) 136 fb_issues = [i for i in issues if "fallback" in i.message.lower()] 137 assert len(fb_issues) == 0 138 139 def test_valid_fallback_list(self): 140 """List-form fallback_model (chain) should validate when every entry has provider+model.""" 141 issues = validate_config_structure({ 142 "fallback_model": [ 143 {"provider": "openrouter", "model": "anthropic/claude-sonnet-4"}, 144 {"provider": "anthropic", "model": "claude-sonnet-4-6"}, 145 ], 146 }) 147 fb_issues = [i for i in issues if "fallback" in i.message.lower()] 148 assert len(fb_issues) == 0 149 150 def test_fallback_list_entry_missing_provider(self): 151 issues = validate_config_structure({ 152 "fallback_model": [ 153 {"provider": "openrouter", "model": "anthropic/claude-sonnet-4"}, 154 {"model": "claude-sonnet-4-6"}, 155 ], 156 }) 157 assert any("fallback_model[1]" in i.message and "provider" in i.message for i in issues) 158 159 def test_fallback_list_entry_missing_model(self): 160 issues = validate_config_structure({ 161 "fallback_model": [ 162 {"provider": "openrouter"}, 163 ], 164 }) 165 assert any("fallback_model[0]" in i.message and "model" in i.message for i in issues) 166 167 def test_fallback_list_entry_not_a_dict(self): 168 issues = validate_config_structure({ 169 "fallback_model": ["openrouter:anthropic/claude-sonnet-4"], 170 }) 171 assert any("fallback_model[0]" in i.message and "should be a dict" in i.message for i in issues) 172 173 174 class TestMissingModelSection: 175 """Warn when custom_providers exists but model section is missing.""" 176 177 def test_custom_providers_without_model(self): 178 issues = validate_config_structure({ 179 "custom_providers": [ 180 {"name": "test", "base_url": "https://example.com/v1"}, 181 ], 182 }) 183 assert any("no 'model' section" in i.message for i in issues) 184 185 def test_custom_providers_with_model(self): 186 issues = validate_config_structure({ 187 "custom_providers": [ 188 {"name": "test", "base_url": "https://example.com/v1"}, 189 ], 190 "model": {"provider": "custom", "default": "test-model"}, 191 }) 192 # Should not warn about missing model section 193 assert not any("no 'model' section" in i.message for i in issues) 194 195 196 class TestConfigIssueDataclass: 197 """ConfigIssue should be a proper dataclass.""" 198 199 def test_fields(self): 200 issue = ConfigIssue(severity="error", message="test msg", hint="test hint") 201 assert issue.severity == "error" 202 assert issue.message == "test msg" 203 assert issue.hint == "test hint" 204 205 def test_equality(self): 206 a = ConfigIssue("error", "msg", "hint") 207 b = ConfigIssue("error", "msg", "hint") 208 assert a == b