test_eval_cmd.py
1 """ 2 Unit tests for cli/commands/eval_cmd.py 3 4 Tests the eval command including: 5 - Running evaluation with scenario file 6 - Verbose output flag 7 - Logging configuration setup 8 - Error handling (missing scenario file, invalid format) 9 - Result output formatting 10 - Exception handling during evaluation 11 """ 12 13 import os 14 from pathlib import Path 15 16 import pytest 17 from click.testing import CliRunner 18 19 from cli.commands.eval_cmd import eval_cmd 20 21 22 @pytest.fixture 23 def runner(): 24 """Create a Click CLI runner for testing""" 25 return CliRunner() 26 27 28 @pytest.fixture 29 def test_config_file(tmp_path): 30 """Create a temporary test suite config file""" 31 config_file = tmp_path / "test_suite.yaml" 32 config_file.write_text(""" 33 test_suite: 34 name: "Test Suite" 35 scenarios: 36 - name: "Test Scenario" 37 input: "test input" 38 """) 39 return config_file 40 41 42 @pytest.fixture 43 def logging_config_file(tmp_path): 44 """Create a temporary logging config file""" 45 configs_dir = tmp_path / "configs" 46 configs_dir.mkdir() 47 logging_config = configs_dir / "logging_config.yaml" 48 logging_config.write_text(""" 49 [loggers] 50 keys=root 51 52 [handlers] 53 keys=console 54 55 [formatters] 56 keys=simple 57 """) 58 return logging_config 59 60 61 class TestEvalCommand: 62 """Tests for the eval CLI command""" 63 64 def test_eval_command_basic(self, runner, test_config_file, mocker): 65 """Test basic eval command with valid config file""" 66 mock_run_eval = mocker.patch("evaluation.run.main") 67 68 result = runner.invoke(eval_cmd, [str(test_config_file)]) 69 70 assert result.exit_code == 0 71 assert "Starting evaluation" in result.output 72 assert str(test_config_file) in result.output 73 assert "Evaluation completed successfully" in result.output 74 mock_run_eval.assert_called_once_with(str(test_config_file), verbose=False) 75 76 def test_eval_command_with_verbose(self, runner, test_config_file, mocker): 77 """Test eval command with verbose flag""" 78 mock_run_eval = mocker.patch("evaluation.run.main") 79 80 result = runner.invoke(eval_cmd, [str(test_config_file), "--verbose"]) 81 82 assert result.exit_code == 0 83 assert "Starting evaluation" in result.output 84 mock_run_eval.assert_called_once_with(str(test_config_file), verbose=True) 85 86 def test_eval_command_with_verbose_short_flag(self, runner, test_config_file, mocker): 87 """Test eval command with -v short flag""" 88 mock_run_eval = mocker.patch("evaluation.run.main") 89 90 result = runner.invoke(eval_cmd, [str(test_config_file), "-v"]) 91 92 assert result.exit_code == 0 93 mock_run_eval.assert_called_once_with(str(test_config_file), verbose=True) 94 95 def test_eval_command_missing_file(self, runner, tmp_path): 96 """Test eval command with non-existent config file""" 97 non_existent = tmp_path / "nonexistent.yaml" 98 99 result = runner.invoke(eval_cmd, [str(non_existent)]) 100 101 assert result.exit_code != 0 102 assert "does not exist" in result.output or "Error" in result.output 103 104 def test_eval_command_with_logging_config(self, runner, test_config_file, logging_config_file, mocker): 105 """Test eval command sets logging config path when it exists""" 106 mock_run_eval = mocker.patch("evaluation.run.main") 107 108 # Change to directory with logging config 109 original_cwd = Path.cwd() 110 os.chdir(logging_config_file.parent.parent) 111 112 try: 113 result = runner.invoke(eval_cmd, [str(test_config_file)]) 114 115 assert result.exit_code == 0 116 # Verify LOGGING_CONFIG_PATH was set 117 assert "LOGGING_CONFIG_PATH" in os.environ or mock_run_eval.called 118 mock_run_eval.assert_called_once() 119 finally: 120 os.chdir(original_cwd) 121 # Clean up environment 122 if "LOGGING_CONFIG_PATH" in os.environ: 123 del os.environ["LOGGING_CONFIG_PATH"] 124 125 def test_eval_command_without_logging_config(self, runner, test_config_file, tmp_path, mocker): 126 """Test eval command when logging config doesn't exist""" 127 mock_run_eval = mocker.patch("evaluation.run.main") 128 129 # Change to directory without logging config 130 original_cwd = Path.cwd() 131 os.chdir(tmp_path) 132 133 try: 134 result = runner.invoke(eval_cmd, [str(test_config_file)]) 135 136 assert result.exit_code == 0 137 # Should still work without logging config 138 mock_run_eval.assert_called_once() 139 finally: 140 os.chdir(original_cwd) 141 142 def test_eval_command_evaluation_exception(self, runner, test_config_file, mocker): 143 """Test eval command handles evaluation exceptions""" 144 mock_run_eval = mocker.patch( 145 "evaluation.run.main", 146 side_effect=Exception("Evaluation failed") 147 ) 148 mock_error_exit = mocker.patch("cli.commands.eval_cmd.error_exit", side_effect=SystemExit(1)) 149 150 result = runner.invoke(eval_cmd, [str(test_config_file)]) 151 152 assert result.exit_code == 1 153 mock_error_exit.assert_called_once() 154 assert "An error occurred during evaluation" in mock_error_exit.call_args[0][0] 155 assert "Evaluation failed" in mock_error_exit.call_args[0][0] 156 157 def test_eval_command_runtime_error(self, runner, test_config_file, mocker): 158 """Test eval command handles runtime errors""" 159 mock_run_eval = mocker.patch( 160 "evaluation.run.main", 161 side_effect=RuntimeError("Runtime error occurred") 162 ) 163 mock_error_exit = mocker.patch("cli.commands.eval_cmd.error_exit", side_effect=SystemExit(1)) 164 165 result = runner.invoke(eval_cmd, [str(test_config_file)]) 166 167 assert result.exit_code == 1 168 mock_error_exit.assert_called_once() 169 assert "Runtime error occurred" in mock_error_exit.call_args[0][0] 170 171 def test_eval_command_value_error(self, runner, test_config_file, mocker): 172 """Test eval command handles value errors""" 173 mock_run_eval = mocker.patch( 174 "evaluation.run.main", 175 side_effect=ValueError("Invalid configuration") 176 ) 177 mock_error_exit = mocker.patch("cli.commands.eval_cmd.error_exit", side_effect=SystemExit(1)) 178 179 result = runner.invoke(eval_cmd, [str(test_config_file)]) 180 181 assert result.exit_code == 1 182 mock_error_exit.assert_called_once() 183 assert "Invalid configuration" in mock_error_exit.call_args[0][0] 184 185 def test_eval_command_file_not_found_error(self, runner, test_config_file, mocker): 186 """Test eval command handles file not found errors""" 187 mock_run_eval = mocker.patch( 188 "evaluation.run.main", 189 side_effect=FileNotFoundError("Config file not found") 190 ) 191 mock_error_exit = mocker.patch("cli.commands.eval_cmd.error_exit", side_effect=SystemExit(1)) 192 193 result = runner.invoke(eval_cmd, [str(test_config_file)]) 194 195 assert result.exit_code == 1 196 mock_error_exit.assert_called_once() 197 198 def test_eval_command_output_formatting(self, runner, test_config_file, mocker): 199 """Test eval command output formatting""" 200 mock_run_eval = mocker.patch("evaluation.run.main") 201 202 result = runner.invoke(eval_cmd, [str(test_config_file)]) 203 204 # Check for styled output 205 assert "Starting evaluation" in result.output 206 assert "Evaluation completed successfully" in result.output 207 # Verify the config path is displayed 208 assert str(test_config_file) in result.output 209 210 def test_eval_command_requires_path_argument(self, runner): 211 """Test eval command requires test suite config path""" 212 result = runner.invoke(eval_cmd, []) 213 214 assert result.exit_code != 0 215 assert "Missing argument" in result.output or "Error" in result.output 216 217 def test_eval_command_path_must_be_file(self, runner, tmp_path): 218 """Test eval command requires path to be a file, not directory""" 219 directory = tmp_path / "test_dir" 220 directory.mkdir() 221 222 result = runner.invoke(eval_cmd, [str(directory)]) 223 224 assert result.exit_code != 0 225 # Click should reject directory when dir_okay=False 226 227 def test_eval_command_resolves_path(self, runner, test_config_file, mocker): 228 """Test eval command resolves relative paths""" 229 mock_run_eval = mocker.patch("evaluation.run.main") 230 231 # Use relative path 232 original_cwd = Path.cwd() 233 os.chdir(test_config_file.parent) 234 235 try: 236 result = runner.invoke(eval_cmd, [test_config_file.name]) 237 238 assert result.exit_code == 0 239 # Should be called with resolved absolute path 240 called_path = mock_run_eval.call_args[0][0] 241 assert Path(called_path).is_absolute() 242 finally: 243 os.chdir(original_cwd) 244 245 def test_eval_command_logging_config_absolute_path(self, runner, test_config_file, logging_config_file, mocker): 246 """Test eval command converts logging config to absolute path""" 247 mock_run_eval = mocker.patch("evaluation.run.main") 248 249 original_cwd = Path.cwd() 250 os.chdir(logging_config_file.parent.parent) 251 252 try: 253 result = runner.invoke(eval_cmd, [str(test_config_file)]) 254 255 assert result.exit_code == 0 256 # If LOGGING_CONFIG_PATH was set, it should be absolute 257 if "LOGGING_CONFIG_PATH" in os.environ: 258 assert Path(os.environ["LOGGING_CONFIG_PATH"]).is_absolute() 259 finally: 260 os.chdir(original_cwd) 261 if "LOGGING_CONFIG_PATH" in os.environ: 262 del os.environ["LOGGING_CONFIG_PATH"] 263 264 def test_eval_command_multiple_verbose_flags(self, runner, test_config_file, mocker): 265 """Test eval command with multiple verbose flags (should still work)""" 266 mock_run_eval = mocker.patch("evaluation.run.main") 267 268 result = runner.invoke(eval_cmd, [str(test_config_file), "-v", "-v"]) 269 270 # Should still work (verbose is a boolean flag) 271 assert result.exit_code == 0 272 mock_run_eval.assert_called_once_with(str(test_config_file), verbose=True) 273 274 def test_eval_command_success_message_color(self, runner, test_config_file, mocker): 275 """Test eval command shows success message in green""" 276 mock_run_eval = mocker.patch("evaluation.run.main") 277 278 result = runner.invoke(eval_cmd, [str(test_config_file)]) 279 280 assert result.exit_code == 0 281 # Success message should be present 282 assert "Evaluation completed successfully" in result.output 283 284 def test_eval_command_starting_message_color(self, runner, test_config_file, mocker): 285 """Test eval command shows starting message in blue""" 286 mock_run_eval = mocker.patch("evaluation.run.main") 287 288 result = runner.invoke(eval_cmd, [str(test_config_file)]) 289 290 assert result.exit_code == 0 291 # Starting message should be present 292 assert "Starting evaluation" in result.output