1  {
 2      "agents": [
 3          "examples/agents/a2a_agents_example.yaml",
 4          "examples/agents/a2a_multimodal_example.yaml",
 5          "examples/agents/orchestrator_example.yaml"
 6      ],
 7      "broker": {
 8          "SOLACE_BROKER_URL_VAR": "SOLACE_BROKER_URL",
 9          "SOLACE_BROKER_USERNAME_VAR": "SOLACE_BROKER_USERNAME",
10          "SOLACE_BROKER_PASSWORD_VAR": "SOLACE_BROKER_PASSWORD",
11          "SOLACE_BROKER_VPN_VAR": "SOLACE_BROKER_VPN"
12      },
13      "llm_models": [
14          {
15              "name": "azure-gpt-4o",
16              "env": {
17                  "LLM_SERVICE_PLANNING_MODEL_NAME": "openai/azure-gpt-4o",
18                  "LLM_SERVICE_ENDPOINT_VAR": "LLM_SERVICE_ENDPOINT",
19                  "LLM_SERVICE_API_KEY_VAR": "LLM_SERVICE_API_KEY"
20              }
21          },
22          {
23              "name": "gemini-3-pro-preview",
24              "env": {
25                  "LLM_SERVICE_PLANNING_MODEL_NAME": "openai/gemini-3-pro-preview",
26                  "LLM_SERVICE_ENDPOINT_VAR": "LLM_SERVICE_ENDPOINT",
27                  "LLM_SERVICE_API_KEY_VAR": "LLM_SERVICE_API_KEY"
28              }
29          },
30          {
31              "name": "gemini-3-flash-preview",
32              "env": {
33                  "LLM_SERVICE_PLANNING_MODEL_NAME": "openai/gemini-3-flash-preview",
34                  "LLM_SERVICE_ENDPOINT_VAR": "LLM_SERVICE_ENDPOINT",
35                  "LLM_SERVICE_API_KEY_VAR": "LLM_SERVICE_API_KEY",
36                  "MAX_TOKENS": "8000"
37              }
38          }
39      ],
40      "results_dir_name": "sam-local-eval-test",
41      "runs": 3,
42      "workers": 4,
43      "test_cases": [
44          "tests/evaluation/test_cases/convert_html_to_md.test.json",
45          "tests/evaluation/test_cases/convert_pdf_to_md.test.json",
46          "tests/evaluation/test_cases/create_diagram.test.json",
47          "tests/evaluation/test_cases/file_artifact.test.json",
48          "tests/evaluation/test_cases/filter_csv_employees_by_age_and_country.test.json",
49          "tests/evaluation/test_cases/generate_employee_csv.test.json",
50          "tests/evaluation/test_cases/generate_sales_report.test.json",
51          "tests/evaluation/test_cases/hello_world.test.json",
52          "tests/evaluation/test_cases/web_search_solace_summary.test.json"
53      ],
54      "evaluation_settings": {
55          "tool_match": {
56              "enabled": true
57          },
58          "response_match": {
59              "enabled": true
60          },
61          "llm_evaluator": {
62              "enabled": true,
63              "env": {
64                  "LLM_SERVICE_PLANNING_MODEL_NAME": "openai/gemini-3-pro-preview",
65                  "LLM_SERVICE_ENDPOINT_VAR": "LLM_SERVICE_ENDPOINT",
66                  "LLM_SERVICE_API_KEY_VAR": "LLM_SERVICE_API_KEY"
67              }
68          }
69      }
70  }