1  {
 2    "agents": [
 3      "examples/agents/orchestrator_example.yaml",
 4      "examples/workflows/level1_simple_sequential.yaml",
 5      "examples/workflows/level2_parallel_execution.yaml",
 6      "examples/workflows/level3_conditional_branching.yaml",
 7      "examples/workflows/level4_map_iteration.yaml",
 8      "examples/workflows/level5_loop_polling.yaml",
 9      "examples/workflows/level6_workflow_composition.yaml",
10      "examples/workflows/level7_error_handling.yaml"
11    ],
12    "broker": {
13      "SOLACE_BROKER_URL_VAR": "SOLACE_BROKER_URL",
14      "SOLACE_BROKER_USERNAME_VAR": "SOLACE_BROKER_USERNAME",
15      "SOLACE_BROKER_PASSWORD_VAR": "SOLACE_BROKER_PASSWORD",
16      "SOLACE_BROKER_VPN_VAR": "SOLACE_BROKER_VPN"
17    },
18    "llm_models": [
19      {
20        "name": "claude-4-sonnet",
21        "env": {
22          "LLM_SERVICE_PLANNING_MODEL_NAME_VAR": "LLM_SERVICE_PLANNING_MODEL_NAME",
23          "LLM_SERVICE_ENDPOINT_VAR": "LLM_SERVICE_ENDPOINT",
24          "LLM_SERVICE_API_KEY_VAR": "LLM_SERVICE_API_KEY"
25        }
26      }
27    ],
28    "results_dir_name": "workflow-eval",
29    "runs": 1,
30    "workers": 2,
31    "test_cases": [
32      "tests/evaluation/test_cases/workflows/level1_simple_sequential.test.json",
33      "tests/evaluation/test_cases/workflows/level2_parallel_execution.test.json",
34      "tests/evaluation/test_cases/workflows/level3_conditional_small.test.json",
35      "tests/evaluation/test_cases/workflows/level3_conditional_medium.test.json",
36      "tests/evaluation/test_cases/workflows/level3_conditional_large.test.json",
37      "tests/evaluation/test_cases/workflows/level4_map_iteration.test.json",
38      "tests/evaluation/test_cases/workflows/level5_loop_polling.test.json",
39      "tests/evaluation/test_cases/workflows/level6_workflow_composition.test.json",
40      "tests/evaluation/test_cases/workflows/level7_success_path.test.json",
41      "tests/evaluation/test_cases/workflows/level7_failure_path.test.json"
42    ],
43    "evaluation_settings": {
44      "tool_match": {
45        "enabled": true
46      },
47      "response_match": {
48        "enabled": true
49      },
50      "llm_evaluator": {
51        "enabled": true,
52        "env": {
53          "LLM_SERVICE_PLANNING_MODEL_NAME_VAR": "LLM_SERVICE_PLANNING_MODEL_NAME",
54          "LLM_SERVICE_ENDPOINT_VAR": "LLM_SERVICE_ENDPOINT",
55          "LLM_SERVICE_API_KEY_VAR": "LLM_SERVICE_API_KEY"
56        }
57      }
58    }
59  }