/ tests / evaluation / test_cases / workflows / level7_failure_path.test.json
level7_failure_path.test.json
 1  {
 2    "test_case_id": "workflow_level7_failure_path",
 3    "category": "Workflows",
 4    "description": "Tests error handling workflow in failure mode. The workflow fails and triggers on_failure and always exit handlers.",
 5    "query": "Please use the ErrorHandlingWorkflow with mode 'failure', and render the output file.",
 6    "target_agent": "OrchestratorAgent",
 7    "wait_time": 300,
 8    "evaluation": {
 9      "expected_tools": ["workflow_ErrorHandlingWorkflow"],
10      "expected_response": "The ErrorHandlingWorkflow completed with status: failed. The workflow encountered a simulated failure, then triggered the failure alert and cleanup handlers. Error: simulated failure, Failure alert sent, Cleanup performed",
11      "criterion": "Evaluate if the orchestrator successfully delegated to ErrorHandlingWorkflow in failure mode, the workflow failed as expected, and the on_failure and always exit handlers were executed (failure alert + cleanup)."
12    }
13  }