1  {
 2    "name": "Customer Support Bot Eval",
 3    "type": "chat",
 4    "endpoint": "https://your-api.com/api/chat",
 5    "method": "POST",
 6    "headers": {
 7      "Content-Type": "application/json"
 8    },
 9    "request_template": {
10      "tenant": "{{tenant}}",
11      "message": "{{message}}",
12      "history": "{{history}}"
13    },
14    "response_field": "response",
15    "session_id_field": "session_id",
16    "defaults": {
17      "tenant": "default"
18    },
19    "threshold": 80,
20    "good_behaviors": [
21      "Greets the user",
22      "Asks clarifying questions",
23      "Provides specific answers with numbers",
24      "Offers to help further"
25    ],
26    "bad_behaviors": [
27      "Says 'I don't know' about core product features",
28      "Makes up information",
29      "Gives responses longer than 3 sentences"
30    ],
31    "scenarios": [
32      {
33        "name": "greeting",
34        "messages": ["Hello"],
35        "criteria": [
36          { "type": "contains", "value": "help", "description": "Should offer help" },
37          { "type": "max_sentences", "value": 3, "description": "Concise greeting" }
38        ]
39      },
40      {
41        "name": "pricing_question",
42        "messages": [
43          "Hi there",
44          "How much does your product cost?"
45        ],
46        "criteria": [
47          { "type": "regex", "value": "\\$\\d+", "description": "Mentions a price" },
48          { "type": "not_contains", "value": "I don't know", "description": "Knows pricing" },
49          { "type": "max_sentences", "value": 4, "description": "Concise answer" }
50        ]
51      },
52      {
53        "name": "unqualified_prospect",
54        "messages": [
55          "Hi, I'm a student doing research on your industry",
56          "I don't have any budget"
57        ],
58        "criteria": [
59          { "type": "not_contains", "value": "book a call", "description": "No booking push" },
60          { "type": "contains", "value": "free", "description": "Suggests free resources" }
61        ]
62      },
63      {
64        "name": "feature_question",
65        "messages": [
66          "Does your product integrate with Salesforce?"
67        ],
68        "criteria": [
69          { "type": "not_contains", "value": "I'm not sure", "description": "Knows integrations" },
70          { "type": "max_sentences", "value": 3, "description": "Concise answer" }
71        ]
72      },
73      {
74        "name": "hostile_prospect",
75        "messages": [
76          "Your product looks terrible",
77          "Why would anyone pay for this?"
78        ],
79        "criteria": [
80          { "type": "not_contains", "value": "sorry", "description": "Doesn't over-apologize" },
81          { "type": "max_sentences", "value": 3, "description": "Stays concise under pressure" }
82        ]
83      }
84    ]
85  }