1  /**
  2   * Triage Agent Known Errors Unit Tests
  3   *
  4   * Tests for the known error database functionality:
  5   * - checkKnownErrorDatabase()
  6   * - normalizeErrorMessage()
  7   * - calculateSimilarity()
  8   * - Integration with classifyErrorTask()
  9   */
 10  
 11  import { test, describe, beforeEach, afterEach } from 'node:test';
 12  import assert from 'node:assert';
 13  import Database from 'better-sqlite3';
 14  import { TriageAgent } from '../../src/agents/triage.js';
 15  import { resetDb as resetBaseDb } from '../../src/agents/base-agent.js';
 16  import { resetDb as resetTaskDb } from '../../src/agents/utils/task-manager.js';
 17  import { resetDb as resetMessageDb } from '../../src/agents/utils/message-manager.js';
 18  import fs from 'fs/promises';
 19  
 20  // Use temporary file database for tests
 21  let db;
 22  let agent;
 23  const TEST_DB_PATH = './tests/agents/test-triage-known-errors.db';
 24  
 25  beforeEach(async () => {
 26    // Close any existing database connection
 27    if (db) {
 28      try {
 29        db.close();
 30      } catch (e) {
 31        // Ignore
 32      }
 33    }
 34  
 35    // Remove existing test database if it exists
 36    try {
 37      await fs.unlink(TEST_DB_PATH);
 38    } catch (e) {
 39      // Ignore if file doesn't exist
 40    }
 41  
 42    // Wait a bit for file system
 43    await new Promise(resolve => setTimeout(resolve, 50));
 44  
 45    // Create temporary test database
 46    db = new Database(TEST_DB_PATH);
 47    process.env.DATABASE_PATH = TEST_DB_PATH;
 48    // Point TEL_DB_PATH at the test DB so tel.agent_tasks queries work via self-attach.
 49    process.env.TEL_DB_PATH = TEST_DB_PATH;
 50  
 51    // Create tables
 52    db.exec(`
 53      CREATE TABLE agent_tasks (
 54        id INTEGER PRIMARY KEY AUTOINCREMENT,
 55        task_type TEXT NOT NULL,
 56        assigned_to TEXT NOT NULL,
 57        created_by TEXT,
 58        status TEXT DEFAULT 'pending',
 59        priority INTEGER DEFAULT 5,
 60        context_json TEXT,
 61        result_json TEXT,
 62        parent_task_id INTEGER,
 63        error_message TEXT,
 64        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
 65        started_at DATETIME,
 66        completed_at DATETIME,
 67        retry_count INTEGER DEFAULT 0
 68      );
 69  
 70      CREATE TABLE agent_messages (
 71        id INTEGER PRIMARY KEY AUTOINCREMENT,
 72        task_id INTEGER,
 73        from_agent TEXT NOT NULL,
 74        to_agent TEXT NOT NULL,
 75        message_type TEXT,
 76        content TEXT NOT NULL,
 77        metadata_json TEXT,
 78        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
 79        read_at DATETIME
 80      );
 81  
 82      CREATE TABLE agent_logs (
 83        id INTEGER PRIMARY KEY AUTOINCREMENT,
 84        task_id INTEGER,
 85        agent_name TEXT NOT NULL,
 86        log_level TEXT,
 87        message TEXT,
 88        data_json TEXT,
 89        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
 90      );
 91  
 92      CREATE TABLE agent_state (
 93        agent_name TEXT PRIMARY KEY,
 94        last_active DATETIME DEFAULT CURRENT_TIMESTAMP,
 95        current_task_id INTEGER,
 96        status TEXT DEFAULT 'idle',
 97        metrics_json TEXT
 98      );
 99  
100      CREATE TABLE agent_outcomes (
101        id INTEGER PRIMARY KEY AUTOINCREMENT,
102        task_id INTEGER NOT NULL,
103        agent_name TEXT NOT NULL,
104        task_type TEXT NOT NULL,
105        outcome TEXT NOT NULL,
106        context_json TEXT,
107        result_json TEXT,
108        duration_ms INTEGER,
109        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
110      );
111  
112      CREATE TABLE IF NOT EXISTS cron_locks (
113        lock_key TEXT PRIMARY KEY,
114        description TEXT,
115        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
116      );
117    `);
118  
119    // Initialize agent
120    agent = new TriageAgent();
121    await agent.initialize();
122  });
123  
124  afterEach(async () => {
125    // Restore env vars
126    delete process.env.TEL_DB_PATH;
127  
128    // Reset all database connections first
129    resetBaseDb();
130    resetTaskDb();
131    resetMessageDb();
132  
133    if (db) {
134      db.close();
135    }
136    // Clean up test database
137    try {
138      await fs.unlink(TEST_DB_PATH);
139    } catch (e) {
140      // Ignore if file doesn't exist
141    }
142  });
143  
144  describe('TriageAgent - normalizeErrorMessage()', () => {
145    test('removes line and column numbers', () => {
146      const input = 'TypeError at file.js:179:45';
147      const normalized = agent.normalizeErrorMessage(input);
148  
149      assert.ok(!normalized.includes(':179'));
150      assert.ok(!normalized.includes(':45'));
151      assert.ok(normalized.includes('file.js'));
152    });
153  
154    test('removes file paths but keeps filename', () => {
155      const input = 'Error in /home/user/project/src/scoring.js';
156      const normalized = agent.normalizeErrorMessage(input);
157  
158      assert.ok(!normalized.includes('/home/user/project/src/'));
159      assert.ok(normalized.includes('scoring.js'));
160    });
161  
162    test('normalizes IDs', () => {
163      const input = 'Site id=12345 failed with task_id: 67890';
164      const normalized = agent.normalizeErrorMessage(input);
165  
166      assert.ok(normalized.includes('id=id'));
167      assert.ok(normalized.includes('task_id=id'));
168      assert.ok(!normalized.includes('12345'));
169      assert.ok(!normalized.includes('67890'));
170    });
171  
172    test('removes timestamps', () => {
173      const input = 'Error at 2024-01-15T10:30:45 failed';
174      const normalized = agent.normalizeErrorMessage(input);
175  
176      assert.ok(normalized.includes('timestamp'));
177      assert.ok(!normalized.includes('2024-01-15'));
178    });
179  
180    test('normalizes URLs', () => {
181      const input = 'Failed to fetch https://api.example.com/v1/data';
182      const normalized = agent.normalizeErrorMessage(input);
183  
184      assert.ok(normalized.includes('url'));
185      assert.ok(!normalized.includes('example.com'));
186    });
187  
188    test('normalizes large numbers', () => {
189      const input = 'Memory exceeded 1234567 bytes';
190      const normalized = agent.normalizeErrorMessage(input);
191  
192      assert.ok(normalized.includes('num'));
193      assert.ok(!normalized.includes('1234567'));
194    });
195  
196    test('converts to lowercase', () => {
197      const input = 'TypeError: Cannot Read Property';
198      const normalized = agent.normalizeErrorMessage(input);
199  
200      assert.strictEqual(normalized, normalized.toLowerCase());
201    });
202  
203    test('normalizes whitespace', () => {
204      const input = 'Error    with   multiple     spaces';
205      const normalized = agent.normalizeErrorMessage(input);
206  
207      assert.ok(!normalized.includes('  '));
208      assert.strictEqual(normalized.trim(), normalized);
209    });
210  
211    test('handles stack traces', () => {
212      const message = 'TypeError: Cannot read property';
213      const stack = 'at score.js:179:45\nat processTask.js:23:10';
214      const normalized = agent.normalizeErrorMessage(message, stack);
215  
216      assert.ok(!normalized.includes(':179'));
217      assert.ok(!normalized.includes(':23'));
218      assert.ok(normalized.includes('score.js'));
219      assert.ok(normalized.includes('processtask.js'));
220    });
221  });
222  
223  describe('TriageAgent - calculateSimilarity()', () => {
224    test('returns 1.0 for identical errors', () => {
225      const error = 'typeerror cannot read property score of null';
226      const similarity = agent.calculateSimilarity(error, error);
227  
228      assert.strictEqual(similarity, 1.0);
229    });
230  
231    test('returns 0.0 for completely different errors', () => {
232      const error1 = 'network timeout error';
233      const error2 = 'database constraint violation';
234      const similarity = agent.calculateSimilarity(error1, error2);
235  
236      assert.ok(similarity < 0.3); // Should be very low
237    });
238  
239    test('returns high similarity for similar errors', () => {
240      const error1 = 'typeerror cannot read property score of null at file.js';
241      const error2 = 'typeerror cannot read property score of null at other.js';
242      const similarity = agent.calculateSimilarity(error1, error2);
243  
244      assert.ok(similarity >= 0.7); // Should be considered similar
245    });
246  
247    test('handles word order differences', () => {
248      const error1 = 'database unique constraint failed';
249      const error2 = 'unique constraint failed database';
250      const similarity = agent.calculateSimilarity(error1, error2);
251  
252      // Jaccard similarity is 1.0 but Levenshtein reduces the combined score
253      // Implementation uses 50/50 blend, so same-word different-order scores ~0.7+
254      assert.ok(similarity >= 0.7, `Similarity ${similarity} should be >= 0.7`);
255    });
256  
257    test('returns partial similarity for overlapping errors', () => {
258      const error1 = 'api error rate limit exceeded timeout';
259      const error2 = 'api error rate limit exceeded';
260      const similarity = agent.calculateSimilarity(error1, error2);
261  
262      assert.ok(similarity > 0.5 && similarity < 1.0);
263    });
264  });
265  
266  describe('TriageAgent - checkKnownErrorDatabase()', () => {
267    test('returns null when no completed fix_bug tasks exist', async () => {
268      const result = agent.checkKnownErrorDatabase('New error never seen before');
269  
270      assert.strictEqual(result, null);
271    });
272  
273    test('returns null when no similar errors found', async () => {
274      // Create a completed fix_bug task with different error
275      db.prepare(
276        `
277        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
278        VALUES ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
279      `
280      ).run(
281        JSON.stringify({
282          error_message: 'Network timeout error ENOTFOUND',
283          error_type: 'network',
284        }),
285        JSON.stringify({
286          fix_description: 'Added retry logic',
287          files_changed: ['src/scrape.js'],
288        })
289      );
290  
291      // Query with completely different error
292      const result = agent.checkKnownErrorDatabase('Database UNIQUE constraint failed');
293  
294      assert.strictEqual(result, null);
295    });
296  
297    test('finds similar error with normalized matching', async () => {
298      // Create a completed fix_bug task
299      db.prepare(
300        `
301        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
302        VALUES ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
303      `
304      ).run(
305        JSON.stringify({
306          error_message: 'TypeError: Cannot read property score of null at scoring.js',
307          stack_trace: 'at processScore (src/scoring.js:179)',
308          error_type: 'null_pointer',
309        }),
310        JSON.stringify({
311          fix_description: 'Added null check: score?.value || 0',
312          files_changed: ['src/scoring.js'],
313          summary: 'Fixed null pointer by adding optional chaining',
314        })
315      );
316  
317      // Query with similar error (different line number and file)
318      const result = agent.checkKnownErrorDatabase(
319        'TypeError: Cannot read property score of null at rescoring.js',
320        'at processScore (src/rescoring.js:234)'
321      );
322  
323      assert.ok(result !== null);
324      assert.ok(result.similarity >= 0.7);
325      assert.strictEqual(result.error_type, 'null_pointer');
326      assert.ok(result.fix_description.includes('null check'));
327    });
328  
329    test('returns highest similarity match when multiple exist', async () => {
330      // Create two completed tasks with different similarities
331      db.prepare(
332        `
333        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
334        VALUES
335          ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP),
336          ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
337      `
338      ).run(
339        // Less similar error
340        JSON.stringify({
341          error_message: 'TypeError: Cannot read property of null',
342          error_type: 'null_pointer',
343        }),
344        JSON.stringify({
345          fix_description: 'Generic null check',
346        }),
347        // More similar error
348        JSON.stringify({
349          error_message: 'TypeError: Cannot read property "score" of null in scoring',
350          error_type: 'null_pointer',
351        }),
352        JSON.stringify({
353          fix_description: 'Specific score null check',
354        })
355      );
356  
357      const result = agent.checkKnownErrorDatabase(
358        'TypeError: Cannot read property "score" of null in scoring module'
359      );
360  
361      assert.ok(result !== null);
362      assert.ok(result.fix_description.includes('Specific score'));
363    });
364  
365    test('requires 70% similarity threshold', async () => {
366      // Create a completed task
367      db.prepare(
368        `
369        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
370        VALUES ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
371      `
372      ).run(
373        JSON.stringify({
374          error_message: 'Database UNIQUE constraint failed on sites.domain',
375          error_type: 'database',
376        }),
377        JSON.stringify({
378          fix_description: 'Added INSERT OR IGNORE',
379        })
380      );
381  
382      // Query with partially similar error (should be below 70%)
383      const result = agent.checkKnownErrorDatabase('Database error occurred');
384  
385      assert.strictEqual(result, null); // Should not match due to low similarity
386    });
387  
388    test('handles malformed JSON gracefully', async () => {
389      // Insert task with invalid JSON
390      db.prepare(
391        `
392        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
393        VALUES ('fix_bug', 'developer', 'completed', 'invalid json', 'invalid json', CURRENT_TIMESTAMP)
394      `
395      ).run();
396  
397      // Should not throw, should return null
398      const result = agent.checkKnownErrorDatabase('Any error');
399  
400      assert.strictEqual(result, null);
401    });
402  
403    test('includes all relevant fix data in result', async () => {
404      db.prepare(
405        `
406        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
407        VALUES ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
408      `
409      ).run(
410        JSON.stringify({
411          error_message: 'API rate limit exceeded status 429',
412          error_type: 'api_error',
413        }),
414        JSON.stringify({
415          fix_description: 'Implemented exponential backoff',
416          files_changed: ['src/utils/api-client.js', 'src/scrape.js'],
417          summary: 'Added rate limiting',
418        })
419      );
420  
421      const result = agent.checkKnownErrorDatabase('API rate limit exceeded status 429');
422  
423      assert.ok(result !== null);
424      assert.ok(result.task_id > 0);
425      assert.strictEqual(result.error_type, 'api_error');
426      assert.ok(result.fix_description.includes('backoff'));
427      assert.ok(Array.isArray(result.files_changed));
428      assert.strictEqual(result.files_changed.length, 2);
429      assert.ok(result.completed_at);
430      assert.ok(result.similarity >= 0.7);
431    });
432  });
433  
434  describe('TriageAgent - classifyErrorTask() with known errors', () => {
435    test('routes known error with lower priority', async () => {
436      // Create a completed fix_bug task
437      db.prepare(
438        `
439        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at, created_at)
440        VALUES ('fix_bug', 'developer', 'completed', ?, ?, datetime('now','-2 hours'), datetime('now','-2 hours'))
441      `
442      ).run(
443        JSON.stringify({
444          error_message: 'TypeError: Cannot read property "score" of null',
445          error_type: 'null_pointer',
446        }),
447        JSON.stringify({
448          fix_description: 'Added null check with optional chaining',
449          files_changed: ['src/score.js'], // Must be non-empty to trigger routing (not dismiss)
450        })
451      );
452  
453      // Create a classify_error task with similar error
454      const taskId = db
455        .prepare(
456          `
457        INSERT INTO agent_tasks (task_type, assigned_to, status, priority, context_json)
458        VALUES ('classify_error', 'triage', 'pending', 5, ?)
459      `
460        )
461        .run(
462          JSON.stringify({
463            error_message: 'TypeError: Cannot read property "score" of null at line 200',
464            stage: 'scoring',
465            frequency: 1,
466          })
467        ).lastInsertRowid;
468  
469      // Get the task
470      const task = db.prepare('SELECT * FROM agent_tasks WHERE id = ?').get(taskId);
471      task.context_json = JSON.parse(task.context_json);
472  
473      // Process it
474      await agent.classifyErrorTask(task);
475  
476      // Verify developer task was created with priority 5
477      const devTasks = db
478        .prepare(
479          `
480        SELECT * FROM agent_tasks
481        WHERE assigned_to = 'developer' AND parent_task_id = ?
482      `
483        )
484        .all(taskId);
485  
486      assert.strictEqual(devTasks.length, 1);
487      assert.strictEqual(devTasks[0].task_type, 'fix_bug');
488      assert.strictEqual(devTasks[0].priority, 5); // Lower priority for known errors
489  
490      const devContext = JSON.parse(devTasks[0].context_json);
491      assert.strictEqual(devContext.severity, 'low'); // Known errors are low severity
492      assert.ok(devContext.known_fix); // Should include known fix data
493      assert.ok(devContext.known_fix.fix_description);
494    });
495  
496    test('logs when known error is detected', async () => {
497      // Create a completed fix_bug task with more specific error message
498      db.prepare(
499        `
500        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
501        VALUES ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
502      `
503      ).run(
504        JSON.stringify({
505          error_message: 'UNIQUE constraint failed: sites.domain when inserting site record',
506          error_type: 'database',
507          stack_trace: 'at insertSite (src/serps.js:123)',
508        }),
509        JSON.stringify({
510          fix_description: 'Use INSERT OR IGNORE',
511        })
512      );
513  
514      const taskId = db
515        .prepare(
516          `
517        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json)
518        VALUES ('classify_error', 'triage', 'pending', ?)
519      `
520        )
521        .run(
522          JSON.stringify({
523            error_message: 'UNIQUE constraint failed: sites.domain when inserting duplicate',
524            stack_trace: 'at insertSite (src/serps.js:456)',
525            stage: 'serps',
526          })
527        ).lastInsertRowid;
528  
529      const task = db.prepare('SELECT * FROM agent_tasks WHERE id = ?').get(taskId);
530      task.context_json = JSON.parse(task.context_json);
531  
532      await agent.classifyErrorTask(task);
533  
534      // Check logs
535      const logs = db
536        .prepare(
537          `
538        SELECT * FROM agent_logs
539        WHERE agent_name = 'triage' AND task_id = ?
540      `
541        )
542        .all(taskId);
543  
544      const knownErrorLog = logs.find(log => log.message.includes('Known error'));
545      assert.ok(knownErrorLog, 'Should log known error detection');
546  
547      const logData = JSON.parse(knownErrorLog.data_json);
548      assert.ok(logData.known_fix_task_id);
549      assert.ok(logData.similarity >= 0.7);
550    });
551  
552    test('falls back to normal classification when no known error found', async () => {
553      // Create a classify_error task with new error
554      const taskId = db
555        .prepare(
556          `
557        INSERT INTO agent_tasks (task_type, assigned_to, status, priority, context_json)
558        VALUES ('classify_error', 'triage', 'pending', 5, ?)
559      `
560        )
561        .run(
562          JSON.stringify({
563            error_message: 'Network ENOTFOUND api.production-service.io',
564            stage: 'serps',
565            frequency: 1,
566          })
567        ).lastInsertRowid;
568  
569      const task = db.prepare('SELECT * FROM agent_tasks WHERE id = ?').get(taskId);
570      task.context_json = JSON.parse(task.context_json);
571  
572      await agent.classifyErrorTask(task);
573  
574      // Verify normal classification occurred
575      const architectTasks = db
576        .prepare(
577          `
578        SELECT * FROM agent_tasks
579        WHERE assigned_to = 'architect' AND parent_task_id = ?
580      `
581        )
582        .all(taskId);
583  
584      assert.strictEqual(architectTasks.length, 1); // Network errors go to architect
585      // Priority calculation: basePriority 6 + high severity boost 2 + serps stage boost 2 = 10
586      assert.strictEqual(architectTasks[0].priority, 10);
587  
588      const context = JSON.parse(architectTasks[0].context_json);
589      assert.strictEqual(context.error_type, 'network');
590      assert.ok(!context.known_fix); // No known fix
591    });
592  
593    test('completes triage task with known_fix in result', async () => {
594      // Create a completed fix_bug task
595      db.prepare(
596        `
597        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, result_json, completed_at)
598        VALUES ('fix_bug', 'developer', 'completed', ?, ?, CURRENT_TIMESTAMP)
599      `
600      ).run(
601        JSON.stringify({
602          error_message: 'Configuration error: OPENROUTER_API_KEY missing',
603          error_type: 'configuration',
604        }),
605        JSON.stringify({
606          fix_description: 'Set OPENROUTER_API_KEY in .env file',
607          files_changed: ['.env'], // Non-empty so it routes (not dismisses as known_operational)
608        })
609      );
610  
611      const taskId = db
612        .prepare(
613          `
614        INSERT INTO agent_tasks (task_type, assigned_to, status, context_json)
615        VALUES ('classify_error', 'triage', 'pending', ?)
616      `
617        )
618        .run(
619          JSON.stringify({
620            error_message: 'Configuration error: OPENROUTER_API_KEY missing',
621            stage: 'scoring',
622          })
623        ).lastInsertRowid;
624  
625      const task = db.prepare('SELECT * FROM agent_tasks WHERE id = ?').get(taskId);
626      task.context_json = JSON.parse(task.context_json);
627  
628      await agent.classifyErrorTask(task);
629  
630      // Verify triage task completed with known_fix
631      const completedTask = db.prepare('SELECT * FROM agent_tasks WHERE id = ?').get(taskId);
632      assert.strictEqual(completedTask.status, 'completed');
633  
634      const result = JSON.parse(completedTask.result_json);
635      assert.strictEqual(result.classification, 'known_error');
636      assert.strictEqual(result.severity, 'low');
637      assert.ok(result.known_fix);
638      assert.ok(result.known_fix.fix_description);
639    });
640  });