1  /**
  2   * SLO Tracker Tests
  3   *
  4   * Tests for Service-Level Objective tracking functionality
  5   */
  6  
  7  import { test } from 'node:test';
  8  import assert from 'node:assert';
  9  import Database from 'better-sqlite3';
 10  import fs from 'fs';
 11  import {
 12    SLO_DEFINITIONS,
 13    calculateStagePerformance,
 14    checkSLOCompliance,
 15    getSLOSummary,
 16    resetDb,
 17  } from '../../src/agents/utils/slo-tracker.js';
 18  
 19  // Test database path
 20  const TEST_DB_PATH = './db/test-slo-tracker.db';
 21  
 22  /**
 23   * Set up test database with schema
 24   */
 25  function setupTestDb() {
 26    // Remove existing test db
 27    if (fs.existsSync(TEST_DB_PATH)) {
 28      fs.unlinkSync(TEST_DB_PATH);
 29    }
 30  
 31    const db = new Database(TEST_DB_PATH);
 32  
 33    // Create minimal schema for testing
 34    db.exec(`
 35      CREATE TABLE sites (
 36        id INTEGER PRIMARY KEY AUTOINCREMENT,
 37        domain TEXT NOT NULL,
 38        status TEXT NOT NULL,
 39        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
 40        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
 41      );
 42  
 43      CREATE TABLE site_status (
 44        id INTEGER PRIMARY KEY AUTOINCREMENT,
 45        site_id INTEGER NOT NULL REFERENCES sites(id),
 46        status TEXT NOT NULL,
 47        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
 48      );
 49  
 50      CREATE INDEX idx_site_status_site_created ON site_status(site_id, created_at);
 51    `);
 52  
 53    return db;
 54  }
 55  
 56  /**
 57   * Insert test data with specific transition durations
 58   */
 59  // Format date for SQLite (space-separated, not T-format, to avoid lexicographic comparison bugs)
 60  function toSQLiteDate(date) {
 61    return date
 62      .toISOString()
 63      .replace('T', ' ')
 64      .replace(/\.\d{3}Z$/, '');
 65  }
 66  
 67  function insertTestData(db, fromStage, toStage, durationMinutes) {
 68    // Create site
 69    const siteResult = db
 70      .prepare('INSERT INTO sites (domain, status) VALUES (?, ?)')
 71      .run(`test-${Date.now()}.com`, toStage);
 72  
 73    const siteId = siteResult.lastInsertRowid;
 74  
 75    // Calculate timestamps
 76    const now = new Date();
 77    const startTime = new Date(now.getTime() - durationMinutes * 60 * 1000);
 78  
 79    // Insert status transitions (use SQLite format to avoid ISO 'T' comparison bugs)
 80    db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
 81      siteId,
 82      fromStage,
 83      toSQLiteDate(startTime)
 84    );
 85  
 86    db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
 87      siteId,
 88      toStage,
 89      toSQLiteDate(now)
 90    );
 91  
 92    return siteId;
 93  }
 94  
 95  test('SLO Tracker - SLO Definitions', async t => {
 96    await t.test('should have defined SLOs', () => {
 97      assert.ok(Array.isArray(SLO_DEFINITIONS), 'SLO_DEFINITIONS should be an array');
 98      assert.ok(SLO_DEFINITIONS.length > 0, 'Should have at least one SLO defined');
 99    });
100  
101    await t.test('should have required SLO fields', () => {
102      for (const slo of SLO_DEFINITIONS) {
103        assert.ok(slo.stage_name, 'SLO should have stage_name');
104        assert.ok(slo.description, 'SLO should have description');
105        assert.ok(typeof slo.target_percentile === 'number', 'target_percentile should be a number');
106        assert.ok(
107          typeof slo.target_duration_minutes === 'number',
108          'target_duration_minutes should be a number'
109        );
110        assert.ok(typeof slo.lookback_hours === 'number', 'lookback_hours should be a number');
111      }
112    });
113  
114    await t.test('should have expected SLOs', () => {
115      const stageNames = SLO_DEFINITIONS.map(slo => slo.stage_name);
116      assert.ok(stageNames.includes('serps_to_assets'), 'Should have serps_to_assets SLO');
117      assert.ok(stageNames.includes('assets_to_scored'), 'Should have assets_to_scored SLO');
118      assert.ok(stageNames.includes('scored_to_rescored'), 'Should have scored_to_rescored SLO');
119    });
120  });
121  
122  test('SLO Tracker - calculateStagePerformance', async t => {
123    let db;
124  
125    t.beforeEach(() => {
126      process.env.DATABASE_PATH = TEST_DB_PATH;
127      db = setupTestDb();
128      resetDb(); // Reset the module's db connection
129    });
130  
131    t.afterEach(() => {
132      if (db) {
133        db.close();
134      }
135      if (fs.existsSync(TEST_DB_PATH)) {
136        fs.unlinkSync(TEST_DB_PATH);
137      }
138      delete process.env.DATABASE_PATH;
139      resetDb();
140    });
141  
142    await t.test('should return empty result when no data', () => {
143      const result = calculateStagePerformance('found', 'assets_captured', 24);
144  
145      assert.strictEqual(result.totalSites, 0);
146      assert.deepStrictEqual(result.durations, []);
147      assert.strictEqual(result.p50, null);
148      assert.strictEqual(result.p95, null);
149      assert.strictEqual(result.p99, null);
150    });
151  
152    await t.test('should calculate performance with single site', () => {
153      // Insert site with 30 minute transition
154      insertTestData(db, 'found', 'assets_captured', 30);
155  
156      const result = calculateStagePerformance('found', 'assets_captured', 24);
157  
158      assert.strictEqual(result.totalSites, 1);
159      assert.strictEqual(result.durations.length, 1);
160      assert.ok(Math.abs(result.p50 - 30) < 1, 'P50 should be ~30 minutes');
161      assert.ok(Math.abs(result.p95 - 30) < 1, 'P95 should be ~30 minutes');
162    });
163  
164    await t.test('should calculate performance with multiple sites', () => {
165      // Insert sites with varying durations: 10, 20, 30, 40, 50 minutes
166      for (const duration of [10, 20, 30, 40, 50]) {
167        insertTestData(db, 'assets_captured', 'prog_scored', duration);
168      }
169  
170      const result = calculateStagePerformance('assets_captured', 'prog_scored', 24);
171  
172      assert.strictEqual(result.totalSites, 5);
173      assert.strictEqual(result.durations.length, 5);
174      assert.ok(Math.abs(result.p50 - 30) < 2, 'P50 should be ~30 minutes');
175      assert.ok(Math.abs(result.p95 - 50) <= 3, 'P95 should be ~50 minutes');
176    });
177  
178    await t.test('should respect lookback window', () => {
179      // Insert old data (25 hours ago) - should be excluded
180      const oldSite = db
181        .prepare('INSERT INTO sites (domain, status) VALUES (?, ?)')
182        .run('old.com', 'prog_scored');
183      const oldTime = new Date(Date.now() - 25 * 60 * 60 * 1000);
184  
185      db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
186        oldSite.lastInsertRowid,
187        'assets_captured',
188        toSQLiteDate(new Date(oldTime.getTime() - 30 * 60 * 1000))
189      );
190      db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
191        oldSite.lastInsertRowid,
192        'prog_scored',
193        toSQLiteDate(oldTime)
194      );
195  
196      // Insert recent data (1 hour ago)
197      insertTestData(db, 'assets_captured', 'prog_scored', 30);
198  
199      const result = calculateStagePerformance('assets_captured', 'prog_scored', 24);
200  
201      assert.strictEqual(result.totalSites, 1, 'Should only include recent data');
202    });
203  
204    await t.test('should handle sites with multiple transitions', () => {
205      // Site transitions: found -> assets_captured -> scored
206      const siteId = insertTestData(db, 'found', 'assets_captured', 60);
207  
208      // Add another transition for the same site
209      const now = new Date();
210      db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
211        siteId,
212        'prog_scored',
213        now.toISOString()
214      );
215  
216      // Query for assets_captured -> scored should work
217      const result = calculateStagePerformance('assets_captured', 'prog_scored', 24);
218      assert.strictEqual(result.totalSites, 1);
219    });
220  });
221  
222  test('SLO Tracker - checkSLOCompliance', async t => {
223    let db;
224  
225    t.beforeEach(() => {
226      process.env.DATABASE_PATH = TEST_DB_PATH;
227      db = setupTestDb();
228      resetDb();
229    });
230  
231    t.afterEach(() => {
232      if (db) {
233        db.close();
234      }
235      if (fs.existsSync(TEST_DB_PATH)) {
236        fs.unlinkSync(TEST_DB_PATH);
237      }
238      delete process.env.DATABASE_PATH;
239      resetDb();
240    });
241  
242    await t.test('should return no violations when no data', () => {
243      const violations = checkSLOCompliance();
244      assert.strictEqual(violations.length, 0);
245    });
246  
247    await t.test('should return no violations when within SLO', () => {
248      // serps_to_assets SLO: 95% within 60 minutes
249      // Insert 20 sites all within 50 minutes
250      for (let i = 0; i < 20; i++) {
251        insertTestData(db, 'found', 'assets_captured', 50);
252      }
253  
254      const violations = checkSLOCompliance();
255  
256      // Find serps_to_assets violations
257      const serpsViolations = violations.filter(v => v.slo.stage_name === 'serps_to_assets');
258      assert.strictEqual(serpsViolations.length, 0, 'Should have no violations');
259    });
260  
261    await t.test('should detect violations when exceeding SLO', () => {
262      // serps_to_assets SLO: 95% within 60 minutes
263      // Insert 20 sites all taking 120 minutes (2x target)
264      for (let i = 0; i < 20; i++) {
265        insertTestData(db, 'found', 'assets_captured', 120);
266      }
267  
268      const violations = checkSLOCompliance();
269  
270      // Find serps_to_assets violations
271      const serpsViolations = violations.filter(v => v.slo.stage_name === 'serps_to_assets');
272      assert.strictEqual(serpsViolations.length, 1, 'Should have one violation');
273  
274      const violation = serpsViolations[0];
275      assert.strictEqual(violation.slo.stage_name, 'serps_to_assets');
276      assert.ok(violation.actual.p95 > 60, 'P95 should exceed 60 minutes');
277      assert.ok(violation.violation_severity, 'Should have severity');
278    });
279  
280    await t.test('should calculate violation severity correctly', () => {
281      // Test critical severity (3x over target)
282      for (let i = 0; i < 20; i++) {
283        insertTestData(db, 'found', 'assets_captured', 180); // 3x the 60min target
284      }
285  
286      const violations = checkSLOCompliance();
287      const serpsViolations = violations.filter(v => v.slo.stage_name === 'serps_to_assets');
288  
289      if (serpsViolations.length > 0) {
290        const violation = serpsViolations[0];
291        assert.ok(
292          ['high', 'critical'].includes(violation.violation_severity),
293          'Should have high or critical severity for 3x over'
294        );
295      }
296    });
297  
298    await t.test('should include violation details', () => {
299      // Create violation
300      for (let i = 0; i < 20; i++) {
301        insertTestData(db, 'assets_captured', 'prog_scored', 90); // 3x the 30min target
302      }
303  
304      const violations = checkSLOCompliance();
305      const assetsViolations = violations.filter(v => v.slo.stage_name === 'assets_to_scored');
306  
307      if (assetsViolations.length > 0) {
308        const violation = assetsViolations[0];
309  
310        assert.ok(violation.slo, 'Should have slo object');
311        assert.ok(violation.actual, 'Should have actual object');
312        assert.ok(violation.violation_description, 'Should have description');
313        assert.ok(typeof violation.actual.totalSites === 'number', 'Should have totalSites');
314        assert.ok(typeof violation.actual.p50 === 'number', 'Should have p50');
315        assert.ok(typeof violation.actual.p95 === 'number', 'Should have p95');
316      }
317    });
318  
319    await t.test('should check multiple SLOs independently', () => {
320      // Create violation for serps_to_assets only
321      for (let i = 0; i < 20; i++) {
322        insertTestData(db, 'found', 'assets_captured', 150);
323      }
324  
325      // Create compliant data for assets_to_scored
326      for (let i = 0; i < 20; i++) {
327        insertTestData(db, 'assets_captured', 'prog_scored', 15);
328      }
329  
330      const violations = checkSLOCompliance();
331  
332      const serpsViolations = violations.filter(v => v.slo.stage_name === 'serps_to_assets');
333      const assetsViolations = violations.filter(v => v.slo.stage_name === 'assets_to_scored');
334  
335      assert.strictEqual(serpsViolations.length, 1, 'Should have serps violation');
336      assert.strictEqual(assetsViolations.length, 0, 'Should not have assets violation');
337    });
338  });
339  
340  test('SLO Tracker - getSLOSummary', async t => {
341    let db;
342  
343    t.beforeEach(() => {
344      process.env.DATABASE_PATH = TEST_DB_PATH;
345      db = setupTestDb();
346      resetDb();
347    });
348  
349    t.afterEach(() => {
350      if (db) {
351        db.close();
352      }
353      if (fs.existsSync(TEST_DB_PATH)) {
354        fs.unlinkSync(TEST_DB_PATH);
355      }
356      delete process.env.DATABASE_PATH;
357      resetDb();
358    });
359  
360    await t.test('should return summary with no violations', () => {
361      const summary = getSLOSummary();
362  
363      assert.ok(summary.total_slos > 0, 'Should have total SLOs');
364      assert.strictEqual(summary.violations, 0, 'Should have no violations');
365      assert.strictEqual(summary.compliance_rate, 100, 'Should be 100% compliant');
366      assert.ok(Array.isArray(summary.violations_detail), 'Should have violations_detail array');
367    });
368  
369    await t.test('should return summary with violations', () => {
370      // Create multiple violations
371      for (let i = 0; i < 20; i++) {
372        insertTestData(db, 'found', 'assets_captured', 150);
373        insertTestData(db, 'assets_captured', 'prog_scored', 90);
374      }
375  
376      const summary = getSLOSummary();
377  
378      assert.ok(summary.violations > 0, 'Should have violations');
379      assert.ok(summary.compliance_rate < 100, 'Should not be 100% compliant');
380      assert.strictEqual(
381        summary.violations,
382        summary.violations_detail.length,
383        'Violations count should match detail length'
384      );
385    });
386  
387    await t.test('should calculate compliance rate correctly', () => {
388      // Create exactly 1 violation out of 3 SLOs
389      for (let i = 0; i < 20; i++) {
390        insertTestData(db, 'found', 'assets_captured', 150); // Violation
391      }
392  
393      const summary = getSLOSummary();
394  
395      assert.strictEqual(summary.total_slos, 3, 'Should have 3 SLOs');
396      assert.strictEqual(summary.violations, 1, 'Should have 1 violation');
397  
398      const expectedRate = ((3 - 1) / 3) * 100;
399      assert.ok(
400        Math.abs(summary.compliance_rate - expectedRate) < 1,
401        `Compliance rate should be ~${expectedRate}%`
402      );
403    });
404  });
405  
406  test('SLO Tracker - Integration', async t => {
407    let db;
408  
409    t.beforeEach(() => {
410      process.env.DATABASE_PATH = TEST_DB_PATH;
411      db = setupTestDb();
412      resetDb();
413    });
414  
415    t.afterEach(() => {
416      if (db) {
417        db.close();
418      }
419      if (fs.existsSync(TEST_DB_PATH)) {
420        fs.unlinkSync(TEST_DB_PATH);
421      }
422      delete process.env.DATABASE_PATH;
423      resetDb();
424    });
425  
426    await t.test('should track realistic pipeline flow', () => {
427      // Simulate realistic pipeline: 100 sites flowing through
428      const now = Date.now();
429  
430      for (let i = 0; i < 100; i++) {
431        const siteResult = db
432          .prepare('INSERT INTO sites (domain, status) VALUES (?, ?)')
433          .run(`site-${i}.com`, 'prog_scored');
434        const siteId = siteResult.lastInsertRowid;
435  
436        // found -> assets_captured (45 minutes)
437        const foundTime = new Date(now - 120 * 60 * 1000);
438        const assetsTime = new Date(now - 75 * 60 * 1000);
439        const scoredTime = new Date(now - 60 * 60 * 1000);
440  
441        db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
442          siteId,
443          'found',
444          foundTime.toISOString()
445        );
446        db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
447          siteId,
448          'assets_captured',
449          assetsTime.toISOString()
450        );
451        db.prepare('INSERT INTO site_status (site_id, status, created_at) VALUES (?, ?, ?)').run(
452          siteId,
453          'prog_scored',
454          scoredTime.toISOString()
455        );
456      }
457  
458      const summary = getSLOSummary();
459  
460      assert.ok(summary.total_slos > 0, 'Should have SLOs');
461      assert.ok(summary.compliance_rate >= 0, 'Should have compliance rate');
462  
463      const violations = checkSLOCompliance();
464      assert.ok(Array.isArray(violations), 'Should return violations array');
465    });
466  
467    await t.test('should handle edge case: exactly at SLO threshold', () => {
468      // Insert sites at 59 minutes - safely within the 60-minute target for serps_to_assets
469      // (Using 59 instead of exactly 60 to avoid floating-point rounding issues)
470      for (let i = 0; i < 20; i++) {
471        insertTestData(db, 'found', 'assets_captured', 59);
472      }
473  
474      const violations = checkSLOCompliance();
475      const serpsViolations = violations.filter(v => v.slo.stage_name === 'serps_to_assets');
476  
477      // At or under threshold should not violate
478      assert.strictEqual(serpsViolations.length, 0, 'Should not violate at exact threshold');
479    });
480  
481    await t.test('should handle mixed performance data', () => {
482      // 90% fast (30 min), 10% slow (150 min)
483      for (let i = 0; i < 18; i++) {
484        insertTestData(db, 'found', 'assets_captured', 30);
485      }
486      for (let i = 0; i < 2; i++) {
487        insertTestData(db, 'found', 'assets_captured', 150);
488      }
489  
490      const result = calculateStagePerformance('found', 'assets_captured', 24);
491      assert.strictEqual(result.totalSites, 20);
492  
493      // P95 should be influenced by the slow ones
494      assert.ok(result.p95 > 30, 'P95 should be higher than fast sites');
495    });
496  });