1  /**
  2   * Process Guardian Tests - Augmented
  3   *
  4   * Tests the Tier 1 process guardian that runs as a direct cron function.
  5   * Verifies pipeline service checks, clearance cycle detection, circuit breaker
  6   * monitoring, status file writing, and full integration run.
  7   */
  8  
  9  import { test, mock, beforeEach, afterEach } from 'node:test';
 10  import assert from 'node:assert/strict';
 11  import Database from 'better-sqlite3';
 12  import { fileURLToPath } from 'url';
 13  import { dirname, join } from 'path';
 14  import { unlinkSync, existsSync } from 'fs';
 15  
 16  // Mock dotenv
 17  mock.module('dotenv', {
 18    defaultExport: { config: () => {} },
 19    namedExports: { config: () => {} },
 20  });
 21  
 22  // Mock child_process execSync to control systemctl output
 23  const execSyncMock = mock.fn();
 24  mock.module('child_process', {
 25    namedExports: {
 26      execSync: execSyncMock,
 27    },
 28  });
 29  
 30  // Mock writeFileSync so we can capture /tmp/watchdog-status.txt writes
 31  let _capturedStatusFile = '';
 32  const writeFileSyncMock = mock.fn((path, content) => {
 33    if (path.includes('watchdog-status')) {
 34      _capturedStatusFile = content;
 35    }
 36  });
 37  import * as realFs from 'fs';
 38  mock.module('fs', {
 39    namedExports: {
 40      ...realFs,
 41      writeFileSync: writeFileSyncMock,
 42    },
 43  });
 44  
 45  const __filename = fileURLToPath(import.meta.url);
 46  const __dirname = dirname(__filename);
 47  const projectRoot = join(__dirname, '../..');
 48  const TEST_DB_PATH = join(projectRoot, 'db/test-process-guardian-aug.db');
 49  
 50  function setupTestDatabase() {
 51    const db = new Database(TEST_DB_PATH);
 52    db.exec(`
 53      CREATE TABLE IF NOT EXISTS system_health (
 54        id INTEGER PRIMARY KEY AUTOINCREMENT,
 55        check_type TEXT NOT NULL,
 56        status TEXT NOT NULL CHECK (status IN ('ok', 'warning', 'critical')),
 57        details TEXT,
 58        action_taken TEXT,
 59        created_at TEXT DEFAULT (datetime('now'))
 60      );
 61      CREATE TABLE IF NOT EXISTS agent_tasks (
 62        id INTEGER PRIMARY KEY AUTOINCREMENT,
 63        task_type TEXT NOT NULL,
 64        assigned_to TEXT NOT NULL,
 65        created_by TEXT,
 66        priority INTEGER DEFAULT 5,
 67        status TEXT DEFAULT 'pending',
 68        context_json TEXT,
 69        parent_task_id INTEGER,
 70        result_json TEXT,
 71        error_message TEXT,
 72        retry_count INTEGER DEFAULT 0,
 73        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 74        started_at TIMESTAMP,
 75        completed_at TIMESTAMP
 76      );
 77      CREATE TABLE IF NOT EXISTS settings (
 78        key TEXT PRIMARY KEY,
 79        value TEXT,
 80        description TEXT,
 81        updated_at TEXT DEFAULT (datetime('now'))
 82      );
 83    `);
 84    return db;
 85  }
 86  
 87  function cleanupTestDatabase() {
 88    if (existsSync(TEST_DB_PATH)) {
 89      try {
 90        unlinkSync(TEST_DB_PATH);
 91      } catch {
 92        /* ignore */
 93      }
 94    }
 95  }
 96  
 97  // ═══════════════════════════════════════════
 98  // Core database tests (no external dependencies)
 99  // ═══════════════════════════════════════════
100  
101  test('Process Guardian', async t => {
102    let db;
103  
104    beforeEach(() => {
105      cleanupTestDatabase();
106      db = setupTestDatabase();
107      process.env.DATABASE_PATH = TEST_DB_PATH;
108      execSyncMock.mock.resetCalls();
109      writeFileSyncMock.mock.resetCalls();
110      _capturedStatusFile = '';
111    });
112  
113    afterEach(() => {
114      if (db?.open) db.close();
115      cleanupTestDatabase();
116      delete process.env.DATABASE_PATH;
117    });
118  
119    await t.test('runProcessGuardian is exported as an async function', async () => {
120      const mod = await import('../../src/cron/process-guardian.js');
121      assert.ok(
122        typeof mod.runProcessGuardian === 'function',
123        'runProcessGuardian should be exported'
124      );
125    });
126  
127    await t.test('system_health table accepts health check records', () => {
128      db.prepare(
129        `INSERT INTO system_health (check_type, status, details, action_taken)
130         VALUES (?, ?, ?, ?)`
131      ).run('pipeline_service', 'ok', JSON.stringify({ service_status: 'active' }), null);
132  
133      db.prepare(
134        `INSERT INTO system_health (check_type, status, details, action_taken)
135         VALUES (?, ?, ?, ?)`
136      ).run(
137        'circuit_breaker',
138        'critical',
139        JSON.stringify({ breaker_open_errors_last_hour: 5 }),
140        null
141      );
142  
143      const rows = db.prepare('SELECT * FROM system_health ORDER BY id').all();
144      assert.equal(rows.length, 2);
145      assert.equal(rows[0].check_type, 'pipeline_service');
146      assert.equal(rows[0].status, 'ok');
147      assert.equal(rows[1].check_type, 'circuit_breaker');
148      assert.equal(rows[1].status, 'critical');
149    });
150  
151    await t.test('circuit breaker detection counts breaker-open errors', () => {
152      for (let i = 0; i < 5; i++) {
153        db.prepare(
154          `INSERT INTO agent_tasks (task_type, assigned_to, context_json, created_at)
155           VALUES (?, ?, ?, datetime('now', '-30 minutes'))`
156        ).run('fix_bug', 'developer', JSON.stringify({ error: 'Breaker is open' }));
157      }
158  
159      const row = db
160        .prepare(
161          `SELECT COUNT(*) as count FROM agent_tasks
162         WHERE created_at >= datetime('now', '-1 hour')
163           AND context_json LIKE '%Breaker is open%'`
164        )
165        .get();
166  
167      assert.equal(row.count, 5);
168      assert.ok(row.count > 3, 'Should detect circuit breaker firing');
169    });
170  
171    await t.test('clearance cycle detection uses previous system_health record', () => {
172      db.prepare(
173        `INSERT INTO system_health (check_type, status, details, created_at)
174         VALUES (?, ?, ?, datetime('now', '-2 minutes'))`
175      ).run('clearance_cycle', 'ok', JSON.stringify({ clearance_running: true, was_running: false }));
176  
177      const lastRow = db
178        .prepare(
179          `SELECT details FROM system_health
180         WHERE check_type = 'clearance_cycle'
181         ORDER BY created_at DESC LIMIT 1`
182        )
183        .get();
184  
185      const details = JSON.parse(lastRow.details);
186      assert.equal(details.clearance_running, true);
187    });
188  
189    await t.test('old system_health records can be cleaned up', () => {
190      for (let i = 0; i < 5; i++) {
191        db.prepare(
192          `INSERT INTO system_health (check_type, status, details, created_at)
193           VALUES (?, ?, ?, datetime('now', '-10 days'))`
194        ).run('pipeline_service', 'ok', '{}');
195      }
196  
197      db.prepare(
198        `INSERT INTO system_health (check_type, status, details)
199         VALUES (?, ?, ?)`
200      ).run('pipeline_service', 'ok', '{}');
201  
202      const result = db
203        .prepare(`DELETE FROM system_health WHERE created_at < datetime('now', '-7 days')`)
204        .run();
205      assert.equal(result.changes, 5);
206  
207      const remaining = db.prepare('SELECT COUNT(*) as count FROM system_health').get();
208      assert.equal(remaining.count, 1);
209    });
210  
211    await t.test('circuit breaker threshold is 3 errors', () => {
212      // 3 errors - should be ok (threshold is > 3, not >= 3)
213      for (let i = 0; i < 3; i++) {
214        db.prepare(
215          `INSERT INTO agent_tasks (task_type, assigned_to, context_json, created_at)
216           VALUES (?, ?, ?, datetime('now', '-10 minutes'))`
217        ).run('fix_bug', 'developer', JSON.stringify({ error: 'Breaker is open' }));
218      }
219  
220      const row = db
221        .prepare(
222          `SELECT COUNT(*) as count FROM agent_tasks
223         WHERE created_at >= datetime('now', '-1 hour')
224           AND context_json LIKE '%Breaker is open%'`
225        )
226        .get();
227  
228      assert.equal(row.count, 3);
229      assert.ok(row.count <= 3, 'At threshold, should not be critical');
230    });
231  
232    await t.test('circuit breaker checks error_message column too', () => {
233      db.prepare(
234        `INSERT INTO agent_tasks (task_type, assigned_to, error_message, created_at)
235         VALUES (?, ?, ?, datetime('now', '-5 minutes'))`
236      ).run('fix_bug', 'developer', 'Breaker is open for agent');
237  
238      const row = db
239        .prepare(
240          `SELECT COUNT(*) as count FROM agent_tasks
241         WHERE created_at >= datetime('now', '-1 hour')
242           AND error_message LIKE '%Breaker is open%'`
243        )
244        .get();
245  
246      assert.equal(row.count, 1);
247    });
248  
249    await t.test('circuit breaker ignores errors older than 1 hour', () => {
250      for (let i = 0; i < 10; i++) {
251        db.prepare(
252          `INSERT INTO agent_tasks (task_type, assigned_to, context_json, created_at)
253           VALUES (?, ?, ?, datetime('now', '-90 minutes'))`
254        ).run('fix_bug', 'developer', JSON.stringify({ error: 'Breaker is open' }));
255      }
256  
257      const row = db
258        .prepare(
259          `SELECT COUNT(*) as count FROM agent_tasks
260         WHERE created_at >= datetime('now', '-1 hour')
261           AND context_json LIKE '%Breaker is open%'`
262        )
263        .get();
264  
265      assert.equal(row.count, 0, 'Old errors should not be counted');
266    });
267  
268    await t.test('clearance cycle records both was_running and clearance_running state', () => {
269      // Simulate: last check had clearance running, current check has it stopped
270      db.prepare(
271        `INSERT INTO system_health (check_type, status, details, created_at)
272         VALUES (?, ?, ?, datetime('now', '-1 minute'))`
273      ).run('clearance_cycle', 'ok', JSON.stringify({ clearance_running: true, was_running: false }));
274  
275      // Now verify next check would detect transition
276      const lastRow = db
277        .prepare(
278          `SELECT details FROM system_health
279         WHERE check_type = 'clearance_cycle'
280         ORDER BY created_at DESC LIMIT 1`
281        )
282        .get();
283  
284      const last = JSON.parse(lastRow.details);
285      assert.equal(last.clearance_running, true);
286      assert.equal(last.was_running, false);
287    });
288  
289    await t.test('system_health stores action_taken for pipeline restarts', () => {
290      db.prepare(
291        `INSERT INTO system_health (check_type, status, details, action_taken)
292         VALUES (?, ?, ?, ?)`
293      ).run(
294        'pipeline_service',
295        'warning',
296        JSON.stringify({ service_status: 'inactive' }),
297        'restarted_pipeline'
298      );
299  
300      const row = db
301        .prepare("SELECT * FROM system_health WHERE action_taken = 'restarted_pipeline'")
302        .get();
303  
304      assert.ok(row);
305      assert.equal(row.status, 'warning');
306      assert.equal(row.action_taken, 'restarted_pipeline');
307    });
308  
309    await t.test('full runProcessGuardian returns correct summary structure', async () => {
310      // Mock execSync: pipeline is active, no clearance running
311      execSyncMock.mock.mockImplementation(cmd => {
312        if (cmd.includes('is-active')) return 'active\n';
313        if (cmd.includes('ps aux')) return 'some processes\n';
314        return '\n';
315      });
316  
317      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
318      const summary = await runProcessGuardian();
319  
320      assert.ok(summary, 'should return a summary');
321      assert.ok(typeof summary.ran_at === 'string', 'ran_at should be ISO string');
322      assert.ok(typeof summary.duration_seconds === 'number', 'duration_seconds should be a number');
323      assert.equal(summary.checks_run, 6, 'should run 6 checks');
324      assert.ok(typeof summary.ok === 'number');
325      assert.ok(typeof summary.warnings === 'number');
326      assert.ok(typeof summary.critical === 'number');
327      assert.ok(Array.isArray(summary.results), 'results should be an array');
328      assert.equal(summary.results.length, 6);
329    });
330  
331    await t.test('runProcessGuardian writes watchdog status file', async () => {
332      execSyncMock.mock.mockImplementation(cmd => {
333        if (cmd.includes('is-active')) return 'active\n';
334        if (cmd.includes('ps aux')) return 'some output\n';
335        return '\n';
336      });
337  
338      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
339      await runProcessGuardian();
340  
341      assert.ok(writeFileSyncMock.mock.calls.length > 0, 'should write status file');
342      const watchdogCall = writeFileSyncMock.mock.calls.find(c =>
343        c.arguments[0].includes('watchdog-status')
344      );
345      assert.ok(watchdogCall, 'should write to watchdog-status.txt');
346      assert.ok(typeof watchdogCall.arguments[1] === 'string');
347      assert.ok(watchdogCall.arguments[1].includes('Process Guardian'));
348    });
349  
350    await t.test('runProcessGuardian status file includes pipeline status', async () => {
351      execSyncMock.mock.mockImplementation(cmd => {
352        if (cmd.includes('is-active')) return 'active\n';
353        if (cmd.includes('ps aux')) return 'output\n';
354        return '\n';
355      });
356  
357      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
358      await runProcessGuardian();
359  
360      const watchdogCall = writeFileSyncMock.mock.calls.find(c =>
361        c.arguments[0].includes('watchdog-status')
362      );
363      const statusContent = watchdogCall?.arguments[1] || '';
364      assert.ok(statusContent.includes('Pipeline'), 'status file should mention Pipeline');
365      assert.ok(
366        statusContent.includes('Circuit breaker'),
367        'status file should mention circuit breaker'
368      );
369    });
370  
371    // ════════════════════════════════════════════════
372    // Error path tests - pipeline service restart (lines 59-76)
373    // ════════════════════════════════════════════════
374  
375    await t.test('pipeline inactive triggers restart and returns warning status', async () => {
376      execSyncMock.mock.mockImplementation(cmd => {
377        if (cmd.includes('is-active')) {
378          const err = new Error('inactive');
379          err.stdout = 'inactive';
380          throw err;
381        }
382        if (cmd.includes('restart')) return 'ok\n';
383        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'some output\n';
384        return '\n';
385      });
386  
387      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
388      const summary = await runProcessGuardian();
389  
390      const pipelineResult = summary.results.find(r => r.check === 'pipeline_service');
391      assert.ok(pipelineResult);
392      assert.equal(pipelineResult.status, 'warning');
393      assert.equal(pipelineResult.actionTaken, 'restarted_pipeline');
394      assert.equal(pipelineResult.serviceStatus, 'inactive');
395    });
396  
397    await t.test('pipeline restart failure sets critical status', async () => {
398      execSyncMock.mock.mockImplementation(cmd => {
399        if (cmd.includes('is-active')) {
400          const err = new Error('inactive');
401          err.stdout = 'inactive';
402          throw err;
403        }
404        if (cmd.includes('restart') && cmd.includes('333method-pipeline')) {
405          throw new Error('Failed: unit not found');
406        }
407        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'some output\n';
408        return '\n';
409      });
410  
411      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
412      const summary = await runProcessGuardian();
413  
414      const pipelineResult = summary.results.find(r => r.check === 'pipeline_service');
415      assert.ok(pipelineResult);
416      assert.equal(pipelineResult.status, 'critical');
417      assert.ok(pipelineResult.actionTaken.includes('restart_failed'));
418    });
419  
420    await t.test('pipeline execSync throws with empty stdout uses inactive fallback', async () => {
421      execSyncMock.mock.mockImplementation(cmd => {
422        if (cmd.includes('is-active')) {
423          const err = new Error('Command failed');
424          err.stdout = '';
425          throw err;
426        }
427        if (cmd.includes('restart')) return '\n';
428        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
429        return '\n';
430      });
431  
432      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
433      const summary = await runProcessGuardian();
434  
435      const pipelineResult = summary.results.find(r => r.check === 'pipeline_service');
436      assert.ok(pipelineResult);
437      assert.equal(pipelineResult.serviceStatus, 'inactive');
438    });
439  
440    // ════════════════════════════════════════════════
441    // Error path tests - clearance cycle (lines 102-137)
442    // ════════════════════════════════════════════════
443  
444    await t.test('clearance cycle ps aux failure makes clearanceRunning false', async () => {
445      execSyncMock.mock.mockImplementation(cmd => {
446        if (cmd.includes('is-active')) return 'active\n';
447        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) {
448          throw new Error('ps: command not found');
449        }
450        return '\n';
451      });
452  
453      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
454      const summary = await runProcessGuardian();
455  
456      const clearanceResult = summary.results.find(r => r.check === 'clearance_cycle');
457      assert.ok(clearanceResult);
458      assert.equal(clearanceResult.clearanceRunning, false);
459      assert.equal(clearanceResult.status, 'ok');
460    });
461  
462    await t.test('clearance cycle detects run-clearance-cycle.sh in ps output', async () => {
463      execSyncMock.mock.mockImplementation(cmd => {
464        if (cmd.includes('is-active')) return 'active\n';
465        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) {
466          return 'root  5678  /bin/bash /tmp/run-clearance-cycle.sh\n';
467        }
468        return '\n';
469      });
470  
471      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
472      const summary = await runProcessGuardian();
473  
474      const clearanceResult = summary.results.find(r => r.check === 'clearance_cycle');
475      assert.ok(clearanceResult);
476      assert.equal(clearanceResult.clearanceRunning, true);
477    });
478  
479    await t.test('clearance cycle restarts pipeline after clearance finishes', async () => {
480      db.prepare(
481        "INSERT INTO system_health (check_type, status, details, created_at) VALUES (?, ?, ?, datetime('now', '-1 minute'))"
482      ).run('clearance_cycle', 'ok', JSON.stringify({ clearance_running: true }));
483  
484      execSyncMock.mock.mockImplementation(cmd => {
485        if (cmd.includes('is-active')) return 'active\n';
486        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) {
487          return 'root  1234  node server.js\n';
488        }
489        if (cmd.includes('restart')) return '\n';
490        return '\n';
491      });
492  
493      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
494      const summary = await runProcessGuardian();
495  
496      const clearanceResult = summary.results.find(r => r.check === 'clearance_cycle');
497      assert.ok(clearanceResult);
498      assert.equal(clearanceResult.actionTaken, 'restarted_pipeline_after_clearance');
499      assert.equal(clearanceResult.status, 'ok');
500    });
501  
502    await t.test('clearance cycle restart failure sets warning with action', async () => {
503      db.prepare(
504        "INSERT INTO system_health (check_type, status, details, created_at) VALUES (?, ?, ?, datetime('now', '-1 minute'))"
505      ).run('clearance_cycle', 'ok', JSON.stringify({ clearance_running: true }));
506  
507      execSyncMock.mock.mockImplementation(cmd => {
508        if (cmd.includes('is-active')) return 'active\n';
509        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) {
510          return 'root  1234  node server.js\n';
511        }
512        if (cmd.includes('restart')) {
513          throw new Error('restart failed: permission denied');
514        }
515        return '\n';
516      });
517  
518      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
519      const summary = await runProcessGuardian();
520  
521      const clearanceResult = summary.results.find(r => r.check === 'clearance_cycle');
522      assert.ok(clearanceResult);
523      assert.equal(clearanceResult.status, 'warning');
524      assert.ok(clearanceResult.actionTaken.includes('clearance_restart_failed'));
525    });
526  
527    await t.test('clearance cycle handles invalid JSON in previous record gracefully', async () => {
528      db.prepare(
529        "INSERT INTO system_health (check_type, status, details, created_at) VALUES (?, ?, ?, datetime('now', '-1 minute'))"
530      ).run('clearance_cycle', 'ok', 'INVALID JSON {{{');
531  
532      execSyncMock.mock.mockImplementation(cmd => {
533        if (cmd.includes('is-active')) return 'active\n';
534        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'some output\n';
535        return '\n';
536      });
537  
538      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
539      const summary = await runProcessGuardian();
540  
541      assert.ok(summary);
542      const clearanceResult = summary.results.find(r => r.check === 'clearance_cycle');
543      assert.ok(clearanceResult);
544      assert.equal(clearanceResult.status, 'ok');
545    });
546  
547    // ════════════════════════════════════════════════
548    // Circuit breaker critical path (lines 261-262)
549    // ════════════════════════════════════════════════
550  
551    await t.test('circuit breaker critical with more than 3 errors in last hour', async () => {
552      for (let i = 0; i < 5; i++) {
553        db.prepare(
554          "INSERT INTO agent_tasks (task_type, assigned_to, error_message, created_at) VALUES (?, ?, ?, datetime('now', '-10 minutes'))"
555        ).run('fix_bug', 'developer', 'Breaker is open');
556      }
557  
558      execSyncMock.mock.mockImplementation(cmd => {
559        if (cmd.includes('is-active')) return 'active\n';
560        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
561        return '\n';
562      });
563  
564      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
565      const summary = await runProcessGuardian();
566  
567      const cbResult = summary.results.find(r => r.check === 'circuit_breaker');
568      assert.ok(cbResult);
569      assert.equal(cbResult.status, 'critical');
570      assert.equal(cbResult.breaker_open_errors_last_hour, 5);
571      assert.equal(summary.critical, 1);
572    });
573  
574    await t.test('circuit breaker checks context_json and result_json columns', async () => {
575      db.prepare(
576        "INSERT INTO agent_tasks (task_type, assigned_to, context_json, created_at) VALUES (?, ?, ?, datetime('now', '-5 minutes'))"
577      ).run('fix_bug', 'developer', JSON.stringify({ error: 'Breaker is open' }));
578  
579      db.prepare(
580        "INSERT INTO agent_tasks (task_type, assigned_to, result_json, created_at) VALUES (?, ?, ?, datetime('now', '-5 minutes'))"
581      ).run('fix_bug', 'developer', JSON.stringify({ result: 'Breaker is open timeout' }));
582  
583      execSyncMock.mock.mockImplementation(cmd => {
584        if (cmd.includes('is-active')) return 'active\n';
585        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
586        return '\n';
587      });
588  
589      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
590      const summary = await runProcessGuardian();
591  
592      const cbResult = summary.results.find(r => r.check === 'circuit_breaker');
593      assert.ok(cbResult);
594      assert.equal(cbResult.breaker_open_errors_last_hour, 2);
595    });
596  
597    // ════════════════════════════════════════════════
598    // Status file content tests (lines 192-193, 223-224, 285-287)
599    // ════════════════════════════════════════════════
600  
601    await t.test('status file shows stopped icon when pipeline inactive', async () => {
602      execSyncMock.mock.mockImplementation(cmd => {
603        if (cmd.includes('is-active')) {
604          const err = new Error('inactive');
605          err.stdout = 'inactive';
606          throw err;
607        }
608        if (cmd.includes('restart')) return '\n';
609        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
610        return '\n';
611      });
612  
613      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
614      await runProcessGuardian();
615  
616      const watchdogCall = writeFileSyncMock.mock.calls.find(c =>
617        c.arguments[0].includes('watchdog-status')
618      );
619      const content = watchdogCall?.arguments[1] || '';
620      assert.ok(content.includes('stopped'), 'should show stopped when pipeline inactive');
621    });
622  
623    await t.test('status file includes Issues section when circuit breaker is critical', async () => {
624      for (let i = 0; i < 5; i++) {
625        db.prepare(
626          "INSERT INTO agent_tasks (task_type, assigned_to, error_message, created_at) VALUES (?, ?, ?, datetime('now', '-5 minutes'))"
627        ).run('fix_bug', 'developer', 'Breaker is open');
628      }
629  
630      execSyncMock.mock.mockImplementation(cmd => {
631        if (cmd.includes('is-active')) return 'active\n';
632        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
633        return '\n';
634      });
635  
636      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
637      await runProcessGuardian();
638  
639      const watchdogCall = writeFileSyncMock.mock.calls.find(c =>
640        c.arguments[0].includes('watchdog-status')
641      );
642      const content = watchdogCall?.arguments[1] || '';
643      assert.ok(
644        content.includes('Issues'),
645        'status file should include Issues section when critical'
646      );
647      assert.ok(content.includes('circuit_breaker'), 'should mention which check failed');
648    });
649  
650    await t.test('status file includes action taken when pipeline restarted', async () => {
651      execSyncMock.mock.mockImplementation(cmd => {
652        if (cmd.includes('is-active')) {
653          const err = new Error('inactive');
654          err.stdout = 'inactive';
655          throw err;
656        }
657        if (cmd.includes('restart')) return '\n';
658        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
659        return '\n';
660      });
661  
662      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
663      await runProcessGuardian();
664  
665      const watchdogCall = writeFileSyncMock.mock.calls.find(c =>
666        c.arguments[0].includes('watchdog-status')
667      );
668      const content = watchdogCall?.arguments[1] || '';
669      assert.ok(content.includes('restarted_pipeline'), 'should show action taken in status file');
670    });
671  
672    await t.test('summary ok+warning+critical always sums to 6', async () => {
673      execSyncMock.mock.mockImplementation(cmd => {
674        if (cmd.includes('is-active')) return 'active\n';
675        if (cmd.includes('ps aux') || cmd.includes('/run/current-system')) return 'output\n';
676        return '\n';
677      });
678  
679      const { runProcessGuardian } = await import('../../src/cron/process-guardian.js');
680      const summary = await runProcessGuardian();
681  
682      assert.equal(summary.ok + summary.warnings + summary.critical, 6);
683    });
684  });