1  /**
   2   * Agent Runner Coverage Tests
   3   *
   4   * Tests for src/agents/runner.js:
   5   * - runAgentCycle (sequential execution with agent mocks)
   6   * - getAgentStats (database queries)
   7   * - checkCircuitBreakers (circuit breaker logic)
   8   * - checkNightmareScenarios (alert detection)
   9   * - createSummaryCommit (git operations, mocked)
  10   * - runCompleteAgentCycle (orchestration)
  11   * - calculateTaskLimit (autoscaling logic via runCompleteAgentCycle with parallel mode)
  12   *
  13   * Strategy: Set DATABASE_PATH to a temp file BEFORE importing runner.js,
  14   * so the module-level `db` connects to our test database.
  15   * We reset the DB contents between tests by truncating tables.
  16   */
  17  
  18  import { describe, test, beforeEach, afterEach, mock } from 'node:test';
  19  import assert from 'node:assert/strict';
  20  import Database from 'better-sqlite3';
  21  import path from 'path';
  22  import { fileURLToPath } from 'url';
  23  import { unlinkSync, existsSync } from 'fs';
  24  
  25  const __filename = fileURLToPath(import.meta.url);
  26  const __dirname = path.dirname(__filename);
  27  
  28  // ── Set DATABASE_PATH BEFORE importing runner.js ───────────────────────────
  29  const TEST_DB_PATH = path.join('/tmp', `test-runner-coverage-${process.pid}.db`);
  30  process.env.DATABASE_PATH = TEST_DB_PATH;
  31  process.env.AGENT_SYSTEM_ENABLED = 'true';
  32  process.env.AGENT_AUTO_COMMIT = 'false';
  33  process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
  34  process.env.AGENT_PARALLEL_EXECUTION = 'false';
  35  process.env.AGENT_IMMEDIATE_INVOCATION = 'false';
  36  process.env.AGENT_REALTIME_NOTIFICATIONS = 'false';
  37  
  38  // ── Test DB schema ─────────────────────────────────────────────────────────
  39  const TEST_SCHEMA = `
  40    CREATE TABLE IF NOT EXISTS agent_tasks (
  41      id INTEGER PRIMARY KEY AUTOINCREMENT,
  42      task_type TEXT NOT NULL,
  43      assigned_to TEXT NOT NULL,
  44      created_by TEXT,
  45      status TEXT DEFAULT 'pending',
  46      priority INTEGER DEFAULT 5,
  47      context_json TEXT,
  48      result_json TEXT,
  49      error_message TEXT,
  50      created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
  51      started_at DATETIME,
  52      completed_at DATETIME,
  53      retry_count INTEGER DEFAULT 0
  54    );
  55  
  56    CREATE TABLE IF NOT EXISTS agent_state (
  57      agent_name TEXT PRIMARY KEY,
  58      last_active DATETIME DEFAULT CURRENT_TIMESTAMP,
  59      current_task_id INTEGER,
  60      status TEXT DEFAULT 'idle',
  61      metrics_json TEXT
  62    );
  63  
  64    CREATE TABLE IF NOT EXISTS agent_logs (
  65      id INTEGER PRIMARY KEY AUTOINCREMENT,
  66      task_id INTEGER,
  67      agent_name TEXT NOT NULL,
  68      log_level TEXT,
  69      message TEXT,
  70      data_json TEXT,
  71      metadata_json TEXT,
  72      created_at DATETIME DEFAULT CURRENT_TIMESTAMP
  73    );
  74  `;
  75  
  76  // Create the test DB and schema before any mocks (delete stale file first)
  77  if (existsSync(TEST_DB_PATH)) unlinkSync(TEST_DB_PATH);
  78  const setupDb = new Database(TEST_DB_PATH);
  79  setupDb.exec(TEST_SCHEMA);
  80  setupDb.close();
  81  
  82  // ── Mock child_process BEFORE importing runner.js ──────────────────────────
  83  let gitStatusOutput = '';
  84  let gitBranchOutput = 'main';
  85  let gitCommitOutput = 'abc1234';
  86  let gitCheckoutShouldFail = false;
  87  let gitPushShouldFail = false;
  88  
  89  const execSyncMock = mock.fn(cmd => {
  90    if (cmd.includes('git status --porcelain')) return gitStatusOutput;
  91    if (cmd.includes('git rev-parse --abbrev-ref HEAD')) return gitBranchOutput;
  92    if (cmd.includes('git rev-parse --short HEAD')) return gitCommitOutput;
  93    if (cmd.includes('git checkout') || cmd.includes('git checkout -B')) {
  94      if (gitCheckoutShouldFail) throw new Error('git checkout failed');
  95      gitBranchOutput = 'autofix';
  96      return '';
  97    }
  98    if (cmd.includes('git push')) {
  99      if (gitPushShouldFail) throw new Error('git push failed');
 100      return '';
 101    }
 102    if (cmd.includes('git add')) return '';
 103    if (cmd.includes('git commit')) return '';
 104    return '';
 105  });
 106  
 107  mock.module('child_process', {
 108    namedExports: { execSync: execSyncMock },
 109  });
 110  
 111  // ── Mock agent classes ─────────────────────────────────────────────────────
 112  // Shared mutable state for agent task counts
 113  const agentTaskCounts = {
 114    monitor: 2,
 115    triage: 1,
 116    developer: 0,
 117    qa: 0,
 118    security: 0,
 119    architect: 0,
 120  };
 121  
 122  // Track which agents should throw
 123  const agentShouldThrow = {
 124    monitor: false,
 125    triage: false,
 126    developer: false,
 127    qa: false,
 128    security: false,
 129    architect: false,
 130  };
 131  
 132  function resetAgentBehavior(counts = {}, throws = {}) {
 133    agentTaskCounts.monitor = counts.monitor ?? 0;
 134    agentTaskCounts.triage = counts.triage ?? 0;
 135    agentTaskCounts.developer = counts.developer ?? 0;
 136    agentTaskCounts.qa = counts.qa ?? 0;
 137    agentTaskCounts.security = counts.security ?? 0;
 138    agentTaskCounts.architect = counts.architect ?? 0;
 139  
 140    agentShouldThrow.monitor = throws.monitor ?? false;
 141    agentShouldThrow.triage = throws.triage ?? false;
 142    agentShouldThrow.developer = throws.developer ?? false;
 143    agentShouldThrow.qa = throws.qa ?? false;
 144    agentShouldThrow.security = throws.security ?? false;
 145    agentShouldThrow.architect = throws.architect ?? false;
 146  }
 147  
 148  function makeMockClass(agentName) {
 149    return class {
 150      constructor() {
 151        this.agentName = agentName;
 152      }
 153      pollTasks(_limit) {
 154        const shouldThrow = Object.prototype.hasOwnProperty.call(agentShouldThrow, agentName)
 155          ? agentShouldThrow[agentName]
 156          : false;
 157        const taskCount = Object.prototype.hasOwnProperty.call(agentTaskCounts, agentName)
 158          ? agentTaskCounts[agentName]
 159          : 0;
 160  
 161        if (shouldThrow) {
 162          return Promise.reject(new Error(`${agentName} agent crash`));
 163        }
 164        return Promise.resolve(taskCount);
 165      }
 166    };
 167  }
 168  
 169  mock.module('../../src/agents/monitor.js', {
 170    namedExports: { MonitorAgent: makeMockClass('monitor') },
 171  });
 172  mock.module('../../src/agents/triage.js', {
 173    namedExports: { TriageAgent: makeMockClass('triage') },
 174  });
 175  mock.module('../../src/agents/developer.js', {
 176    namedExports: { DeveloperAgent: makeMockClass('developer') },
 177  });
 178  mock.module('../../src/agents/qa.js', {
 179    namedExports: { QAAgent: makeMockClass('qa') },
 180  });
 181  mock.module('../../src/agents/security.js', {
 182    namedExports: { SecurityAgent: makeMockClass('security') },
 183  });
 184  mock.module('../../src/agents/architect.js', {
 185    namedExports: { ArchitectAgent: makeMockClass('architect') },
 186  });
 187  
 188  // ── Import runner AFTER all mocks are registered ──────────────────────────
 189  const { runAgentCycle, getAgentStats, checkCircuitBreakers, runCompleteAgentCycle } =
 190    await import('../../src/agents/runner.js');
 191  
 192  // ── Helper: direct DB connection for test setup ───────────────────────────
 193  let testDb;
 194  
 195  function clearTables() {
 196    testDb.exec('DELETE FROM agent_tasks');
 197    testDb.exec('DELETE FROM agent_state');
 198    testDb.exec('DELETE FROM agent_logs');
 199  
 200    // Re-insert agent state rows
 201    const agents = ['monitor', 'triage', 'developer', 'qa', 'security', 'architect'];
 202    for (const name of agents) {
 203      testDb
 204        .prepare(`INSERT OR IGNORE INTO agent_state (agent_name, status) VALUES (?, 'idle')`)
 205        .run(name);
 206    }
 207  }
 208  
 209  beforeEach(() => {
 210    testDb = new Database(TEST_DB_PATH);
 211    process.env.AGENT_SYSTEM_ENABLED = 'true';
 212    process.env.AGENT_AUTO_COMMIT = 'false';
 213    process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
 214    process.env.AGENT_PARALLEL_EXECUTION = 'false';
 215    process.env.AGENT_CIRCUIT_BREAKER_THRESHOLD = '0.3';
 216  
 217    gitStatusOutput = '';
 218    gitBranchOutput = 'main';
 219    gitCommitOutput = 'abc1234';
 220    gitCheckoutShouldFail = false;
 221    gitPushShouldFail = false;
 222  
 223    clearTables();
 224    resetAgentBehavior();
 225    execSyncMock.mock.resetCalls();
 226  });
 227  
 228  afterEach(() => {
 229    testDb.close();
 230  });
 231  
 232  // Cleanup after all tests
 233  afterEach(async () => {});
 234  
 235  // ── runAgentCycle ──────────────────────────────────────────────────────────
 236  describe('runAgentCycle - sequential execution', () => {
 237    test('returns disabled result when AGENT_SYSTEM_ENABLED=false', async () => {
 238      process.env.AGENT_SYSTEM_ENABLED = 'false';
 239      const result = await runAgentCycle();
 240      assert.equal(result.enabled, false);
 241      assert.equal(result.processed, 0);
 242    });
 243  
 244    test('returns summary with enabled=true when system is active', async () => {
 245      resetAgentBehavior({ monitor: 3, triage: 1 });
 246      const result = await runAgentCycle({ tasksPerAgent: 5 });
 247  
 248      assert.equal(result.enabled, true);
 249      assert.ok(result.timestamp);
 250      assert.ok(typeof result.agents === 'object');
 251      assert.equal(result.total_processed, 4);
 252      assert.equal(result.errors.length, 0);
 253    });
 254  
 255    test('records per-agent processed count and status', async () => {
 256      resetAgentBehavior({ monitor: 2, developer: 1 });
 257      const result = await runAgentCycle({ tasksPerAgent: 3 });
 258  
 259      assert.equal(result.agents.monitor.processed, 2);
 260      assert.equal(result.agents.monitor.status, 'success');
 261      assert.ok(result.agents.monitor.duration_ms >= 0);
 262      assert.equal(result.agents.developer.processed, 1);
 263      assert.equal(result.total_processed, 3);
 264    });
 265  
 266    test('continues running agents when one throws an error', async () => {
 267      resetAgentBehavior({ monitor: 2, triage: 1 }, { developer: true });
 268      const result = await runAgentCycle({ tasksPerAgent: 5 });
 269  
 270      assert.ok(result.agents.developer);
 271      assert.equal(result.agents.developer.status, 'failed');
 272      assert.equal(result.agents.developer.error, 'developer agent crash');
 273      assert.equal(result.errors.length, 1);
 274      assert.equal(result.errors[0].agent, 'developer');
 275  
 276      // Other agents should still have succeeded
 277      assert.equal(result.agents.monitor.status, 'success');
 278      assert.equal(result.agents.triage.status, 'success');
 279    });
 280  
 281    test('verbose=true completes without error when tasks processed', async () => {
 282      resetAgentBehavior({ monitor: 1 });
 283      const result = await runAgentCycle({ tasksPerAgent: 5, verbose: true });
 284      assert.equal(result.enabled, true);
 285      assert.equal(result.agents.monitor.processed, 1);
 286    });
 287  
 288    test('verbose=true completes without error when no tasks', async () => {
 289      resetAgentBehavior(); // all 0
 290      const result = await runAgentCycle({ tasksPerAgent: 5, verbose: true });
 291      assert.equal(result.total_processed, 0);
 292      assert.equal(result.enabled, true);
 293    });
 294  
 295    test('defaults to tasksPerAgent=5 when not specified', async () => {
 296      resetAgentBehavior({ monitor: 1 });
 297      const result = await runAgentCycle();
 298      assert.equal(result.enabled, true);
 299      assert.ok(result.agents.monitor);
 300    });
 301  
 302    test('handles multiple agent failures gracefully', async () => {
 303      resetAgentBehavior({}, { monitor: true, developer: true, qa: true });
 304      const result = await runAgentCycle({ tasksPerAgent: 5 });
 305  
 306      assert.equal(result.errors.length, 3);
 307      assert.equal(result.agents.monitor.status, 'failed');
 308      assert.equal(result.agents.developer.status, 'failed');
 309      assert.equal(result.agents.qa.status, 'failed');
 310  
 311      // Other agents should still succeed
 312      assert.equal(result.agents.triage.status, 'success');
 313      assert.equal(result.agents.security.status, 'success');
 314      assert.equal(result.agents.architect.status, 'success');
 315    });
 316  });
 317  
 318  // ── getAgentStats ──────────────────────────────────────────────────────────
 319  describe('getAgentStats - database queries', () => {
 320    test('returns empty agents array when no tasks in last 24h', () => {
 321      const stats = getAgentStats();
 322      assert.ok(Array.isArray(stats.agents));
 323      assert.equal(stats.agents.length, 0);
 324      assert.ok(stats.overall);
 325      assert.equal(stats.overall.total, 0);
 326      assert.equal(stats.overall.success_rate, 0);
 327      assert.equal(stats.overall.failure_rate, 0);
 328    });
 329  
 330    test('calculates correct success_rate and failure_rate', () => {
 331      const now = new Date().toISOString();
 332      // 3 completed, 1 failed for developer
 333      for (let i = 0; i < 3; i++) {
 334        testDb
 335          .prepare(
 336            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at, completed_at, started_at)
 337           VALUES ('fix_bug', 'developer', 'completed', ?, ?, ?)`
 338          )
 339          .run(now, now, now);
 340      }
 341      testDb
 342        .prepare(
 343          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 344         VALUES ('fix_bug', 'developer', 'failed', ?)`
 345        )
 346        .run(now);
 347  
 348      const stats = getAgentStats();
 349      const devStats = stats.agents.find(a => a.agent === 'developer');
 350      assert.ok(devStats, 'developer stats should exist');
 351      assert.equal(devStats.total, 4);
 352      assert.equal(devStats.completed, 3);
 353      assert.equal(devStats.failed, 1);
 354      assert.ok(Math.abs(devStats.success_rate - 0.75) < 0.01);
 355      assert.ok(Math.abs(devStats.failure_rate - 0.25) < 0.01);
 356    });
 357  
 358    test('counts pending, running, and blocked tasks', () => {
 359      const now = new Date().toISOString();
 360      testDb
 361        .prepare(
 362          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 363         VALUES ('run_tests', 'qa', 'pending', ?)`
 364        )
 365        .run(now);
 366      testDb
 367        .prepare(
 368          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 369         VALUES ('run_tests', 'qa', 'running', ?)`
 370        )
 371        .run(now);
 372      testDb
 373        .prepare(
 374          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 375         VALUES ('run_tests', 'qa', 'blocked', ?)`
 376        )
 377        .run(now);
 378  
 379      const stats = getAgentStats();
 380      const qaStats = stats.agents.find(a => a.agent === 'qa');
 381      assert.ok(qaStats);
 382      assert.equal(qaStats.total, 3);
 383      assert.equal(qaStats.pending, 1);
 384      assert.equal(qaStats.running, 1);
 385      assert.equal(qaStats.blocked, 1);
 386    });
 387  
 388    test('overall stats aggregate across multiple agents', () => {
 389      const now = new Date().toISOString();
 390      testDb
 391        .prepare(
 392          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 393         VALUES ('fix_bug', 'developer', 'completed', ?)`
 394        )
 395        .run(now);
 396      testDb
 397        .prepare(
 398          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 399         VALUES ('run_tests', 'qa', 'failed', ?)`
 400        )
 401        .run(now);
 402  
 403      const stats = getAgentStats();
 404      assert.equal(stats.overall.total, 2);
 405      assert.equal(stats.overall.completed, 1);
 406      assert.equal(stats.overall.failed, 1);
 407      assert.ok(Math.abs(stats.overall.success_rate - 0.5) < 0.01);
 408      assert.ok(Math.abs(stats.overall.failure_rate - 0.5) < 0.01);
 409    });
 410  
 411    test('excludes tasks older than 24 hours', () => {
 412      testDb
 413        .prepare(
 414          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 415         VALUES ('fix_bug', 'developer', 'completed', datetime('now', '-2 days'))`
 416        )
 417        .run();
 418  
 419      const stats = getAgentStats();
 420      assert.equal(stats.overall.total, 0);
 421    });
 422  
 423    test('returns success_rate=0 when all tasks are pending (no completed)', () => {
 424      const now = new Date().toISOString();
 425      testDb
 426        .prepare(
 427          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 428         VALUES ('fix_bug', 'developer', 'pending', ?)`
 429        )
 430        .run(now);
 431  
 432      const stats = getAgentStats();
 433      const devStats = stats.agents.find(a => a.agent === 'developer');
 434      assert.ok(devStats);
 435      assert.equal(devStats.success_rate, 0);
 436      assert.equal(devStats.failure_rate, 0);
 437    });
 438  });
 439  
 440  // ── checkCircuitBreakers ───────────────────────────────────────────────────
 441  describe('checkCircuitBreakers - circuit breaker logic', () => {
 442    test('returns empty array when no agents have high failure rate', () => {
 443      const blocked = checkCircuitBreakers();
 444      assert.ok(Array.isArray(blocked));
 445      assert.equal(blocked.length, 0);
 446    });
 447  
 448    test('triggers circuit breaker when failure rate > 30% with >= 10 tasks', () => {
 449      const now = new Date().toISOString();
 450      // 8 failed + 2 completed = 80% failure rate, 10 total
 451      for (let i = 0; i < 8; i++) {
 452        testDb
 453          .prepare(
 454            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 455           VALUES ('fix_bug', 'developer', 'failed', ?)`
 456          )
 457          .run(now);
 458      }
 459      for (let i = 0; i < 2; i++) {
 460        testDb
 461          .prepare(
 462            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 463           VALUES ('fix_bug', 'developer', 'completed', ?)`
 464          )
 465          .run(now);
 466      }
 467  
 468      const blocked = checkCircuitBreakers();
 469      const devBlocked = blocked.find(b => b.agent === 'developer');
 470      assert.ok(devBlocked, 'developer should be blocked');
 471      assert.ok(devBlocked.failure_rate > 0.3);
 472      assert.equal(devBlocked.total_tasks, 10);
 473    });
 474  
 475    test('does NOT trigger circuit breaker when fewer than 10 tasks total', () => {
 476      const now = new Date().toISOString();
 477      // 5 failed (100% failure rate), but < 10 total
 478      for (let i = 0; i < 5; i++) {
 479        testDb
 480          .prepare(
 481            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 482           VALUES ('fix_bug', 'developer', 'failed', ?)`
 483          )
 484          .run(now);
 485      }
 486  
 487      const blocked = checkCircuitBreakers();
 488      const devBlocked = blocked.find(b => b.agent === 'developer');
 489      assert.equal(devBlocked, undefined);
 490    });
 491  
 492    test('auto-recovers when cooldown expired and failure rate dropped', () => {
 493      // Set developer to blocked state with trigger time 2 hours ago (cooldown is 30 min)
 494      const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString();
 495      testDb
 496        .prepare(
 497          `INSERT OR REPLACE INTO agent_state (agent_name, status, metrics_json)
 498         VALUES ('developer', 'blocked', ?)`
 499        )
 500        .run(JSON.stringify({ circuit_breaker_triggered_at: twoHoursAgo }));
 501  
 502      // Add recent tasks with LOW failure rate (10% failure rate = 0.1 <= threshold 0.3)
 503      // so that developer appears in stats.agents AND failure_rate is below threshold
 504      const now = new Date().toISOString();
 505      for (let i = 0; i < 9; i++) {
 506        testDb
 507          .prepare(
 508            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at, completed_at, started_at)
 509           VALUES ('fix_bug', 'developer', 'completed', ?, ?, ?)`
 510          )
 511          .run(now, now, now);
 512      }
 513      testDb
 514        .prepare(
 515          `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 516         VALUES ('fix_bug', 'developer', 'failed', ?)`
 517        )
 518        .run(now);
 519  
 520      checkCircuitBreakers();
 521  
 522      // Check that developer was auto-recovered (runner sets to 'idle' on recovery)
 523      const devState = testDb
 524        .prepare('SELECT status FROM agent_state WHERE agent_name = ?')
 525        .get('developer');
 526      assert.equal(devState.status, 'idle');
 527    });
 528  
 529    test('does NOT auto-recover when cooldown has not expired', () => {
 530      // Set developer to blocked state with trigger time 5 minutes ago (within 30 min cooldown)
 531      const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString();
 532      testDb
 533        .prepare(
 534          `INSERT OR REPLACE INTO agent_state (agent_name, status, metrics_json)
 535         VALUES ('developer', 'blocked', ?)`
 536        )
 537        .run(JSON.stringify({ circuit_breaker_triggered_at: fiveMinutesAgo }));
 538  
 539      // Add recent tasks with LOW failure rate (so developer appears in stats)
 540      // but cooldown hasn't expired yet
 541      const now = new Date().toISOString();
 542      for (let i = 0; i < 10; i++) {
 543        testDb
 544          .prepare(
 545            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at, completed_at, started_at)
 546           VALUES ('fix_bug', 'developer', 'completed', ?, ?, ?)`
 547          )
 548          .run(now, now, now);
 549      }
 550  
 551      checkCircuitBreakers();
 552  
 553      // Developer should still be blocked (cooldown not expired)
 554      const devState = testDb
 555        .prepare('SELECT status FROM agent_state WHERE agent_name = ?')
 556        .get('developer');
 557      assert.notEqual(devState.status, 'active');
 558    });
 559  
 560    test('does NOT auto-recover when failure rate still above threshold', () => {
 561      // Blocked 2 hours ago (cooldown expired) but still failing at high rate
 562      const twoHoursAgo = new Date(Date.now() - 2 * 60 * 60 * 1000).toISOString();
 563      testDb
 564        .prepare(
 565          `INSERT OR REPLACE INTO agent_state (agent_name, status, metrics_json)
 566         VALUES ('developer', 'blocked', ?)`
 567        )
 568        .run(JSON.stringify({ circuit_breaker_triggered_at: twoHoursAgo }));
 569  
 570      // Add recent failures (80% failure rate > 30% threshold)
 571      const now = new Date().toISOString();
 572      for (let i = 0; i < 8; i++) {
 573        testDb
 574          .prepare(
 575            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 576           VALUES ('fix_bug', 'developer', 'failed', ?)`
 577          )
 578          .run(now);
 579      }
 580      for (let i = 0; i < 2; i++) {
 581        testDb
 582          .prepare(
 583            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at, completed_at, started_at)
 584           VALUES ('fix_bug', 'developer', 'completed', ?, ?, ?)`
 585          )
 586          .run(now, now, now);
 587      }
 588  
 589      checkCircuitBreakers();
 590  
 591      // Should still be blocked since failure rate is high - not active
 592      const devState = testDb
 593        .prepare('SELECT status FROM agent_state WHERE agent_name = ?')
 594        .get('developer');
 595      assert.notEqual(devState.status, 'active');
 596    });
 597  
 598    test('handles agent with no metrics_json gracefully', () => {
 599      testDb
 600        .prepare(
 601          `INSERT OR REPLACE INTO agent_state (agent_name, status, metrics_json)
 602         VALUES ('developer', 'blocked', NULL)`
 603        )
 604        .run();
 605  
 606      // Should not crash
 607      const blocked = checkCircuitBreakers();
 608      assert.ok(Array.isArray(blocked));
 609    });
 610  
 611    test('respects custom threshold from AGENT_CIRCUIT_BREAKER_THRESHOLD env var', () => {
 612      process.env.AGENT_CIRCUIT_BREAKER_THRESHOLD = '0.5'; // 50% threshold
 613  
 614      const now = new Date().toISOString();
 615      // 6 failed, 4 completed = 60% failure rate - should trigger at 50% threshold
 616      for (let i = 0; i < 6; i++) {
 617        testDb
 618          .prepare(
 619            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 620           VALUES ('fix_bug', 'developer', 'failed', ?)`
 621          )
 622          .run(now);
 623      }
 624      for (let i = 0; i < 4; i++) {
 625        testDb
 626          .prepare(
 627            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 628           VALUES ('fix_bug', 'developer', 'completed', ?)`
 629          )
 630          .run(now);
 631      }
 632  
 633      const blocked = checkCircuitBreakers();
 634      const devBlocked = blocked.find(b => b.agent === 'developer');
 635      assert.ok(devBlocked, 'should be blocked with 60% failure and 50% threshold');
 636  
 637      process.env.AGENT_CIRCUIT_BREAKER_THRESHOLD = '0.3';
 638    });
 639  
 640    test('uses custom cooldown from AGENT_CIRCUIT_BREAKER_COOLDOWN env var', () => {
 641      process.env.AGENT_CIRCUIT_BREAKER_COOLDOWN = '120'; // 2 hour cooldown
 642  
 643      // Trigger time 90 minutes ago (within 2h cooldown, so should NOT auto-recover)
 644      const ninetyMinAgo = new Date(Date.now() - 90 * 60 * 1000).toISOString();
 645      testDb
 646        .prepare(
 647          `INSERT OR REPLACE INTO agent_state (agent_name, status, metrics_json)
 648         VALUES ('developer', 'blocked', ?)`
 649        )
 650        .run(JSON.stringify({ circuit_breaker_triggered_at: ninetyMinAgo }));
 651  
 652      // Add recent tasks with LOW failure rate (so developer appears in stats)
 653      const now = new Date().toISOString();
 654      for (let i = 0; i < 10; i++) {
 655        testDb
 656          .prepare(
 657            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at, completed_at, started_at)
 658           VALUES ('fix_bug', 'developer', 'completed', ?, ?, ?)`
 659          )
 660          .run(now, now, now);
 661      }
 662  
 663      checkCircuitBreakers();
 664  
 665      // Should NOT auto-recover since 90 min < 120 min cooldown
 666      const devState = testDb
 667        .prepare('SELECT status FROM agent_state WHERE agent_name = ?')
 668        .get('developer');
 669      assert.notEqual(devState.status, 'active');
 670  
 671      process.env.AGENT_CIRCUIT_BREAKER_COOLDOWN = '30';
 672    });
 673  });
 674  
 675  // ── runCompleteAgentCycle ──────────────────────────────────────────────────
 676  describe('runCompleteAgentCycle - orchestration', () => {
 677    test('aborts when critical nightmare scenario detected (cost runaway)', async () => {
 678      // Inject >66 log entries to push cost above $1/hour (66.7 * 0.015 = $1.0)
 679      for (let i = 0; i < 70; i++) {
 680        testDb
 681          .prepare(
 682            `INSERT INTO agent_logs (agent_name, log_level, message, created_at)
 683           VALUES ('monitor', 'INFO', 'agent initialized', datetime('now', '-30 minutes'))`
 684          )
 685          .run();
 686      }
 687  
 688      const result = await runCompleteAgentCycle();
 689      assert.equal(result.aborted, true);
 690      assert.ok(result.alerts);
 691      assert.ok(result.alerts.some(a => a.type === 'cost_runaway'));
 692    });
 693  
 694    test('runs sequential cycle when parallel flags are false', async () => {
 695      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
 696      process.env.AGENT_PARALLEL_EXECUTION = 'false';
 697      resetAgentBehavior({ monitor: 1, triage: 0, developer: 0, qa: 0, security: 0, architect: 0 });
 698  
 699      const result = await runCompleteAgentCycle({ tasksPerAgent: 3 });
 700      assert.equal(result.enabled, true);
 701      assert.equal(result.total_processed, 1);
 702    });
 703  
 704    test('runs parallel cycle when AGENT_RUN_TYPES_IN_PARALLEL=true', async () => {
 705      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'true';
 706      resetAgentBehavior({ monitor: 2, triage: 1 });
 707  
 708      const result = await runCompleteAgentCycle({ tasksPerAgent: 3 });
 709      assert.equal(result.enabled, true);
 710      // Parallel run should have processed both agents
 711      assert.ok(result.total_processed >= 0);
 712  
 713      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
 714    });
 715  
 716    test('runs parallel cycle when AGENT_PARALLEL_EXECUTION=true', async () => {
 717      process.env.AGENT_PARALLEL_EXECUTION = 'true';
 718      resetAgentBehavior({ monitor: 1 });
 719  
 720      const result = await runCompleteAgentCycle();
 721      assert.equal(result.enabled, true);
 722  
 723      process.env.AGENT_PARALLEL_EXECUTION = 'false';
 724    });
 725  
 726    test('includes circuit_breakers in summary when agent is blocked by high failure rate', async () => {
 727      const now = new Date().toISOString();
 728      // Trigger circuit breaker for developer
 729      for (let i = 0; i < 8; i++) {
 730        testDb
 731          .prepare(
 732            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 733           VALUES ('fix_bug', 'developer', 'failed', ?)`
 734          )
 735          .run(now);
 736      }
 737      for (let i = 0; i < 2; i++) {
 738        testDb
 739          .prepare(
 740            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 741           VALUES ('fix_bug', 'developer', 'completed', ?)`
 742          )
 743          .run(now);
 744      }
 745  
 746      resetAgentBehavior();
 747      const result = await runCompleteAgentCycle();
 748  
 749      if (result.circuit_breakers) {
 750        assert.ok(Array.isArray(result.circuit_breakers));
 751        const devCb = result.circuit_breakers.find(b => b.agent === 'developer');
 752        assert.ok(devCb);
 753      }
 754    });
 755  
 756    test('includes post-run alerts in summary', async () => {
 757      // Create stale task buildup for post-run nightmare detection
 758      for (let i = 0; i < 55; i++) {
 759        testDb
 760          .prepare(
 761            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 762           VALUES ('fix_bug', 'developer', 'pending', datetime('now', '-2 hours'))`
 763          )
 764          .run();
 765      }
 766  
 767      resetAgentBehavior();
 768      const result = await runCompleteAgentCycle();
 769  
 770      if (!result.aborted && result.alerts) {
 771        assert.ok(Array.isArray(result.alerts));
 772      }
 773    });
 774  
 775    test('creates summary commit when AGENT_AUTO_COMMIT=true with changes', async () => {
 776      process.env.AGENT_AUTO_COMMIT = 'true';
 777      process.env.AGENT_AUTO_COMMIT_BRANCH = 'autofix';
 778      gitStatusOutput = ' M src/test.js';
 779      gitBranchOutput = 'autofix';
 780      gitCommitOutput = 'def5678';
 781  
 782      resetAgentBehavior();
 783      const result = await runCompleteAgentCycle();
 784  
 785      if (!result.aborted) {
 786        assert.equal(result.commit, 'def5678');
 787        assert.equal(result.branch, 'autofix');
 788      }
 789  
 790      process.env.AGENT_AUTO_COMMIT = 'false';
 791    });
 792  
 793    test('skips commit when AGENT_AUTO_COMMIT is not set to true', async () => {
 794      process.env.AGENT_AUTO_COMMIT = 'false';
 795      resetAgentBehavior();
 796  
 797      const result = await runCompleteAgentCycle();
 798      assert.equal(result.commit, undefined);
 799    });
 800  
 801    test('skips commit when git status shows no changes', async () => {
 802      process.env.AGENT_AUTO_COMMIT = 'true';
 803      gitStatusOutput = ''; // no changes
 804  
 805      resetAgentBehavior();
 806      const result = await runCompleteAgentCycle();
 807      if (!result.aborted) {
 808        assert.equal(result.commit, undefined);
 809      }
 810  
 811      process.env.AGENT_AUTO_COMMIT = 'false';
 812    });
 813  
 814    test('switches to autofix branch when on different branch', async () => {
 815      process.env.AGENT_AUTO_COMMIT = 'true';
 816      process.env.AGENT_AUTO_COMMIT_BRANCH = 'autofix';
 817      gitStatusOutput = ' M src/test.js';
 818      gitBranchOutput = 'main'; // Different from autofix
 819      gitCommitOutput = 'newcommit';
 820  
 821      resetAgentBehavior();
 822      const result = await runCompleteAgentCycle();
 823  
 824      if (!result.aborted) {
 825        // Git checkout should have been called
 826        const calls = execSyncMock.mock.calls.map(c => c.arguments[0]);
 827        const hasCheckout = calls.some(cmd => cmd.includes('checkout'));
 828        assert.ok(hasCheckout, 'Should have called git checkout');
 829      }
 830  
 831      process.env.AGENT_AUTO_COMMIT = 'false';
 832    });
 833  
 834    test('returns null commit when git checkout fails', async () => {
 835      process.env.AGENT_AUTO_COMMIT = 'true';
 836      gitStatusOutput = ' M src/test.js';
 837      gitBranchOutput = 'main'; // triggers checkout
 838      gitCheckoutShouldFail = true;
 839  
 840      resetAgentBehavior();
 841      const result = await runCompleteAgentCycle();
 842  
 843      if (!result.aborted) {
 844        assert.equal(result.commit, undefined);
 845      }
 846  
 847      process.env.AGENT_AUTO_COMMIT = 'false';
 848    });
 849  
 850    test('handles git push failure gracefully when AGENT_AUTO_PUSH=true', async () => {
 851      process.env.AGENT_AUTO_COMMIT = 'true';
 852      process.env.AGENT_AUTO_PUSH = 'true';
 853      process.env.AGENT_AUTO_COMMIT_BRANCH = 'autofix';
 854      gitStatusOutput = ' M src/test.js';
 855      gitBranchOutput = 'autofix';
 856      gitCommitOutput = 'push123';
 857      gitPushShouldFail = true;
 858  
 859      resetAgentBehavior();
 860      // Should not throw even when push fails
 861      const result = await runCompleteAgentCycle();
 862      if (!result.aborted) {
 863        assert.equal(result.commit, 'push123'); // commit still created, push failed
 864      }
 865  
 866      process.env.AGENT_AUTO_COMMIT = 'false';
 867      process.env.AGENT_AUTO_PUSH = 'false';
 868    });
 869  });
 870  
 871  // ── Nightmare scenario detection ───────────────────────────────────────────
 872  describe('checkNightmareScenarios (via runCompleteAgentCycle pre-check)', () => {
 873    test('detects task loop when same fix_bug context repeated > 5 times in 1 hour', async () => {
 874      const ctx = JSON.stringify({ error: 'same error loop', file: 'test.js' });
 875      for (let i = 0; i < 6; i++) {
 876        testDb
 877          .prepare(
 878            `INSERT INTO agent_tasks (task_type, assigned_to, status, context_json, created_at)
 879           VALUES ('fix_bug', 'developer', 'pending', ?, datetime('now', '-30 minutes'))`
 880          )
 881          .run(ctx);
 882      }
 883  
 884      resetAgentBehavior();
 885      const result = await runCompleteAgentCycle();
 886  
 887      // Should not crash; may or may not abort depending on severity
 888      assert.ok(result !== undefined);
 889    });
 890  
 891    test('detects stale task buildup when > 50 tasks pending more than 1 hour', async () => {
 892      for (let i = 0; i < 55; i++) {
 893        testDb
 894          .prepare(
 895            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 896           VALUES ('fix_bug', 'developer', 'pending', datetime('now', '-2 hours'))`
 897          )
 898          .run();
 899      }
 900  
 901      resetAgentBehavior();
 902      const result = await runCompleteAgentCycle();
 903  
 904      // stale_task_buildup is 'high' severity (not critical), so pre-check doesn't abort
 905      // but it may appear in post-alerts
 906      assert.ok(result !== undefined);
 907    });
 908  
 909    test('detects circuit breaker cascade when 3+ agents are blocked', async () => {
 910      const agentNames = ['developer', 'qa', 'security'];
 911      for (const name of agentNames) {
 912        testDb
 913          .prepare(`INSERT OR REPLACE INTO agent_state (agent_name, status) VALUES (?, 'blocked')`)
 914          .run(name);
 915      }
 916  
 917      resetAgentBehavior();
 918      const result = await runCompleteAgentCycle();
 919  
 920      // circuit_breaker_cascade is 'critical' - should abort
 921      if (result.aborted) {
 922        const hasCascade =
 923          result.alerts && result.alerts.some(a => a.type === 'circuit_breaker_cascade');
 924        assert.ok(hasCascade);
 925      }
 926    });
 927  
 928    test('emergency shutdown when hourly cost > $5', async () => {
 929      // Need > 333 log entries (333 * 0.015 = $4.995, 334 * 0.015 = $5.01)
 930      for (let i = 0; i < 340; i++) {
 931        testDb
 932          .prepare(
 933            `INSERT INTO agent_logs (agent_name, log_level, message, created_at)
 934           VALUES ('monitor', 'INFO', 'agent initialized', datetime('now', '-30 minutes'))`
 935          )
 936          .run();
 937      }
 938  
 939      process.env.AGENT_SYSTEM_ENABLED = 'true';
 940      resetAgentBehavior();
 941      const result = await runCompleteAgentCycle();
 942  
 943      assert.equal(result.aborted, true);
 944      // Emergency shutdown should have set AGENT_SYSTEM_ENABLED to false
 945      assert.equal(process.env.AGENT_SYSTEM_ENABLED, 'false');
 946  
 947      // Restore for subsequent tests
 948      process.env.AGENT_SYSTEM_ENABLED = 'true';
 949    });
 950  });
 951  
 952  // ── Autoscaling via parallel mode ──────────────────────────────────────────
 953  describe('calculateTaskLimit autoscaling (via parallel runCompleteAgentCycle)', () => {
 954    test('uses base limit when queue depth is normal (< 10)', async () => {
 955      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'true';
 956      process.env.AGENT_MAX_TASKS_PER_CYCLE = '5';
 957  
 958      resetAgentBehavior({ monitor: 0 });
 959      const result = await runCompleteAgentCycle({ verbose: false });
 960      assert.equal(result.enabled, true);
 961  
 962      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
 963    });
 964  
 965    test('autoscales to maxLimit=20 when queue depth > 20', async () => {
 966      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'true';
 967      process.env.AGENT_MAX_TASKS_PER_CYCLE = '5';
 968  
 969      // Insert 25 pending developer tasks
 970      for (let i = 0; i < 25; i++) {
 971        testDb
 972          .prepare(
 973            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
 974           VALUES ('fix_bug', 'developer', 'pending', datetime('now'))`
 975          )
 976          .run();
 977      }
 978  
 979      resetAgentBehavior({ developer: 5 });
 980      const result = await runCompleteAgentCycle({ verbose: true });
 981  
 982      if (!result.aborted && result.agents) {
 983        // developer task_limit should be 20 (maxLimit)
 984        if (result.agents.developer && result.agents.developer.task_limit !== undefined) {
 985          assert.equal(result.agents.developer.task_limit, 20);
 986        }
 987      }
 988  
 989      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
 990    });
 991  
 992    test('autoscales to 2x base when queue depth is 11-20', async () => {
 993      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'true';
 994      process.env.AGENT_MAX_TASKS_PER_CYCLE = '5';
 995  
 996      // Insert 15 pending qa tasks
 997      for (let i = 0; i < 15; i++) {
 998        testDb
 999          .prepare(
1000            `INSERT INTO agent_tasks (task_type, assigned_to, status, created_at)
1001           VALUES ('run_tests', 'qa', 'pending', datetime('now'))`
1002          )
1003          .run();
1004      }
1005  
1006      resetAgentBehavior({ qa: 3 });
1007      const result = await runCompleteAgentCycle();
1008  
1009      if (!result.aborted && result.agents) {
1010        if (result.agents.qa && result.agents.qa.task_limit !== undefined) {
1011          // task_limit should be min(5*2, 20) = 10
1012          assert.equal(result.agents.qa.task_limit, 10);
1013        }
1014      }
1015  
1016      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
1017    });
1018  
1019    test('parallel run handles agent failure gracefully', async () => {
1020      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'true';
1021  
1022      resetAgentBehavior({ monitor: 1 }, { developer: true });
1023      const result = await runCompleteAgentCycle();
1024  
1025      if (!result.aborted && result.agents) {
1026        const agentNames = Object.keys(result.agents);
1027        assert.ok(agentNames.length > 0);
1028      }
1029  
1030      process.env.AGENT_RUN_TYPES_IN_PARALLEL = 'false';
1031    });
1032  });