1  /**
  2   * Tests for src/cron.js — runCron execution path (database-driven v2)
  3   *
  4   * Covers:
  5   * - runCron: circuit breaker check, global lock, loadJobs, shouldRun, handler dispatch
  6   * - checkAndClearStaleLock: fresh lock (skip), stale lock (clear), no lock (proceed)
  7   * - intervalToMinutes: minutes/hours/days/weeks conversion
  8   * - generateSummary: null result, .summary, .manual, legacy fields
  9   * - logTaskStart, logTaskComplete, logTaskFailed
 10   * - executeCommand: command-type job dispatch
 11   *
 12   * Uses mock.module() to stub all external modules so handler bodies don't make real API calls.
 13   *
 14   * NOTE: requires --experimental-test-module-mocks
 15   */
 16  
 17  import { test, describe, mock, after } from 'node:test';
 18  import assert from 'node:assert/strict';
 19  import Database from 'better-sqlite3';
 20  import { createPgMock } from '../helpers/pg-mock.js';
 21  
 22  process.env.NODE_ENV = 'test';
 23  process.env.LOGS_DIR = '/tmp/test-logs';
 24  
 25  // ── Shared in-memory SQLite db for all tests ──────────────────────────────────
 26  const db = new Database(':memory:');
 27  db.exec(`
 28    CREATE TABLE IF NOT EXISTS sites (
 29      id INTEGER PRIMARY KEY AUTOINCREMENT,
 30      domain TEXT NOT NULL DEFAULT 'test.com',
 31      status TEXT DEFAULT 'found',
 32      score REAL,
 33      error_message TEXT,
 34      updated_at TEXT DEFAULT (datetime('now'))
 35    );
 36    CREATE TABLE IF NOT EXISTS messages (
 37      id INTEGER PRIMARY KEY AUTOINCREMENT,
 38      site_id INTEGER,
 39      direction TEXT NOT NULL DEFAULT 'outbound',
 40      approval_status TEXT,
 41      delivery_status TEXT,
 42      read_at TEXT,
 43      created_at TEXT DEFAULT (datetime('now')),
 44      updated_at TEXT DEFAULT (datetime('now')),
 45      message_type TEXT DEFAULT 'outreach',
 46      raw_payload TEXT
 47    );
 48    CREATE TABLE IF NOT EXISTS keywords (
 49      id INTEGER PRIMARY KEY AUTOINCREMENT,
 50      keyword TEXT NOT NULL,
 51      status TEXT DEFAULT 'pending'
 52    );
 53    CREATE TABLE IF NOT EXISTS settings (
 54      key TEXT PRIMARY KEY, value TEXT, description TEXT,
 55      updated_at TEXT DEFAULT (datetime('now'))
 56    );
 57    CREATE TABLE IF NOT EXISTS cron_jobs (
 58      id INTEGER PRIMARY KEY AUTOINCREMENT,
 59      name TEXT NOT NULL UNIQUE,
 60      task_key TEXT NOT NULL UNIQUE,
 61      description TEXT,
 62      enabled INTEGER NOT NULL DEFAULT 1,
 63      handler_type TEXT NOT NULL DEFAULT 'function',
 64      handler_value TEXT,
 65      interval_value INTEGER NOT NULL DEFAULT 5,
 66      interval_unit TEXT NOT NULL DEFAULT 'minutes',
 67      timeout_seconds INTEGER,
 68      priority INTEGER DEFAULT 5,
 69      critical INTEGER DEFAULT 1,
 70      last_run_at TEXT,
 71      created_at TEXT DEFAULT (datetime('now')),
 72      updated_at TEXT DEFAULT (datetime('now'))
 73    );
 74    CREATE TABLE IF NOT EXISTS cron_job_logs (
 75      id INTEGER PRIMARY KEY AUTOINCREMENT,
 76      job_name TEXT NOT NULL,
 77      started_at TEXT NOT NULL DEFAULT (datetime('now')),
 78      finished_at TEXT,
 79      status TEXT NOT NULL DEFAULT 'running',
 80      summary TEXT,
 81      full_log TEXT,
 82      error_message TEXT,
 83      items_processed INTEGER DEFAULT 0,
 84      items_failed INTEGER DEFAULT 0
 85    );
 86    CREATE TABLE IF NOT EXISTS cron_locks (
 87      lock_key TEXT PRIMARY KEY,
 88      description TEXT,
 89      updated_at TEXT DEFAULT (datetime('now'))
 90    );
 91    CREATE TABLE IF NOT EXISTS pipeline_control (key TEXT PRIMARY KEY, value TEXT);
 92    CREATE TABLE IF NOT EXISTS agent_tasks (
 93      id INTEGER PRIMARY KEY AUTOINCREMENT,
 94      task_type TEXT NOT NULL, assigned_to TEXT,
 95      priority INTEGER DEFAULT 5, status TEXT DEFAULT 'pending',
 96      context_json TEXT, result_json TEXT, error_message TEXT,
 97      created_at TEXT DEFAULT (datetime('now')),
 98      updated_at TEXT DEFAULT (datetime('now'))
 99    );
100    CREATE TABLE IF NOT EXISTS llm_usage (
101      id INTEGER PRIMARY KEY AUTOINCREMENT,
102      site_id INTEGER, stage TEXT NOT NULL,
103      provider TEXT NOT NULL DEFAULT 'openrouter',
104      model TEXT NOT NULL DEFAULT 'test-model',
105      prompt_tokens INTEGER NOT NULL DEFAULT 100,
106      completion_tokens INTEGER NOT NULL DEFAULT 50,
107      total_tokens INTEGER NOT NULL DEFAULT 150,
108      estimated_cost REAL DEFAULT 0.001,
109      created_at TEXT DEFAULT (datetime('now'))
110    );
111    CREATE TABLE IF NOT EXISTS site_status (
112      id INTEGER PRIMARY KEY AUTOINCREMENT,
113      site_id INTEGER NOT NULL, status TEXT,
114      created_at TEXT DEFAULT (datetime('now'))
115    );
116    CREATE TABLE IF NOT EXISTS human_review_queue (
117      id INTEGER PRIMARY KEY AUTOINCREMENT,
118      file TEXT, reason TEXT, type TEXT,
119      priority TEXT DEFAULT 'medium', status TEXT DEFAULT 'pending',
120      created_at TEXT DEFAULT (datetime('now'))
121    );
122    CREATE TABLE IF NOT EXISTS dashboard_cache (
123      cache_key TEXT PRIMARY KEY, cache_value TEXT NOT NULL,
124      expires_at TEXT NOT NULL, updated_at TEXT NOT NULL DEFAULT (datetime('now'))
125    );
126  `);
127  
128  // ── Mock db.js to route through shared in-memory SQLite ──────────────────────
129  mock.module('../../src/utils/db.js', { namedExports: createPgMock(db) });
130  
131  // ── Mock all external dependencies BEFORE importing cron.js ──────────────────
132  
133  mock.module('../../src/utils/sync-email-events.js', {
134    namedExports: { syncEmailEvents: async () => ({ synced: 2, errors: 0 }) },
135  });
136  mock.module('../../src/utils/sync-unsubscribes.js', {
137    namedExports: { syncUnsubscribes: async () => ({ synced: 1, errors: 0 }) },
138  });
139  mock.module('../../src/inbound/sms.js', {
140    namedExports: {
141      pollInboundSMS: async () => ({ processed: 0, new_messages: 0 }),
142      setupWebhookServer: async () => {},
143    },
144  });
145  mock.module('../../src/inbound/email.js', {
146    namedExports: { pollInboundEmails: async () => ({ processed: 0, stored: 0, unmatched: 0 }) },
147  });
148  mock.module('../../src/inbound/processor.js', {
149    namedExports: { processAllReplies: async () => ({ sms: { sent: 0 }, email: { sent: 0 } }) },
150  });
151  mock.module('../../src/cron/poll-free-scans.js', {
152    namedExports: { pollFreeScans: async () => ({ processed: 0, inserted: 0, failed: 0 }) },
153  });
154  mock.module('../../src/cron/poll-purchases.js', {
155    namedExports: { pollPurchases: async () => ({ processed: 0, successful: 0 }) },
156  });
157  mock.module('../../src/cron/process-purchases.js', {
158    namedExports: {
159      processPendingPurchases: async () => ({ processed: 0, delivered: 0, failed: 0 }),
160    },
161  });
162  mock.module('../../src/cron/precompute-dashboard.js', {
163    namedExports: {
164      precomputeDashboard: async () => ({
165        summary: 'Dashboard precomputed',
166        details: {},
167        metrics: {},
168      }),
169    },
170  });
171  mock.module('../../src/cron/process-guardian.js', {
172    namedExports: {
173      runProcessGuardian: async () => ({
174        checks_run: 6,
175        ok: 6,
176        warnings: 0,
177        critical: 0,
178        duration_seconds: 0.1,
179        results: [],
180      }),
181    },
182  });
183  mock.module('../../src/cron/process-reaper.js', {
184    namedExports: {
185      runProcessReaper: async () => ({
186        zombie_count: 0,
187        free_mem_mb: 512,
188        swap_pct: 0,
189        stale_processes_killed: 0,
190        duration_seconds: 0.1,
191      }),
192    },
193  });
194  mock.module('../../src/cron/cleanup-test-dbs.js', {
195    namedExports: { runCleanupTestDbs: () => ({ deleted: 0, freed_kb: 0 }) },
196  });
197  mock.module('../../src/cron/pipeline-status-monitor.js', {
198    namedExports: {
199      runPipelineStatusMonitor: async () => ({
200        summary: 'Pipeline ok',
201        checks_run: 3,
202        duration_seconds: 0.1,
203        actions: [],
204      }),
205    },
206  });
207  mock.module('../../src/cron/classify-unknown-errors.js', {
208    namedExports: {
209      classifyUnknownErrors: async () => ({
210        sites_retried: 0,
211        outreaches_retried: 0,
212        patterns_applied: 0,
213      }),
214    },
215  });
216  mock.module('../../src/agents/utils/task-manager.js', {
217    namedExports: {
218      createAgentTask: async () => 1,
219      findDuplicateTask: async () => null,
220    },
221  });
222  mock.module('../../src/utils/log-rotator.js', {
223    namedExports: { rotateLogs: () => ({}) },
224  });
225  mock.module('../../src/utils/rate-limit-scheduler.js', {
226    namedExports: {
227      getSkipStages: () => new Set(),
228      getRateLimitStatus: () => [],
229      setRateLimit: () => {},
230    },
231  });
232  mock.module('../../src/utils/load-env.js', {
233    namedExports: {},
234  });
235  
236  // Import AFTER mocks (schema already created above)
237  const { default: cronModule } = await import('../../src/cron.js');
238  
239  after(() => {
240    try { db.close(); } catch { /* ignore */ }
241  });
242  
243  // ── Helpers ───────────────────────────────────────────────────────────────────
244  
245  function seedJob(overrides = {}) {
246    const defaults = {
247      name: 'Test Job',
248      task_key: 'syncEmailEvents',
249      enabled: 1,
250      handler_type: 'function',
251      handler_value: null,
252      interval_value: 1,
253      interval_unit: 'minutes',
254      timeout_seconds: 30,
255      critical: 0,
256      last_run_at: null,
257    };
258    const job = { ...defaults, ...overrides };
259    db.prepare(
260      `INSERT OR REPLACE INTO cron_jobs
261       (name, task_key, enabled, handler_type, handler_value, interval_value, interval_unit, timeout_seconds, critical, last_run_at)
262       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
263    ).run(
264      job.name, job.task_key, job.enabled, job.handler_type, job.handler_value,
265      job.interval_value, job.interval_unit, job.timeout_seconds, job.critical, job.last_run_at
266    );
267  }
268  
269  function clearJobs() {
270    db.prepare('DELETE FROM cron_jobs').run();
271    db.prepare('DELETE FROM cron_job_logs').run();
272    db.prepare('DELETE FROM cron_locks').run();
273  }
274  
275  function getLogs(whereClause = '') {
276    return db.prepare(`SELECT * FROM cron_job_logs ${whereClause}`).all();
277  }
278  
279  // ── Tests ─────────────────────────────────────────────────────────────────────
280  
281  describe('runCron — no jobs in database', () => {
282    test('runs without error when no enabled jobs exist', async () => {
283      clearJobs();
284      const { runCron } = cronModule;
285      await assert.doesNotReject(() => runCron());
286    });
287  });
288  
289  describe('runCron — circuit breaker disabled', () => {
290    test('skips all jobs when circuit breaker is disabled', async () => {
291      clearJobs();
292      db.prepare(
293        "INSERT OR REPLACE INTO settings (key, value) VALUES ('cron_circuit_breaker_enabled', 'false')"
294      ).run();
295      seedJob({ name: 'Email Sync', task_key: 'syncEmailEvents' });
296  
297      const { runCron } = cronModule;
298      await assert.doesNotReject(() => runCron());
299  
300      const logs = getLogs();
301      assert.equal(logs.length, 0, 'No logs should exist when circuit breaker disabled');
302  
303      db.prepare("DELETE FROM settings WHERE key = 'cron_circuit_breaker_enabled'").run();
304    });
305  });
306  
307  describe('runCron — function handler job runs', () => {
308    test('runs a function-type job and writes cron_job_logs success entry', async () => {
309      clearJobs();
310      seedJob({ name: 'Email Sync', task_key: 'syncEmailEvents', last_run_at: null });
311  
312      const { runCron } = cronModule;
313      await assert.doesNotReject(() => runCron());
314  
315      const logs = getLogs("WHERE status = 'success'");
316      assert.ok(logs.length >= 1, `Expected ≥1 success log, got ${logs.length}`);
317      assert.ok(logs[0].summary, 'Should have a summary');
318      assert.ok(logs[0].finished_at, 'Should have a finished_at timestamp');
319    });
320  });
321  
322  describe('runCron — job already ran recently (shouldRun=false)', () => {
323    test('skips job that ran 1 second ago (within interval)', async () => {
324      clearJobs();
325      const recentTime = new Date(Date.now() - 1000).toISOString().replace('T', ' ').slice(0, 19);
326      seedJob({
327        name: 'Email Sync',
328        task_key: 'syncEmailEvents',
329        interval_value: 5,
330        interval_unit: 'minutes',
331        last_run_at: recentTime,
332      });
333  
334      const { runCron } = cronModule;
335      await assert.doesNotReject(() => runCron());
336  
337      const logs = getLogs();
338      assert.equal(logs.length, 0, 'Should skip job that ran recently');
339    });
340  });
341  
342  describe('runCron — handler throws error (logTaskFailed path)', () => {
343    test('logs failed status when handler throws', async () => {
344      clearJobs();
345      seedJob({
346        name: 'Unknown Handler',
347        task_key: 'nonExistentHandler',
348        critical: 0,
349        last_run_at: null,
350      });
351  
352      const origExit = process.exit;
353      process.exit = () => {};
354  
355      const { runCron } = cronModule;
356      await assert.doesNotReject(() => runCron());
357  
358      process.exit = origExit;
359  
360      const logs = getLogs("WHERE status = 'failed'");
361      assert.ok(logs.length >= 1, `Expected ≥1 failed log, got ${logs.length}`);
362      assert.ok(
363        logs[0].error_message?.includes('not found'),
364        `error_message: ${logs[0].error_message}`
365      );
366    });
367  });
368  
369  describe('runCron — critical failure calls process.exit(1)', () => {
370    test('calls process.exit(1) when critical job fails', async () => {
371      clearJobs();
372      seedJob({
373        name: 'Critical Job',
374        task_key: 'nonExistentCritical',
375        critical: 1,
376        last_run_at: null,
377      });
378  
379      let exitCode = null;
380      const origExit = process.exit;
381      process.exit = code => { exitCode = code; };
382  
383      const { runCron } = cronModule;
384      await assert.doesNotReject(() => runCron());
385  
386      process.exit = origExit;
387      assert.equal(exitCode, 1, 'Should call process.exit(1) for critical failures');
388    });
389  });
390  
391  describe('runCron — global lock prevents re-entry', () => {
392    test('second runCron while lock is held exits early without running jobs', async () => {
393      clearJobs();
394      seedJob({ name: 'Email Sync', task_key: 'syncEmailEvents', last_run_at: null });
395      db.prepare(
396        `INSERT OR REPLACE INTO cron_locks (lock_key, description, updated_at)
397         VALUES ('cron_runner_global_lock', 'held by test', datetime('now'))`
398      ).run();
399  
400      const { runCron } = cronModule;
401      await assert.doesNotReject(() => runCron());
402  
403      const logs = getLogs();
404      db.prepare("DELETE FROM cron_locks WHERE lock_key = 'cron_runner_global_lock'").run();
405      assert.equal(logs.length, 0, 'Jobs should not run when global lock is held');
406    });
407  });
408  
409  describe('runCron — stale global lock is cleared', () => {
410    test('clears stale lock (>10min) and runs jobs normally', async () => {
411      clearJobs();
412      seedJob({ name: 'Email Sync', task_key: 'syncEmailEvents', last_run_at: null });
413      db.prepare(
414        `INSERT OR REPLACE INTO cron_locks (lock_key, description, updated_at)
415         VALUES ('cron_runner_global_lock', 'stale lock', datetime('now', '-11 minutes'))`
416      ).run();
417  
418      const { runCron } = cronModule;
419      await assert.doesNotReject(() => runCron());
420  
421      const logs = getLogs("WHERE status = 'success'");
422      assert.ok(logs.length >= 1, `Should have run job after clearing stale lock: ${logs.length}`);
423    });
424  });
425  
426  describe('runCron — multiple jobs (hours/days/weeks intervals)', () => {
427    test('seeds and runs multiple jobs with different interval units', async () => {
428      clearJobs();
429      seedJob({ name: 'Job1', task_key: 'syncEmailEvents', interval_value: 1, interval_unit: 'minutes', last_run_at: null });
430      seedJob({ name: 'Job2', task_key: 'syncUnsubscribes', interval_value: 1, interval_unit: 'hours', last_run_at: null });
431      seedJob({ name: 'Job3', task_key: 'pollPurchases', interval_value: 1, interval_unit: 'days', last_run_at: null });
432  
433      const { runCron } = cronModule;
434      await assert.doesNotReject(() => runCron());
435  
436      const logs = getLogs("WHERE status = 'success'");
437      assert.ok(logs.length >= 3, `Expected ≥3 success logs, got ${logs.length}`);
438    });
439  });
440  
441  describe('HANDLERS registry', () => {
442    test('HANDLERS object is exported and has expected function keys', () => {
443      const { HANDLERS } = cronModule;
444      assert.equal(typeof HANDLERS, 'object', 'HANDLERS should be an object');
445      const expectedKeys = [
446        'syncEmailEvents',
447        'syncUnsubscribes',
448        'pollInboundSMS',
449        'pollPurchases',
450        'processGuardian',
451        'processReaper',
452        'classifyUnknownErrors',
453        'databaseMaintenance',
454      ];
455      for (const key of expectedKeys) {
456        assert.ok(typeof HANDLERS[key] === 'function', `HANDLERS.${key} should be a function`);
457      }
458    });
459  });
460  
461  describe('runCron — job with weeks interval', () => {
462    test('does not run weekly job that ran 1 minute ago', async () => {
463      clearJobs();
464      const recentTime = new Date(Date.now() - 60000).toISOString().replace('T', ' ').slice(0, 19);
465      seedJob({
466        name: 'Weekly Job',
467        task_key: 'syncEmailEvents',
468        interval_value: 1,
469        interval_unit: 'weeks',
470        last_run_at: recentTime,
471      });
472  
473      const { runCron } = cronModule;
474      await assert.doesNotReject(() => runCron());
475  
476      const logs = getLogs();
477      assert.equal(logs.length, 0, 'Weekly job should be skipped (ran only 1min ago)');
478    });
479  });
480  
481  describe('generateSummary — via logTaskComplete paths', () => {
482    test('databaseMaintenance handler returns structured result with summary', async () => {
483      const { HANDLERS } = cronModule;
484      const result = await HANDLERS.databaseMaintenance();
485      assert.ok(typeof result === 'object', 'should return object');
486      assert.ok(typeof result.summary === 'string', 'should have summary string');
487      assert.ok(typeof result.metrics === 'object', 'should have metrics');
488    });
489  
490    test('cleanupTestDbs handler returns structured result', async () => {
491      const { HANDLERS } = cronModule;
492      const result = await HANDLERS.cleanupTestDbs();
493      assert.ok(typeof result === 'object', 'should return object');
494      assert.ok(typeof result.summary === 'string', 'should have summary string');
495      assert.ok('deleted' in result.metrics, 'should have deleted in metrics');
496    });
497  
498    test('processGuardian handler returns structured result', async () => {
499      const { HANDLERS } = cronModule;
500      const result = await HANDLERS.processGuardian();
501      assert.ok(typeof result === 'object', 'should return object');
502      assert.ok(typeof result.summary === 'string', 'should have summary string');
503      assert.ok(typeof result.metrics === 'object', 'should have metrics');
504      assert.ok('checks_run' in result.metrics, 'should have checks_run in metrics');
505    });
506  
507    test('processReaper handler returns structured result', async () => {
508      const { HANDLERS } = cronModule;
509      const result = await HANDLERS.processReaper();
510      assert.ok(typeof result === 'object', 'should return object');
511      assert.ok(typeof result.summary === 'string', 'should have summary string');
512      assert.ok('zombie_count' in result.metrics, 'should have zombie_count in metrics');
513    });
514  
515    test('pollFreeScans handler returns structured result', async () => {
516      const { HANDLERS } = cronModule;
517      const result = await HANDLERS.pollFreeScans();
518      assert.ok(typeof result === 'object', 'should return object');
519      assert.ok(typeof result.summary === 'string', 'should have summary string');
520      assert.ok('processed' in result.metrics, 'should have processed in metrics');
521    });
522  
523    test('pollInboundEmails handler returns structured result', async () => {
524      const { HANDLERS } = cronModule;
525      const result = await HANDLERS.pollInboundEmails();
526      assert.ok(typeof result === 'object');
527      assert.ok(typeof result.summary === 'string');
528      assert.ok('processed' in result.metrics);
529    });
530  
531    test('sendPendingReplies handler returns structured result', async () => {
532      const { HANDLERS } = cronModule;
533      const result = await HANDLERS.sendPendingReplies();
534      assert.ok(typeof result === 'object');
535      assert.ok(typeof result.summary === 'string');
536      assert.ok('sms_sent' in result.metrics);
537      assert.ok('email_sent' in result.metrics);
538    });
539  
540    test('processPurchases handler returns structured result', async () => {
541      const { HANDLERS } = cronModule;
542      const result = await HANDLERS.processPurchases();
543      assert.ok(typeof result === 'object');
544      assert.ok(typeof result.summary === 'string');
545      assert.ok('processed' in result.metrics);
546      assert.ok('delivered' in result.metrics);
547      assert.ok('failed' in result.metrics);
548    });
549  
550    test('classifyUnknownErrors handler returns structured result', async () => {
551      const { HANDLERS } = cronModule;
552      const result = await HANDLERS.classifyUnknownErrors();
553      assert.ok(typeof result === 'object');
554      assert.ok(typeof result.summary === 'string');
555      assert.ok('sites_retried' in result.metrics);
556    });
557  
558    test('precomputeDashboard handler returns result', async () => {
559      const { HANDLERS } = cronModule;
560      const result = await HANDLERS.precomputeDashboard();
561      assert.ok(result !== null && result !== undefined);
562    });
563  
564    test('pipelineStatusMonitor handler returns structured result', async () => {
565      const { HANDLERS } = cronModule;
566      const result = await HANDLERS.pipelineStatusMonitor();
567      assert.ok(typeof result === 'object');
568      assert.ok(typeof result.summary === 'string');
569      assert.ok('checks_run' in result.metrics);
570    });
571  });