1  /**
  2   * Unit Tests for Proposals Stage
  3   *
  4   * Tests runProposalsStage() covering:
  5   * - No template countries → early return (lines 120-129)
  6   * - Blocklist filtering marks sites as 'ignore' (lines 164-176)
  7   * - Batch processing succeeds (lines 182-217)
  8   * - Error logging for failed sites (lines 208-212)
  9   * - Stage-level catch block (lines 218-220)
 10   * - No sites needing proposals → early return (lines 154-162)
 11   * - Rework processing when rework count > 0 (lines 91-94)
 12   * - Re-queue enriched sites with unknown country (lines 100-114)
 13   *
 14   * Run with:
 15   *   NODE_ENV=test LOGS_DIR=/tmp/test-logs DATABASE_PATH=/tmp/test-sites.db \
 16   *   node --experimental-test-module-mocks --test tests/stages/proposals.test.js
 17   */
 18  
 19  import { test, describe, mock, beforeEach } from 'node:test';
 20  import assert from 'node:assert';
 21  import { createPgMock } from '../helpers/pg-mock.js'; // eslint-disable-line no-unused-vars
 22  
 23  // ============================================================================
 24  // MUTABLE STUBS — mutated per-test via beforeEach
 25  // ============================================================================
 26  
 27  // DB query results
 28  let mockReworkCount = 0;
 29  let mockRequeueChanges = 0;
 30  let mockSites = [];
 31  
 32  // Track DB calls
 33  const dbCalls = { updates: [], runs: [] };
 34  
 35  // Template countries — controls whether proposals can proceed
 36  let mockTemplateCountries = ['AU', 'US', 'GB'];
 37  
 38  // Blocklist stub
 39  let stubCheckBlocklist = (_domain, _cc) => null;
 40  
 41  // Generator stubs
 42  const stubGenerateTemplateProposals = mock.fn(async _siteId => ({
 43    variants: [{ type: 'email' }],
 44    contactCount: 1,
 45  }));
 46  const stubGenerateLLMProposals = mock.fn(async _siteId => ({
 47    variants: [{ type: 'email' }],
 48    contactCount: 1,
 49  }));
 50  const stubProcessReworkRequests = mock.fn(async () => {});
 51  
 52  // processBatch stub — by default succeeds for all items
 53  let stubProcessBatch = mock.fn(async (items, processor, _opts) => {
 54    const results = [];
 55    const errors = [];
 56    for (let i = 0; i < items.length; i++) {
 57      try {
 58        const r = await processor(items[i], i);
 59        results.push(r);
 60      } catch (err) {
 61        errors.push(err);
 62      }
 63    }
 64    return { results, errors };
 65  });
 66  
 67  // Retry handler stubs
 68  const stubRecordFailure = mock.fn(() => {});
 69  const stubResetRetries = mock.fn(() => {});
 70  
 71  // ============================================================================
 72  // MockDatabase
 73  // ============================================================================
 74  
 75  class MockDatabase {
 76    constructor(_path) {
 77      this._closed = false;
 78    }
 79  
 80    pragma() {
 81      return undefined;
 82    }
 83  
 84    prepare(sql) {
 85      const trimmed = sql.trim();
 86      return {
 87        all: (...args) => {
 88          // Main site query
 89          if (trimmed.includes("'enriched'") && trimmed.includes('score >=')) {
 90            return mockSites;
 91          }
 92          return [];
 93        },
 94        get: (...args) => {
 95          // Rework count
 96          if (trimmed.includes('rework')) {
 97            return { cnt: mockReworkCount };
 98          }
 99          // Retry count for recordFailure
100          if (trimmed.includes('retry_count')) {
101            return { retry_count: 0 };
102          }
103          return null;
104        },
105        run: (...args) => {
106          // Re-queue enriched sites with unknown country
107          if (trimmed.includes('country_code IS NULL') && trimmed.includes('UPDATE sites')) {
108            dbCalls.runs.push({ sql: trimmed, args });
109            return { changes: mockRequeueChanges };
110          }
111          // Blocklist ignore update
112          if (trimmed.includes("status = 'ignored'")) {
113            dbCalls.updates.push({ sql: trimmed, args });
114            return { changes: 1 };
115          }
116          dbCalls.runs.push({ sql: trimmed, args });
117          return { changes: 0, lastInsertRowid: 0 };
118        },
119      };
120    }
121  
122    close() {
123      this._closed = true;
124    }
125  }
126  
127  // ============================================================================
128  // MOCK MODULES — must come before dynamic imports
129  // ============================================================================
130  
131  mock.module('better-sqlite3', {
132    defaultExport: MockDatabase,
133  });
134  
135  mock.module('../../src/proposal-generator-v2.js', {
136    namedExports: {
137      generateProposalVariants: (...args) => stubGenerateLLMProposals(...args),
138      processReworkQueue: (...args) => stubProcessReworkRequests(...args),
139    },
140  });
141  
142  mock.module('../../src/proposal-generator-templates.js', {
143    namedExports: {
144      generateProposalVariants: (...args) => stubGenerateTemplateProposals(...args),
145    },
146  });
147  
148  mock.module('../../src/utils/logger.js', {
149    defaultExport: class MockLogger {
150      constructor() {}
151      success() {}
152      info() {}
153      error() {}
154      warn() {}
155      debug() {}
156    },
157  });
158  
159  mock.module('../../src/utils/summary-generator.js', {
160    namedExports: {
161      generateStageCompletion: () => {},
162      displayProgress: () => {},
163    },
164  });
165  
166  mock.module('../../src/utils/error-handler.js', {
167    namedExports: {
168      processBatch: (...args) => stubProcessBatch(...args),
169    },
170    defaultExport: {
171      processBatch: (...args) => stubProcessBatch(...args),
172    },
173  });
174  
175  mock.module('../../src/utils/site-filters.js', {
176    namedExports: {
177      checkBlocklist: (...args) => stubCheckBlocklist(...args),
178    },
179  });
180  
181  mock.module('../../src/utils/retry-handler.js', {
182    namedExports: {
183      recordFailure: (...args) => stubRecordFailure(...args),
184      resetRetries: (...args) => stubResetRetries(...args),
185    },
186  });
187  
188  // Mock db.js — proposals.js uses run/getOne/getAll from db.js (not better-sqlite3 directly)
189  mock.module('../../src/utils/db.js', {
190    namedExports: {
191      getAll: async (sql, _params) => {
192        const trimmed = sql.trim();
193        if (trimmed.includes("'enriched'") && trimmed.includes('score >=')) {
194          return mockSites;
195        }
196        return [];
197      },
198      getOne: async (sql, _params) => {
199        const trimmed = sql.trim();
200        if (trimmed.includes('rework')) {
201          return { cnt: mockReworkCount };
202        }
203        if (trimmed.includes('retry_count')) {
204          return { retry_count: 0 };
205        }
206        return null;
207      },
208      run: async (sql, params) => {
209        const trimmed = sql.trim();
210        if (trimmed.includes('country_code IS NULL') && trimmed.includes('UPDATE sites')) {
211          dbCalls.runs.push({ sql: trimmed, args: params });
212          return { changes: mockRequeueChanges };
213        }
214        if (trimmed.includes("status = 'ignored'")) {
215          dbCalls.updates.push({ sql: trimmed, args: params });
216          return { changes: 1 };
217        }
218        dbCalls.runs.push({ sql: trimmed, args: params });
219        return { changes: 0, lastInsertRowid: null };
220      },
221      query: async () => ({ rows: [], rowCount: 0 }),
222      withTransaction: async fn => {
223        const fakeClient = { query: async () => ({ rows: [], rowCount: 0 }) };
224        return await fn(fakeClient);
225      },
226    },
227  });
228  
229  // Mock fs functions used by getTemplateCountries
230  mock.module('fs', {
231    namedExports: {
232      existsSync: p => {
233        // Templates dir exists if we have template countries
234        if (p.includes('data/templates')) {
235          return mockTemplateCountries.length > 0;
236        }
237        return false;
238      },
239      readdirSync: (p, _opts) => {
240        if (p.includes('data/templates')) {
241          return mockTemplateCountries.map(cc => ({
242            name: cc.toLowerCase(),
243            isDirectory: () => true,
244          }));
245        }
246        return [];
247      },
248      readFileSync: () => '{}',
249      writeFileSync: () => {},
250      mkdirSync: () => {},
251      unlinkSync: () => {},
252    },
253  });
254  
255  // ============================================================================
256  // Import module under test AFTER all mocks
257  // ============================================================================
258  
259  const { runProposalsStage } = await import('../../src/stages/proposals.js');
260  
261  // ============================================================================
262  // TEST HELPERS
263  // ============================================================================
264  
265  beforeEach(() => {
266    mockReworkCount = 0;
267    mockRequeueChanges = 0;
268    mockSites = [];
269    mockTemplateCountries = ['AU', 'US', 'GB'];
270    dbCalls.updates = [];
271    dbCalls.runs = [];
272  
273    stubCheckBlocklist = (_domain, _cc) => null;
274  
275    stubGenerateTemplateProposals.mock.resetCalls();
276    stubGenerateLLMProposals.mock.resetCalls();
277    stubProcessReworkRequests.mock.resetCalls();
278    stubRecordFailure.mock.resetCalls();
279    stubResetRetries.mock.resetCalls();
280    stubProcessBatch.mock.resetCalls();
281  
282    // Reset processBatch to default success behavior
283    stubProcessBatch = mock.fn(async (items, processor, _opts) => {
284      const results = [];
285      const errors = [];
286      for (let i = 0; i < items.length; i++) {
287        try {
288          const r = await processor(items[i], i);
289          results.push(r);
290        } catch (err) {
291          errors.push(err);
292        }
293      }
294      return { results, errors };
295    });
296  });
297  
298  // ============================================================================
299  // TESTS
300  // ============================================================================
301  
302  describe('runProposalsStage', () => {
303    // ── Lines 120-129: No template countries → early return ──────────────────
304    test('returns early with zero counts when no template countries exist', async () => {
305      mockTemplateCountries = [];
306  
307      const result = await runProposalsStage();
308  
309      assert.deepStrictEqual(result, {
310        processed: 0,
311        succeeded: 0,
312        failed: 0,
313        skipped: 0,
314        duration: result.duration, // dynamic
315      });
316      assert.ok(result.duration >= 0);
317    });
318  
319    // ── Lines 154-162: No sites needing proposals ────────────────────────────
320    test('returns early when no sites need proposals', async () => {
321      mockSites = []; // No enriched sites
322  
323      const result = await runProposalsStage();
324  
325      assert.equal(result.processed, 0);
326      assert.equal(result.succeeded, 0);
327      assert.equal(result.failed, 0);
328      assert.equal(result.skipped, 0);
329    });
330  
331    // ── Lines 91-94: Rework processing ──────────────────────────────────────
332    test('calls processReworkRequests when rework count > 0', async () => {
333      mockReworkCount = 3;
334      mockSites = []; // No new sites, but rework should still run
335  
336      await runProposalsStage();
337  
338      assert.equal(stubProcessReworkRequests.mock.callCount(), 1);
339    });
340  
341    test('does not call processReworkRequests when rework count is 0', async () => {
342      mockReworkCount = 0;
343      mockSites = [];
344  
345      await runProposalsStage();
346  
347      assert.equal(stubProcessReworkRequests.mock.callCount(), 0);
348    });
349  
350    // ── Lines 100-114: Re-queue enriched sites with unknown country ─────────
351    test('logs info when re-queuing sites with unknown country', async () => {
352      mockRequeueChanges = 5;
353      mockSites = [];
354  
355      const result = await runProposalsStage();
356  
357      // Should still return zero processed (no sites matched main query)
358      assert.equal(result.processed, 0);
359    });
360  
361    // ── Lines 164-176: Blocklist filtering ──────────────────────────────────
362    test('marks blocklisted sites as ignore in database', async () => {
363      mockSites = [
364        {
365          id: 1,
366          domain: 'yelp.com',
367          url: 'https://yelp.com',
368          score: 50,
369          grade: 'D',
370          keyword: 'plumber',
371          country_code: 'US',
372        },
373        {
374          id: 2,
375          domain: 'good-site.com',
376          url: 'https://good-site.com',
377          score: 55,
378          grade: 'D',
379          keyword: 'plumber',
380          country_code: 'AU',
381        },
382      ];
383  
384      stubCheckBlocklist = (domain, _cc) => {
385        if (domain === 'yelp.com') return { reason: 'Business directory' };
386        return null;
387      };
388  
389      const result = await runProposalsStage();
390  
391      // Blocklist update should have been called for yelp.com
392      const ignoreUpdates = dbCalls.updates.filter(u => u.sql.includes("status = 'ignored'"));
393      assert.ok(
394        ignoreUpdates.length >= 1,
395        'Should have at least one ignore update for blocklisted site'
396      );
397    });
398  
399    // ── Lines 178-179: Ignored count logging ────────────────────────────────
400    test('reports count of ignored blocklisted sites', async () => {
401      mockSites = [
402        {
403          id: 1,
404          domain: 'directory.com',
405          url: 'https://directory.com',
406          score: 40,
407          grade: 'F',
408          keyword: 'test',
409          country_code: 'AU',
410        },
411        {
412          id: 2,
413          domain: 'another-dir.com',
414          url: 'https://another-dir.com',
415          score: 45,
416          grade: 'F',
417          keyword: 'test',
418          country_code: 'US',
419        },
420      ];
421  
422      stubCheckBlocklist = () => ({ reason: 'Franchise site' });
423  
424      const result = await runProposalsStage();
425  
426      // Both sites blocked but still processed through the batch
427      assert.equal(result.processed, 2);
428    });
429  
430    // ── Lines 192-200: Successful batch processing ──────────────────────────
431    test('processes sites and counts successes', async () => {
432      mockSites = [
433        {
434          id: 10,
435          domain: 'site-a.com',
436          url: 'https://site-a.com',
437          score: 60,
438          grade: 'D',
439          keyword: 'dentist',
440          country_code: 'AU',
441        },
442        {
443          id: 11,
444          domain: 'site-b.com',
445          url: 'https://site-b.com',
446          score: 70,
447          grade: 'C',
448          keyword: 'dentist',
449          country_code: 'AU',
450        },
451      ];
452  
453      const result = await runProposalsStage();
454  
455      assert.equal(result.processed, 2);
456      assert.equal(result.succeeded, 2);
457      assert.equal(result.failed, 0);
458    });
459  
460    // ── Lines 202-212: Error logging for failed sites ───────────────────────
461    test('counts errors from failed batch processing', async () => {
462      mockSites = [
463        {
464          id: 20,
465          domain: 'fail-site.com',
466          url: 'https://fail-site.com',
467          score: 50,
468          grade: 'D',
469          keyword: 'plumber',
470          country_code: 'AU',
471        },
472      ];
473  
474      // Make processBatch return an error
475      stubProcessBatch = mock.fn(async (items, _processor, _opts) => {
476        return {
477          results: [],
478          errors: [{ item: { url: 'https://fail-site.com' }, error: { message: 'LLM timeout' } }],
479        };
480      });
481  
482      const result = await runProposalsStage();
483  
484      assert.equal(result.processed, 1);
485      assert.equal(result.succeeded, 0);
486      assert.equal(result.failed, 1);
487    });
488  
489    test('handles errors without item or error properties gracefully', async () => {
490      mockSites = [
491        {
492          id: 21,
493          domain: 'fail2.com',
494          url: 'https://fail2.com',
495          score: 55,
496          grade: 'D',
497          keyword: 'test',
498          country_code: 'US',
499        },
500      ];
501  
502      // Error object without item/error fields — tests toString() fallback on line 210
503      stubProcessBatch = mock.fn(async () => ({
504        results: [],
505        errors: ['raw string error'],
506      }));
507  
508      const result = await runProposalsStage();
509  
510      assert.equal(result.failed, 1);
511      assert.equal(result.succeeded, 0);
512    });
513  
514    // ── Lines 214-216: Duration and stage completion ────────────────────────
515    test('returns duration in stats', async () => {
516      mockSites = [
517        {
518          id: 30,
519          domain: 'fast.com',
520          url: 'https://fast.com',
521          score: 65,
522          grade: 'D',
523          keyword: 'test',
524          country_code: 'GB',
525        },
526      ];
527  
528      const result = await runProposalsStage();
529  
530      assert.ok('duration' in result, 'Should have duration field');
531      assert.ok(result.duration >= 0, 'Duration should be non-negative');
532    });
533  
534    // ── Lines 218-220: Stage-level catch block ──────────────────────────────
535    test('throws and propagates error when stage fails catastrophically', async () => {
536      mockSites = [
537        {
538          id: 40,
539          domain: 'crash.com',
540          url: 'https://crash.com',
541          score: 40,
542          grade: 'F',
543          keyword: 'test',
544          country_code: 'AU',
545        },
546      ];
547  
548      // Make processBatch throw (not return errors, but actually throw)
549      stubProcessBatch = mock.fn(async () => {
550        throw new Error('Database connection lost');
551      });
552  
553      await assert.rejects(
554        () => runProposalsStage(),
555        err => {
556          assert.ok(err.message.includes('Database connection lost'));
557          return true;
558        }
559      );
560    });
561  
562    // ── Options: concurrency and limit ──────────────────────────────────────
563    test('passes concurrency option to processBatch', async () => {
564      mockSites = [
565        {
566          id: 50,
567          domain: 'conc.com',
568          url: 'https://conc.com',
569          score: 60,
570          grade: 'D',
571          keyword: 'test',
572          country_code: 'AU',
573        },
574      ];
575  
576      await runProposalsStage({ concurrency: 5 });
577  
578      assert.equal(stubProcessBatch.mock.callCount(), 1);
579      const callArgs = stubProcessBatch.mock.calls[0].arguments;
580      assert.equal(callArgs[2].concurrency, 5);
581    });
582  
583    test('uses default concurrency of 2 when not specified', async () => {
584      mockSites = [
585        {
586          id: 51,
587          domain: 'default-conc.com',
588          url: 'https://default-conc.com',
589          score: 60,
590          grade: 'D',
591          keyword: 'test',
592          country_code: 'AU',
593        },
594      ];
595  
596      await runProposalsStage();
597  
598      const callArgs = stubProcessBatch.mock.calls[0].arguments;
599      assert.equal(callArgs[2].concurrency, 2);
600    });
601  
602    // ── Score range options ─────────────────────────────────────────────────
603    test('respects minScore and maxScore options', async () => {
604      mockSites = [];
605  
606      const result = await runProposalsStage({ minScore: 10, maxScore: 50 });
607  
608      assert.equal(result.processed, 0); // No sites, but the query ran with the filters
609    });
610  });