1  /**
  2   * Integration Tests for Process Pipeline
  3   * Tests full end-to-end processing including browser automation
  4   */
  5  
  6  import { describe, test, before, after } from 'node:test';
  7  import assert from 'node:assert';
  8  import Database from 'better-sqlite3';
  9  import { join, dirname } from 'path';
 10  import { fileURLToPath } from 'url';
 11  import { existsSync, statSync, rmSync } from 'fs';
 12  import { processSingleUrl } from '../../src/process.js';
 13  import { getScoreJson } from '../../src/utils/score-storage.js';
 14  import { initTestDb } from '../../src/utils/test-db.js';
 15  
 16  const __filename = fileURLToPath(import.meta.url);
 17  const __dirname = dirname(__filename);
 18  const projectRoot = join(__dirname, '../..');
 19  
 20  const TEST_URL = 'https://www.loremipsum.de/';
 21  const TEST_DOMAIN = 'loremipsum.de';
 22  
 23  describe('Process Integration Tests', () => {
 24    let db;
 25  
 26    before(() => {
 27      const dbPath = process.env.DATABASE_PATH || join(projectRoot, 'db/sites.db');
 28      db = initTestDb(dbPath);
 29    });
 30  
 31    after(() => {
 32      if (db) {
 33        // Get screenshot path and site ID before deleting record
 34        const site = db
 35          .prepare('SELECT id, screenshot_path FROM sites WHERE domain = ?')
 36          .get(TEST_DOMAIN);
 37  
 38        if (site) {
 39          // Delete related records in tables without CASCADE
 40          db.prepare('DELETE FROM llm_usage WHERE site_id = ?').run(site.id);
 41          db.prepare('DELETE FROM error_fix_history WHERE site_id = ?').run(site.id);
 42  
 43          // Delete site record (will cascade to outreaches, site_status, prompt_feedback)
 44          db.prepare('DELETE FROM sites WHERE id = ?').run(site.id);
 45  
 46          // Clean up screenshot files if they exist
 47          if (site.screenshot_path) {
 48            const screenshotDir = join(projectRoot, site.screenshot_path);
 49            if (existsSync(screenshotDir)) {
 50              rmSync(screenshotDir, { recursive: true, force: true });
 51              console.log(`\n🧹 Cleaned up test data for ${TEST_DOMAIN} (DB record + screenshots)`);
 52            }
 53          } else {
 54            console.log(`\n🧹 Cleaned up test data for ${TEST_DOMAIN} (DB record only)`);
 55          }
 56        }
 57  
 58        db.close();
 59      }
 60    });
 61  
 62    test('should delete and recreate database record from browser', async () => {
 63      // Step 1: Delete existing record if it exists
 64      const deleteStmt = db.prepare('DELETE FROM sites WHERE domain = ?');
 65      deleteStmt.run(TEST_DOMAIN);
 66  
 67      // Verify deletion
 68      const afterDelete = db.prepare('SELECT * FROM sites WHERE domain = ?').get(TEST_DOMAIN);
 69      assert.strictEqual(afterDelete, undefined, 'Record should be deleted');
 70  
 71      // Step 2: Process the URL (capture from browser and score)
 72      await processSingleUrl(db, TEST_URL);
 73  
 74      // Step 3: Verify the record was created with all expected data
 75      const site = db.prepare('SELECT * FROM sites WHERE domain = ?').get(TEST_DOMAIN);
 76  
 77      assert.ok(site, 'Site record should exist');
 78      assert.strictEqual(site.domain, TEST_DOMAIN, 'Domain should match');
 79      assert.strictEqual(site.landing_page_url, TEST_URL, 'URL should match');
 80      assert.strictEqual(site.keyword, 'manual', 'Keyword should be "manual"');
 81      assert.strictEqual(site.status, 'prog_scored', 'Status should be "prog_scored"');
 82  
 83      // Verify screenshot_path exists and points to valid directory
 84      assert.ok(site.screenshot_path, 'Screenshot path should exist');
 85      const screenshotDir = join(projectRoot, site.screenshot_path);
 86      assert.ok(existsSync(screenshotDir), 'Screenshot directory should exist');
 87  
 88      // Verify screenshot files were captured
 89      const screenshotFiles = [
 90        'desktop_above.jpg',
 91        'desktop_above_uncropped.jpg',
 92        'desktop_below.jpg',
 93        'desktop_below_uncropped.jpg',
 94        'mobile_above.jpg',
 95        'mobile_above_uncropped.jpg',
 96      ];
 97  
 98      for (const file of screenshotFiles) {
 99        const filePath = join(screenshotDir, file);
100        assert.ok(existsSync(filePath), `Screenshot file ${file} should exist`);
101        const stats = statSync(filePath);
102        assert.ok(stats.size > 1000, `Screenshot ${file} should have reasonable size`);
103      }
104  
105      // Verify HTML was captured (stored on filesystem, 'fs' sentinel in DB)
106      assert.ok(site.html_dom === 'fs', 'HTML DOM should have filesystem sentinel');
107  
108      // Verify HTTP status
109      assert.ok(site.http_status_code, 'HTTP status code should exist');
110      assert.strictEqual(site.http_status_code, 200, 'Should be HTTP 200');
111  
112      // Verify scoring data (stored on filesystem since migration 121)
113      const scoreJson = getScoreJson(site.id);
114      assert.ok(scoreJson, 'Score JSON should exist on filesystem');
115      const scoreData = JSON.parse(scoreJson);
116      assert.ok(scoreData, 'Score data should be valid JSON');
117      assert.ok(scoreData.overall_calculation, 'Should have overall calculation');
118  
119      // Verify score
120      assert.ok(site.score !== null, 'Score should exist');
121      assert.ok(typeof site.score === 'number', 'Score should be a number');
122  
123      // Verify timestamps
124      assert.ok(site.created_at, 'Created timestamp should exist');
125      assert.ok(site.updated_at, 'Updated timestamp should exist');
126  
127      console.log(`\n✅ Successfully processed ${TEST_DOMAIN}`);
128      console.log(`   Score: ${site.score} (${site.grade})`);
129      console.log(`   Status: ${site.status}`);
130      console.log(`   Screenshots: 6 files in ${site.screenshot_path}`);
131    });
132  
133    test('should verify screenshot data integrity', async () => {
134      const site = db.prepare('SELECT * FROM sites WHERE domain = ?').get(TEST_DOMAIN);
135  
136      assert.ok(site, 'Site should exist from previous test');
137      assert.ok(site.screenshot_path, 'Screenshot path should exist');
138  
139      const screenshotDir = join(projectRoot, site.screenshot_path);
140      const screenshotFiles = [
141        'desktop_above.jpg',
142        'desktop_below.jpg',
143        'mobile_above.jpg',
144        'desktop_above_uncropped.jpg',
145        'desktop_below_uncropped.jpg',
146        'mobile_above_uncropped.jpg',
147      ];
148  
149      for (const filename of screenshotFiles) {
150        const filePath = join(screenshotDir, filename);
151  
152        // Verify file exists
153        assert.ok(existsSync(filePath), `${filename} should exist`);
154  
155        // Verify file has reasonable size
156        const stats = statSync(filePath);
157        assert.ok(
158          stats.size > 1000,
159          `${filename} should have substantial data (${stats.size} bytes)`
160        );
161  
162        // Verify JPEG magic number by reading first 2 bytes
163        const { readFileSync } = await import('fs');
164        const buffer = readFileSync(filePath);
165        assert.strictEqual(buffer[0], 0xff, `${filename} should start with JPEG header`);
166        assert.strictEqual(buffer[1], 0xd8, `${filename} should have valid JPEG SOI`);
167      }
168  
169      console.log(`\n✅ All ${screenshotFiles.length} screenshot files are valid JPEG images`);
170    });
171  
172    test('should verify score structure', async () => {
173      const site = db.prepare('SELECT * FROM sites WHERE domain = ?').get(TEST_DOMAIN);
174  
175      assert.ok(site, 'Site should exist');
176  
177      // score_json stored on filesystem since migration 121
178      const scoreJson = getScoreJson(site.id);
179      assert.ok(scoreJson, 'Score JSON should exist on filesystem');
180      const scoreData = JSON.parse(scoreJson);
181  
182      // Verify required score structure
183      assert.ok(scoreData.overall_calculation, 'Should have overall_calculation');
184      assert.ok(
185        scoreData.overall_calculation.conversion_score !== undefined,
186        'Should have conversion_score'
187      );
188      assert.ok(scoreData.overall_calculation.letter_grade, 'Should have letter_grade');
189  
190      // Verify score is in expected range (0-100)
191      const score = scoreData.overall_calculation.conversion_score;
192      assert.ok(score >= 0 && score <= 100, `Score ${score} should be between 0-100`);
193  
194      // Verify grade exists
195      const grade = scoreData.overall_calculation.letter_grade;
196      assert.ok(grade, 'Should have letter grade');
197  
198      console.log(`\n✅ Score structure is valid`);
199      console.log(`   Score: ${score} (${grade})`);
200    });
201  });