mvp.js
1 #!/usr/bin/env node 2 3 /** 4 * MVP Pipeline Orchestration 5 * Full end-to-end pipeline: SERP → Score → Generate Proposals → Send Outreach → Process Replies 6 */ 7 8 import { join, dirname } from 'path'; 9 import { fileURLToPath } from 'url'; 10 import Logger from './utils/logger.js'; 11 import { scrapeSERP } from './scrape.js'; 12 import { captureScreenshots } from './capture.js'; 13 import { scoreSite } from './score.js'; 14 import { generateProposals } from './proposal-generator-v2.js'; 15 import { bulkUpdateOutreachContacts } from './contacts/prioritize.js'; 16 import { sendEmail } from './outreach/email.js'; 17 import { sendSMS } from './outreach/sms.js'; 18 import { submitContactForm } from './outreach/form.js'; 19 import { pollAllChannels, processAllReplies } from './inbound/processor.js'; 20 import './utils/load-env.js'; 21 import { run, getOne, getAll, query, withTransaction, closePool, getPool } from './utils/db.js'; 22 23 const __filename = fileURLToPath(import.meta.url); 24 const __dirname = dirname(__filename); 25 const projectRoot = join(__dirname, '..'); 26 27 const logger = new Logger('MVP'); 28 29 /** 30 * Run POC pipeline for a keyword (SERP → Capture → Score) 31 */ 32 async function runPOCForKeyword(keyword) { 33 logger.info(`Running POC pipeline for keyword: "${keyword}"`); 34 35 // Scrape SERP 36 const serpResults = await scrapeSERP(keyword); 37 logger.success(`Found ${serpResults.length} results for "${keyword}"`); 38 39 // Capture screenshots and score each site 40 let scored = 0; 41 for (const result of serpResults) { 42 try { 43 // Capture screenshots 44 await captureScreenshots(result.domain, result.url); 45 46 // Score the site 47 await scoreSite(result.domain); 48 49 scored++; 50 logger.success(`Scored ${result.domain} (${scored}/${serpResults.length})`); 51 } catch (error) { 52 logger.error(`Failed to process ${result.domain}`, error); 53 } 54 } 55 56 logger.success(`POC complete: ${scored}/${serpResults.length} sites scored`); 57 return scored; 58 } 59 60 /** 61 * Filter sites scoring B- to E 62 */ 63 async function getScoredSites(minScore = 0, maxScore = 82) { 64 return await getAll( 65 `SELECT id, domain, conversion_score, conversion_score_json 66 FROM sites 67 WHERE conversion_score IS NOT NULL 68 AND conversion_score >= $1 69 AND conversion_score <= $2 70 AND processing_status = 'scored' 71 ORDER BY conversion_score DESC`, 72 [minScore, maxScore] 73 ); 74 } 75 76 /** 77 * Generate proposals for filtered sites 78 */ 79 async function generateProposalsForSites(sites) { 80 logger.info(`Generating proposals for ${sites.length} sites...`); 81 82 let generated = 0; 83 let failed = 0; 84 85 for (const site of sites) { 86 try { 87 await generateProposals(site.id); 88 generated++; 89 logger.success(`Generated proposals for ${site.domain} (${generated}/${sites.length})`); 90 } catch (error) { 91 logger.error(`Failed to generate proposals for ${site.domain}`, error); 92 failed++; 93 } 94 } 95 96 logger.success(`Proposals generated: ${generated}/${sites.length} (${failed} failed)`); 97 return { generated, failed }; 98 } 99 100 /** 101 * Send outreach via available channels 102 */ 103 async function sendOutreach(limit = null) { 104 logger.info('Sending outreach...'); 105 106 // First, update contact URIs for all pending outreaches 107 const updateResults = bulkUpdateOutreachContacts(limit); 108 logger.success( 109 `Contact URIs updated: ${updateResults.succeeded}/${updateResults.total} sites, ${updateResults.totalOutreachesUpdated} outreaches` 110 ); 111 112 // Get outreaches ready to send 113 const outreaches = await getAll( 114 `SELECT id, site_id, contact_method, contact_uri, message_body AS proposal_text, subject_line, variant_number 115 FROM messages 116 WHERE direction = 'outbound' 117 AND approval_status = 'pending' 118 AND contact_uri != 'PENDING_CONTACT_EXTRACTION' 119 ${limit ? `LIMIT ${parseInt(limit, 10)}` : ''}` 120 ); 121 122 logger.info(`Sending ${outreaches.length} outreaches...`); 123 124 const results = { 125 total: outreaches.length, 126 sent: { email: 0, sms: 0, form: 0 }, 127 failed: 0, 128 }; 129 130 for (const outreach of outreaches) { 131 try { 132 switch (outreach.contact_method) { 133 case 'email': 134 await sendEmail( 135 outreach.id, 136 outreach.contact_uri, 137 outreach.subject_line, 138 outreach.proposal_text 139 ); 140 results.sent.email++; 141 break; 142 143 case 'sms': 144 await sendSMS(outreach.id, outreach.proposal_text, outreach.contact_uri); 145 results.sent.sms++; 146 break; 147 148 case 'form': 149 await submitContactForm( 150 outreach.id, 151 outreach.contact_uri, 152 outreach.subject_line, 153 outreach.proposal_text 154 ); 155 results.sent.form++; 156 break; 157 158 default: 159 logger.warn(`Unsupported contact method: ${outreach.contact_method}`); 160 results.failed++; 161 } 162 163 logger.success( 164 `Sent via ${outreach.contact_method} to ${outreach.contact_uri} (outreach #${outreach.id})` 165 ); 166 } catch (error) { 167 logger.error(`Failed to send outreach #${outreach.id}`, error); 168 results.failed++; 169 } 170 } 171 172 logger.success( 173 `Outreach complete: ${results.sent.email} emails, ${results.sent.sms} SMS, ${results.sent.form} forms sent (${results.failed} failed)` 174 ); 175 176 return results; 177 } 178 179 /** 180 * Process inbound replies from prospects 181 */ 182 async function processReplies() { 183 logger.info('Processing inbound replies...'); 184 185 try { 186 // Poll all channels for new messages 187 const pollResults = await pollAllChannels(); 188 const totalReceived = pollResults.sms.stored + pollResults.email.stored; 189 logger.success( 190 `Received ${totalReceived} new messages (SMS: ${pollResults.sms.stored}, Email: ${pollResults.email.stored})` 191 ); 192 193 // Process any pending operator replies 194 const replyResults = await processAllReplies(); 195 const totalSent = replyResults.sms.sent + replyResults.email.sent; 196 const totalFailed = replyResults.sms.failed + replyResults.email.failed; 197 198 if (totalSent > 0) { 199 logger.success(`Sent ${totalSent} operator replies (${totalFailed} failed)`); 200 } 201 202 return { 203 received: totalReceived, 204 sent: totalSent, 205 failed: totalFailed, 206 }; 207 } catch (error) { 208 logger.error('Failed to process replies', error); 209 return { 210 received: 0, 211 sent: 0, 212 failed: 0, 213 error: error.message, 214 }; 215 } 216 } 217 218 /** 219 * Full MVP pipeline 220 */ 221 async function runMVP(keyword, options = {}) { 222 const { 223 skipPOC = false, 224 skipProposals = false, 225 skipOutreach = false, 226 skipReplies = false, 227 minScore = 0, 228 maxScore = 82, 229 limit = null, 230 } = options; 231 232 logger.info(`Starting MVP pipeline for keyword: "${keyword}"`); 233 234 try { 235 // Step 1: Run POC pipeline (SERP → Score) 236 if (!skipPOC) { 237 await runPOCForKeyword(keyword); 238 } else { 239 logger.info('Skipping POC phase (using existing scores)'); 240 } 241 242 // Step 2: Filter sites scoring B- to E 243 const sites = await getScoredSites(minScore, maxScore); 244 logger.info(`Found ${sites.length} sites scoring between ${minScore}-${maxScore}`); 245 246 if (sites.length === 0) { 247 logger.warn('No sites in score range, exiting'); 248 return { success: false, reason: 'no_sites_in_range' }; 249 } 250 251 // Step 3: Generate proposals 252 if (!skipProposals) { 253 await generateProposalsForSites(sites); 254 } else { 255 logger.info('Skipping proposal generation (using existing proposals)'); 256 } 257 258 // Step 4: Send outreach 259 if (!skipOutreach) { 260 await sendOutreach(limit); 261 } else { 262 logger.info('Skipping outreach (proposals generated but not sent)'); 263 } 264 265 // Step 5: Process inbound replies 266 if (!skipReplies) { 267 await processReplies(); 268 } else { 269 logger.info('Skipping replies (no inbound processing)'); 270 } 271 272 logger.success('✅ MVP pipeline complete!'); 273 return { success: true }; 274 } catch (error) { 275 logger.error('MVP pipeline failed', error); 276 return { success: false, error: error.message }; 277 } 278 } 279 280 // CLI functionality 281 if (import.meta.url === `file://${process.argv[1]}`) { 282 const command = process.argv[2]; 283 284 if (command === 'run') { 285 const keyword = process.argv[3]; 286 287 if (!keyword) { 288 console.error( 289 'Usage: node src/mvp.js run <keyword> [--skip-poc] [--skip-proposals] [--skip-outreach] [--limit N]' 290 ); 291 process.exit(1); 292 } 293 294 // Parse options 295 const options = { 296 skipPOC: process.argv.includes('--skip-poc'), 297 skipProposals: process.argv.includes('--skip-proposals'), 298 skipOutreach: process.argv.includes('--skip-outreach'), 299 skipReplies: process.argv.includes('--skip-replies'), 300 }; 301 302 const limitIndex = process.argv.indexOf('--limit'); 303 if (limitIndex !== -1) { 304 options.limit = parseInt(process.argv[limitIndex + 1], 10); 305 } 306 307 runMVP(keyword, options) 308 .then(result => { 309 if (result.success) { 310 console.log('\n✅ MVP pipeline completed successfully\n'); 311 process.exit(0); 312 } else { 313 console.error(`\n❌ MVP pipeline failed: ${result.reason || result.error}\n`); 314 process.exit(1); 315 } 316 }) 317 .catch(error => { 318 console.error(`\n❌ Fatal error: ${error.message}\n`); 319 process.exit(1); 320 }); 321 } else if (command === 'poc') { 322 const keyword = process.argv[3]; 323 324 if (!keyword) { 325 console.error('Usage: node src/mvp.js poc <keyword>'); 326 process.exit(1); 327 } 328 329 runPOCForKeyword(keyword) 330 .then(scored => { 331 console.log(`\n✅ POC complete: ${scored} sites scored\n`); 332 process.exit(0); 333 }) 334 .catch(error => { 335 console.error(`\n❌ POC failed: ${error.message}\n`); 336 process.exit(1); 337 }); 338 } else if (command === 'propose') { 339 const minScore = parseInt(process.argv[3], 10) || 0; 340 const maxScore = parseInt(process.argv[4], 10) || 82; 341 342 getScoredSites(minScore, maxScore) 343 .then(sites => { 344 console.log(`\nFound ${sites.length} sites scoring ${minScore}-${maxScore}\n`); 345 return generateProposalsForSites(sites); 346 }) 347 .then(result => { 348 console.log(`\n✅ Proposals generated: ${result.generated} (${result.failed} failed)\n`); 349 process.exit(0); 350 }) 351 .catch(error => { 352 console.error(`\n❌ Proposal generation failed: ${error.message}\n`); 353 process.exit(1); 354 }); 355 } else if (command === 'send') { 356 const limit = process.argv[3] ? parseInt(process.argv[3], 10) : null; 357 358 sendOutreach(limit) 359 .then(result => { 360 console.log( 361 `\n✅ Outreach sent: ${result.sent.email + result.sent.sms + result.sent.form} total\n` 362 ); 363 process.exit(0); 364 }) 365 .catch(error => { 366 console.error(`\n❌ Outreach failed: ${error.message}\n`); 367 process.exit(1); 368 }); 369 } else if (command === 'replies') { 370 processReplies() 371 .then(result => { 372 console.log(`\n✅ Replies processed: ${result.received} received, ${result.sent} sent\n`); 373 process.exit(0); 374 }) 375 .catch(error => { 376 console.error(`\n❌ Reply processing failed: ${error.message}\n`); 377 process.exit(1); 378 }); 379 } else { 380 console.log('MVP Pipeline Orchestration'); 381 console.log(''); 382 console.log('Usage:'); 383 console.log(' run <keyword> [options] - Run full MVP pipeline'); 384 console.log(' poc <keyword> - Run POC phase only (SERP → Score)'); 385 console.log(' propose [min] [max] - Generate proposals for scored sites'); 386 console.log(' send [limit] - Send pending outreaches'); 387 console.log(' replies - Poll and process inbound replies'); 388 console.log(''); 389 console.log('Options for "run":'); 390 console.log(' --skip-poc - Skip POC phase (use existing scores)'); 391 console.log(' --skip-proposals - Skip proposal generation'); 392 console.log(' --skip-outreach - Skip sending outreach'); 393 console.log(' --skip-replies - Skip inbound reply processing'); 394 console.log(' --limit N - Limit outreach to N messages'); 395 console.log(''); 396 console.log('Examples:'); 397 console.log(' node src/mvp.js run "plumber sydney"'); 398 console.log(' node src/mvp.js run "plumber sydney" --skip-poc --limit 5'); 399 console.log(' node src/mvp.js poc "electrician melbourne"'); 400 console.log(' node src/mvp.js propose 0 82'); 401 console.log(' node src/mvp.js send 10'); 402 console.log(' node src/mvp.js replies'); 403 console.log(''); 404 process.exit(1); 405 } 406 } 407 408 export default { 409 runMVP, 410 runPOCForKeyword, 411 getScoredSites, 412 generateProposalsForSites, 413 sendOutreach, 414 processReplies, 415 };