1  #!/usr/bin/env node
  2  
  3  /**
  4   * Weekly Learning Report Cron Job
  5   * Analyzes prompt feedback patterns and flags low-performing prompts for human review
  6   * Adds approximately weekly (not exact schedule) via cron_jobs table
  7   */
  8  
  9  import { run } from './../utils/db.js';
 10  import Logger from '../utils/logger.js';
 11  import { generatePromptRecommendations } from '../utils/prompt-learning.js';
 12  
 13  const logger = new Logger('WeeklyLearningReport');
 14  
 15  // Approval rate threshold for flagging (70%)
 16  const APPROVAL_THRESHOLD = 70;
 17  
 18  // Prompt files to analyze
 19  const PROMPT_FILES = [
 20    'PROPOSAL.md',
 21    'CONVERSION-SCORING-VISION.md',
 22    'CONVERSION-SCORING-NOVIS.md',
 23    'CLASSIFICATION.md',
 24    'FORM-FIELD-DETECTION',
 25  ];
 26  
 27  /**
 28   * Add item to human review queue
 29   */
 30  async function addToReviewQueue(promptFile, reason, recommendations) {
 31    try {
 32      const recommendationText = recommendations
 33        .map(
 34          (rec, i) =>
 35            `${i + 1}. [${rec.priority.toUpperCase()}] ${rec.issue}\n   → ${rec.recommendation}`
 36        )
 37        .join('\n\n');
 38  
 39      await run(
 40        `INSERT INTO human_review_queue (file, reason, type, priority, status)
 41         VALUES ($1, $2, $3, $4, 'pending')`,
 42        [
 43          `prompts/${promptFile}`,
 44          `${reason}\n\n${recommendationText}`,
 45          'prompt_quality',
 46          'medium',
 47        ]
 48      );
 49  
 50      logger.info(`Added ${promptFile} to human review queue`);
 51    } catch (err) {
 52      logger.error(`Failed to add ${promptFile} to review queue: ${err.message}`);
 53    }
 54  }
 55  
 56  /**
 57   * Log cron job execution
 58   */
 59  async function logCronExecution(status, details) {
 60    try {
 61      await run(
 62        `INSERT INTO ops.cron_jobs (job_name, last_run, status, details)
 63         VALUES ('weekly-learning-report', NOW(), $1, $2)
 64         ON CONFLICT (job_name) DO UPDATE SET
 65           last_run = NOW(),
 66           status   = EXCLUDED.status,
 67           details  = EXCLUDED.details`,
 68        [status, details]
 69      );
 70    } catch (err) {
 71      logger.error(`Failed to log cron execution: ${err.message}`);
 72    }
 73  }
 74  
 75  /**
 76   * Run weekly learning report
 77   */
 78  async function runWeeklyReport() {
 79    const startTime = Date.now();
 80  
 81    logger.info('Starting weekly learning report...');
 82  
 83    const results = {
 84      analyzed: 0,
 85      flagged: 0,
 86      skipped: 0,
 87      errors: [],
 88    };
 89  
 90    // Analyze each prompt file
 91    for (const promptFile of PROMPT_FILES) {
 92      try {
 93        logger.info(`Analyzing ${promptFile}...`);
 94  
 95        const analysis = generatePromptRecommendations(promptFile);
 96        results.analyzed++;
 97  
 98        // Check if data is available
 99        if (!analysis || !analysis.stats || analysis.stats.total === 0) {
100          logger.warn(`No feedback data available for ${promptFile}`);
101          results.skipped++;
102          continue;
103        }
104  
105        const { stats, recommendations } = analysis;
106        const { approvalRate } = stats;
107  
108        logger.info(`${promptFile}: ${approvalRate}% approval rate (${stats.total} samples)`);
109  
110        // Flag low-performing prompts
111        if (approvalRate < APPROVAL_THRESHOLD && recommendations.length > 0) {
112          const reason =
113            `Low approval rate: ${approvalRate}% (target: ${APPROVAL_THRESHOLD}%+)\n` +
114            `Total feedback: ${stats.total}\n` +
115            `Approved: ${stats.approved}, Rework: ${stats.rework}, Rejected: ${stats.rejected}\n\n` +
116            `This prompt needs review and improvement.`;
117  
118          await addToReviewQueue(promptFile, reason, recommendations);
119          results.flagged++;
120  
121          logger.warn(`Flagged ${promptFile} for review (${approvalRate}% approval)`);
122        } else if (approvalRate >= APPROVAL_THRESHOLD) {
123          logger.success(`${promptFile} performing well (${approvalRate}% approval)`);
124        }
125      } catch (err) {
126        logger.error(`Failed to analyze ${promptFile}: ${err.message}`);
127        results.errors.push(`${promptFile}: ${err.message}`);
128      }
129    }
130  
131    const duration = Date.now() - startTime;
132  
133    // Log execution
134    const details = JSON.stringify({
135      analyzed: results.analyzed,
136      flagged: results.flagged,
137      skipped: results.skipped,
138      errors: results.errors.length,
139      duration_ms: duration,
140    });
141  
142    await logCronExecution('completed', details);
143  
144    // Summary
145    logger.info('Weekly Learning Report Summary:');
146    logger.info(`   Analyzed: ${results.analyzed} prompts`);
147    logger.info(`   Flagged for review: ${results.flagged} prompts`);
148    logger.info(`   Skipped (no data): ${results.skipped} prompts`);
149    logger.info(`   Errors: ${results.errors.length}`);
150    logger.info(`   Duration: ${(duration / 1000).toFixed(2)}s`);
151  
152    if (results.flagged > 0) {
153      logger.warn(
154        `${results.flagged} prompts flagged for human review. Check the Human Review dashboard.`
155      );
156    }
157  
158    if (results.errors.length > 0) {
159      logger.error('Errors encountered:');
160      results.errors.forEach(err => logger.error(`  - ${err}`));
161    }
162  
163    return results;
164  }
165  
166  // Run if executed directly
167  if (import.meta.url === `file://${process.argv[1]}`) {
168    runWeeklyReport()
169      .then(() => {
170        logger.success('Weekly learning report completed');
171        process.exit(0);
172      })
173      .catch(err => {
174        logger.error(`Weekly learning report failed: ${err.message}`);
175        process.exit(1);
176      });
177  }
178  
179  export default runWeeklyReport;