ai-hallucination-detector.js
1 #!/usr/bin/env node 2 3 /** 4 * AI Hallucination Detection & Validation Engine 5 * 6 * Advanced system for detecting and preventing AI-generated false claims in CV content. 7 * This system provides comprehensive validation using multiple detection strategies: 8 * 9 * - Quantitative claim validation against GitHub metrics 10 * - Timeline coherence analysis 11 * - Generic language detection (signs of AI generation) 12 * - Impossible claim detection (physics/logic violations) 13 * - Consistency verification across content sections 14 * - External fact-checking integration 15 * 16 * Part of the CV Enhancement Pipeline's quality assurance layer. 17 * 18 * @author Adrian Wedd 19 * @version 2.1.0 20 */ 21 22 const fs = require('fs').promises; 23 const path = require('path'); 24 const _crypto = require('crypto'); 25 26 /** 27 * Advanced AI Hallucination Detection System 28 * 29 * Multi-layered validation engine that ensures AI-generated content 30 * maintains factual accuracy and professional credibility 31 */ 32 class AIHallucinationDetector { 33 constructor() { 34 // Determine the correct data directory path 35 const currentDir = process.cwd(); 36 if (currentDir.includes('.github/scripts')) { 37 this.dataDir = path.join(currentDir, '../../data'); 38 } else { 39 this.dataDir = path.join(currentDir, 'data'); 40 } 41 this.cacheDir = path.join(this.dataDir, 'validation-cache'); 42 this.detectionResults = { 43 overall_confidence: 0, 44 validation_timestamp: new Date().toISOString(), 45 detection_layers: { 46 quantitative_validation: { passed: 0, failed: 0, warnings: [] }, 47 timeline_coherence: { passed: 0, failed: 0, warnings: [] }, 48 generic_language: { score: 0, flags: [] }, 49 impossible_claims: { detected: [], severity: 'none' }, 50 consistency_check: { coherent: true, conflicts: [] } 51 }, 52 flagged_content: [], 53 recommendations: [], 54 urgent_reviews: [] 55 }; 56 57 // AI hallucination patterns and thresholds 58 this.hallucinationPatterns = { 59 // Impossible technical claims 60 impossible_performance: [ 61 /(\d+)00%\s+(?:improvement|increase|boost)/gi, // >1000% improvements 62 /(\d{4,})\s*x\s+(?:faster|quicker|speedier)/gi, // Implausible speed increases 63 /reduced.*from\s+hours?\s+to\s+seconds?/gi, // Impossible time reductions 64 /(\d+)\s*billion\s+(?:users?|requests?)/gi // Impossible scale claims 65 ], 66 67 // Generic AI language patterns 68 generic_ai_phrases: [ 69 /leveraging\s+(?:cutting-edge|state-of-the-art|advanced)/gi, 70 /innovative\s+solutions?\s+that\s+deliver/gi, 71 /robust\s+and\s+scalable\s+(?:architecture|system)/gi, 72 /seamlessly\s+integrat(?:ed?|ing)/gi, 73 /comprehensive\s+(?:framework|solution|approach)/gi, 74 /end-to-end\s+(?:solution|implementation)/gi 75 ], 76 77 // Timeline impossibilities 78 timeline_violations: [ 79 /within\s+(\d+)\s+(?:day|week)s?\s+.*(?:complete|built|developed)/gi, 80 /single\s+day.*(?:architected|designed|built)/gi, 81 /overnight.*(?:transformation|migration|rebuild)/gi 82 ], 83 84 // Quantitative exaggerations 85 suspicious_metrics: [ 86 /(\d+)%\s+(?:reduction|decrease).*(?:latency|time|cost)/gi, 87 /saved\s+\$?(\d+(?:,\d+)*(?:\.\d+)?)\s*(?:million|billion)?/gi, 88 /increased.*by\s+(\d+)%/gi, 89 /(\d+)x\s+(?:more\s+)?(?:efficient|faster|better)/gi 90 ], 91 92 // AI meta-commentary patterns (leaked prompt artifacts) 93 meta_commentary: [ 94 /\bAI[- ]enhanced\b/gi, 95 /\boptimized\s+for\b/gi, 96 /\benhanced\s+by\s+AI\b/gi, 97 /\bleveraging\s+AI\b/gi, 98 /\[NOTE:/gi, 99 /\[TODO:/gi, 100 /\bTODO:/gi, 101 /\bNOTE:/gi, 102 /\[AI\b/gi, 103 /\bgenerated\s+by\s+(?:AI|Claude|GPT|LLM)/gi, 104 /\boptimized\s+by\s+(?:AI|Claude|GPT|LLM)/gi, 105 /\bas\s+an\s+AI\s+(?:language\s+)?model\b/gi, 106 /\bI(?:'m| am)\s+an?\s+AI\b/gi, 107 /\bprompt\s+(?:engineering|instructions?|template)\b/gi, 108 /\bhere(?:'s| is)\s+(?:a|an|the)\s+(?:revised|updated|improved|enhanced)\s+version\b/gi, 109 /\bI've\s+(?:revised|updated|improved|enhanced|rewritten)\b/gi, 110 /\bplease\s+(?:review|note|see)\s+(?:the\s+)?(?:following|below|above)\b/gi 111 ] 112 }; 113 114 // Credibility scoring weights 115 this.scoringWeights = { 116 quantitative_accuracy: 0.35, 117 timeline_coherence: 0.25, 118 generic_language_penalty: 0.15, 119 impossible_claims_penalty: 0.25 120 }; 121 } 122 123 /** 124 * Main hallucination detection pipeline 125 */ 126 async detectHallucinations() { 127 console.log('🛡️ **AI HALLUCINATION DETECTION INITIATED**'); 128 console.log('🔍 Multi-layer validation of AI-generated content...'); 129 console.log(''); 130 131 try { 132 // Ensure cache directory exists 133 await this.ensureCacheDirectory(); 134 135 // Load data sources 136 const aiContent = await this.loadAIEnhancements(); 137 const githubData = await this.loadGitHubData(); 138 const cvData = await this.loadCVData(); 139 140 // Even if no AI enhancements exist, validate the base CV directly 141 if (!aiContent || Object.keys(aiContent).length === 0) { 142 console.log('⚠️ No AI-enhanced content found — validating base CV directly'); 143 144 if (!cvData || Object.keys(cvData).length === 0) { 145 console.log('⚠️ No base CV data found either — nothing to validate'); 146 return this.generateEmptyReport(); 147 } 148 149 // Run validation layers against the base CV data itself 150 console.log('📊 **DETECTION LAYERS ANALYSIS (base CV)**'); 151 152 await this.detectGenericLanguage(cvData); 153 await this.detectImpossibleClaims(cvData); 154 await this.detectMetaCommentary(cvData); 155 await this.validateBaseCVIntegrity(cvData); 156 157 this.calculateOverallConfidenceForBaseCVOnly(); 158 await this.generateRecommendations(); 159 await this.saveDetectionResults(); 160 this.displayValidationSummary(); 161 162 return this.detectionResults; 163 } 164 165 console.log('📊 **DETECTION LAYERS ANALYSIS**'); 166 167 // Layer 1: Quantitative Validation 168 await this.validateQuantitativeClaims(aiContent, githubData); 169 170 // Layer 2: Timeline Coherence Analysis 171 await this.analyzeTimelineCoherence(aiContent, cvData); 172 173 // Layer 3: Generic Language Detection 174 await this.detectGenericLanguage(aiContent); 175 176 // Layer 4: Impossible Claims Detection 177 await this.detectImpossibleClaims(aiContent); 178 179 // Layer 5: Consistency Verification 180 await this.verifyConsistency(aiContent, cvData); 181 182 // Layer 6: Meta-commentary detection 183 await this.detectMetaCommentary(aiContent); 184 185 // Also validate the base CV if it exists 186 if (cvData && Object.keys(cvData).length > 0) { 187 await this.validateBaseCVIntegrity(cvData); 188 } 189 190 // Calculate overall confidence score 191 this.calculateOverallConfidence(); 192 193 // Generate recommendations 194 await this.generateRecommendations(); 195 196 // Save results 197 await this.saveDetectionResults(); 198 199 // Display summary 200 this.displayValidationSummary(); 201 202 return this.detectionResults; 203 204 } catch (error) { 205 console.error('❌ Hallucination detection failed:', error.message); 206 throw error; 207 } 208 } 209 210 /** 211 * Layer 1: Validate quantitative claims against actual data 212 */ 213 async validateQuantitativeClaims(aiContent, githubData) { 214 console.log('1️⃣ Validating quantitative claims...'); 215 216 const claims = this.extractQuantitativeClaims(aiContent); 217 let validClaims = 0; 218 let invalidClaims = 0; 219 220 for (const claim of claims) { 221 const validation = await this.validateClaimAgainstData(claim, githubData); 222 223 if (validation.isValid) { 224 validClaims++; 225 console.log(` ✅ Valid: ${claim.text}`); 226 } else { 227 invalidClaims++; 228 console.log(` ❌ Invalid: ${claim.text} (Actual: ${validation.actualValue})`); 229 230 this.detectionResults.flagged_content.push({ 231 type: 'quantitative_discrepancy', 232 content: claim.text, 233 expected: claim.value, 234 actual: validation.actualValue, 235 severity: validation.severity 236 }); 237 } 238 } 239 240 this.detectionResults.detection_layers.quantitative_validation = { 241 passed: validClaims, 242 failed: invalidClaims, 243 accuracy_rate: validClaims / (validClaims + invalidClaims) || 0 244 }; 245 246 console.log(` 📊 Quantitative validation: ${validClaims} valid, ${invalidClaims} invalid`); 247 } 248 249 /** 250 * Layer 2: Analyze timeline coherence and logical flow 251 */ 252 async analyzeTimelineCoherence(aiContent, cvData) { 253 console.log('2️⃣ Analyzing timeline coherence...'); 254 255 const timelineEvents = this.extractTimelineEvents(aiContent, cvData); 256 const violations = []; 257 258 // Check for impossible timeframes 259 for (const event of timelineEvents) { 260 if (this.isTimelineViolation(event)) { 261 violations.push({ 262 type: 'impossible_timeframe', 263 description: event.description, 264 timeframe: event.timeframe, 265 violation: 'Insufficient time for claimed accomplishment' 266 }); 267 } 268 } 269 270 // Check chronological consistency 271 const chronologyIssues = this.checkChronology(timelineEvents); 272 violations.push(...chronologyIssues); 273 274 this.detectionResults.detection_layers.timeline_coherence = { 275 passed: timelineEvents.length - violations.length, 276 failed: violations.length, 277 violations: violations 278 }; 279 280 console.log(` ⏰ Timeline analysis: ${violations.length} violations detected`); 281 282 if (violations.length > 0) { 283 this.detectionResults.flagged_content.push(...violations); 284 } 285 } 286 287 /** 288 * Layer 3: Detect generic AI language patterns 289 */ 290 async detectGenericLanguage(aiContent) { 291 console.log('3️⃣ Detecting generic AI language patterns...'); 292 293 const textContent = this.extractAllText(aiContent); 294 let genericScore = 0; 295 const detectedPatterns = []; 296 297 // Check generic AI patterns 298 const genericPatterns = this.hallucinationPatterns.generic_ai_phrases || []; 299 for (const pattern of genericPatterns) { 300 const matches = textContent.match(pattern) || []; 301 if (matches.length > 0) { 302 genericScore += matches.length * 10; // Each match adds 10 to generic score 303 detectedPatterns.push({ 304 pattern: pattern.source, 305 matches: matches.length, 306 examples: matches.slice(0, 3) 307 }); 308 } 309 } 310 311 this.detectionResults.detection_layers.generic_language = { 312 score: genericScore, 313 threshold: 50, // Above 50 is concerning 314 flags: detectedPatterns 315 }; 316 317 console.log(` 🤖 Generic language score: ${genericScore}/100 (lower is better)`); 318 319 if (genericScore > 50) { 320 this.detectionResults.flagged_content.push({ 321 type: 'generic_ai_language', 322 score: genericScore, 323 patterns: detectedPatterns, 324 severity: genericScore > 80 ? 'high' : 'medium' 325 }); 326 } 327 } 328 329 /** 330 * Layer 4: Detect impossible or highly implausible claims 331 */ 332 async detectImpossibleClaims(aiContent) { 333 console.log('4️⃣ Detecting impossible claims...'); 334 335 const textContent = this.extractAllText(aiContent); 336 const impossibleClaims = []; 337 338 // Check each category of impossible patterns 339 for (const [category, patterns] of Object.entries(this.hallucinationPatterns)) { 340 if (category === 'generic_ai_phrases' || category === 'meta_commentary') continue; 341 342 if (Array.isArray(patterns)) { 343 for (const pattern of patterns) { 344 const matches = textContent.match(pattern) || []; 345 for (const match of matches) { 346 impossibleClaims.push({ 347 category: category, 348 claim: match, 349 severity: this.assessClaimSeverity(match, category) 350 }); 351 } 352 } 353 } 354 } 355 356 this.detectionResults.detection_layers.impossible_claims = { 357 detected: impossibleClaims, 358 count: impossibleClaims.length, 359 severity: this.getOverallSeverity(impossibleClaims) 360 }; 361 362 console.log(` 🚨 Impossible claims detected: ${impossibleClaims.length}`); 363 364 if (impossibleClaims.length > 0) { 365 this.detectionResults.flagged_content.push({ 366 type: 'impossible_claims', 367 claims: impossibleClaims, 368 severity: 'high' 369 }); 370 } 371 } 372 373 /** 374 * Layer 5: Verify consistency across content sections 375 */ 376 async verifyConsistency(aiContent, cvData) { 377 console.log('5️⃣ Verifying content consistency...'); 378 379 const conflicts = []; 380 381 // Check for conflicting claims across sections 382 const extractedData = { 383 experience_years: this.extractExperienceYears(aiContent), 384 skill_counts: this.extractSkillCounts(aiContent), 385 project_counts: this.extractProjectCounts(aiContent) 386 }; 387 388 // Cross-reference with base CV data 389 for (const [key, aiValues] of Object.entries(extractedData)) { 390 const baseValue = this.getBaseValue(cvData, key); 391 if (baseValue && this.hasSignificantDiscrepancy(aiValues, baseValue)) { 392 conflicts.push({ 393 type: 'data_inconsistency', 394 field: key, 395 ai_values: aiValues, 396 base_value: baseValue, 397 discrepancy: this.calculateDiscrepancy(aiValues, baseValue) 398 }); 399 } 400 } 401 402 this.detectionResults.detection_layers.consistency_check = { 403 coherent: conflicts.length === 0, 404 conflicts: conflicts, 405 consistency_score: Math.max(0, 100 - (conflicts.length * 20)) 406 }; 407 408 console.log(` 🔄 Consistency check: ${conflicts.length} conflicts found`); 409 410 if (conflicts.length > 0) { 411 this.detectionResults.flagged_content.push(...conflicts); 412 } 413 } 414 415 /** 416 * Layer 6: Detect AI meta-commentary that leaked into content 417 */ 418 async detectMetaCommentary(content) { 419 console.log('6️⃣ Detecting AI meta-commentary leakage...'); 420 421 const textContent = this.extractAllText(content); 422 const metaFlags = []; 423 424 const metaPatterns = this.hallucinationPatterns.meta_commentary || []; 425 for (const pattern of metaPatterns) { 426 const matches = textContent.match(pattern) || []; 427 for (const match of matches) { 428 metaFlags.push({ 429 type: 'meta_commentary', 430 match: match, 431 pattern: pattern.source, 432 severity: 'high' 433 }); 434 } 435 } 436 437 if (metaFlags.length > 0) { 438 console.log(` 🗯️ Meta-commentary leaks found: ${metaFlags.length}`); 439 this.detectionResults.flagged_content.push({ 440 type: 'meta_commentary', 441 flags: metaFlags, 442 count: metaFlags.length, 443 severity: 'high' 444 }); 445 } else { 446 console.log(' 🗯️ No meta-commentary leaks detected'); 447 } 448 } 449 450 /** 451 * Validate the base CV itself for hallucination patterns, independent of AI enhancements 452 */ 453 async validateBaseCVIntegrity(cvData) { 454 console.log('7️⃣ Validating base CV integrity...'); 455 456 const textContent = this.extractAllText(cvData); 457 const issues = []; 458 459 // Check for generic AI language in base CV 460 const genericPatterns = this.hallucinationPatterns.generic_ai_phrases || []; 461 for (const pattern of genericPatterns) { 462 const matches = textContent.match(pattern) || []; 463 for (const match of matches) { 464 issues.push({ 465 type: 'base_cv_generic_language', 466 match: match, 467 severity: 'medium' 468 }); 469 } 470 } 471 472 // Check for impossible claims in base CV 473 for (const [category, patterns] of Object.entries(this.hallucinationPatterns)) { 474 if (category === 'generic_ai_phrases' || category === 'meta_commentary') continue; 475 if (!Array.isArray(patterns)) continue; 476 for (const pattern of patterns) { 477 const matches = textContent.match(pattern) || []; 478 for (const match of matches) { 479 issues.push({ 480 type: 'base_cv_impossible_claim', 481 category: category, 482 match: match, 483 severity: 'high' 484 }); 485 } 486 } 487 } 488 489 // Check for meta-commentary in base CV 490 const metaPatterns = this.hallucinationPatterns.meta_commentary || []; 491 for (const pattern of metaPatterns) { 492 const matches = textContent.match(pattern) || []; 493 for (const match of matches) { 494 issues.push({ 495 type: 'base_cv_meta_commentary', 496 match: match, 497 severity: 'high' 498 }); 499 } 500 } 501 502 // Check for future dates in experience 503 const currentYear = new Date().getFullYear(); 504 if (cvData.experience && Array.isArray(cvData.experience)) { 505 for (const exp of cvData.experience) { 506 if (exp.period) { 507 const yearMatches = exp.period.match(/\b(20\d{2})\b/g); 508 if (yearMatches) { 509 for (const yr of yearMatches) { 510 if (parseInt(yr, 10) > currentYear + 1) { 511 issues.push({ 512 type: 'base_cv_future_date', 513 field: `experience: ${exp.position}`, 514 value: exp.period, 515 severity: 'high' 516 }); 517 } 518 } 519 } 520 } 521 } 522 } 523 524 if (issues.length > 0) { 525 console.log(` 📄 Base CV issues found: ${issues.length}`); 526 this.detectionResults.flagged_content.push({ 527 type: 'base_cv_integrity', 528 issues: issues, 529 count: issues.length, 530 severity: issues.some(i => i.severity === 'high') ? 'high' : 'medium' 531 }); 532 } else { 533 console.log(' 📄 Base CV integrity check passed'); 534 } 535 } 536 537 /** 538 * Calculate overall confidence score using weighted metrics 539 */ 540 calculateOverallConfidence() { 541 const layers = this.detectionResults.detection_layers; 542 543 // Calculate individual layer scores (0-100) 544 // When no quantitative claims exist, treat as clean (100) not failed (0) 545 const quantRate = layers.quantitative_validation.accuracy_rate; 546 const quantScore = (layers.quantitative_validation.passed === 0 && layers.quantitative_validation.failed === 0) 547 ? 100 : (quantRate * 100); 548 const timelineScore = layers.timeline_coherence.failed === 0 ? 100 : 549 Math.max(0, 100 - (layers.timeline_coherence.failed * 25)); 550 const genericPenalty = Math.min(50, layers.generic_language.score); 551 const impossiblePenalty = layers.impossible_claims.count * 30; 552 const consistencyScore = layers.consistency_check.consistency_score; 553 554 // Apply weighted scoring 555 const overallScore = Math.max(0, Math.min(100, 556 (quantScore * this.scoringWeights.quantitative_accuracy) + 557 (timelineScore * this.scoringWeights.timeline_coherence) + 558 (consistencyScore * this.scoringWeights.quantitative_accuracy) - 559 (genericPenalty * this.scoringWeights.generic_language_penalty) - 560 (impossiblePenalty * this.scoringWeights.impossible_claims_penalty) 561 )); 562 563 // Additional penalty for meta-commentary (hard penalty) 564 const metaFlags = this.detectionResults.flagged_content.filter( 565 f => f.type === 'meta_commentary' 566 ); 567 const metaCount = metaFlags.reduce((sum, f) => sum + (f.count || 0), 0); 568 const metaPenalty = metaCount * 10; 569 570 this.detectionResults.overall_confidence = Math.max(0, Math.round(overallScore - metaPenalty)); 571 572 console.log(''); 573 console.log(`🎯 **OVERALL CONFIDENCE SCORE: ${this.detectionResults.overall_confidence}/100**`); 574 console.log(''); 575 } 576 577 /** 578 * Calculate confidence when only base CV is being validated (no AI enhancements) 579 */ 580 calculateOverallConfidenceForBaseCVOnly() { 581 const genericPenalty = Math.min(50, this.detectionResults.detection_layers.generic_language.score); 582 const impossiblePenalty = (this.detectionResults.detection_layers.impossible_claims.count || 0) * 30; 583 584 // Count meta-commentary and base CV integrity issues 585 const metaFlags = this.detectionResults.flagged_content.filter( 586 f => f.type === 'meta_commentary' 587 ); 588 const metaCount = metaFlags.reduce((sum, f) => sum + (f.count || 0), 0); 589 const metaPenalty = metaCount * 10; 590 591 const baseCVIssues = this.detectionResults.flagged_content.filter( 592 f => f.type === 'base_cv_integrity' 593 ); 594 const baseCVHighCount = baseCVIssues.reduce((sum, f) => { 595 if (!f.issues) return sum; 596 return sum + f.issues.filter(i => i.severity === 'high').length; 597 }, 0); 598 const baseCVPenalty = baseCVHighCount * 15; 599 600 const overallScore = Math.max(0, 100 - genericPenalty - impossiblePenalty - metaPenalty - baseCVPenalty); 601 this.detectionResults.overall_confidence = Math.round(overallScore); 602 603 console.log(''); 604 console.log(`🎯 **OVERALL CONFIDENCE SCORE: ${this.detectionResults.overall_confidence}/100**`); 605 console.log(''); 606 } 607 608 /** 609 * Generate actionable recommendations based on detection results 610 */ 611 async generateRecommendations() { 612 const recommendations = []; 613 const urgentReviews = []; 614 615 // Critical issues requiring immediate attention 616 if (this.detectionResults.overall_confidence < 70) { 617 urgentReviews.push({ 618 priority: 'critical', 619 message: 'Low confidence score requires immediate content review', 620 action: 'Manual review and fact-checking of all AI-generated content' 621 }); 622 } 623 624 // Specific recommendations based on detection results 625 const flaggedCount = this.detectionResults.flagged_content.length; 626 if (flaggedCount > 5) { 627 recommendations.push({ 628 category: 'content_quality', 629 message: `${flaggedCount} flagged items require attention`, 630 action: 'Review and correct flagged content before deployment' 631 }); 632 } 633 634 // Generic language recommendations 635 const genericScore = this.detectionResults.detection_layers.generic_language.score; 636 if (genericScore > 50) { 637 recommendations.push({ 638 category: 'authenticity', 639 message: 'High generic language score indicates AI-generated feel', 640 action: 'Revise content to be more specific and personal' 641 }); 642 } 643 644 // Quantitative accuracy recommendations 645 const quantAccuracy = this.detectionResults.detection_layers.quantitative_validation.accuracy_rate; 646 if (quantAccuracy < 0.8) { 647 recommendations.push({ 648 category: 'accuracy', 649 message: 'Low quantitative accuracy detected', 650 action: 'Verify all numerical claims against GitHub data' 651 }); 652 } 653 654 // Meta-commentary recommendations 655 const metaFlags = this.detectionResults.flagged_content.filter(f => f.type === 'meta_commentary'); 656 if (metaFlags.length > 0) { 657 urgentReviews.push({ 658 priority: 'high', 659 message: 'AI meta-commentary detected in content — prompt artifacts have leaked through', 660 action: 'Remove all AI meta-commentary references before deployment' 661 }); 662 } 663 664 this.detectionResults.recommendations = recommendations; 665 this.detectionResults.urgent_reviews = urgentReviews; 666 } 667 668 /** 669 * Helper methods for claim extraction and validation 670 */ 671 extractQuantitativeClaims(aiContent) { 672 const claims = []; 673 const patterns = [ 674 /(\d+)\s*(?:years?|months?)\s+(?:of\s+)?experience/gi, 675 /(\d+)\+?\s*(?:projects?|repositories?|systems?)/gi, 676 /(\d+)\s*(?:programming\s+)?languages?/gi, 677 /(\d+)%\s+(?:improvement|increase|reduction|faster)/gi, 678 /(\d+)x\s+(?:faster|more|better|improved)/gi 679 ]; 680 681 const textContent = this.extractAllText(aiContent); 682 683 for (const pattern of patterns) { 684 let match; 685 while ((match = pattern.exec(textContent)) !== null) { 686 claims.push({ 687 text: match[0], 688 value: parseInt(match[1]), 689 type: this.classifyClaimType(match[0]) 690 }); 691 } 692 } 693 694 return claims; 695 } 696 697 async validateClaimAgainstData(claim, githubData) { 698 const actualValue = this.getActualValue(claim.type, githubData); 699 const tolerance = this.getToleranceForClaimType(claim.type); 700 701 const isValid = Math.abs(claim.value - actualValue) <= tolerance; 702 703 return { 704 isValid, 705 actualValue, 706 severity: isValid ? 'none' : (Math.abs(claim.value - actualValue) > tolerance * 2 ? 'high' : 'medium') 707 }; 708 } 709 710 /** 711 * Helper methods (implementation details) 712 */ 713 extractAllText(content) { 714 if (typeof content === 'string') return content; 715 return JSON.stringify(content, null, 2); 716 } 717 718 classifyClaimType(claimText) { 719 const lower = claimText.toLowerCase(); 720 if (lower.includes('year') || lower.includes('month')) return 'experience'; 721 if (lower.includes('project') || lower.includes('repository')) return 'projects'; 722 if (lower.includes('language')) return 'languages'; 723 if (lower.includes('%')) return 'performance'; 724 return 'general'; 725 } 726 727 getActualValue(claimType, githubData) { 728 switch (claimType) { 729 case 'projects': return githubData?.repositories?.total_count || githubData?.summary?.total_repos || 0; 730 case 'languages': return githubData?.languages?.length || githubData?.summary?.languages?.length || 0; 731 default: return 0; 732 } 733 } 734 735 getToleranceForClaimType(claimType) { 736 switch (claimType) { 737 case 'experience': return 6; // 6 months tolerance 738 case 'projects': return 3; // 3 project tolerance 739 case 'languages': return 2; // 2 language tolerance 740 case 'performance': return 20; // 20% tolerance 741 default: return 1; 742 } 743 } 744 745 /** 746 * Extract timeline events from AI content and CV data 747 */ 748 extractTimelineEvents(aiContent, cvData) { 749 const events = []; 750 751 // Extract from CV experience entries 752 const experiences = cvData?.experience || aiContent?.experience || []; 753 for (const exp of experiences) { 754 if (!exp.period) continue; 755 const parsed = this.parsePeriod(exp.period); 756 events.push({ 757 description: `${exp.position || 'Unknown'} at ${exp.company || 'Unknown'}`, 758 timeframe: exp.period, 759 startYear: parsed.startYear, 760 endYear: parsed.endYear, 761 position: exp.position || '', 762 source: 'experience' 763 }); 764 } 765 766 // Extract from projects 767 const projects = cvData?.projects || aiContent?.projects || []; 768 for (const proj of projects) { 769 if (!proj.period) continue; 770 const parsed = this.parsePeriod(proj.period); 771 events.push({ 772 description: `Project: ${proj.name || 'Unknown'}`, 773 timeframe: proj.period, 774 startYear: parsed.startYear, 775 endYear: parsed.endYear, 776 source: 'project' 777 }); 778 } 779 780 // Extract from achievements 781 const achievements = cvData?.achievements || aiContent?.achievements || []; 782 for (const ach of achievements) { 783 if (!ach.date) continue; 784 const parsed = this.parsePeriod(ach.date); 785 events.push({ 786 description: `Achievement: ${ach.title || 'Unknown'}`, 787 timeframe: ach.date, 788 startYear: parsed.startYear, 789 endYear: parsed.endYear, 790 source: 'achievement' 791 }); 792 } 793 794 // Extract timeline claims from text content 795 const textContent = this.extractAllText(aiContent); 796 const timelinePatterns = this.hallucinationPatterns.timeline_violations || []; 797 for (const pattern of timelinePatterns) { 798 const matches = textContent.match(pattern) || []; 799 for (const match of matches) { 800 events.push({ 801 description: match, 802 timeframe: match, 803 startYear: null, 804 endYear: null, 805 source: 'text_claim', 806 flagged: true 807 }); 808 } 809 } 810 811 return events; 812 } 813 814 /** 815 * Parse a period string like "2018 - Present" into start/end years 816 */ 817 parsePeriod(periodStr) { 818 const currentYear = new Date().getFullYear(); 819 const years = periodStr.match(/\b(20\d{2}|19\d{2})\b/g); 820 const isPresent = /present|current|now/i.test(periodStr); 821 822 let startYear = null; 823 let endYear = null; 824 825 if (years && years.length >= 1) { 826 startYear = parseInt(years[0], 10); 827 } 828 if (years && years.length >= 2) { 829 endYear = parseInt(years[1], 10); 830 } else if (isPresent) { 831 endYear = currentYear; 832 } else if (startYear) { 833 endYear = startYear; 834 } 835 836 return { startYear, endYear }; 837 } 838 839 /** 840 * Check if a timeline event is a violation (impossible timeframe) 841 */ 842 isTimelineViolation(event) { 843 const currentYear = new Date().getFullYear(); 844 845 // Events extracted from timeline_violations patterns are inherently flagged 846 if (event.flagged) return true; 847 848 // Start date in the far future 849 if (event.startYear && event.startYear > currentYear + 1) return true; 850 851 // End date before start date 852 if (event.startYear && event.endYear && event.endYear < event.startYear) return true; 853 854 // Unreasonably old start date for a tech career (before 1990) 855 if (event.startYear && event.startYear < 1990) return true; 856 857 return false; 858 } 859 860 /** 861 * Check chronological consistency across all timeline events 862 */ 863 checkChronology(timelineEvents) { 864 const issues = []; 865 const currentYear = new Date().getFullYear(); 866 867 // Check for overlapping full-time positions 868 // Skip overlaps where one role is self-employment (Director/Founder/Owner/Consultant) 869 const selfEmploymentPattern = /\b(?:director|founder|owner|co-founder|consultant|freelance|self-employed)\b/i; 870 const fullTimeExperiences = timelineEvents.filter(e => e.source === 'experience'); 871 for (let i = 0; i < fullTimeExperiences.length; i++) { 872 for (let j = i + 1; j < fullTimeExperiences.length; j++) { 873 const a = fullTimeExperiences[i]; 874 const b = fullTimeExperiences[j]; 875 876 // Skip if we cannot parse years 877 if (!a.startYear || !a.endYear || !b.startYear || !b.endYear) continue; 878 879 // Allow concurrent roles when one is self-employment/directorship 880 if (selfEmploymentPattern.test(a.position || '') || selfEmploymentPattern.test(b.position || '')) continue; 881 882 // Check for overlap (allowing 1 year overlap for transitions) 883 const overlapStart = Math.max(a.startYear, b.startYear); 884 const overlapEnd = Math.min(a.endYear, b.endYear); 885 if (overlapEnd - overlapStart > 1) { 886 issues.push({ 887 type: 'chronology_overlap', 888 description: `Significant overlap between "${a.description}" and "${b.description}"`, 889 timeframe: `${a.timeframe} vs ${b.timeframe}`, 890 violation: 'More than 1 year of overlapping full-time positions' 891 }); 892 } 893 } 894 } 895 896 // Check that achievements fall within reasonable time 897 const achievementEvents = timelineEvents.filter(e => e.source === 'achievement'); 898 for (const ach of achievementEvents) { 899 if (ach.endYear && ach.endYear > currentYear + 1) { 900 issues.push({ 901 type: 'future_achievement', 902 description: ach.description, 903 timeframe: ach.timeframe, 904 violation: 'Achievement date is in the future' 905 }); 906 } 907 } 908 909 return issues; 910 } 911 912 /** 913 * Assess the severity of a single claim 914 */ 915 assessClaimSeverity(claimText, category) { 916 const text = claimText.toLowerCase(); 917 918 // Impossible performance claims are always high severity 919 if (category === 'impossible_performance') return 'high'; 920 921 // Timeline violations are high severity 922 if (category === 'timeline_violations') return 'high'; 923 924 // Suspicious metrics: severity depends on the magnitude 925 if (category === 'suspicious_metrics') { 926 const numMatch = claimText.match(/(\d+)/); 927 if (numMatch) { 928 const num = parseInt(numMatch[1], 10); 929 if (num > 500) return 'high'; 930 if (num > 100) return 'medium'; 931 } 932 // Dollar amounts with million/billion 933 if (/million|billion/i.test(text)) return 'high'; 934 return 'medium'; 935 } 936 937 return 'medium'; 938 } 939 940 /** 941 * Determine the overall severity from a list of claims 942 */ 943 getOverallSeverity(claims) { 944 if (claims.length === 0) return 'none'; 945 if (claims.some(c => c.severity === 'high')) return 'high'; 946 if (claims.some(c => c.severity === 'medium')) return 'medium'; 947 return 'low'; 948 } 949 950 /** 951 * Extract experience years mentioned in AI content 952 */ 953 extractExperienceYears(aiContent) { 954 const text = this.extractAllText(aiContent); 955 const matches = text.match(/(\d+)\+?\s*(?:years?)\s+(?:of\s+)?experience/gi) || []; 956 const values = []; 957 for (const match of matches) { 958 const num = match.match(/(\d+)/); 959 if (num) values.push(parseInt(num[1], 10)); 960 } 961 return values; 962 } 963 964 /** 965 * Extract skill counts mentioned in AI content 966 */ 967 extractSkillCounts(aiContent) { 968 const text = this.extractAllText(aiContent); 969 const matches = text.match(/(\d+)\+?\s*(?:skills?|technologies?|tools?|languages?)/gi) || []; 970 const values = []; 971 for (const match of matches) { 972 const num = match.match(/(\d+)/); 973 if (num) values.push(parseInt(num[1], 10)); 974 } 975 return values; 976 } 977 978 /** 979 * Extract project counts mentioned in AI content 980 */ 981 extractProjectCounts(aiContent) { 982 const text = this.extractAllText(aiContent); 983 const matches = text.match(/(\d+)\+?\s*(?:projects?|repositories?|applications?|systems?)/gi) || []; 984 const values = []; 985 for (const match of matches) { 986 const num = match.match(/(\d+)/); 987 if (num) values.push(parseInt(num[1], 10)); 988 } 989 return values; 990 } 991 992 /** 993 * Get the base value from CV data for a given key 994 */ 995 getBaseValue(cvData, key) { 996 if (!cvData) return null; 997 998 switch (key) { 999 case 'experience_years': { 1000 // Calculate from the earliest experience entry to now 1001 const experiences = cvData.experience || []; 1002 if (experiences.length === 0) return null; 1003 let earliest = new Date().getFullYear(); 1004 for (const exp of experiences) { 1005 if (exp.period) { 1006 const years = exp.period.match(/\b(20\d{2}|19\d{2})\b/g); 1007 if (years) { 1008 const yr = parseInt(years[0], 10); 1009 if (yr < earliest) earliest = yr; 1010 } 1011 } 1012 } 1013 return new Date().getFullYear() - earliest; 1014 } 1015 case 'skill_counts': { 1016 const skills = cvData.skills || []; 1017 return skills.length; 1018 } 1019 case 'project_counts': { 1020 const projects = cvData.projects || []; 1021 return projects.length; 1022 } 1023 default: 1024 return null; 1025 } 1026 } 1027 1028 /** 1029 * Check if there is a significant discrepancy between AI values and base value 1030 */ 1031 hasSignificantDiscrepancy(aiValues, baseValue) { 1032 if (!Array.isArray(aiValues) || aiValues.length === 0) return false; 1033 if (baseValue === null || baseValue === undefined) return false; 1034 1035 for (const val of aiValues) { 1036 const diff = Math.abs(val - baseValue); 1037 // More than 50% discrepancy or more than 5 absolute difference 1038 if (diff > Math.max(baseValue * 0.5, 5)) { 1039 return true; 1040 } 1041 } 1042 return false; 1043 } 1044 1045 /** 1046 * Calculate the discrepancy between AI values and base value 1047 */ 1048 calculateDiscrepancy(aiValues, baseValue) { 1049 if (!Array.isArray(aiValues) || aiValues.length === 0) return 0; 1050 if (baseValue === 0) return aiValues[0] || 0; 1051 1052 // Return the maximum discrepancy percentage 1053 let maxDiscrepancy = 0; 1054 for (const val of aiValues) { 1055 const pct = Math.abs(val - baseValue) / baseValue * 100; 1056 if (pct > maxDiscrepancy) maxDiscrepancy = pct; 1057 } 1058 return Math.round(maxDiscrepancy); 1059 } 1060 1061 /** 1062 * Data loading methods 1063 */ 1064 async loadAIEnhancements() { 1065 try { 1066 const enhancementsPath = path.join(this.dataDir, 'ai-enhancements.json'); 1067 const content = await fs.readFile(enhancementsPath, 'utf8'); 1068 return JSON.parse(content); 1069 } catch { 1070 console.warn('⚠️ AI enhancements data not found'); 1071 return {}; 1072 } 1073 } 1074 1075 async loadGitHubData() { 1076 try { 1077 const summaryPath = path.join(this.dataDir, 'activity-summary.json'); 1078 const summary = JSON.parse(await fs.readFile(summaryPath, 'utf8')); 1079 1080 // Load detailed activity data if available 1081 const latestActivity = summary.data_files?.latest_activity; 1082 if (latestActivity) { 1083 const detailedPath = path.join(this.dataDir, 'activity', latestActivity); 1084 const detailed = JSON.parse(await fs.readFile(detailedPath, 'utf8')); 1085 return { summary, detailed }; 1086 } 1087 1088 return { summary }; 1089 } catch { 1090 console.warn('⚠️ GitHub data not found'); 1091 return {}; 1092 } 1093 } 1094 1095 async loadCVData() { 1096 try { 1097 const cvPath = path.join(this.dataDir, 'base-cv.json'); 1098 const content = await fs.readFile(cvPath, 'utf8'); 1099 return JSON.parse(content); 1100 } catch { 1101 console.warn('⚠️ Base CV data not found'); 1102 return {}; 1103 } 1104 } 1105 1106 /** 1107 * Utility methods 1108 */ 1109 async ensureCacheDirectory() { 1110 try { 1111 await fs.mkdir(this.cacheDir, { recursive: true }); 1112 } catch { 1113 // Directory already exists or creation failed 1114 } 1115 } 1116 1117 generateEmptyReport() { 1118 return { 1119 overall_confidence: 100, 1120 validation_timestamp: new Date().toISOString(), 1121 message: 'No content to validate (no AI enhancements and no base CV)', 1122 detection_layers: {}, 1123 flagged_content: [], 1124 recommendations: [] 1125 }; 1126 } 1127 1128 async saveDetectionResults() { 1129 const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); 1130 const resultsPath = path.join(this.dataDir, `validation-report-${timestamp}.json`); 1131 1132 await fs.writeFile(resultsPath, JSON.stringify(this.detectionResults, null, 2), 'utf8'); 1133 1134 // Also save as latest report 1135 const latestPath = path.join(this.dataDir, 'latest-validation-report.json'); 1136 await fs.writeFile(latestPath, JSON.stringify(this.detectionResults, null, 2), 'utf8'); 1137 1138 console.log(`💾 Validation report saved: ${resultsPath}`); 1139 } 1140 1141 displayValidationSummary() { 1142 console.log('📋 **VALIDATION SUMMARY**'); 1143 console.log('========================'); 1144 console.log(`🎯 Overall Confidence: ${this.detectionResults.overall_confidence}/100`); 1145 console.log(`🚨 Flagged Items: ${this.detectionResults.flagged_content.length}`); 1146 console.log(`⚠️ Recommendations: ${this.detectionResults.recommendations.length}`); 1147 console.log(`🔥 Urgent Reviews: ${this.detectionResults.urgent_reviews.length}`); 1148 1149 if (this.detectionResults.overall_confidence >= 90) { 1150 console.log('✅ EXCELLENT: Content has high credibility'); 1151 } else if (this.detectionResults.overall_confidence >= 70) { 1152 console.log('⚠️ GOOD: Minor issues detected, review recommended'); 1153 } else { 1154 console.log('🚨 CRITICAL: Significant issues detected, immediate review required'); 1155 } 1156 1157 console.log(''); 1158 } 1159 } 1160 1161 /** 1162 * Main execution function 1163 */ 1164 async function main() { 1165 const detector = new AIHallucinationDetector(); 1166 1167 try { 1168 const results = await detector.detectHallucinations(); 1169 1170 // Exit with error code if critical issues detected 1171 if (results.overall_confidence < 70) { 1172 console.error('🚨 CRITICAL VALIDATION FAILURES DETECTED'); 1173 process.exit(1); 1174 } 1175 1176 console.log('✅ AI Hallucination detection completed successfully'); 1177 process.exit(0); 1178 1179 } catch (error) { 1180 console.error('❌ AI Hallucination detection failed:', error.message); 1181 process.exit(1); 1182 } 1183 } 1184 1185 // Run if called directly 1186 if (require.main === module) { 1187 main(); 1188 } 1189 1190 module.exports = { AIHallucinationDetector };