test-xml-prompt-integration.js
1 #!/usr/bin/env node 2 3 /** 4 * XML Prompt Integration Test Suite 5 * 6 * Tests the integration of XML-structured prompts with few-shot learning 7 * to validate Issues #96 and #97 implementation quality. 8 * 9 * Test Categories: 10 * - XML prompt construction validation 11 * - Few-shot example integration 12 * - Response validation and quality scoring 13 * - Integration with existing claude-enhancer.js 14 * - Performance and fallback testing 15 * 16 * @author Adrian Wedd 17 * @version 2.1.0 18 */ 19 20 const { AdvancedXMLPromptConstructor } = require('./enhancer-modules/advanced-xml-prompt-constructor'); 21 const { XMLFewShotIntegrator } = require('./enhancer-modules/xml-few-shot-integrator'); 22 23 /** 24 * Test suite for XML prompt engineering integration 25 */ 26 class XMLPromptIntegrationTestSuite { 27 constructor() { 28 this.xmlConstructor = new AdvancedXMLPromptConstructor(); 29 this.xmlIntegrator = new XMLFewShotIntegrator(); 30 this.testResults = { 31 total_tests: 0, 32 passed_tests: 0, 33 failed_tests: 0, 34 test_details: [] 35 }; 36 } 37 38 /** 39 * Run comprehensive test suite 40 */ 41 async runTestSuite() { 42 console.log('🧪 **XML PROMPT INTEGRATION TEST SUITE**'); 43 console.log('Testing Implementation of Issues #96 (Few-Shot Prompting) and #97 (XML Structuring)'); 44 console.log(''); 45 46 try { 47 // Initialize components 48 await this.xmlConstructor.initialize(); 49 await this.xmlIntegrator.initialize(); 50 51 console.log('📋 Running Test Categories:'); 52 console.log(' 1. XML Prompt Construction Validation'); 53 console.log(' 2. Few-Shot Example Integration'); 54 console.log(' 3. Response Validation and Quality Scoring'); 55 console.log(' 4. Integration with Existing System'); 56 console.log(' 5. Performance and Fallback Testing'); 57 console.log(''); 58 59 await this.testXMLPromptConstruction(); 60 await this.testFewShotExampleIntegration(); 61 await this.testResponseValidation(); 62 await this.testSystemIntegration(); 63 await this.testPerformanceAndFallbacks(); 64 65 this.printTestSummary(); 66 return this.testResults; 67 68 } catch (error) { 69 console.error('❌ Test suite execution failed:', error.message); 70 throw error; 71 } 72 } 73 74 /** 75 * Test XML prompt construction functionality 76 */ 77 async testXMLPromptConstruction() { 78 console.log('🔨 Testing XML Prompt Construction...'); 79 80 // Test 1: Professional Summary XML Construction 81 await this.runTest('Professional Summary XML Prompt Construction', async () => { 82 const contextData = { 83 currentContent: "AI Engineer with experience in machine learning.", 84 activityScore: 75, 85 languages: ['Python', 'JavaScript', 'TypeScript'], 86 repositories: 15, 87 specialization: 'AI/ML engineering', 88 evidence: [{ observation: 'Strong GitHub activity', inference: 'Technical competency', confidence: 'high' }] 89 }; 90 91 const result = await this.xmlConstructor.constructXMLPrompt('professional-summary', contextData, 'balanced'); 92 93 this.assert(result.prompt.includes('<prompt_engineering_framework>'), 'XML structure root element present'); 94 this.assert(result.prompt.includes('<few_shot_learning>'), 'Few-shot learning section present'); 95 this.assert(result.prompt.includes('<evidence_base>'), 'Evidence base section present'); 96 this.assert(result.metadata.prompt_type === 'professional-summary', 'Correct prompt type in metadata'); 97 this.assert(result.metadata.creativity_level === 'balanced', 'Correct creativity level in metadata'); 98 }); 99 100 // Test 2: Skills Enhancement XML Construction 101 await this.runTest('Skills Enhancement XML Prompt Construction', async () => { 102 const contextData = { 103 currentContent: ['Python', 'Machine Learning', 'Docker'], 104 activityScore: 65, 105 languages: ['Python', 'Go'], 106 repositories: 8, 107 specialization: 'ML systems' 108 }; 109 110 const result = await this.xmlConstructor.constructXMLPrompt('skills-enhancement', contextData, 'creative'); 111 112 this.assert(result.prompt.includes('<candidate_analysis>'), 'Candidate analysis section present'); 113 this.assert(result.prompt.includes('<enhancement_requirements>'), 'Enhancement requirements section present'); 114 this.assert(result.metadata.creativity_level === 'creative', 'Creative level properly set'); 115 }); 116 117 // Test 3: Experience Enhancement XML Construction 118 await this.runTest('Experience Enhancement XML Prompt Construction', async () => { 119 const contextData = { 120 currentContent: [{ position: 'Software Engineer', company: 'TechCorp', achievements: ['Built ML pipeline'] }], 121 activityScore: 80, 122 languages: ['Python', 'Rust', 'JavaScript'], 123 repositories: 20 124 }; 125 126 const result = await this.xmlConstructor.constructXMLPrompt('experience-enhancement', contextData, 'innovative'); 127 128 this.assert(result.prompt.includes('<current_content>'), 'Current content section present'); 129 this.assert(result.prompt.includes('<response_instructions>'), 'Response instructions present'); 130 this.assert(result.metadata.few_shot_examples_used >= 0, 'Few-shot examples metadata present'); 131 }); 132 133 console.log('✅ XML Prompt Construction tests completed'); 134 } 135 136 /** 137 * Test few-shot example integration 138 */ 139 async testFewShotExampleIntegration() { 140 console.log('📚 Testing Few-Shot Example Integration...'); 141 142 // Test 4: Few-Shot Examples Loading 143 await this.runTest('Few-Shot Examples Loading and Structure', async () => { 144 const stats = this.xmlConstructor.getStats(); 145 146 this.assert(stats.few_shot_examples > 0, 'Few-shot examples loaded'); 147 this.assert(stats.supported_prompt_types.includes('professional-summary'), 'Professional summary examples available'); 148 149 // Test example structure 150 const contextData = { activityScore: 70, languages: ['Python'] }; 151 const result = await this.xmlConstructor.constructXMLPrompt('professional-summary', contextData, 'balanced'); 152 153 this.assert(result.prompt.includes('<example id='), 'Actual examples included in prompt'); 154 this.assert(result.prompt.includes('<input>'), 'Example input structure present'); 155 this.assert(result.prompt.includes('<expected_output>'), 'Example output structure present'); 156 }); 157 158 // Test 5: Creativity Level Example Selection 159 await this.runTest('Creativity Level Example Selection', async () => { 160 const contextData = { activityScore: 65, languages: ['Python', 'JavaScript'] }; 161 162 const conservativeResult = await this.xmlConstructor.constructXMLPrompt('professional-summary', contextData, 'conservative'); 163 const innovativeResult = await this.xmlConstructor.constructXMLPrompt('professional-summary', contextData, 'innovative'); 164 165 // Both should have examples, but potentially different ones 166 this.assert(conservativeResult.prompt.includes('<few_shot_learning>'), 'Conservative examples present'); 167 this.assert(innovativeResult.prompt.includes('<few_shot_learning>'), 'Innovative examples present'); 168 this.assert(conservativeResult.metadata.creativity_level === 'conservative', 'Conservative metadata correct'); 169 this.assert(innovativeResult.metadata.creativity_level === 'innovative', 'Innovative metadata correct'); 170 }); 171 172 console.log('✅ Few-Shot Example Integration tests completed'); 173 } 174 175 /** 176 * Test response validation and quality scoring 177 */ 178 async testResponseValidation() { 179 console.log('🎯 Testing Response Validation and Quality Scoring...'); 180 181 // Test 6: Valid Response Validation 182 await this.runTest('Valid Response Validation', async () => { 183 const validResponse = { 184 enhanced_summary: "Senior AI Engineer with 5+ years developing production machine learning systems, demonstrated through 15+ ML projects and consistent GitHub contributions across Python, JavaScript, and TypeScript ecosystems.", 185 key_differentiators: ["Production ML expertise", "Multi-language proficiency", "Consistent delivery"], 186 technical_positioning: "Senior AI Engineer positioned for technical leadership roles", 187 confidence_score: 0.9 188 }; 189 190 const validation = this.xmlConstructor.validateOutput(validResponse, 'professional-summary'); 191 192 this.assert(validation.valid === true, 'Valid response passes validation'); 193 this.assert(validation.score > 0.7, 'Quality score above threshold'); 194 this.assert(validation.errors.length === 0, 'No validation errors for valid response'); 195 }); 196 197 // Test 7: Invalid Response Handling 198 await this.runTest('Invalid Response Handling', async () => { 199 const invalidResponse = { 200 // Missing required fields 201 enhanced_summary: "Too short", 202 confidence_score: 0.5 203 }; 204 205 const validation = this.xmlConstructor.validateOutput(invalidResponse, 'professional-summary'); 206 207 this.assert(validation.valid === false, 'Invalid response fails validation'); 208 this.assert(validation.errors.length > 0, 'Validation errors detected'); 209 this.assert(validation.score < 0.8, 'Quality score reflects issues'); 210 }); 211 212 // Test 8: Quality Criteria Enforcement 213 await this.runTest('Quality Criteria Enforcement', async () => { 214 const responseWithGenericTerms = { 215 enhanced_summary: "Cutting-edge AI Engineer with seamlessly integrated innovative solutions", 216 key_differentiators: ["Generic expertise", "Synergistic approaches"], 217 technical_positioning: "Disruptive technology leader", 218 confidence_score: 0.8 219 }; 220 221 const validation = this.xmlConstructor.validateOutput(responseWithGenericTerms, 'professional-summary'); 222 223 this.assert(validation.warnings.length > 0, 'Generic terms trigger warnings'); 224 this.assert(validation.score < 0.9, 'Quality score penalized for generic language'); 225 }); 226 227 console.log('✅ Response Validation tests completed'); 228 } 229 230 /** 231 * Test integration with existing system 232 */ 233 async testSystemIntegration() { 234 console.log('🔗 Testing System Integration...'); 235 236 // Test 9: XML Integrator Initialization 237 await this.runTest('XML Integrator Initialization', async () => { 238 const stats = this.xmlIntegrator.getStats(); 239 240 this.assert(stats.initialized === true, 'XML integrator properly initialized'); 241 this.assert(stats.xml_constructor_stats.initialized === true, 'XML constructor initialized'); 242 this.assert(stats.enhancement_types_supported.length > 0, 'Enhancement types supported'); 243 }); 244 245 // Test 10: Professional Summary Integration 246 await this.runTest('Professional Summary Integration', async () => { 247 const cvData = { 248 professional_summary: "AI Engineer with machine learning experience." 249 }; 250 const activityMetrics = { 251 total_repos: 10, 252 top_languages: ['Python', 'JavaScript'], 253 total_stars: 25 254 }; 255 256 const result = await this.xmlIntegrator.enhanceProfessionalSummaryXML(cvData, activityMetrics, 'balanced'); 257 258 this.assert(result.xmlPrompt.length > 0, 'XML prompt generated'); 259 this.assert(result.enhancementType === 'xml-few-shot', 'Correct enhancement type'); 260 this.assert(result.quality_expected > 0.7, 'Quality expectation set'); 261 this.assert(result.contextData.activityScore >= 0, 'Activity score calculated'); 262 }); 263 264 console.log('✅ System Integration tests completed'); 265 } 266 267 /** 268 * Test performance and fallback mechanisms 269 */ 270 async testPerformanceAndFallbacks() { 271 console.log('⚡ Testing Performance and Fallback Mechanisms...'); 272 273 // Test 11: Performance Metrics Tracking 274 await this.runTest('Performance Metrics Tracking', async () => { 275 // Reset metrics 276 this.xmlIntegrator.resetMetrics(); 277 278 // Generate some prompts 279 const cvData = { professional_summary: "Test summary" }; 280 const activityMetrics = { total_repos: 5, top_languages: ['Python'] }; 281 282 await this.xmlIntegrator.enhanceProfessionalSummaryXML(cvData, activityMetrics, 'balanced'); 283 await this.xmlIntegrator.enhanceSkillsSectionXML(cvData, activityMetrics, 'creative'); 284 285 const metrics = this.xmlIntegrator.getPerformanceMetrics(); 286 287 this.assert(metrics.prompts_constructed >= 2, 'Prompts constructed metric tracked'); 288 this.assert(metrics.success_rate >= 0, 'Success rate calculated'); 289 this.assert(typeof metrics.quality_improvement_rate === 'number', 'Quality improvement rate tracked'); 290 }); 291 292 // Test 12: Fallback Mechanism 293 await this.runTest('Fallback Mechanism Functionality', async () => { 294 // Create a scenario that should trigger fallback 295 const result = this.xmlIntegrator.createFallbackPrompt('professional-summary', {}, {}, 'balanced'); 296 297 this.assert(result.enhancementType === 'xml-fallback', 'Fallback type correctly set'); 298 this.assert(result.metadata.fallback_used === true, 'Fallback flag set in metadata'); 299 this.assert(result.xmlPrompt.includes('<enhancement_task>'), 'Basic XML structure in fallback'); 300 this.assert(result.quality_expected < 0.8, 'Lower quality expectation for fallback'); 301 }); 302 303 // Test 13: Memory and Resource Management 304 await this.runTest('Memory and Resource Management', async () => { 305 const initialStats = this.xmlConstructor.getStats(); 306 307 // Generate multiple prompts to test resource management 308 for (let i = 0; i < 5; i++) { 309 const contextData = { 310 currentContent: `Test content ${i}`, 311 activityScore: 60 + i * 5, 312 languages: ['Python', 'JavaScript'] 313 }; 314 await this.xmlConstructor.constructXMLPrompt('professional-summary', contextData, 'balanced'); 315 } 316 317 const finalStats = this.xmlConstructor.getStats(); 318 319 // Resource usage should be reasonable 320 this.assert(finalStats.initialized === true, 'Constructor remains initialized'); 321 this.assert(finalStats.few_shot_examples === initialStats.few_shot_examples, 'Examples count stable'); 322 }); 323 324 console.log('✅ Performance and Fallback tests completed'); 325 } 326 327 /** 328 * Run individual test with error handling 329 */ 330 async runTest(testName, testFunction) { 331 this.testResults.total_tests++; 332 333 try { 334 await testFunction(); 335 this.testResults.passed_tests++; 336 this.testResults.test_details.push({ name: testName, status: 'PASSED', error: null }); 337 console.log(` ✅ ${testName}`); 338 } catch (error) { 339 this.testResults.failed_tests++; 340 this.testResults.test_details.push({ name: testName, status: 'FAILED', error: error.message }); 341 console.log(` ❌ ${testName}: ${error.message}`); 342 } 343 } 344 345 /** 346 * Assert condition with descriptive error 347 */ 348 assert(condition, message) { 349 if (!condition) { 350 throw new Error(`Assertion failed: ${message}`); 351 } 352 } 353 354 /** 355 * Print comprehensive test summary 356 */ 357 printTestSummary() { 358 console.log(''); 359 console.log('🎯 **TEST SUITE SUMMARY**'); 360 console.log(`Total Tests: ${this.testResults.total_tests}`); 361 console.log(`Passed: ${this.testResults.passed_tests} ✅`); 362 console.log(`Failed: ${this.testResults.failed_tests} ❌`); 363 console.log(`Success Rate: ${((this.testResults.passed_tests / this.testResults.total_tests) * 100).toFixed(1)}%`); 364 console.log(''); 365 366 if (this.testResults.failed_tests > 0) { 367 console.log('❌ **FAILED TESTS:**'); 368 this.testResults.test_details 369 .filter(test => test.status === 'FAILED') 370 .forEach(test => { 371 console.log(` - ${test.name}: ${test.error}`); 372 }); 373 console.log(''); 374 } 375 376 console.log('📊 **IMPLEMENTATION STATUS:**'); 377 console.log(`Issue #97 (XML Tag Structuring): ${this.testResults.passed_tests >= 3 ? '✅ IMPLEMENTED' : '❌ NEEDS WORK'}`); 378 console.log(`Issue #96 (Few-Shot Prompting): ${this.testResults.passed_tests >= 5 ? '✅ IMPLEMENTED' : '❌ NEEDS WORK'}`); 379 console.log(`Integration Quality: ${this.testResults.passed_tests >= 10 ? '✅ HIGH' : this.testResults.passed_tests >= 7 ? '⚠️ MEDIUM' : '❌ LOW'}`); 380 console.log(''); 381 382 if (this.testResults.passed_tests === this.testResults.total_tests) { 383 console.log('🎉 **ALL TESTS PASSED!** XML prompt engineering integration is ready for production use.'); 384 } else { 385 console.log(`⚠️ ${this.testResults.failed_tests} test(s) failed. Review implementation before deployment.`); 386 } 387 } 388 } 389 390 /** 391 * Main execution function 392 */ 393 async function main() { 394 try { 395 const testSuite = new XMLPromptIntegrationTestSuite(); 396 const results = await testSuite.runTestSuite(); 397 398 // Exit with appropriate code 399 process.exit(results.failed_tests > 0 ? 1 : 0); 400 401 } catch (error) { 402 console.error('❌ Test suite execution failed:', error.message); 403 process.exit(1); 404 } 405 } 406 407 // Execute if called directly 408 if (require.main === module) { 409 main().catch(console.error); 410 } 411 412 module.exports = { XMLPromptIntegrationTestSuite };