verify-enhanced-chunking.sh
1 #!/bin/bash 2 3 # Script to verify the Enhanced Adaptive Chunking System 4 # This script will: 5 # 1. Run all tests for the Enhanced Adaptive Chunking System 6 # 2. Verify that all mocks are replaced by real implementations 7 # 3. Ensure all features are included in the binary build 8 # 4. Perform integration testing of all components 9 10 set -e 11 12 echo "Enhanced Adaptive Chunking System Verification" 13 echo "==============================================" 14 15 # Create verification directory 16 VERIFY_DIR="verification-results/$(date +%Y%m%d-%H%M%S)" 17 mkdir -p "$VERIFY_DIR" 18 mkdir -p "$VERIFY_DIR/test-results" 19 mkdir -p "$VERIFY_DIR/build-results" 20 mkdir -p "$VERIFY_DIR/integration-results" 21 22 # Function to run tests for a package 23 run_tests() { 24 local package=$1 25 local output_file="$VERIFY_DIR/test-results/$(basename $package).txt" 26 27 echo "Running tests for $package..." 28 if go test -v "./$package" > "$output_file" 2>&1; then 29 echo " ✓ Tests passed for $package" 30 return 0 31 else 32 echo " ✗ Tests failed for $package" 33 echo " See $output_file for details" 34 return 1 35 fi 36 } 37 38 # Function to check for mock implementations 39 check_mocks() { 40 local dir=$1 41 local output_file="$VERIFY_DIR/test-results/mock-check.txt" 42 43 echo "Checking for mock implementations in $dir..." 44 45 # Find files containing "mock" in their name or content 46 find "$dir" -type f -name "*.go" | xargs grep -l "mock" > "$output_file" 2>/dev/null || true 47 48 # Count the number of files 49 local count=$(wc -l < "$output_file") 50 51 if [ "$count" -eq 0 ]; then 52 echo " ✓ No mock implementations found in $dir" 53 return 0 54 else 55 echo " ⚠ Found $count files with potential mock implementations in $dir" 56 echo " See $output_file for details" 57 return 1 58 fi 59 } 60 61 # Function to build the binary 62 build_binary() { 63 local output_file="$VERIFY_DIR/build-results/build.txt" 64 65 echo "Building binary..." 66 if go build -o "$VERIFY_DIR/build-results/keepsync-cli" ./cmd/keepsync-cli > "$output_file" 2>&1; then 67 echo " ✓ Binary built successfully" 68 return 0 69 else 70 echo " ✗ Binary build failed" 71 echo " See $output_file for details" 72 return 1 73 fi 74 } 75 76 # Function to verify features in binary 77 verify_features() { 78 local binary="$VERIFY_DIR/build-results/keepsync-cli" 79 local output_file="$VERIFY_DIR/build-results/features.txt" 80 81 echo "Verifying features in binary..." 82 83 # List of features to verify 84 local features=( 85 "EnhancedAdaptiveChunker" 86 "DifferentialChunker" 87 "Deduplicator" 88 "AdaptiveCompressor" 89 "EnhancedMetricsCollector" 90 ) 91 92 # Check each feature 93 local failed=0 94 for feature in "${features[@]}"; do 95 echo " Checking for $feature..." 96 if strings "$binary" | grep -q "$feature"; then 97 echo " ✓ $feature found in binary" 98 else 99 echo " ✗ $feature not found in binary" 100 failed=1 101 fi 102 done 103 104 if [ "$failed" -eq 0 ]; then 105 echo " ✓ All features verified in binary" 106 return 0 107 else 108 echo " ✗ Some features not found in binary" 109 return 1 110 fi 111 } 112 113 # Function to run integration tests 114 run_integration_tests() { 115 local output_dir="$VERIFY_DIR/integration-results" 116 117 echo "Running integration tests..." 118 119 # Create test files 120 echo " Creating test files..." 121 mkdir -p "$output_dir/test-files" 122 dd if=/dev/urandom bs=1M count=1 of="$output_dir/test-files/small.dat" 2>/dev/null 123 dd if=/dev/urandom bs=1M count=10 of="$output_dir/test-files/medium.dat" 2>/dev/null 124 125 # Test enhanced adaptive chunking 126 echo " Testing enhanced adaptive chunking..." 127 if go run ./cmd/keepsync-cli/enhanced-chunking-demo/main.go \ 128 --input "$output_dir/test-files/medium.dat" \ 129 --output "$output_dir/chunked.dat" \ 130 > "$output_dir/enhanced-chunking.txt" 2>&1; then 131 echo " ✓ Enhanced adaptive chunking test passed" 132 else 133 echo " ✗ Enhanced adaptive chunking test failed" 134 echo " See $output_dir/enhanced-chunking.txt for details" 135 return 1 136 fi 137 138 # Test differential chunking 139 echo " Testing differential chunking..." 140 # Create modified file 141 cp "$output_dir/test-files/medium.dat" "$output_dir/test-files/medium-modified.dat" 142 dd if=/dev/urandom bs=100K count=1 of="$output_dir/test-files/medium-modified.dat" seek=5 conv=notrunc 2>/dev/null 143 144 if go run ./cmd/keepsync-cli/diff-chunking-tool/diff-chunking-test.go \ 145 --original "$output_dir/test-files/medium.dat" \ 146 --modified "$output_dir/test-files/medium-modified.dat" \ 147 --output "$output_dir/diff-result.dat" \ 148 --stats "$output_dir/diff-stats.json" \ 149 > "$output_dir/differential-chunking.txt" 2>&1; then 150 echo " ✓ Differential chunking test passed" 151 else 152 echo " ✗ Differential chunking test failed" 153 echo " See $output_dir/differential-chunking.txt for details" 154 return 1 155 fi 156 157 # Verify the result 158 if cmp -s "$output_dir/test-files/medium-modified.dat" "$output_dir/diff-result.dat"; then 159 echo " ✓ Differential chunking verification passed" 160 else 161 echo " ✗ Differential chunking verification failed" 162 return 1 163 fi 164 165 # Test deduplication 166 echo " Testing deduplication..." 167 # Create duplicate files 168 cp "$output_dir/test-files/small.dat" "$output_dir/test-files/small-duplicate.dat" 169 170 # Create deduplication test script 171 cat > "$output_dir/deduplication-test.go" << 'EOF' 172 package main 173 174 import ( 175 "context" 176 "flag" 177 "fmt" 178 "os" 179 180 "keepSync/cmd/keepsync-cli/services" 181 ) 182 183 func main() { 184 // Parse command-line flags 185 inputDir := flag.String("input-dir", "", "Input directory containing files to deduplicate") 186 outputDir := flag.String("output-dir", "", "Output directory for deduplicated files") 187 flag.Parse() 188 189 if *inputDir == "" || *outputDir == "" { 190 fmt.Println("Error: input-dir and output-dir flags are required") 191 flag.Usage() 192 os.Exit(1) 193 } 194 195 // Create deduplicator 196 options := services.DefaultDeduplicationOptions() 197 options.ContentStoreDir = *outputDir + "/content-store" 198 deduplicator, err := services.NewDeduplicator(options) 199 if err != nil { 200 fmt.Printf("Error creating deduplicator: %v\n", err) 201 os.Exit(1) 202 } 203 204 // Create context 205 ctx := context.Background() 206 207 // Get list of files 208 files, err := os.ReadDir(*inputDir) 209 if err != nil { 210 fmt.Printf("Error reading directory: %v\n", err) 211 os.Exit(1) 212 } 213 214 // Deduplicate files 215 for _, file := range files { 216 if file.IsDir() { 217 continue 218 } 219 220 filePath := *inputDir + "/" + file.Name() 221 fmt.Printf("Deduplicating %s...\n", filePath) 222 223 // Deduplicate file 224 manifest, err := deduplicator.DeduplicateFile(ctx, filePath) 225 if err != nil { 226 fmt.Printf("Error deduplicating file: %v\n", err) 227 os.Exit(1) 228 } 229 230 // Print deduplication statistics 231 fmt.Printf(" Original size: %d bytes\n", manifest.Size) 232 fmt.Printf(" Total chunks: %d\n", manifest.TotalChunks) 233 fmt.Printf(" New chunks: %d\n", manifest.NewChunks) 234 fmt.Printf(" Existing chunks: %d\n", manifest.ExistingChunks) 235 fmt.Printf(" Deduplication ratio: %.2f%%\n", float64(manifest.ExistingChunks)/float64(manifest.TotalChunks)*100) 236 } 237 } 238 EOF 239 240 # Run deduplication test 241 if go run "$output_dir/deduplication-test.go" \ 242 --input-dir "$output_dir/test-files" \ 243 --output-dir "$output_dir/deduplicated" \ 244 > "$output_dir/deduplication.txt" 2>&1; then 245 echo " ✓ Deduplication test passed" 246 else 247 echo " ✗ Deduplication test failed" 248 echo " See $output_dir/deduplication.txt for details" 249 return 1 250 fi 251 252 # Test adaptive compression 253 echo " Testing adaptive compression..." 254 # Create compression test script 255 cat > "$output_dir/compression-test.go" << 'EOF' 256 package main 257 258 import ( 259 "context" 260 "flag" 261 "fmt" 262 "io/ioutil" 263 "os" 264 265 "keepSync/cmd/keepsync-cli/services" 266 ) 267 268 func main() { 269 // Parse command-line flags 270 inputFile := flag.String("input", "", "Input file to compress") 271 outputFile := flag.String("output", "", "Output file for compressed data") 272 flag.Parse() 273 274 if *inputFile == "" || *outputFile == "" { 275 fmt.Println("Error: input and output flags are required") 276 flag.Usage() 277 os.Exit(1) 278 } 279 280 // Create compressor 281 options := services.DefaultAdaptiveCompressionOptions() 282 compressor := services.NewAdaptiveCompressor(options) 283 284 // Create context 285 ctx := context.Background() 286 287 // Read input file 288 data, err := ioutil.ReadFile(*inputFile) 289 if err != nil { 290 fmt.Printf("Error reading input file: %v\n", err) 291 os.Exit(1) 292 } 293 294 // Compress data 295 compressed, result, err := compressor.Compress(ctx, data) 296 if err != nil { 297 fmt.Printf("Error compressing data: %v\n", err) 298 os.Exit(1) 299 } 300 301 // Write compressed data to output file 302 err = ioutil.WriteFile(*outputFile, compressed, 0644) 303 if err != nil { 304 fmt.Printf("Error writing output file: %v\n", err) 305 os.Exit(1) 306 } 307 308 // Print compression statistics 309 fmt.Printf("Compression Statistics:\n") 310 fmt.Printf(" Original size: %d bytes\n", len(data)) 311 fmt.Printf(" Compressed size: %d bytes\n", len(compressed)) 312 fmt.Printf(" Compression ratio: %.2f%%\n", float64(len(compressed))/float64(len(data))*100) 313 fmt.Printf(" Algorithm: %s\n", result.Algorithm) 314 fmt.Printf(" Level: %s\n", result.Level) 315 fmt.Printf(" Duration: %s\n", result.Duration) 316 } 317 EOF 318 319 # Run compression test 320 if go run "$output_dir/compression-test.go" \ 321 --input "$output_dir/test-files/medium.dat" \ 322 --output "$output_dir/compressed.dat" \ 323 > "$output_dir/compression.txt" 2>&1; then 324 echo " ✓ Adaptive compression test passed" 325 else 326 echo " ✗ Adaptive compression test failed" 327 echo " See $output_dir/compression.txt for details" 328 return 1 329 fi 330 331 # Verify all tests passed 332 echo " ✓ All integration tests passed" 333 return 0 334 } 335 336 # Main verification process 337 echo "Running unit tests..." 338 run_tests "cmd/keepsync-cli/services" || exit 1 339 340 echo "Checking for mock implementations..." 341 check_mocks "cmd/keepsync-cli/services" || echo " ⚠ Some mock implementations found, consider replacing them" 342 343 echo "Building binary..." 344 build_binary || exit 1 345 346 echo "Verifying features in binary..." 347 verify_features || echo " ⚠ Some features not found in binary, consider adding them" 348 349 echo "Running integration tests..." 350 run_integration_tests || exit 1 351 352 echo "Verification completed successfully!" 353 echo "Results are available in $VERIFY_DIR"