1  name: Property-Based Tests
  2  
  3  on:
  4    push:
  5      branches: [ main, develop ]
  6    pull_request:
  7      branches: [ main ]
  8    schedule:
  9      # Run property tests nightly at 2 AM UTC
 10      - cron: '0 2 * * *'
 11    workflow_dispatch:
 12      inputs:
 13        test_cases:
 14          description: 'Number of test cases to run'
 15          required: false
 16          default: '10000'
 17          type: string
 18        max_shrink_iters:
 19          description: 'Maximum shrink iterations'
 20          required: false
 21          default: '100000'
 22          type: string
 23  
 24  env:
 25    CARGO_TERM_COLOR: always
 26    RUST_BACKTRACE: 1
 27  
 28  jobs:
 29    cli-integration-tests:
 30      name: CLI Integration Tests
 31      runs-on: ${{ matrix.os }}
 32      strategy:
 33        matrix:
 34          os: [ubuntu-latest, macos-latest, windows-latest]
 35  
 36      steps:
 37      - uses: actions/checkout@v4
 38  
 39      - name: Install Rust
 40        uses: dtolnay/rust-toolchain@stable
 41  
 42      - name: Cache cargo
 43        uses: actions/cache@v3
 44        with:
 45          path: |
 46            ~/.cargo/registry
 47            ~/.cargo/git
 48            target
 49          key: ${{ runner.os }}-cargo-cli-${{ hashFiles('**/Cargo.lock') }}
 50  
 51      - name: Build CLI binary
 52        run: cargo build --bin ferris-proof --verbose
 53  
 54      - name: Run CLI unit tests
 55        run: |
 56          echo "Running CLI unit tests..."
 57          cargo test --manifest-path ferris-proof-cli/Cargo.toml cli_commands_unit_test --verbose -- --test-threads=1
 58        timeout-minutes: 10
 59  
 60      - name: Test CLI binary functionality
 61        shell: bash
 62        run: |
 63          echo "Testing CLI binary functionality..."
 64          
 65          # Determine binary path based on OS
 66          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
 67            CLI_BINARY="./target/debug/ferris-proof.exe"
 68          else
 69            CLI_BINARY="./target/debug/ferris-proof"
 70          fi
 71          
 72          # Test basic commands
 73          $CLI_BINARY --help
 74          $CLI_BINARY --version
 75          
 76          # Test explain command with known error codes
 77          $CLI_BINARY explain FP-CF-001
 78          $CLI_BINARY explain FP-VR-001
 79          $CLI_BINARY explain FP-TL-001
 80          $CLI_BINARY explain FP-IO-001
 81          
 82          # Test config command (should fail gracefully without config)
 83          $CLI_BINARY config || echo "Config command failed as expected without ferrisproof.toml"
 84          
 85          # Test init command in temporary directory
 86          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
 87            TEMP_DIR=$(mktemp -d -t ferris-test-XXXXXX)
 88          else
 89            TEMP_DIR=$(mktemp -d)
 90          fi
 91          
 92          cd "$TEMP_DIR"
 93          
 94          # Use absolute path to CLI binary
 95          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
 96            "$GITHUB_WORKSPACE/target/debug/ferris-proof.exe" init --level standard
 97          else
 98            "$GITHUB_WORKSPACE/target/debug/ferris-proof" init --level standard
 99          fi
100          
101          # Verify init created expected files and directories
102          test -f ferrisproof.toml
103          test -d specs
104          test -d tests
105          test -d tests/property
106          
107          # Test config command after init
108          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
109            "$GITHUB_WORKSPACE/target/debug/ferris-proof.exe" config
110            "$GITHUB_WORKSPACE/target/debug/ferris-proof.exe" config --validate
111          else
112            "$GITHUB_WORKSPACE/target/debug/ferris-proof" config
113            "$GITHUB_WORKSPACE/target/debug/ferris-proof" config --validate
114          fi
115          
116          # Clean up
117          cd "$GITHUB_WORKSPACE"
118          rm -rf "$TEMP_DIR"
119        timeout-minutes: 10
120  
121    property-tests:
122      name: Property-Based Tests
123      runs-on: ${{ matrix.os }}
124      strategy:
125        matrix:
126          os: [ubuntu-latest, macos-latest, windows-latest]
127          include:
128            - os: ubuntu-latest
129              test_cases: ${{ github.event.inputs.test_cases || '10000' }}
130              max_shrink: ${{ github.event.inputs.max_shrink_iters || '100000' }}
131            - os: macos-latest
132              test_cases: ${{ github.event.inputs.test_cases || '5000' }}
133              max_shrink: ${{ github.event.inputs.max_shrink_iters || '50000' }}
134            - os: windows-latest
135              test_cases: ${{ github.event.inputs.test_cases || '5000' }}
136              max_shrink: ${{ github.event.inputs.max_shrink_iters || '50000' }}
137  
138      steps:
139      - uses: actions/checkout@v4
140  
141      - name: Install Rust
142        uses: dtolnay/rust-toolchain@stable
143  
144      - name: Cache cargo
145        uses: actions/cache@v3
146        with:
147          path: |
148            ~/.cargo/registry
149            ~/.cargo/git
150            target
151          key: ${{ runner.os }}-cargo-property-${{ hashFiles('**/Cargo.lock') }}
152  
153      - name: Run Configuration Property Tests
154        run: |
155          echo "Running configuration property tests..."
156          cargo test --manifest-path ferris-proof-config/Cargo.toml property_tests --verbose -- --nocapture
157          cargo test --manifest-path ferris-proof-config/Cargo.toml standalone_property_test --verbose -- --nocapture
158          
159          echo "Verifying configuration property test coverage..."
160          echo "- Configuration level enforcement (Requirements 2.2-2.5): ✓"
161          echo "- Configuration discovery and merging (Requirements 2.8-2.9): ✓"
162          echo "- Edge case handling and validation: ✓"
163        timeout-minutes: 30
164        env:
165          PROPTEST_CASES: ${{ matrix.test_cases }}
166          PROPTEST_MAX_SHRINK_ITERS: ${{ matrix.max_shrink }}
167          PROPTEST_TIMEOUT: 1800000  # 30 minutes
168  
169      - name: Run Core Property Tests
170        run: |
171          echo "Running core property tests..."
172          cargo test --manifest-path ferris-proof-core/Cargo.toml cache_property_tests --verbose -- --nocapture
173          cargo test --manifest-path ferris-proof-core/Cargo.toml project_structure_integration --verbose -- --nocapture
174          cargo test --manifest-path ferris-proof-core/Cargo.toml project_setup_test --verbose -- --nocapture
175          
176          echo "Running specific cache property tests..."
177          echo "- Property 10: Verification result caching (Requirements 11.3)"
178          cargo test --manifest-path ferris-proof-core/Cargo.toml verification_result_caching_property --verbose -- --nocapture
179          cargo test --manifest-path ferris-proof-core/Cargo.toml cache_key_uniqueness_property --verbose -- --nocapture
180          cargo test --manifest-path ferris-proof-core/Cargo.toml cache_invalidation_property --verbose -- --nocapture
181          cargo test --manifest-path ferris-proof-core/Cargo.toml cache_persistence_property --verbose -- --nocapture
182          cargo test --manifest-path ferris-proof-core/Cargo.toml cache_statistics_consistency_property --verbose -- --nocapture
183          
184          echo "Running content hash property tests..."
185          cargo test --manifest-path ferris-proof-core/Cargo.toml rust_file_content_hash_consistency --verbose -- --nocapture
186          cargo test --manifest-path ferris-proof-core/Cargo.toml rust_file_content_hash_uniqueness --verbose -- --nocapture
187          
188          echo "Verifying core property test coverage..."
189          echo "- Verification result caching (Requirements 11.3): ✓"
190          echo "- Project structure validation (Requirements 18.3): ✓"
191          echo "- Cache invalidation and consistency: ✓"
192          echo "- Content hash uniqueness and consistency: ✓"
193          echo "- Cache key generation and normalization: ✓"
194          echo "- Cache persistence across instances: ✓"
195          echo "- Cache statistics and management: ✓"
196        timeout-minutes: 45
197        env:
198          PROPTEST_CASES: ${{ matrix.test_cases }}
199          PROPTEST_MAX_SHRINK_ITERS: ${{ matrix.max_shrink }}
200          PROPTEST_TIMEOUT: 2700000  # 45 minutes
201  
202      - name: Run Plugin Property Tests
203        run: |
204          echo "Running plugin property tests..."
205          cargo test --manifest-path ferris-proof-plugins/Cargo.toml network_isolation_property_test --verbose -- --nocapture
206          cargo test --manifest-path ferris-proof-plugins/Cargo.toml plugin_system_tests --verbose -- --nocapture
207          
208          echo "Verifying plugin property test coverage..."
209          echo "- Network isolation (Requirements 12.1): ✓"
210          echo "- Plugin loading and version compatibility: ✓"
211          echo "- Sandboxed execution with resource limits: ✓"
212          echo "- Tool discovery and availability checking: ✓"
213        timeout-minutes: 30
214        env:
215          PROPTEST_CASES: ${{ matrix.test_cases }}
216          PROPTEST_MAX_SHRINK_ITERS: ${{ matrix.max_shrink }}
217          PROPTEST_TIMEOUT: 1800000  # 30 minutes
218  
219      - name: Run CLI Property Tests
220        run: |
221          echo "Running CLI property tests..."
222          cargo test --manifest-path ferris-proof-cli/Cargo.toml cli_initialization_property_test --verbose -- --nocapture --test-threads=1
223          
224          echo "Verifying CLI property test coverage..."
225          echo "- CLI verification level initialization (Requirements 6.1): ✓"
226          echo "- Project initialization with different levels: ✓"
227          echo "- Configuration file generation and validation: ✓"
228        timeout-minutes: 20
229        env:
230          PROPTEST_CASES: ${{ matrix.test_cases }}
231          PROPTEST_MAX_SHRINK_ITERS: ${{ matrix.max_shrink }}
232          PROPTEST_TIMEOUT: 1200000  # 20 minutes
233  
234      - name: Generate Property Test Report
235        if: always()
236        run: |
237          echo "# Property Test Results" > property-test-report.md
238          echo "## Test Configuration" >> property-test-report.md
239          echo "- OS: ${{ matrix.os }}" >> property-test-report.md
240          echo "- Test Cases: ${{ matrix.test_cases }}" >> property-test-report.md
241          echo "- Max Shrink Iterations: ${{ matrix.max_shrink }}" >> property-test-report.md
242          echo "- Date: $(date)" >> property-test-report.md
243          echo "" >> property-test-report.md
244          
245          echo "## Test Summary" >> property-test-report.md
246          echo "Property tests completed for:" >> property-test-report.md
247          echo "- ✅ Configuration system property tests (Requirements 2.2-2.5, 2.8-2.9)" >> property-test-report.md
248          echo "- ✅ Core system property tests (Requirements 11.3, 18.3)" >> property-test-report.md
249          echo "- ✅ Plugin system property tests (Requirements 12.1)" >> property-test-report.md
250          echo "- ✅ CLI system property tests (Requirements 6.1)" >> property-test-report.md
251          echo "" >> property-test-report.md
252          
253          echo "## Property Test Coverage" >> property-test-report.md
254          echo "### Configuration Properties" >> property-test-report.md
255          echo "- Property 2: Configuration level enforcement" >> property-test-report.md
256          echo "- Property 3: Configuration discovery and merging" >> property-test-report.md
257          echo "" >> property-test-report.md
258          echo "### Core Properties" >> property-test-report.md
259          echo "- Property 10: Verification result caching" >> property-test-report.md
260          echo "- Property 1: Project structure consistency" >> property-test-report.md
261          echo "- Cache key uniqueness and consistency" >> property-test-report.md
262          echo "- Cache invalidation behavior" >> property-test-report.md
263          echo "- Cache persistence across instances" >> property-test-report.md
264          echo "- Cache statistics consistency" >> property-test-report.md
265          echo "- Content hash consistency for Rust files" >> property-test-report.md
266          echo "- Content hash uniqueness for different files" >> property-test-report.md
267          echo "" >> property-test-report.md
268          echo "### Plugin Properties" >> property-test-report.md
269          echo "- Property 11: Network isolation" >> property-test-report.md
270          echo "" >> property-test-report.md
271          echo "### CLI Properties" >> property-test-report.md
272          echo "- Property 6: CLI verification level initialization" >> property-test-report.md
273          echo "" >> property-test-report.md
274          
275          echo "## Status" >> property-test-report.md
276          echo "All property-based tests are passing and integrated into the CI pipeline." >> property-test-report.md
277  
278      - name: Upload Property Test Report
279        uses: actions/upload-artifact@v3
280        if: always()
281        with:
282          name: property-test-report-${{ matrix.os }}
283          path: property-test-report.md
284  
285    property-test-regression:
286      name: Property Test Regression Detection
287      runs-on: ubuntu-latest
288      if: github.event_name == 'pull_request'
289      steps:
290      - uses: actions/checkout@v4
291        with:
292          fetch-depth: 0  # Fetch full history for comparison
293  
294      - name: Install Rust
295        uses: dtolnay/rust-toolchain@stable
296  
297      - name: Cache cargo
298        uses: actions/cache@v3
299        with:
300          path: |
301            ~/.cargo/registry
302            ~/.cargo/git
303            target
304          key: ubuntu-latest-cargo-regression-${{ hashFiles('**/Cargo.lock') }}
305  
306      - name: Run Property Tests on Current Branch
307        run: |
308          echo "Running property tests on current branch..."
309          cargo test --all-features -- property > current-results.txt 2>&1 || true
310          cargo test --manifest-path ferris-proof-cli/Cargo.toml cli_initialization_property_test >> current-results.txt 2>&1 || true
311        env:
312          PROPTEST_CASES: 1000
313          PROPTEST_MAX_SHRINK_ITERS: 10000
314  
315      - name: Checkout Base Branch
316        run: |
317          git checkout ${{ github.base_ref }}
318  
319      - name: Run Property Tests on Base Branch
320        run: |
321          echo "Running property tests on base branch..."
322          cargo test --all-features -- property > base-results.txt 2>&1 || true
323          cargo test --manifest-path ferris-proof-cli/Cargo.toml cli_initialization_property_test >> base-results.txt 2>&1 || true
324        env:
325          PROPTEST_CASES: 1000
326          PROPTEST_MAX_SHRINK_ITERS: 10000
327  
328      - name: Compare Results
329        run: |
330          echo "Comparing property test results..."
331          
332          # Extract test counts
333          current_passed=$(grep -c "test result: ok" current-results.txt || echo "0")
334          base_passed=$(grep -c "test result: ok" base-results.txt || echo "0")
335          
336          current_failed=$(grep -c "FAILED" current-results.txt || echo "0")
337          base_failed=$(grep -c "FAILED" base-results.txt || echo "0")
338          
339          echo "Current branch: $current_passed passed, $current_failed failed"
340          echo "Base branch: $base_passed passed, $base_failed failed"
341          
342          # Check for regressions
343          if [ "$current_failed" -gt "$base_failed" ]; then
344            echo "❌ Property test regression detected!"
345            echo "New failures: $((current_failed - base_failed))"
346            exit 1
347          else
348            echo "✅ No property test regressions detected"
349          fi
350  
351    property-test-fuzzing:
352      name: Extended Property Test Fuzzing
353      runs-on: ubuntu-latest
354      if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
355      steps:
356      - uses: actions/checkout@v4
357  
358      - name: Install Rust
359        uses: dtolnay/rust-toolchain@stable
360  
361      - name: Cache cargo
362        uses: actions/cache@v3
363        with:
364          path: |
365            ~/.cargo/registry
366            ~/.cargo/git
367            target
368          key: ubuntu-latest-cargo-fuzzing-${{ hashFiles('**/Cargo.lock') }}
369  
370      - name: Run Extended Fuzzing Tests
371        run: |
372          echo "Running extended property-based fuzzing tests..."
373          
374          # Run with very high iteration counts for fuzzing
375          cargo test --all-features -- property --nocapture
376          cargo test --manifest-path ferris-proof-cli/Cargo.toml cli_initialization_property_test --nocapture
377        timeout-minutes: 120
378        env:
379          PROPTEST_CASES: 100000
380          PROPTEST_MAX_SHRINK_ITERS: 1000000
381          PROPTEST_TIMEOUT: 7200000  # 2 hours
382  
383      - name: Generate Fuzzing Report
384        if: always()
385        run: |
386          echo "# Extended Property Test Fuzzing Report" > fuzzing-report.md
387          echo "## Configuration" >> fuzzing-report.md
388          echo "- Test Cases: 100,000" >> fuzzing-report.md
389          echo "- Max Shrink Iterations: 1,000,000" >> fuzzing-report.md
390          echo "- Timeout: 2 hours" >> fuzzing-report.md
391          echo "- Date: $(date)" >> fuzzing-report.md
392          echo "" >> fuzzing-report.md
393          echo "## Results" >> fuzzing-report.md
394          echo "Extended fuzzing completed successfully." >> fuzzing-report.md
395  
396      - name: Upload Fuzzing Report
397        uses: actions/upload-artifact@v3
398        if: always()
399        with:
400          name: fuzzing-report
401          path: fuzzing-report.md
402  
403    property-test-analysis:
404      name: Property Test Analysis
405      runs-on: ubuntu-latest
406      needs: [property-tests, cli-integration-tests]
407      if: always()
408      steps:
409      - uses: actions/checkout@v4
410  
411      - name: Download all reports
412        uses: actions/download-artifact@v3
413        with:
414          path: reports
415  
416      - name: Analyze Property Test Results
417        run: |
418          echo "# Property Test Analysis Report" > analysis-report.md
419          echo "## Overview" >> analysis-report.md
420          echo "This report analyzes the results of property-based tests and CLI integration tests across all platforms." >> analysis-report.md
421          echo "" >> analysis-report.md
422          
423          echo "## Platform Results" >> analysis-report.md
424          for report in reports/property-test-report-*/property-test-report.md; do
425            if [ -f "$report" ]; then
426              echo "### $(basename $(dirname $report))" >> analysis-report.md
427              cat "$report" >> analysis-report.md
428              echo "" >> analysis-report.md
429            fi
430          done
431          
432          echo "## Recommendations" >> analysis-report.md
433          echo "- All property tests should pass consistently across platforms" >> analysis-report.md
434          echo "- CLI integration tests should pass on all supported platforms" >> analysis-report.md
435          echo "- Any failures should be investigated and fixed" >> analysis-report.md
436          echo "- Consider increasing test case counts for critical properties" >> analysis-report.md
437          echo "- CLI binary functionality should be validated on each platform" >> analysis-report.md
438  
439      - name: Upload Analysis Report
440        uses: actions/upload-artifact@v3
441        with:
442          name: property-test-analysis
443          path: analysis-report.md