benchmark.yml
1 name: Core Benchmarks 2 3 on: 4 push: 5 branches: 6 - master 7 pull_request: 8 types: [ synchronize, labeled ] 9 branches: 10 - master 11 workflow_dispatch: 12 13 jobs: 14 synthesis: 15 strategy: 16 matrix: 17 config: [basic, full] 18 name: Run synthesis benchmarks 19 if: > 20 github.ref == 'refs/heads/master' || 21 ( 22 github.event_name == 'pull_request' && 23 ( 24 (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'benchmark')) || 25 (github.event.action == 'labeled' && github.event.label.name == 'benchmark') 26 ) 27 ) 28 runs-on: ubuntu-24.04 29 timeout-minutes: 45 30 container: ghcr.io/kuznia-rdzeni/amaranth-synth:ecp5-2025.01.31 31 steps: 32 - uses: actions/checkout@v4 33 34 - name: Set ownership (Github Actions workaround) 35 run: | 36 # https://github.com/actions/runner/issues/2033 37 chown -R $(id -u):$(id -g) $PWD 38 39 - name: Set up Python 40 uses: actions/setup-python@v5 41 with: 42 python-version: '3.13' 43 44 - name: Install dependencies 45 run: | 46 python3 -m venv venv 47 . venv/bin/activate 48 python3 -m pip install --upgrade pip 49 python3 -m pip install ".[dev]" 50 51 - name: Synthesize 52 run: | 53 . venv/bin/activate 54 PYTHONHASHSEED=0 ./scripts/synthesize.py --verbose --strip-debug --config ${{ matrix.config }} 55 56 - name: Print synthesis information 57 run: cat ./build/top.tim 58 59 - name: Collect Benchmark information 60 run: | 61 . venv/bin/activate 62 ./scripts/parse_benchmark_info.py -o synth-benchmark-${{ matrix.config }}.json 63 cat ./synth-benchmark-${{ matrix.config }}.json 64 65 - uses: actions/upload-artifact@v4 66 with: 67 name: synth_benchmark_results-${{ matrix.config }} 68 path: synth-benchmark-${{ matrix.config }}.json 69 70 71 build-perf-benchmarks: 72 name: Build performance benchmarks 73 runs-on: ubuntu-latest 74 if: > 75 github.ref == 'refs/heads/master' || 76 ( 77 github.event_name == 'pull_request' && 78 ( 79 (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'benchmark')) || 80 (github.event.action == 'labeled' && github.event.label.name == 'benchmark') 81 ) 82 ) 83 container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2024.12.07 84 steps: 85 - name: Checkout 86 uses: actions/checkout@v4 87 with: 88 submodules: recursive 89 90 - name: Build embench 91 run: cd test/external/embench && make 92 93 - uses: actions/upload-artifact@v4 94 with: 95 name: "embench" 96 path: | 97 test/external/embench/build 98 99 run-perf-benchmarks: 100 name: Run performance benchmarks 101 runs-on: ubuntu-22.04 # older version for compatibility with Docker image 102 timeout-minutes: 40 103 container: ghcr.io/kuznia-rdzeni/verilator:v5.032-2026.04.15 104 needs: build-perf-benchmarks 105 steps: 106 - name: Checkout 107 uses: actions/checkout@v4 108 109 - name: Set ownership (Github Actions workaround) 110 run: | 111 # https://github.com/actions/runner/issues/2033 112 chown -R $(id -u):$(id -g) $PWD 113 114 - name: Set up Python 115 uses: actions/setup-python@v5 116 with: 117 python-version: '3.13' 118 119 - name: Install dependencies 120 run: | 121 python3 -m venv venv 122 . venv/bin/activate 123 python3 -m pip install --upgrade pip 124 python3 -m pip install ".[dev]" 125 126 - name: Generate Verilog 127 run: | 128 . venv/bin/activate 129 PYTHONHASHSEED=0 TRANSACTRON_VERBOSE=1 ./scripts/gen_verilog.py --verbose --config full 130 131 - uses: actions/download-artifact@v4 132 with: 133 name: "embench" 134 path: test/external/embench/build 135 136 - name: Run benchmarks 137 run: | 138 . venv/bin/activate 139 scripts/run_benchmarks.py -o perf_benchmark.json --summary perf.md 140 cat perf.md >> $GITHUB_STEP_SUMMARY 141 142 - uses: actions/upload-artifact@v4 143 with: 144 name: perf_benchmark_results 145 path: perf_benchmark.json 146 147 pr-summary: 148 name: Create benchmark summary for a pull request 149 if: ${{ github.event_name == 'pull_request' }} 150 runs-on: ubuntu-latest 151 timeout-minutes: 5 152 needs: [run-perf-benchmarks, synthesis] 153 steps: 154 - name: Checkout 155 uses: actions/checkout@v4 156 157 - name: Set ownership (Github Actions workaround) 158 run: | 159 # https://github.com/actions/runner/issues/2033 160 chown -R $(id -u):$(id -g) $PWD 161 162 - name: Set up Python 163 uses: actions/setup-python@v5 164 with: 165 python-version: '3.13' 166 167 - name: Install dependencies 168 run: | 169 python3 -m venv venv 170 . venv/bin/activate 171 python3 -m pip install --upgrade pip 172 python3 -m pip install ".[dev]" 173 174 - name: Get baseline benchmarks results 175 uses: actions/cache@v4 176 env: 177 cache-name: benchmark-results 178 with: 179 key: ${{ env.cache-name }}-${{ runner.os }}-${{ github.event.pull_request.base.sha }} 180 path: | 181 perf_benchmark.json 182 synth-benchmark-basic.json 183 synth-benchmark-full.json 184 185 - uses: actions/download-artifact@v4 186 with: 187 name: perf_benchmark_results 188 path: artifacts 189 - uses: actions/download-artifact@v4 190 with: 191 name: synth_benchmark_results-basic 192 path: artifacts 193 - uses: actions/download-artifact@v4 194 with: 195 name: synth_benchmark_results-full 196 path: artifacts 197 198 - name: Save PR number 199 run: | 200 mkdir -p ./summary 201 echo ${{ github.event.number }} > ./summary/pr_number 202 203 - name: Create comment 204 id: createComment 205 run: | 206 . venv/bin/activate 207 { 208 echo "## Benchmarks summary" 209 echo "### Performance benchmarks" 210 ci/print_benchmark_summary.py --precision 3 artifacts/perf_benchmark.json perf_benchmark.json 211 echo 212 echo "You can view all the metrics [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." 213 echo "### Synthesis benchmarks (basic)" 214 ci/print_benchmark_summary.py --precision 0 artifacts/synth-benchmark-basic.json synth-benchmark-basic.json 215 echo 216 echo "### Synthesis benchmarks (full)" 217 ci/print_benchmark_summary.py --precision 0 artifacts/synth-benchmark-full.json synth-benchmark-full.json 218 echo 219 } >> summary/comment.md 220 221 - uses: actions/upload-artifact@v4 222 with: 223 name: benchmark-summary 224 path: summary 225 226 publish-benchmarks: 227 name: Publish benchmarks 228 if: github.ref == 'refs/heads/master' 229 runs-on: ubuntu-latest 230 timeout-minutes: 5 231 needs: [run-perf-benchmarks, synthesis] 232 steps: 233 - uses: actions/checkout@v4 234 235 - uses: actions/download-artifact@v4 236 with: 237 name: perf_benchmark_results 238 path: . 239 - uses: actions/download-artifact@v4 240 with: 241 name: synth_benchmark_results-basic 242 path: . 243 - uses: actions/download-artifact@v4 244 with: 245 name: synth_benchmark_results-full 246 path: . 247 248 - name: Put results in a cache 249 uses: actions/cache@v4 250 env: 251 cache-name: benchmark-results 252 with: 253 key: ${{ env.cache-name }}-${{ runner.os }}-${{ github.sha }} 254 path: | 255 perf_benchmark.json 256 synth-benchmark-basic.json 257 synth-benchmark-full.json 258 259 - name: Store benchmark result (IPC) 260 uses: benchmark-action/github-action-benchmark@v1 261 with: 262 name: Performance (IPC) 263 tool: 'customBiggerIsBetter' 264 output-file-path: './perf_benchmark.json' 265 github-token: ${{ secrets.GITHUB_TOKEN }} 266 auto-push: true 267 benchmark-data-dir-path: "dev/benchmark" 268 269 - name: Store synthesis benchmark result (basic) 270 uses: benchmark-action/github-action-benchmark@v1 271 with: 272 name: Fmax and LCs (basic) 273 tool: 'customBiggerIsBetter' 274 output-file-path: './synth-benchmark-basic.json' 275 github-token: ${{ secrets.GITHUB_TOKEN }} 276 auto-push: true 277 benchmark-data-dir-path: "dev/benchmark" 278 279 - name: Store synthesis benchmark result (full) 280 uses: benchmark-action/github-action-benchmark@v1 281 with: 282 name: Fmax and LCs (full) 283 tool: 'customBiggerIsBetter' 284 output-file-path: './synth-benchmark-full.json' 285 github-token: ${{ secrets.GITHUB_TOKEN }} 286 auto-push: true 287 benchmark-data-dir-path: "dev/benchmark"