Nightly Benchmark #52
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Benchmark | |
| # Runs the -Pbenchmark profile nightly and records median-of-3 results | |
| # into benchmarks/history.jsonl. Flags regressions >30% vs the rolling | |
| # 7-run median via workflow annotations — observation only, does NOT | |
| # block anything. The release workflow gate (release.yml) is the | |
| # enforcement layer; this is the trend-tracking layer. | |
| # | |
| # Median-of-3 damps GH runner noise (see benchmarks/README.md for the | |
| # noise-handling rationale). | |
| on: | |
| schedule: | |
| - cron: '0 7 * * *' # 07:00 UTC daily (after property + soak nightlies) | |
| workflow_dispatch: {} # manual trigger for on-demand runs | |
| # Default to read-all at top level; the benchmark job escalates contents:write | |
| # only because it commits history.jsonl back to the benchmark-data branch. | |
| permissions: read-all | |
| jobs: | |
| benchmark: | |
| name: Benchmark + trend check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| permissions: | |
| contents: write # needed to commit history.jsonl back to the repo | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| cache: maven | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: '3.12' | |
| # Release-workflow memory gotcha: benchmarks need a full `mvn install` | |
| # into the local repo first, or the reactor can't resolve the new | |
| # version and surefire silently uses stale XML from prior runs. We | |
| # also delete any pre-existing surefire output so the mtime check | |
| # (post-run) can fail-fast if parsing picks up stale data. | |
| - name: Install reactor to local Maven repo | |
| run: | | |
| mvn -B install -DskipTests --file cycles-protocol-service/pom.xml | |
| rm -rf cycles-protocol-service/cycles-protocol-service-api/target/surefire-reports | |
| - name: Benchmark — trial 1 | |
| run: > | |
| mvn -B test -Pbenchmark | |
| --file cycles-protocol-service/pom.xml | |
| -pl cycles-protocol-service-api | |
| - name: Parse trial 1 | |
| run: | | |
| python3 scripts/parse-benchmarks.py \ | |
| cycles-protocol-service/cycles-protocol-service-api/target/surefire-reports \ | |
| --trial-of 1 > /tmp/trial1.json | |
| cat /tmp/trial1.json | |
| rm -rf cycles-protocol-service/cycles-protocol-service-api/target/surefire-reports | |
| - name: Benchmark — trial 2 | |
| run: > | |
| mvn -B test -Pbenchmark | |
| --file cycles-protocol-service/pom.xml | |
| -pl cycles-protocol-service-api | |
| - name: Parse trial 2 | |
| run: | | |
| python3 scripts/parse-benchmarks.py \ | |
| cycles-protocol-service/cycles-protocol-service-api/target/surefire-reports \ | |
| --trial-of 2 > /tmp/trial2.json | |
| cat /tmp/trial2.json | |
| rm -rf cycles-protocol-service/cycles-protocol-service-api/target/surefire-reports | |
| - name: Benchmark — trial 3 | |
| run: > | |
| mvn -B test -Pbenchmark | |
| --file cycles-protocol-service/pom.xml | |
| -pl cycles-protocol-service-api | |
| - name: Parse trial 3 | |
| run: | | |
| python3 scripts/parse-benchmarks.py \ | |
| cycles-protocol-service/cycles-protocol-service-api/target/surefire-reports \ | |
| --trial-of 3 > /tmp/trial3.json | |
| cat /tmp/trial3.json | |
| - name: Median-aggregate 3 trials | |
| run: | | |
| python3 scripts/median-benchmarks.py /tmp/trial1.json /tmp/trial2.json /tmp/trial3.json \ | |
| > /tmp/nightly.json | |
| echo '--- nightly record ---' | |
| cat /tmp/nightly.json | |
| # History + baseline live on the `benchmark-data` branch, not main. | |
| # Branch protection on main rejects bot pushes; moving telemetry | |
| # data to a dedicated non-protected branch avoids the need for | |
| # bypass configuration or PAT secrets, while keeping the data | |
| # git-tracked and readable via `git fetch origin benchmark-data`. | |
| - name: Fetch benchmark-data branch (worktree) | |
| run: | | |
| git fetch origin benchmark-data:benchmark-data | |
| git worktree add bench-data benchmark-data | |
| - name: Trend check | |
| id: trend | |
| run: | | |
| # Compare this run against the rolling median from the data | |
| # branch (not our current checkout, where the files no longer | |
| # live). Do this BEFORE appending so the current run doesn't | |
| # become part of its own baseline window. | |
| python3 scripts/check-regression.py trend \ | |
| --current /tmp/nightly.json \ | |
| --history bench-data/benchmarks/history.jsonl \ | |
| --window 7 \ | |
| --threshold 0.30 \ | |
| | tee /tmp/trend-summary.md | |
| - name: Attach summary to job | |
| if: always() | |
| run: | | |
| if [ -f /tmp/trend-summary.md ]; then | |
| cat /tmp/trend-summary.md >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Commit history to benchmark-data branch | |
| # Skip on workflow_dispatch re-runs to avoid clutter; only commit | |
| # scheduled nightlies. | |
| if: github.event_name == 'schedule' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| cat /tmp/nightly.json >> bench-data/benchmarks/history.jsonl | |
| echo "" >> bench-data/benchmarks/history.jsonl | |
| cd bench-data | |
| git add benchmarks/history.jsonl | |
| if git diff --cached --quiet; then | |
| echo "no history changes to commit" | |
| exit 0 | |
| fi | |
| git commit -m "chore(bench): nightly benchmark $(date -u +%Y-%m-%d) | |
| [skip ci]" | |
| git push origin benchmark-data |