benchmark #32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: benchmark | |
| # Nightly ASV benchmark trend (advisory; never blocks PRs). Results accumulate | |
| # across runs via the cache so step-detection sees the whole history; a detected | |
| # wall-time regression opens/updates a tracking issue. Deterministic call-count | |
| # regressions are gated separately by test/integration/test_perf_counters.py (blocking, PRs). | |
| on: | |
| schedule: | |
| - cron: "0 7 * * *" # nightly, 07:00 UTC | |
| workflow_dispatch: {} # manual trigger for verification | |
| permissions: | |
| issues: write | |
| jobs: | |
| asv: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # asv needs history to benchmark commits | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| - name: Set up Python 3.13 | |
| run: uv python install 3.13 | |
| - name: Install dependencies | |
| run: uv sync --python 3.13 --group dev --all-extras | |
| - name: Restore accumulated ASV results | |
| uses: actions/cache@v4 | |
| with: | |
| path: benchmarks/results | |
| key: asv-results-${{ github.run_id }} | |
| restore-keys: asv-results- | |
| - name: Configure ASV machine | |
| working-directory: benchmarks | |
| run: uv run asv machine --yes | |
| - name: Run benchmarks for new commits | |
| working-directory: benchmarks | |
| run: uv run asv run NEW --skip-existing --show-stderr || true | |
| - name: Detect regression (HEAD vs previous commit, same runner) | |
| id: detect | |
| working-directory: benchmarks | |
| run: | | |
| # `asv continuous` exits non-zero for BOTH a real regression and a | |
| # benchmark build/run failure, so the exit code alone can't tell them | |
| # apart. Parse the sentinel line, which asv prints only after a | |
| # comparison actually completes. | |
| out=$(uv run asv continuous --factor 1.5 HEAD~1 HEAD --show-stderr 2>&1) || true | |
| echo "$out" | |
| if grep -q "PERFORMANCE DECREASED" <<<"$out"; then | |
| echo "regressed=true" >> "$GITHUB_OUTPUT" | |
| elif grep -qE "BENCHMARKS NOT SIGNIFICANTLY CHANGED|PERFORMANCE INCREASED" <<<"$out"; then | |
| echo "regressed=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "::error::asv continuous did not complete a comparison; benchmark build/run failure (not a regression)" | |
| exit 1 | |
| fi | |
| - name: Open/update regression issue | |
| if: steps.detect.outputs.regressed == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = "Nightly benchmark regression detected"; | |
| const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const body = `ASV flagged a wall-time regression on \`main\`. Workflow run: ${url}`; | |
| const existing = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: "open", | |
| labels: "benchmark-regression", | |
| }); | |
| if (existing.data.length) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: existing.data[0].number, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title, | |
| body, | |
| labels: ["benchmark-regression"], | |
| }); | |
| } |