benchmark.yml

name: Benchmarks

# Out-of-band run of the metric-walk benchmark harness (#1068).
#
# Deliberately not part of per-PR CI: shared runners cannot give
# stable numbers, which is exactly the lesson that produced this
# harness. See docs/development/benchmarking.md.
#
# The complexity-class gate is ratio-based and therefore portable
# across hosts, but a busy runner can still skew a single run, so a
# failure is re-measured with more rounds before it is reported.

on:
  schedule:
    # 07:00 UTC on the 15th of January, April, July, and October.
    # Two weeks after the mutation-testing cron so the two out-of-band
    # jobs do not land on the same day.
    - cron: '0 7 15 1,4,7,10 *'
  workflow_dispatch:

permissions:
  contents: read
  issues: write

concurrency:
  group: benchmark
  cancel-in-progress: false

# All `uses:` below are pinned to commit SHAs to eliminate supply-chain
# risk from action-release spoofing. Bump via Dependabot (see
# .github/dependabot.yml), which rewrites the SHA + version comment.
jobs:
  bench:
    name: Metric walk
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          submodules: recursive

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # stable tip
        with:
          toolchain: stable

      - name: Cache cargo build artifacts
        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2

      # Runs first so its measurements survive even when the gate below
      # fails: a complexity regression is easier to localise with the
      # per-metric corpus numbers in hand.
      - name: Criterion measurements
        run: |
          set -euo pipefail
          mkdir -p target/bench
          cargo bench -p big-code-analysis-bench --bench metric_walk \
            2>&1 | tee target/bench/metric_walk.log

      # The bench target exits 1 for "a probe left its complexity class"
      # and 2 for "the harness could not run" (bad arguments, or a
      # language not compiled in). Only the former is re-measured and
      # reported; a 2 fails the job without filing an issue, because an
      # infrastructure error that reproduces on the retry would
      # otherwise open one claiming a complexity regression.
      - name: Complexity-class gate
        id: gate
        run: |
          set -euo pipefail
          mkdir -p target/bench
          # `status=$?` after an `if` block would read the *if
          # statement's* status, which is 0 whenever no branch ran, so
          # the exit code is captured with `|| status=$?` instead.
          # `pipefail` makes that cargo's code rather than tee's.
          status=0
          cargo bench -p big-code-analysis-bench --bench scaling -- --rounds 11 \
            2>&1 | tee target/bench/scaling.log || status=$?
          if [ "$status" = "0" ]; then
            exit 0
          fi
          if [ "$status" != "1" ]; then
            echo "The harness exited ${status}, which is not a gate failure."
            echo "Treating as an infrastructure error; no issue will be filed."
            exit "$status"
          fi
          echo "A probe exceeded its bound; re-measuring with more rounds"
          echo "before reporting, in case the runner was contended."
          status=0
          cargo bench -p big-code-analysis-bench --bench scaling -- --rounds 21 \
            2>&1 | tee target/bench/scaling-confirm.log || status=$?
          if [ "$status" = "0" ]; then
            echo "The second run passed. Treating the first as measurement noise."
            exit 0
          fi
          if [ "$status" != "1" ]; then
            echo "The confirmation run exited ${status}, not a gate failure."
            # Removed so the issue-filing step below, which keys on this
            # file, cannot mistake an infrastructure error for a
            # confirmed regression.
            rm -f target/bench/scaling-confirm.log
            exit "$status"
          fi
          exit 1

      - name: Upload benchmark output
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: metric-walk-benchmarks
          path: |
            target/bench/
            target/criterion/
          if-no-files-found: warn
          retention-days: 90

      - name: File issue on a confirmed complexity regression
        if: failure() && steps.gate.conclusion == 'failure'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_REPO: ${{ github.repository }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -euo pipefail
          if [ ! -f target/bench/scaling-confirm.log ]; then
            echo "The gate failed before producing a confirmation run;"
            echo "treating as infrastructure failure, not opening an issue."
            exit 0
          fi
          {
            echo "The scheduled \`benchmark.yml\` run measured a metric-walk"
            echo "probe outside its declared complexity bound, twice."
            echo
            echo "- Run: ${RUN_URL}"
            echo
            echo "Reproduce locally with \`make bench-scaling\`. See"
            echo "\`docs/development/benchmarking.md\` for how to read the table"
            echo "and what each probe covers."
            echo
            echo "## Confirmation run"
            echo
            echo '```'
            tail -n 80 target/bench/scaling-confirm.log
            echo '```'
          } > /tmp/benchmark-issue-body.md
          gh issue create \
            --title "Metric walk: complexity bound exceeded (run ${{ github.run_id }})" \
            --label "bug" \
            --body-file /tmp/benchmark-issue-body.md