benchmarks.yml

name: Compiler benchmarking

on:
  pull_request:
  workflow_dispatch:
    inputs:
      llvm_build_type:
        description: "LLVM build type: debug | release"
        required: true
        default: "release"
      compiler_tester_reference_branch:
        description: "compiler-tester branch to use as a benchmark reference"
        required: true
        default: "main"
      compiler_tester_candidate_branch:
        description: "compiler-tester branch to use as a benchmark candidate"
        required: true
        default: "main"
      compiler_llvm_benchmark_mode:
        description: "Mode filter for compiler-llvm benchmarks"
        required: false
        default: "^M^B3"
      compiler_llvm_benchmark_path:
        description: "Path filter for compiler-llvm benchmarks"
        required: false
        default: ""

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  benchmarks:
    strategy:
      matrix:
        type: ["reference", "candidate"]
    name: ${{ matrix.type }}
    runs-on: matterlabs-ci-runner
    container:
      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
      options: -m 110g
    env:
      LLVM_BENCHMARK_MODE: ${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }}
      LLVM_BENCHMARK_PATH: ${{ github.event.inputs.compiler_llvm_benchmark_path || '' }}
    steps:
      - name: Define branches
        run: |
          if [ "${{ matrix.type }}" = "candidate" ]; then
            echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }}" >> $GITHUB_ENV
          else
            echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch || 'main' }}" >> $GITHUB_ENV
          fi
          
          # Just for one-time test, remove after
          echo "COMPILER_TESTER_BRANCH_NAME=${{ github.head_ref}}" >> $GITHUB_ENV

      - name: Checkout compiler-tester
        uses: actions/checkout@v4
        with:
          repository: matter-labs/era-compiler-tester
          ref: ${{ env.COMPILER_TESTER_BRANCH_NAME }}
          submodules: recursive

      - name: Build LLVM
        uses: ./.github/actions/build-llvm
        with:
          debug: ${{ github.event.inputs.llvm_build_type || 'release' }}

      - name: Benchmark
        env:
          RUST_BACKTRACE: full
        run: |
          export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/"
          cargo build --verbose --release --bin 'compiler-tester'
          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
          ./target/release/compiler-tester \
            --zksolc './target-zksolc/release/zksolc' \
            --zkvyper './target-zkvyper/release/zkvyper' \
            --path=${{ env.LLVM_BENCHMARK_PATH }} \
            --mode=${{ env.LLVM_BENCHMARK_MODE }} \
            --benchmark=${{ matrix.type }}.json

      - uses: actions/upload-artifact@v4
        with:
          name: compiler-llvm-${{ matrix.type }}-benchmark
          path: ${{ matrix.type }}.json

  analysis:
    name: "Benchmark comparison"
    runs-on: matterlabs-ci-runner
    container:
      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
    needs: benchmarks
    steps:
      - name: Checking out the compiler-tester repository
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }}
          submodules: recursive

      - uses: actions/download-artifact@v4
        with:
          pattern: compiler-llvm-*
          merge-multiple: true

      - name: Comparing the LLVM framework benchmark results
        run: |
          cargo run --release --bin benchmark-analyzer -- \
            --reference reference.json --candidate candidate.json --output-file result.txt

      - name: Posting the LLVM benchmark results to the summary
        run: |
          printf "Benchmark results:\n" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          cat result.txt >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          cat $GITHUB_STEP_SUMMARY > result.txt

      - name: Posting the LLVM benchmark results to a PR comment
        if: github.event_name == 'pull_request'
        uses: mshick/add-pr-comment@v2
        env:
          GITHUB_TOKEN: ${{ secrets.ZKSYNC_ADMIN_BOT_ORG_REPO_WRITE }}
        with:
          message-path: result.txt