ci: refactor tests and benchmarks

.github/actions/build-llvm/action.yml

@@ -0,0 +1,79 @@
+name: 'Build LLVM'
+description: 'Builds backend LLVM framework'
+inputs:
+  target:
+    description: 'Specific build target triplet.'
+    required: false
+    default: ''
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare environment
+      if: runner.os == 'macOS'
+      run: brew install cmake ninja
+      shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
+
+    - name: Define build target
+      id: build-target
+      if: inputs.target != ''
+      shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
+      run: |
+        rustup target add ${{ inputs.target }}
+        echo "target=--target ${{ inputs.target }}" >> $GITHUB_OUTPUT
+
+    - name: Clone LLVM framework
+      shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
+      run: |
+        cargo install compiler-llvm-builder ${{ steps.build-target.outputs.target }}
+        zkevm-llvm clone
+
+    - name: Define ccache key
+      shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
+      id: ccache-key
+      run: |
+        LLVM_BRANCH="$(git -C ./llvm rev-parse --abbrev-ref HEAD)"
+        LLVM_SHORT_SHA="$(git -C ./llvm rev-parse --short HEAD)"
+        echo "key=llvm-${LLVM_BRANCH}-${LLVM_SHORT_SHA}-${{ runner.os }}-${{ runner.arch }}" | tee -a "${GITHUB_OUTPUT}"
+
+    - name: Prepare ccache installation
+      if: runner.os == 'Linux'
+      shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
+      run: apt update
+
+    - name: Set up compiler cache
+      uses: hendrikmuhs/[email protected]
+      env:
+        CCACHE_BASEDIR: ${{ github.workspace }}
+        CCACHE_NOHASHDIR: "true"
+        CCACHE_COMPILERCHECK: "content"
+      with:
+        key: ${{ steps.ccache-key.outputs.key }}
+        restore-keys: ${{ steps.ccache-key.outputs.key }}
+        append-timestamp: false
+        max-size: "2G"
+        verbose: 2
+        save: ${{ github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/v') }}
+
+    - name: Build LLVM Linux
+      if: ${{ runner.os == 'Linux' }}
+      uses: nick-fields/retry@v2
+      env:
+        CCACHE_BASEDIR: ${{ github.workspace }}
+        CCACHE_NOHASHDIR: "true"
+        CCACHE_COMPILERCHECK: "content"
+        LIBSTDCPP_SOURCE_PATH: "C:/a/_temp/msys64/mingw64/lib/libstdc++.a"
+      with:
+        timeout_minutes: 60
+        max_attempts: 16 # protection mechanism for sporadic dependencies download failure
+        command: zkevm-llvm build --targets 'EVM' --use-ccache
+
+    - name: Build LLVM MacOS and Windows
+      if: ${{ runner.os != 'Linux' }}
+      shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }}
+      env:
+        CCACHE_BASEDIR: ${{ github.workspace }}
+        CCACHE_NOHASHDIR: "true"
+        CCACHE_COMPILERCHECK: "content"
+        LIBSTDCPP_SOURCE_PATH: "C:/a/_temp/msys64/mingw64/lib/libstdc++.a"
+      run: |
+        zkevm-llvm build --targets 'EVM' --use-ccache

.github/workflows/benchmarks.yml

@@ -1,202 +1,126 @@
-name: Benchmarking
+name: Compiler benchmarking
 
 on:
   pull_request:
+  workflow_dispatch:
+    inputs:
+      llvm_build_type:
+        description: "LLVM build type: debug | release"
+        required: true
+        default: "release"
+      compiler_tester_reference_branch:
+        description: "compiler-tester branch to use as a benchmark reference"
+        required: true
+        default: "main"
+      compiler_tester_candidate_branch:
+        description: "compiler-tester branch to use as a benchmark candidate"
+        required: true
+        default: "main"
+      compiler_llvm_benchmark_mode:
+        description: "Mode filter for compiler-llvm benchmarks"
+        required: false
+        default: "^M^B3"
+      compiler_llvm_benchmark_path:
+        description: "Path filter for compiler-llvm benchmarks"
+        required: false
+        default: ""
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  benchmarks_candidate:
-    runs-on: [matterlabs-ci-runner]
+  benchmarks:
+    strategy:
+      matrix:
+        type: ["reference", "candidate"]
+    name: ${{ matrix.type }}
+    runs-on: matterlabs-ci-runner
     container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
       options: -m 110g
+    env:
+      LLVM_BENCHMARK_MODE: ${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }}
+      LLVM_BENCHMARK_PATH: ${{ github.event.inputs.compiler_llvm_benchmark_path || '' }}
     steps:
-      - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1
-      - name: Preparing workspace. Setting environment.
+      - name: Define branches
         run: |
-          echo "BENCHMARK_MODE=${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }}" >> $GITHUB_ENV
-          echo "BENCHMARK_PATH=${{ github.event.inputs.compiler_llvm_benchmark_path || '' }}" >> $GITHUB_ENV
-          echo "CANDIDATE_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch }}" >> $GITHUB_ENV
-
-      - name: Get branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
-
-      - name: Preparing workspace. Checkout compiler-tester candidate (pull request)
-        if: github.event_name == 'pull_request'
+          if [ "${{ matrix.type }}" = "candidate" ]; then
+            echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }}" >> $GITHUB_ENV
+          else
+            echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch || 'main' }}" >> $GITHUB_ENV
+          fi
+          
+          # Just for one-time test, remove after
+          echo "COMPILER_TESTER_BRANCH_NAME=${{ github.head_ref}}" >> $GITHUB_ENV
+
+      - name: Checkout compiler-tester
         uses: actions/checkout@v4
         with:
+          repository: matter-labs/era-compiler-tester
+          ref: ${{ env.COMPILER_TESTER_BRANCH_NAME }}
           submodules: recursive
-          path: compiler-tester
 
-      - name: Preparing workspace. Checkout compiler-tester candidate (workflow dispatch)
-        if: github.event_name == 'workflow_dispatch'
-        uses: actions/checkout@v4
+      - name: Build LLVM
+        uses: ./.github/actions/build-llvm
         with:
-          path: compiler-tester
-          submodules: recursive
-          ref: ${{ env.CANDIDATE_BRANCH_NAME }}
-
-      - name: Testing. Benchmarking LLVM framework candidate.
-        id: compiler_tester_run
-        run: |
-          cd compiler-tester
-          cargo install compiler-llvm-builder
-          /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build --targets 'EVM'
-
-          export RUST_BACKTRACE='full'
-          export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/"
-          cargo build --verbose --release --bin 'compiler-tester'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
-          cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
-
-          ./target/release/compiler-tester \
-            --zksolc './target-zksolc/release/zksolc' \
-            --zkvyper './target-zkvyper/release/zkvyper' \
-            --path=${{ env.BENCHMARK_PATH || '' }} \
-            --mode=${{ env.BENCHMARK_MODE || '' }} \
-            --benchmark='candidate.json'
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: candidate-benchmark
-          path: compiler-tester/candidate.json
-
-  benchmarks_reference:
-    runs-on: [matterlabs-ci-runner]
-    container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-      options: -m 110g
-    steps:
-      - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1
-      - name: Preparing workspace. Setting environment.
-        run: |
-          echo "BENCHMARK_MODE=${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }}" >> $GITHUB_ENV
-          echo "BENCHMARK_PATH=${{ github.event.inputs.compiler_llvm_benchmark_path || '' }}" >> $GITHUB_ENV
-          echo "REFERENCE_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch }}" >> $GITHUB_ENV
-
-      - name: Get branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
+          debug: ${{ github.event.inputs.llvm_build_type || 'release' }}
 
-      - name: Preparing workspace. Checkout compiler-tester reference (pull request)
-        if: github.event_name == 'pull_request'
-        uses: actions/checkout@v4
-        with:
-          path: compiler-tester
-          submodules: recursive
-          ref: ${{ env.BRANCH_NAME }}
-
-      - name: Preparing workspace. Checkout compiler-tester reference (workflow dispatch)
-        if: github.event_name == 'workflow_dispatch'
-        uses: actions/checkout@v4
-        with:
-          path: compiler-tester
-          submodules: recursive
-          ref: ${{ env.REFERENCE_BRANCH_NAME }}
-
-      - name: Testing. Benchmarking LLVM framework reference.
-        id: compiler_tester_run
+      - name: Benchmark
+        env:
+          RUST_BACKTRACE: full
         run: |
-          cd compiler-tester
-          cargo install compiler-llvm-builder
-          /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build --targets 'EVM'
-
-          export RUST_BACKTRACE='full'
           export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/"
           cargo build --verbose --release --bin 'compiler-tester'
           cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/'
           cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/'
-
           ./target/release/compiler-tester \
             --zksolc './target-zksolc/release/zksolc' \
             --zkvyper './target-zkvyper/release/zkvyper' \
-            --path=${{ env.BENCHMARK_PATH || '' }} \
-            --mode=${{ env.BENCHMARK_MODE || '' }} \
-            --benchmark='reference.json'
+            --path=${{ env.LLVM_BENCHMARK_PATH }} \
+            --mode=${{ env.LLVM_BENCHMARK_MODE }} \
+            --benchmark=${{ matrix.type }}.json
 
       - uses: actions/upload-artifact@v4
         with:
-          name: reference-benchmark
-          path: compiler-tester/reference.json
+          name: compiler-llvm-${{ matrix.type }}-benchmark
+          path: ${{ matrix.type }}.json
 
-  benchmarks_analysis:
-    runs-on: [matterlabs-ci-runner]
-    needs: [benchmarks_candidate, benchmarks_reference]
+  analysis:
+    name: "Benchmark comparison"
+    runs-on: matterlabs-ci-runner
     container:
-      image: matterlabs/llvm_runner:latest
-      credentials:
-        username: ${{ secrets.DOCKERHUB_USER }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
+      image: matterlabs/llvm_runner:ubuntu22-llvm15-latest
+    needs: benchmarks
     steps:
-      - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1
-      - name: Get branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV
-
-      - name: Preparing workspace. Checkout compiler-tester candidate (pull request)
-        if: github.event_name == 'pull_request'
+      - name: Checking out the compiler-tester repository
         uses: actions/checkout@v4
         with:
-          path: compiler-tester
+          ref: ${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }}
           submodules: recursive
 
-      - name: Preparing workspace. Checkout compiler-tester candidate (workflow dispatch)
-        if: github.event_name == 'workflow_dispatch'
-        uses: actions/checkout@v4
-        with:
-          path: compiler-tester
-          ref: ${{ env.CANDIDATE_BRANCH_NAME }}
-
       - uses: actions/download-artifact@v4
         with:
-          name: candidate-benchmark
-          path: compiler-tester
+          pattern: compiler-llvm-*
+          merge-multiple: true
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: reference-benchmark
-          path: compiler-tester
-
-      - name: Testing. Comparing LLVM framework results.
-        id: compiler_tester_run
+      - name: Comparing the LLVM framework benchmark results
         run: |
-          cd compiler-tester && \
-          export RUST_BACKTRACE=full && \
-          cargo run --release --bin benchmark-analyzer -- --reference reference.json --candidate candidate.json --output-file result.txt && \
-          chown 1000:1000 result.txt
+          cargo run --release --bin benchmark-analyzer -- \
+            --reference reference.json --candidate candidate.json --output-file result.txt
 
-      - name: Testing. Posting LLVM benchmark results to step summary.
+      - name: Posting the LLVM benchmark results to the summary
         run: |
           printf "Benchmark results:\n" >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
-          cat ./compiler-tester/result.txt >> $GITHUB_STEP_SUMMARY
+          cat result.txt >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
-          cat $GITHUB_STEP_SUMMARY > ./compiler-tester/result.txt
+          cat $GITHUB_STEP_SUMMARY > result.txt
 
-      - name: Testing. Posting LLVM benchmark results to PR comment.
+      - name: Posting the LLVM benchmark results to a PR comment
         if: github.event_name == 'pull_request'
         uses: mshick/add-pr-comment@v2
-        with:
-          message-path: ./compiler-tester/result.txt
-
-      - uses: 8398a7/action-slack@v3
-        with:
-          status: ${{ job.status }}
-          fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message)
         env:
-          SLACK_WEBHOOK_URL: ${{ secrets.COMPILER_CI_MATTERMOST_WEBHOOK }} # required
-        if: ${{ failure() || success() }} # Skip canceled jobs
-
+          GITHUB_TOKEN: ${{ secrets.ZKSYNC_ADMIN_BOT_ORG_REPO_WRITE }}
+        with:
+          message-path: result.txt