From b9b07df60b09f6316c9999ed01ba4f2c7e0df6c0 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Tue, 12 Mar 2024 15:48:23 +0000 Subject: [PATCH] ci: refactor tests and benchmarks --- .github/actions/build-llvm/action.yml | 79 +++++++++ .github/workflows/benchmarks.yml | 228 +++++++++----------------- .github/workflows/tests.yaml | 38 ++--- 3 files changed, 170 insertions(+), 175 deletions(-) create mode 100644 .github/actions/build-llvm/action.yml diff --git a/.github/actions/build-llvm/action.yml b/.github/actions/build-llvm/action.yml new file mode 100644 index 00000000..70b452f8 --- /dev/null +++ b/.github/actions/build-llvm/action.yml @@ -0,0 +1,79 @@ +name: 'Build LLVM' +description: 'Builds backend LLVM framework' +inputs: + target: + description: 'Specific build target triplet.' + required: false + default: '' +runs: + using: "composite" + steps: + - name: Prepare environment + if: runner.os == 'macOS' + run: brew install cmake ninja + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + + - name: Define build target + id: build-target + if: inputs.target != '' + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + run: | + rustup target add ${{ inputs.target }} + echo "target=--target ${{ inputs.target }}" >> $GITHUB_OUTPUT + + - name: Clone LLVM framework + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + run: | + cargo install compiler-llvm-builder ${{ steps.build-target.outputs.target }} + zkevm-llvm clone + + - name: Define ccache key + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + id: ccache-key + run: | + LLVM_BRANCH="$(git -C ./llvm rev-parse --abbrev-ref HEAD)" + LLVM_SHORT_SHA="$(git -C ./llvm rev-parse --short HEAD)" + echo "key=llvm-${LLVM_BRANCH}-${LLVM_SHORT_SHA}-${{ runner.os }}-${{ runner.arch }}" | tee -a "${GITHUB_OUTPUT}" + + - name: Prepare ccache installation + if: runner.os == 'Linux' + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + run: apt update + + - name: Set up compiler cache + uses: hendrikmuhs/ccache-action@v1.2 + env: + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_NOHASHDIR: "true" + CCACHE_COMPILERCHECK: "content" + with: + key: ${{ steps.ccache-key.outputs.key }} + restore-keys: ${{ steps.ccache-key.outputs.key }} + append-timestamp: false + max-size: "2G" + verbose: 2 + save: ${{ github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/v') }} + + - name: Build LLVM Linux + if: ${{ runner.os == 'Linux' }} + uses: nick-fields/retry@v2 + env: + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_NOHASHDIR: "true" + CCACHE_COMPILERCHECK: "content" + LIBSTDCPP_SOURCE_PATH: "C:/a/_temp/msys64/mingw64/lib/libstdc++.a" + with: + timeout_minutes: 60 + max_attempts: 16 # protection mechanism for sporadic dependencies download failure + command: zkevm-llvm build --targets 'EVM' --use-ccache + + - name: Build LLVM MacOS and Windows + if: ${{ runner.os != 'Linux' }} + shell: ${{ runner.os == 'Windows' && 'msys2 {0}' || 'bash' }} + env: + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_NOHASHDIR: "true" + CCACHE_COMPILERCHECK: "content" + LIBSTDCPP_SOURCE_PATH: "C:/a/_temp/msys64/mingw64/lib/libstdc++.a" + run: | + zkevm-llvm build --targets 'EVM' --use-ccache diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b7d8df90..e51852ed 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,202 +1,126 @@ -name: Benchmarking +name: Compiler benchmarking on: pull_request: + workflow_dispatch: + inputs: + llvm_build_type: + description: "LLVM build type: debug | release" + required: true + default: "release" + compiler_tester_reference_branch: + description: "compiler-tester branch to use as a benchmark reference" + required: true + default: "main" + compiler_tester_candidate_branch: + description: "compiler-tester branch to use as a benchmark candidate" + required: true + default: "main" + compiler_llvm_benchmark_mode: + description: "Mode filter for compiler-llvm benchmarks" + required: false + default: "^M^B3" + compiler_llvm_benchmark_path: + description: "Path filter for compiler-llvm benchmarks" + required: false + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - benchmarks_candidate: - runs-on: [matterlabs-ci-runner] + benchmarks: + strategy: + matrix: + type: ["reference", "candidate"] + name: ${{ matrix.type }} + runs-on: matterlabs-ci-runner container: - image: matterlabs/llvm_runner:latest - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + image: matterlabs/llvm_runner:ubuntu22-llvm15-latest options: -m 110g + env: + LLVM_BENCHMARK_MODE: ${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }} + LLVM_BENCHMARK_PATH: ${{ github.event.inputs.compiler_llvm_benchmark_path || '' }} steps: - - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1 - - name: Preparing workspace. Setting environment. + - name: Define branches run: | - echo "BENCHMARK_MODE=${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }}" >> $GITHUB_ENV - echo "BENCHMARK_PATH=${{ github.event.inputs.compiler_llvm_benchmark_path || '' }}" >> $GITHUB_ENV - echo "CANDIDATE_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch }}" >> $GITHUB_ENV - - - name: Get branch name (pull request) - if: github.event_name == 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV - - - name: Preparing workspace. Checkout compiler-tester candidate (pull request) - if: github.event_name == 'pull_request' + if [ "${{ matrix.type }}" = "candidate" ]; then + echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }}" >> $GITHUB_ENV + else + echo "COMPILER_TESTER_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch || 'main' }}" >> $GITHUB_ENV + fi + + # Just for one-time test, remove after + echo "COMPILER_TESTER_BRANCH_NAME=${{ github.head_ref}}" >> $GITHUB_ENV + + - name: Checkout compiler-tester uses: actions/checkout@v4 with: + repository: matter-labs/era-compiler-tester + ref: ${{ env.COMPILER_TESTER_BRANCH_NAME }} submodules: recursive - path: compiler-tester - - name: Preparing workspace. Checkout compiler-tester candidate (workflow dispatch) - if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v4 + - name: Build LLVM + uses: ./.github/actions/build-llvm with: - path: compiler-tester - submodules: recursive - ref: ${{ env.CANDIDATE_BRANCH_NAME }} - - - name: Testing. Benchmarking LLVM framework candidate. - id: compiler_tester_run - run: | - cd compiler-tester - cargo install compiler-llvm-builder - /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build --targets 'EVM' - - export RUST_BACKTRACE='full' - export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/" - cargo build --verbose --release --bin 'compiler-tester' - cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/' - cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/' - - ./target/release/compiler-tester \ - --zksolc './target-zksolc/release/zksolc' \ - --zkvyper './target-zkvyper/release/zkvyper' \ - --path=${{ env.BENCHMARK_PATH || '' }} \ - --mode=${{ env.BENCHMARK_MODE || '' }} \ - --benchmark='candidate.json' - - - uses: actions/upload-artifact@v4 - with: - name: candidate-benchmark - path: compiler-tester/candidate.json - - benchmarks_reference: - runs-on: [matterlabs-ci-runner] - container: - image: matterlabs/llvm_runner:latest - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - options: -m 110g - steps: - - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1 - - name: Preparing workspace. Setting environment. - run: | - echo "BENCHMARK_MODE=${{ github.event.inputs.compiler_llvm_benchmark_mode || '^M^B3' }}" >> $GITHUB_ENV - echo "BENCHMARK_PATH=${{ github.event.inputs.compiler_llvm_benchmark_path || '' }}" >> $GITHUB_ENV - echo "REFERENCE_BRANCH_NAME=${{ github.event.inputs.compiler_tester_reference_branch }}" >> $GITHUB_ENV - - - name: Get branch name (pull request) - if: github.event_name == 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV + debug: ${{ github.event.inputs.llvm_build_type || 'release' }} - - name: Preparing workspace. Checkout compiler-tester reference (pull request) - if: github.event_name == 'pull_request' - uses: actions/checkout@v4 - with: - path: compiler-tester - submodules: recursive - ref: ${{ env.BRANCH_NAME }} - - - name: Preparing workspace. Checkout compiler-tester reference (workflow dispatch) - if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v4 - with: - path: compiler-tester - submodules: recursive - ref: ${{ env.REFERENCE_BRANCH_NAME }} - - - name: Testing. Benchmarking LLVM framework reference. - id: compiler_tester_run + - name: Benchmark + env: + RUST_BACKTRACE: full run: | - cd compiler-tester - cargo install compiler-llvm-builder - /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build --targets 'EVM' - - export RUST_BACKTRACE='full' export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/" cargo build --verbose --release --bin 'compiler-tester' cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-solidity-*/*/Cargo.toml --target-dir './target-zksolc/' cargo build --verbose --release --manifest-path /usr/local/cargo/git/checkouts/era-compiler-vyper-*/*/Cargo.toml --target-dir './target-zkvyper/' - ./target/release/compiler-tester \ --zksolc './target-zksolc/release/zksolc' \ --zkvyper './target-zkvyper/release/zkvyper' \ - --path=${{ env.BENCHMARK_PATH || '' }} \ - --mode=${{ env.BENCHMARK_MODE || '' }} \ - --benchmark='reference.json' + --path=${{ env.LLVM_BENCHMARK_PATH }} \ + --mode=${{ env.LLVM_BENCHMARK_MODE }} \ + --benchmark=${{ matrix.type }}.json - uses: actions/upload-artifact@v4 with: - name: reference-benchmark - path: compiler-tester/reference.json + name: compiler-llvm-${{ matrix.type }}-benchmark + path: ${{ matrix.type }}.json - benchmarks_analysis: - runs-on: [matterlabs-ci-runner] - needs: [benchmarks_candidate, benchmarks_reference] + analysis: + name: "Benchmark comparison" + runs-on: matterlabs-ci-runner container: - image: matterlabs/llvm_runner:latest - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + image: matterlabs/llvm_runner:ubuntu22-llvm15-latest + needs: benchmarks steps: - - uses: AutoModality/action-clean@492b2d2e2e77247bfd0b17eaa89a371b2f3430ee # v1 - - name: Get branch name (pull request) - if: github.event_name == 'pull_request' - shell: bash - run: echo "BRANCH_NAME=$(echo ${GITHUB_BASE_REF} | tr / -)" >> $GITHUB_ENV - - - name: Preparing workspace. Checkout compiler-tester candidate (pull request) - if: github.event_name == 'pull_request' + - name: Checking out the compiler-tester repository uses: actions/checkout@v4 with: - path: compiler-tester + ref: ${{ github.event.inputs.compiler_tester_candidate_branch || 'main' }} submodules: recursive - - name: Preparing workspace. Checkout compiler-tester candidate (workflow dispatch) - if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v4 - with: - path: compiler-tester - ref: ${{ env.CANDIDATE_BRANCH_NAME }} - - uses: actions/download-artifact@v4 with: - name: candidate-benchmark - path: compiler-tester + pattern: compiler-llvm-* + merge-multiple: true - - uses: actions/download-artifact@v4 - with: - name: reference-benchmark - path: compiler-tester - - - name: Testing. Comparing LLVM framework results. - id: compiler_tester_run + - name: Comparing the LLVM framework benchmark results run: | - cd compiler-tester && \ - export RUST_BACKTRACE=full && \ - cargo run --release --bin benchmark-analyzer -- --reference reference.json --candidate candidate.json --output-file result.txt && \ - chown 1000:1000 result.txt + cargo run --release --bin benchmark-analyzer -- \ + --reference reference.json --candidate candidate.json --output-file result.txt - - name: Testing. Posting LLVM benchmark results to step summary. + - name: Posting the LLVM benchmark results to the summary run: | printf "Benchmark results:\n" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - cat ./compiler-tester/result.txt >> $GITHUB_STEP_SUMMARY + cat result.txt >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - cat $GITHUB_STEP_SUMMARY > ./compiler-tester/result.txt + cat $GITHUB_STEP_SUMMARY > result.txt - - name: Testing. Posting LLVM benchmark results to PR comment. + - name: Posting the LLVM benchmark results to a PR comment if: github.event_name == 'pull_request' uses: mshick/add-pr-comment@v2 - with: - message-path: ./compiler-tester/result.txt - - - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) env: - SLACK_WEBHOOK_URL: ${{ secrets.COMPILER_CI_MATTERMOST_WEBHOOK }} # required - if: ${{ failure() || success() }} # Skip canceled jobs - + GITHUB_TOKEN: ${{ secrets.ZKSYNC_ADMIN_BOT_ORG_REPO_WRITE }} + with: + message-path: result.txt diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 75d14bcb..9d88736d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,7 +1,10 @@ -name: Testing +name: Test on: pull_request: + push: + branches: + - main concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -12,31 +15,19 @@ jobs: runs-on: [self-hosted, ci-runner-compiler] timeout-minutes: 720 container: - image: matterlabs/llvm_runner:latest - credentials: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + image: matterlabs/llvm_runner:ubuntu22-llvm15-latest options: -m 110g steps: - # Add safe.directory per https://github.com/actions/checkout/issues/766 - - name: Setting all dirs as safe safe.directory - run: | - git config --system --add safe.directory '*' - git config --global --add safe.directory '*' - - - name: Preparing workspace. Checkout compiler-tester repository. + - name: Checkout uses: actions/checkout@v4 with: - path: compiler-tester submodules: recursive - - name: Testing. Building and running compiler tester. - id: compiler_tester_run - run: | - cd compiler-tester - cargo install compiler-llvm-builder - /usr/local/cargo/bin/zkevm-llvm clone && /usr/local/cargo/bin/zkevm-llvm build --targets 'EVM' + - name: Build LLVM + uses: ./.github/actions/build-llvm + - name: Run tests + run: | export RUST_BACKTRACE='full' export LLVM_SYS_150_PREFIX="$(pwd)/target-llvm/target-final/" cargo build --verbose --release --bin 'compiler-tester' @@ -47,10 +38,11 @@ jobs: --zksolc './target-zksolc/release/zksolc' \ --zkvyper './target-zkvyper/release/zkvyper' - - uses: 8398a7/action-slack@v3 + - name: Send Slack notification + uses: 8398a7/action-slack@v3 + if: ${{ failure() || success() }} # Skip canceled jobs with: status: ${{ job.status }} - fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) + fields: repo,commit,author,action,eventName,ref,workflow,job,took,pullRequest env: - SLACK_WEBHOOK_URL: ${{ secrets.COMPILER_CI_MATTERMOST_WEBHOOK }} # required - if: ${{ failure() || success() }} # Skip canceled jobs + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}