Skip to content

chore: fix mem regression #63752

chore: fix mem regression

chore: fix mem regression #63752

Workflow file for this run

# CI for Aztec. At a high-level, runs ./bootstrap.sh ci in root. See root README.md for more details.
# Only for internal devs. For external devs, see ci3-external.yml.
#
# CAREFUL! We use "exec" a lot to ensure signal propagation to the child process, to allow proper ec2 cleanup.
name: CI3
on:
workflow_dispatch:
push:
tags:
- "v*"
pull_request:
types: [opened, synchronize, reopened, ready_for_review, labeled]
merge_group: {}
concurrency:
# Allow full concurrency for merge-train PRs, one-run-per-branch for everything else.
group: ci3-${{ (startsWith(github.event.pull_request.head.ref, 'merge-train/') && github.run_id) || github.event.merge_group.head_ref || github.ref_name }}
cancel-in-progress: true
jobs:
# The main CI job for Aztec pull requests. It gatekeeps merges to next, intended for the merge queue.
#
# This orchestrates AWS EC2 spot instances then remotely runs ./bootstrap.sh
# The ci-full label enables heavier tests. For other labels, see ci3.sh.
#
# Standard PR) Merge Queue
# | |
# | 1x AMD64 EC2 Spot | | [ 4x AMD64 EC2 Spot ] |
# | [ 1x ARM64 EC2 Spot ] |
# |
# Filters out "accidental"
# successes, but non-determinism
# can still cause flaky tests.
#
# If you suspect a flaky test, look for people recently reporting similar things.
# Otherwise, spend time ensuring it is not your PR. Spend some time attempting to fix it.
# Try not to just increase timeouts. Post either about your fix or your efforts where
# engineers can broadly see it.
ci:
runs-on: ubuntu-latest
# exclusive with ci3-external.yml: never run on forks
# (github.event.pull_request.head.repo.fork resolves to nil if not a pull request)
if: github.event.pull_request.head.repo.fork != true && (github.event.pull_request.draft == false || contains(github.event.pull_request.labels.*.name, 'ci-draft'))
environment: ${{ startsWith(github.ref, 'refs/tags/v') && 'master' || '' }}
permissions:
id-token: write # required for OIDC assume-role with AWS
contents: read
steps:
- name: Remove wakeup label
if: contains(github.event.pull_request.labels.*.name, 'ci-wakeup-pr-after-merge')
env:
GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-wakeup-pr-after-merge --repo ${{ github.repository }} || true
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
# The commit to checkout. We want our actual commit, and not the result of merging the PR to the target.
ref: ${{ github.event.pull_request.head.sha || github.sha }}
# Fetch PR commits depth (we'll deepen by 1 in squash script if needed)
fetch-depth: ${{ github.event.pull_request.commits || 1 }}
persist-credentials: true # Required for bootstrap_ec2's git fetch
- name: Configure AWS credentials (OIDC)
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
aws-region: us-east-2
role-session-name: ci3-${{ github.run_id }}
role-duration-seconds: 7200 # 2h – covers max AWS_SHUTDOWN_TIME (90 min ARM) + 30 min buffer
- name: Determine CI Mode
env:
MERGE_GROUP_BASE_REF: ${{ github.event.merge_group.base_ref }}
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
GITHUB_REF_NAME: ${{ github.ref_name }}
GITHUB_TOKEN: ${{ github.token }}
PR_LABELS_JSON: ${{ toJson(github.event.pull_request.labels.*.name) }}
run: |
# Parse labels from JSON env var to avoid shell injection via label names
mapfile -t LABELS < <(echo "$PR_LABELS_JSON" | jq -r '.[]')
./.github/ci3_labels_to_env.sh "${LABELS[@]}"
- name: Run
env:
CI_USE_SSH: ${{ (vars.CI_USE_SSH == '1' && !contains(github.event.pull_request.labels.*.name, 'ci-ssm')) && '1' || '0' }}
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
# For automatic documentation updates via Claude Code
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
# Nightly test env vars.
EXTERNAL_ETHEREUM_HOSTS: "https://json-rpc.${{ secrets.GCP_SEPOLIA_URL }}?key=${{ secrets.GCP_SEPOLIA_API_KEY }},${{ secrets.INFURA_SEPOLIA_URL }}"
EXTERNAL_ETHEREUM_CONSENSUS_HOST: "https://beacon.${{ secrets.GCP_SEPOLIA_URL }}"
EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY: ${{ secrets.GCP_SEPOLIA_API_KEY }}
EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER: "X-goog-api-key"
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
MERGE_GROUP_BASE_REF: ${{ github.event.merge_group.base_ref }}
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
PR_COMMITS: ${{ github.event.pull_request.commits }}
PR_NUMBER: ${{ github.event.pull_request.number }}
GITHUB_REF_NAME: ${{ github.ref_name }}
GITHUB_ACTOR: ${{ github.actor }}
CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
AWS_OIDC_ROLE_ARN: ${{ secrets.AWS_OIDC_ROLE_ARN }}
# NOTE: $CI_MODE is set in the Determine CI Mode step.
run: ./.github/ci3.sh $CI_MODE
- name: Post-Actions
env:
SHOULD_SQUASH_MERGE: ${{ contains(github.event.pull_request.labels.*.name, 'ci-squash-and-merge') && '1' || '0' }}
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: ./.github/ci3_success.sh
- name: Notify Slack on backport CI failure
if: failure() && startsWith(github.event.pull_request.head.ref, 'backport-to-') && endsWith(github.event.pull_request.head.ref, '-staging')
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
PR_HTML_URL: ${{ github.event.pull_request.html_url }}
PR_NUM: ${{ github.event.pull_request.number }}
PR_TITLE: ${{ github.event.pull_request.title }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
if [ -n "${SLACK_BOT_TOKEN}" ]; then
TEXT="CI3 failed on backport PR: <${PR_HTML_URL}|#${PR_NUM} - ${PR_TITLE}>\n<${RUN_URL}|View Run>"
curl -X POST https://slack.com/api/chat.postMessage \
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-H "Content-type: application/json" \
--data "$(jq -n --arg c "#team-alpha" --arg t "$TEXT" '{channel:$c, text:$t}')"
fi
- name: Upload benchmarks
if: env.SHOULD_UPLOAD_BENCHMARKS == '1'
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
with: &ci_benchmark_args
name: Aztec Benchmarks
benchmark-data-dir-path: "bench/${{ env.BENCH_BRANCH }}"
tool: "customSmallerIsBetter"
output-file-path: ./bench-out/bench.json
gh-repository: github.com/AztecProtocol/benchmark-page-data
github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
auto-push: true
ref: ${{ github.event.pull_request.head.sha || github.sha }}
alert-threshold: "105%"
comment-on-alert: false
fail-on-alert: false
max-items-in-chart: 100
# End-to-end tests that target a network deployment.
# We run this every release (at minimum, nightly), or when explicitly requested.
# This task runs against a real testnet deployment. This uses resources on GCP (not AWS, thank free credit incentives).
# Runs two test sets in parallel.
ci-network-scenario:
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
strategy:
fail-fast: false
matrix:
test_set: ["1", "2"]
# We either run after a release (tag starting with v), or when the ci-network-scenario label is present in a PR.
# We exclude ci-release-pr test tags (v0.0.1-commit.*) which are only for testing the release process.
needs: ci
if: github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false && ((startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-commit.')) || contains(github.event.pull_request.labels.*.name, 'ci-network-scenario'))
steps:
- name: Remove label (one-time use)
if: github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'ci-network-scenario')
env:
GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-network-scenario --repo ${{ github.repository }} || true
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Configure AWS credentials (OIDC)
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
aws-region: us-east-2
role-session-name: ci3-network-scenario-${{ github.run_id }}
role-duration-seconds: 23400 # 6.5h – covers AWS_SHUTDOWN_TIME (360 min) + 30 min buffer
- name: Run Network Scenarios
timeout-minutes: 350
env:
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
# For pushing docker images (only for PR label, otherwise we use the tag)
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
RUN_ID: ${{ github.run_id }}
AWS_SHUTDOWN_TIME: 360
NO_SPOT: 1
GIT_REF: ${{ github.ref }}
GIT_REF_NAME: ${{ github.ref_name }}
HEAD_REF: ${{ github.head_ref || github.ref_name }}
run: |
# For release tags, use the release image; for PRs, omit to build and push to aztecdev
if [[ "$GIT_REF" == refs/tags/v* ]]; then
tag="$GIT_REF_NAME"
tag="${tag#v}"
major_version=$(./ci3/semver major "$tag")
namespace="v${major_version}-scenario"
docker_image="aztecprotocol/aztec:${tag}"
else
# branch name
namespace=pr-$(echo "$HEAD_REF" | sed 's/[^a-z0-9-]/-/g' | cut -c1-20 | sed 's/-*$//')
docker_image=""
fi
echo "NAMESPACE=$namespace" >> $GITHUB_ENV
set -x # print next line
./.github/ci3.sh network-scenarios next-scenario "$namespace" "$docker_image" "${{ matrix.test_set }}"
- name: Cleanup network resources
# Clean up if this is a CI label or nightly.
if: always() && (!startsWith(github.ref, 'refs/tags/v') || contains(github.ref_name, '-nightly.'))
env:
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
NO_SPOT: 1
run: |
./.github/ci3.sh network-teardown next-scenario "${NAMESPACE}-${{ matrix.test_set }}" || true
#############
# Benchmarks
#############
- name: Download deploy benchmarks
if: always() && startsWith(github.ref, 'refs/tags/v')
run: |
if ./ci.sh gh-deploy-bench; then
echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
fi
- name: Upload benchmarks
if: env.ENABLE_DEPLOY_BENCH == '1'
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
with: *ci_benchmark_args
- name: Notify Slack on failure
if: failure() && startsWith(github.ref, 'refs/tags/v')
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
GIT_REF: ${{ github.ref }}
RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
if [ -n "${SLACK_BOT_TOKEN}" ]; then
TEXT="Nightly Spartan Scenario FAILED in ci3.yml (nightly tag ${GIT_REF}: <${RUN_URL}|View Run>"
curl -X POST https://slack.com/api/chat.postMessage \
-H "Authorization: Bearer $SLACK_BOT_TOKEN" \
-H "Content-type: application/json" \
--data "$(jq -n --arg c "#alerts-next-scenario" --arg t "$TEXT" '{channel:$c, text:$t}')"
fi
# Spartan network benchmarks triggered on-demand from a PR label.
# Runs TPS and proving benchmarks in parallel, uploads results to a PR-specific
# path on the benchmark dashboard so nightly (bench/next) is never affected.
# One-time use: label is removed after the job runs.
ci-network-bench:
name: ${{ matrix.bench_type }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- bench_type: benchmark
ci3_cmd: network-bench
scenario: tps-scenario
namespace_suffix: bench
download_cmd: gh-spartan-bench
timeout: 300
- bench_type: proving-benchmark
ci3_cmd: network-proving-bench
scenario: prove-n-tps-fake
namespace_suffix: proving-bench
download_cmd: gh-spartan-proving-bench
timeout: 240
needs: ci
if: github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci-network-bench') && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
steps:
- name: Remove label (one-time use)
env:
GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-network-bench --repo ${{ github.repository }} || true
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Run Network Benchmarks
timeout-minutes: ${{ matrix.timeout }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
# For pushing docker images built from the PR
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
RUN_ID: ${{ github.run_id }}
AWS_SHUTDOWN_TIME: ${{ matrix.timeout }}
NO_SPOT: 1
run: |
namespace=pr-${{ github.event.pull_request.number }}-${{ matrix.namespace_suffix }}
echo "NAMESPACE=$namespace" >> $GITHUB_ENV
set -x
./.github/ci3.sh ${{ matrix.ci3_cmd }} ${{ matrix.scenario }} "$namespace"
- name: Cleanup network resources
if: always()
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
NO_SPOT: 1
run: ./.github/ci3.sh network-teardown ${{ matrix.scenario }} "${NAMESPACE}" || true
- name: Download benchmarks
if: always()
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
if ./ci.sh ${{ matrix.download_cmd }}; then
echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
fi
- name: Upload benchmarks
if: always() && env.ENABLE_DEPLOY_BENCH == '1'
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
with:
name: Spartan
benchmark-data-dir-path: "bench/pr-${{ github.event.pull_request.number }}"
tool: "customSmallerIsBetter"
output-file-path: ./bench-out/bench.json
gh-repository: github.com/AztecProtocol/benchmark-page-data
github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
auto-push: true
ref: ${{ github.event.pull_request.head.sha || github.sha }}
alert-threshold: "120%"
comment-on-alert: false
fail-on-alert: false
max-items-in-chart: 100
# KIND-based e2e tests that run on a local Kubernetes cluster.
# One-time use: label is removed after the job runs.
ci-network-kind:
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
needs: ci
if: github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci-network-kind')
timeout-minutes: 180 # 3 hours for KIND tests
steps:
- name: Remove label
env:
GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-network-kind --repo ${{ github.repository }}
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- name: Configure AWS credentials (OIDC)
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
aws-region: us-east-2
role-session-name: ci3-network-kind-${{ github.run_id }}
role-duration-seconds: 12600 # 3.5h – covers AWS_SHUTDOWN_TIME (180 min) + 30 min buffer
- name: Run KIND Test
env:
GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
RUN_ID: ${{ github.run_id }}
AWS_SHUTDOWN_TIME: 180
run: |
./.github/ci3.sh network-tests-kind