Update syncserver-postgres Docker image version (#2183) #61
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main Workflow - Lint, Build, Test | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - main | |
| tags: | |
| - "**" | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| python-version: | |
| description: "Python version to use" | |
| required: false | |
| default: "3.12" # PY_VER | |
| rust-version: | |
| description: "Rust version to use" | |
| required: false | |
| default: "1.91" # RUST_VER | |
| env: | |
| PYTHON_VERSION: ${{ inputs.python-version || '3.12' }} # PY_VER | |
| RUST_VERSION: ${{ inputs.rust-version || '1.91' }} # RUST_VER | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| permissions: {} # workflow-level default — deny all | |
| jobs: | |
| # Setup Python ====== | |
| python-env: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Cache pip and Poetry virtualenv | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| - name: Install Poetry | |
| run: pip3 install poetry | |
| - name: Install Python dependencies | |
| run: poetry install --with tokenserver-unit-tests,dev --no-interaction --no-ansi | |
| - name: Display Python Version Info | |
| run: | | |
| if [ "$(which python)" != "" ]; then python --version; fi | |
| uname -a | |
| cat /etc/os-release | |
| # Setup Rust ====== | |
| rust-env: | |
| runs-on: ubuntu-latest | |
| permissions: {} | |
| steps: | |
| - name: Cache Rust toolchain | |
| id: cache-rust-toolchain | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.rustup/toolchains | |
| ~/.rustup/update-hashes | |
| key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} | |
| - name: Install Rust toolchain | |
| if: steps.cache-rust-toolchain.outputs.cache-hit != 'true' | |
| run: rustup toolchain install ${{ env.RUST_VERSION }} --component rustfmt --component clippy --component llvm-tools-preview --no-self-update && rustup default ${{ env.RUST_VERSION }} | |
| - name: Display Rust Version Info | |
| shell: bash | |
| run: | | |
| if [ "$(which rustc)" != "" ]; then rustc --version; fi | |
| uname -a | |
| cat /etc/os-release | |
| # Python lint and format checks ====== | |
| python-checks: | |
| needs: python-env | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Restore pip and Poetry virtualenv | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| - name: Install Poetry | |
| run: pip3 install poetry | |
| - name: Python Format Check | |
| run: poetry run ruff format --diff tools | |
| - name: Python Lint Check | |
| run: poetry run ruff check tools | |
| - name: Python Docstring Check | |
| run: poetry run pydocstyle -es --count --config=pyproject.toml tools | |
| - name: Python Security Check | |
| run: poetry run bandit --quiet -r -c pyproject.toml tools | |
| - name: Python Type Check | |
| run: poetry run mypy --config-file=pyproject.toml tools | |
| # Rust lint and format checks ====== | |
| rust-checks: | |
| needs: rust-env | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Restore Rust toolchain | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.rustup/toolchains | |
| ~/.rustup/update-hashes | |
| key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} | |
| - name: Set Rust toolchain | |
| run: rustup default ${{ env.RUST_VERSION }} | |
| - name: Cache cargo-audit | |
| id: cache-cargo-audit | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-audit | |
| key: ${{ runner.os }}-cargo-audit-${{ hashFiles('.github/workflows/main-workflow.yml') }} | |
| - name: Install cargo-audit | |
| if: steps.cache-cargo-audit.outputs.cache-hit != 'true' | |
| run: cargo install --locked cargo-audit | |
| - name: Cache mdbook | |
| id: cache-mdbook | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cargo/bin/mdbook | |
| ~/.cargo/bin/mdbook-mermaid | |
| key: ${{ runner.os }}-mdbook-${{ hashFiles('Makefile') }} | |
| - name: Setup documentation checks | |
| if: steps.cache-mdbook.outputs.cache-hit != 'true' | |
| run: make doc-install-deps | |
| - name: Rust Format Check | |
| run: cargo fmt -- --check | |
| - name: Cargo Audit | |
| run: cargo audit | |
| - name: Documentation checks | |
| run: make doc-test | |
| # Rust clippy lint checker | |
| clippy: | |
| needs: rust-env | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| target: [spanner, mysql, postgres] | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Restore Rust toolchain | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.rustup/toolchains | |
| ~/.rustup/update-hashes | |
| key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} | |
| - name: Set Rust toolchain | |
| run: rustup default ${{ env.RUST_VERSION }} | |
| - name: Rust Clippy ${{ matrix.target }} | |
| run: make clippy_${{ matrix.target }} | |
| # Postgres unit tests ====== | |
| build-and-unit-test-postgres: | |
| needs: [rust-env, python-env] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| checks: write | |
| services: | |
| postgres: | |
| image: postgres:18.0 | |
| env: | |
| POSTGRES_USER: test | |
| POSTGRES_PASSWORD: test | |
| POSTGRES_DB: syncstorage | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd="pg_isready -U test" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=5 | |
| env: | |
| SYNC_SYNCSTORAGE__DATABASE_URL: postgres://test:test@127.0.0.1/syncstorage | |
| SYNC_TOKENSERVER__DATABASE_URL: postgres://test:test@127.0.0.1/tokenserver | |
| SYNC_TOKENSERVER__NODE_TYPE: postgres | |
| RUST_BACKTRACE: 1 | |
| RUST_TEST_THREADS: 1 | |
| CARGO_INCREMENTAL: "0" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Restore Rust toolchain | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.rustup/toolchains | |
| ~/.rustup/update-hashes | |
| key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} | |
| - name: Set Rust toolchain | |
| run: rustup default ${{ env.RUST_VERSION }} | |
| - name: Restore pip and Poetry virtualenv | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| - name: Cache Cargo build artifacts | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-postgres-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-postgres- | |
| - name: Create test results directory | |
| run: mkdir -p workflow/test-results | |
| - name: Install PostgreSQL client | |
| run: sudo apt-get update && sudo apt-get install -y postgresql-client | |
| - name: Create Tokenserver database | |
| run: | | |
| PGPASSWORD=test psql -U test -h 127.0.0.1 -d syncstorage -c 'CREATE DATABASE tokenserver;' | |
| - name: Create version.json | |
| run: | | |
| printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ | |
| "${GITHUB_SHA}" \ | |
| "${GITHUB_REF_NAME}" \ | |
| "${GITHUB_REPOSITORY_OWNER}" \ | |
| "${GITHUB_REPOSITORY_NAME}" \ | |
| "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| > version.json | |
| env: | |
| GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Cache cargo-nextest | |
| id: cache-cargo-nextest-postgres | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-nextest | |
| key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-nextest | |
| if: steps.cache-cargo-nextest-postgres.outputs.cache-hit != 'true' | |
| run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin | |
| - name: Cache cargo-llvm-cov | |
| id: cache-cargo-llvm-cov-postgres | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-llvm-cov | |
| key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-llvm-cov | |
| if: steps.cache-cargo-llvm-cov-postgres.outputs.cache-hit != 'true' | |
| run: cargo install --locked cargo-llvm-cov | |
| - name: Run unit tests with coverage | |
| run: make postgres_test_with_coverage | |
| - name: Run unit tests with coverage (quota enforced) | |
| run: make postgres_test_with_coverage | |
| env: | |
| SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 | |
| - name: Install Poetry | |
| run: pip3 install poetry | |
| - name: Run Postgres utils tests | |
| working-directory: tools/postgres | |
| run: | | |
| poetry install --no-interaction --no-ansi | |
| WORKFLOW=$(echo "${GITHUB_WORKFLOW}" | tr ' ' '-' | tr '[:upper:]' '[:lower:]') | |
| poetry run pytest test_purge_ttl.py -v --junit-xml="../../workflow/test-results/${GITHUB_RUN_NUMBER}__$(date +%s)__$(basename ${GITHUB_REPOSITORY})__${WORKFLOW}__postgres_utils__results.xml" | |
| env: | |
| SYNC_SYNCSTORAGE__DATABASE_URL: postgresql://test:test@127.0.0.1/syncstorage | |
| - name: Publish Test Report | |
| uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 | |
| if: always() | |
| with: | |
| name: Postgres Unit Tests | |
| path: workflow/test-results/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: postgres-test-results | |
| path: workflow/test-results/ | |
| # Upload to GCS on master | |
| - name: Authenticate to Google Cloud | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} | |
| - name: Upload JUnit results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/junit | |
| glob: "*.xml" | |
| parent: false | |
| process_gcloudignore: false | |
| - name: Upload coverage results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage | |
| glob: "*.json" | |
| parent: false | |
| process_gcloudignore: false | |
| # Docker build Postgres Image ====== | |
| build-postgres-image: | |
| runs-on: ubuntu-latest | |
| needs: [rust-env, python-env] | |
| permissions: | |
| contents: read | |
| actions: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Create version.json | |
| run: | | |
| printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ | |
| "${GITHUB_SHA}" \ | |
| "${GITHUB_REF_NAME}" \ | |
| "${GITHUB_REPOSITORY_OWNER}" \ | |
| "${GITHUB_REPOSITORY_NAME}" \ | |
| "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| > version.json | |
| env: | |
| GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Cache Docker image tar | |
| id: cache-postgres-image | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: /tmp/postgres-image.tar | |
| key: ${{ runner.os }}-postgres-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} | |
| - name: Set up Docker Buildx | |
| if: steps.cache-postgres-image.outputs.cache-hit != 'true' | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Build Postgres Docker image | |
| if: steps.cache-postgres-image.outputs.cache-hit != 'true' | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| push: false | |
| tags: app:build | |
| build-args: | | |
| SYNCSTORAGE_DATABASE_BACKEND=postgres | |
| TOKENSERVER_DATABASE_BACKEND=postgres | |
| outputs: type=docker,dest=/tmp/postgres-image.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Upload Docker image artifact | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: postgres-docker-image | |
| path: /tmp/postgres-image.tar | |
| retention-days: 1 | |
| # End-to-end tests for Postgres build ====== | |
| postgres-e2e-tests: | |
| runs-on: ubuntu-latest | |
| needs: build-postgres-image | |
| permissions: | |
| contents: read | |
| checks: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Download Docker image | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 | |
| with: | |
| name: postgres-docker-image | |
| path: /tmp | |
| - name: Load Docker image | |
| run: docker load --input /tmp/postgres-image.tar | |
| - name: Create test results directory | |
| run: mkdir -p workflow/test-results | |
| - name: Run Postgres e2e tests | |
| run: make docker_run_postgres_e2e_tests | |
| env: | |
| SYNCSTORAGE_RS_IMAGE: app:build | |
| - name: Publish E2E Test Report | |
| uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 | |
| if: always() | |
| with: | |
| name: Postgres E2E Tests | |
| path: workflow/test-results/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| - name: Upload e2e test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: postgres-e2e-test-results | |
| path: workflow/test-results/ | |
| # Upload to GCS on master | |
| - name: Authenticate to Google Cloud | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} | |
| - name: Upload e2e test results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/junit | |
| glob: "*.xml" | |
| parent: false | |
| process_gcloudignore: false | |
| # MySQL Unit tests ====== | |
| build-and-unit-test-mysql: | |
| runs-on: ubuntu-latest | |
| needs: [rust-env, python-env] | |
| permissions: | |
| contents: read | |
| checks: write | |
| services: | |
| mysql: | |
| image: mysql:8.0 | |
| env: | |
| MYSQL_ROOT_PASSWORD: password | |
| MYSQL_USER: test | |
| MYSQL_PASSWORD: test | |
| MYSQL_DATABASE: syncstorage | |
| ports: | |
| - 3306:3306 | |
| options: >- | |
| --health-cmd="mysqladmin ping" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=5 | |
| env: | |
| SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:test@127.0.0.1/syncstorage | |
| SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver | |
| SYNC_TOKENSERVER__NODE_TYPE: spanner | |
| RUST_BACKTRACE: 1 | |
| RUST_TEST_THREADS: 1 | |
| CARGO_INCREMENTAL: "0" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Restore Rust toolchain | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.rustup/toolchains | |
| ~/.rustup/update-hashes | |
| key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} | |
| - name: Set Rust toolchain | |
| run: rustup default ${{ env.RUST_VERSION }} | |
| - name: Restore pip and Poetry virtualenv | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| - name: Cache Cargo build artifacts | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-mysql-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-mysql- | |
| - name: Create test results directory | |
| run: mkdir -p workflow/test-results | |
| - name: Install MySQL client | |
| run: sudo apt-get update && sudo apt-get install -y default-mysql-client | |
| - name: Create Tokenserver database | |
| run: | | |
| mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' | |
| mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" | |
| - name: Create version.json | |
| run: | | |
| printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ | |
| "${GITHUB_SHA}" \ | |
| "${GITHUB_REF_NAME}" \ | |
| "${GITHUB_REPOSITORY_OWNER}" \ | |
| "${GITHUB_REPOSITORY_NAME}" \ | |
| "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| > version.json | |
| env: | |
| GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Cache cargo-nextest | |
| id: cache-cargo-nextest-mysql | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-nextest | |
| key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-nextest | |
| if: steps.cache-cargo-nextest-mysql.outputs.cache-hit != 'true' | |
| run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin | |
| - name: Cache cargo-llvm-cov | |
| id: cache-cargo-llvm-cov-mysql | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-llvm-cov | |
| key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-llvm-cov | |
| if: steps.cache-cargo-llvm-cov-mysql.outputs.cache-hit != 'true' | |
| run: cargo install --locked cargo-llvm-cov | |
| - name: Run unit tests with coverage | |
| run: make test_with_coverage | |
| - name: Run unit tests with coverage (quota enforced) | |
| run: make test_with_coverage | |
| env: | |
| SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1 | |
| - name: Publish Test Report | |
| uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 | |
| if: always() | |
| with: | |
| name: MySQL Unit Tests | |
| path: workflow/test-results/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: mysql-test-results | |
| path: workflow/test-results/ | |
| # Upload to GCS on master | |
| - name: Authenticate to Google Cloud | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} | |
| - name: Upload JUnit results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/junit | |
| glob: "*.xml" | |
| parent: false | |
| process_gcloudignore: false | |
| - name: Upload coverage results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage | |
| glob: "*.json" | |
| parent: false | |
| process_gcloudignore: false | |
| # Docker build for MySQL Image | |
| build-mysql-image: | |
| runs-on: ubuntu-latest | |
| needs: [rust-env, python-env] | |
| permissions: | |
| contents: read | |
| actions: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Create version.json | |
| run: | | |
| printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ | |
| "${GITHUB_SHA}" \ | |
| "${GITHUB_REF_NAME}" \ | |
| "${GITHUB_REPOSITORY_OWNER}" \ | |
| "${GITHUB_REPOSITORY_NAME}" \ | |
| "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| > version.json | |
| env: | |
| GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Cache Docker image tar | |
| id: cache-mysql-image | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: /tmp/mysql-image.tar | |
| key: ${{ runner.os }}-mysql-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} | |
| - name: Set up Docker Buildx | |
| if: steps.cache-mysql-image.outputs.cache-hit != 'true' | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Build MySQL Docker image | |
| if: steps.cache-mysql-image.outputs.cache-hit != 'true' | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| push: false | |
| tags: app:build | |
| build-args: | | |
| SYNCSTORAGE_DATABASE_BACKEND=mysql | |
| TOKENSERVER_DATABASE_BACKEND=mysql | |
| outputs: type=docker,dest=/tmp/mysql-image.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Upload Docker image artifact | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: mysql-docker-image | |
| path: /tmp/mysql-image.tar | |
| retention-days: 1 | |
| # End-to-end tests for MySQL ====== | |
| mysql-e2e-tests: | |
| runs-on: ubuntu-latest | |
| needs: build-mysql-image | |
| permissions: | |
| contents: read | |
| checks: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Download Docker image | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 | |
| with: | |
| name: mysql-docker-image | |
| path: /tmp | |
| - name: Load Docker image | |
| run: docker load --input /tmp/mysql-image.tar | |
| - name: Create test results directory | |
| run: mkdir -p workflow/test-results | |
| - name: Run MySQL e2e tests | |
| run: make docker_run_mysql_e2e_tests | |
| env: | |
| SYNCSTORAGE_RS_IMAGE: app:build | |
| - name: Publish E2E Test Report | |
| uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 | |
| if: always() | |
| with: | |
| name: MySQL E2E Tests | |
| path: workflow/test-results/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| - name: Upload e2e test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: mysql-e2e-test-results | |
| path: workflow/test-results/ | |
| # Upload to GCS on master | |
| - name: Authenticate to Google Cloud | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} | |
| - name: Upload e2e test results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/junit | |
| glob: "*.xml" | |
| parent: false | |
| process_gcloudignore: false | |
| # Spanner Unit tests ====== | |
| build-and-unit-test-spanner: | |
| runs-on: ubuntu-latest | |
| needs: [rust-env, python-env] | |
| permissions: | |
| contents: read | |
| checks: write | |
| services: | |
| spanner-emulator: | |
| image: gcr.io/cloud-spanner-emulator/emulator:1.4.0 | |
| ports: | |
| - 9010:9010 | |
| - 9020:9020 | |
| mysql: | |
| image: mysql:8.0 | |
| env: | |
| MYSQL_ROOT_PASSWORD: password | |
| MYSQL_USER: test | |
| MYSQL_PASSWORD: test | |
| MYSQL_DATABASE: syncstorage | |
| ports: | |
| - 3306:3306 | |
| options: >- | |
| --health-cmd="mysqladmin ping" | |
| --health-interval=10s | |
| --health-timeout=5s | |
| --health-retries=3 | |
| env: | |
| # The code expects a spanner URL like: | |
| SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database | |
| RUST_BACKTRACE: 1 | |
| RUST_TEST_THREADS: 1 | |
| CARGO_INCREMENTAL: "0" | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Restore Rust toolchain | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.rustup/toolchains | |
| ~/.rustup/update-hashes | |
| key: ${{ runner.os }}-rust-toolchain-${{ env.RUST_VERSION }} | |
| - name: Set Rust toolchain | |
| run: rustup default ${{ env.RUST_VERSION }} | |
| - name: Restore pip and Poetry virtualenv | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/pypoetry/virtualenvs | |
| key: ${{ runner.os }}-python-${{ hashFiles('pyproject.toml', 'poetry.lock') }} | |
| - name: Cache Cargo build artifacts | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-spanner-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-cargo-spanner- | |
| - name: Create test results directory | |
| run: mkdir -p workflow/test-results | |
| - name: Create version.json | |
| run: | | |
| printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ | |
| "${GITHUB_SHA}" \ | |
| "${GITHUB_REF_NAME}" \ | |
| "${GITHUB_REPOSITORY_OWNER}" \ | |
| "${GITHUB_REPOSITORY_NAME}" \ | |
| "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| > version.json | |
| env: | |
| GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Cache cargo-nextest | |
| id: cache-cargo-nextest-spanner | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-nextest | |
| key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-nextest | |
| if: steps.cache-cargo-nextest-spanner.outputs.cache-hit != 'true' | |
| run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin | |
| - name: Cache cargo-llvm-cov | |
| id: cache-cargo-llvm-cov-spanner | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: ~/.cargo/bin/cargo-llvm-cov | |
| key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install cargo-llvm-cov | |
| if: steps.cache-cargo-llvm-cov-spanner.outputs.cache-hit != 'true' | |
| run: cargo install --locked cargo-llvm-cov | |
| - name: Build workspace (spanner feature) | |
| run: | | |
| # Build with the spanner feature so any compile-time issues surface early | |
| cargo build --workspace --no-default-features --features=syncstorage-db/spanner --features=py_verifier | |
| - name: Wait for Spanner Emulator to be ready | |
| run: | | |
| echo "Waiting for Spanner emulator to be ready..." | |
| for i in {1..30}; do | |
| if curl -s http://localhost:9020/ > /dev/null 2>&1; then | |
| echo "Spanner emulator is ready (REST port 9020 responding)" | |
| break | |
| fi | |
| echo "Attempt $i/30: Spanner emulator not ready yet, waiting..." | |
| sleep 2 | |
| done | |
| # Verify both ports are accessible | |
| if ! curl -s http://localhost:9020/ > /dev/null 2>&1; then | |
| echo "ERROR: Cannot connect to Spanner emulator REST API at localhost:9020" | |
| exit 1 | |
| fi | |
| echo "Spanner emulator is fully ready" | |
| - name: Setup Spanner schema & instance (prepare-spanner.sh) | |
| env: | |
| SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database | |
| SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: http://localhost:9020 | |
| run: | | |
| # prepare-spanner.sh uses the REST API (port 9020) | |
| scripts/prepare-spanner.sh | |
| - name: Create Tokenserver database | |
| run: | | |
| mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' | |
| mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" | |
| - name: Run Spanner unit tests with coverage | |
| env: | |
| SYNC_SYNCSTORAGE__DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database | |
| SYNC_SYNCSTORAGE__SPANNER_EMULATOR_HOST: localhost:9010 | |
| SYNC_TOKENSERVER__DATABASE_URL: mysql://test:test@127.0.0.1/tokenserver | |
| SYNC_TOKENSERVER__NODE_TYPE: spanner | |
| RUST_TEST_THREADS: 1 | |
| run: make spanner_test_with_coverage | |
| - name: Publish Test Report | |
| uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 | |
| if: always() | |
| with: | |
| name: Spanner Unit Tests | |
| path: workflow/test-results/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: spanner-test-results | |
| path: workflow/test-results/ | |
| # Upload to GCS on master | |
| - name: Authenticate to Google Cloud | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} | |
| - name: Upload JUnit results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/junit | |
| glob: "*.xml" | |
| parent: false | |
| process_gcloudignore: false | |
| - name: Upload coverage results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/coverage | |
| glob: "*.json" | |
| parent: false | |
| process_gcloudignore: false | |
| # Docker image build for Spanner | |
| build-spanner-image: | |
| runs-on: ubuntu-latest | |
| needs: [rust-env, python-env] | |
| permissions: | |
| contents: read | |
| actions: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Create version.json | |
| run: | | |
| printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \ | |
| "${GITHUB_SHA}" \ | |
| "${GITHUB_REF_NAME}" \ | |
| "${GITHUB_REPOSITORY_OWNER}" \ | |
| "${GITHUB_REPOSITORY_NAME}" \ | |
| "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ | |
| > version.json | |
| env: | |
| GITHUB_REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| - name: Cache Docker image tar | |
| id: cache-spanner-image | |
| uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: /tmp/spanner-image.tar | |
| key: ${{ runner.os }}-spanner-image-${{ hashFiles('Dockerfile', 'Cargo.lock', '**/*.rs', '**/Cargo.toml', 'tools/**', 'scripts/**') }} | |
| - name: Set up Docker Buildx | |
| if: steps.cache-spanner-image.outputs.cache-hit != 'true' | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 | |
| - name: Build Spanner Docker image (local artifact) | |
| if: steps.cache-spanner-image.outputs.cache-hit != 'true' | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7 | |
| with: | |
| context: . | |
| push: false | |
| tags: app:build | |
| build-args: | | |
| SYNCSTORAGE_DATABASE_BACKEND=spanner | |
| MYSQLCLIENT_PKG=libmysqlclient-dev | |
| outputs: type=docker,dest=/tmp/spanner-image.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Upload Docker image artifact | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: spanner-docker-image | |
| path: /tmp/spanner-image.tar | |
| retention-days: 1 | |
| # End-to-end tests for Spanner ====== | |
| spanner-e2e-tests: | |
| runs-on: ubuntu-latest | |
| needs: build-spanner-image | |
| permissions: | |
| contents: read | |
| checks: write | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| persist-credentials: false | |
| - name: Download Docker image | |
| uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 | |
| with: | |
| name: spanner-docker-image | |
| path: /tmp | |
| - name: Load Docker image | |
| run: docker load --input /tmp/spanner-image.tar | |
| - name: Create test results directory | |
| run: mkdir -p workflow/test-results | |
| - name: Run Spanner e2e tests | |
| run: make docker_run_spanner_e2e_tests | |
| env: | |
| SYNCSTORAGE_RS_IMAGE: app:build | |
| - name: Publish E2E Test Report | |
| uses: dorny/test-reporter@a810f9bf83f2344124a920a7a0a85a6716e791f0 | |
| if: always() | |
| with: | |
| name: Spanner E2E Tests | |
| path: workflow/test-results/*.xml | |
| reporter: java-junit | |
| fail-on-error: false | |
| - name: Upload e2e test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: spanner-e2e-test-results | |
| path: workflow/test-results/ | |
| # Upload to GCS on master | |
| - name: Authenticate to Google Cloud | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| credentials_json: ${{ secrets.ETE_GCLOUD_SERVICE_KEY }} | |
| - name: Upload e2e test results to GCS | |
| if: github.ref == 'refs/heads/master' | |
| uses: google-github-actions/upload-cloud-storage@c0f6160ff80057923ff50e5e567695cea181ec23 # v2 | |
| with: | |
| path: workflow/test-results | |
| destination: ecosystem-test-eng-metrics/syncstorage-rs/junit | |
| glob: "*.xml" | |
| parent: false | |
| process_gcloudignore: false |