Harden validator persistence against disk exhaustion #421
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QSDM Go | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: qsdm-go-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - "QSDM/source/**" | |
| - "QSDM/scripts/**" | |
| - "QSDM/Dockerfile" | |
| - "QSDM/config/**" | |
| - "QSDM/deploy/**" | |
| - ".github/**" | |
| pull_request: | |
| paths: | |
| - "QSDM/source/**" | |
| - "QSDM/scripts/**" | |
| - "QSDM/Dockerfile" | |
| - "QSDM/config/**" | |
| - "QSDM/deploy/**" | |
| - ".github/**" | |
| jobs: | |
| build-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version-file: QSDM/source/go.mod | |
| # Both rebrand-guardrail scripts below (check-no-new-legacy-metrics.sh | |
| # and check-no-collapsed-env-preferred.sh) use ripgrep for the | |
| # filtered file walk. ubuntu-latest doesn't ship rg by default, so | |
| # without this install both scripts exit 2 with | |
| # "ripgrep (rg) is required" -- which is the root cause of the | |
| # build-test failures observed on sessions 72/73/74 (run ids | |
| # 25625305998 / 25626376835 / 25640513232). Install once, reuse. | |
| - name: Install ripgrep (required by rebrand guardrail scripts) | |
| run: sudo apt-get update -y && sudo apt-get install -y ripgrep | |
| # Guardrail for the qsdm_* -> qsdm_* Prometheus metric prefix | |
| # migration. Runs before build so it fails fast on regressions | |
| # without burning compute on a build that we're about to reject. | |
| - name: Check for new legacy qsdm_* metric names | |
| run: bash QSDM/scripts/check-no-new-legacy-metrics.sh | |
| # Sibling guardrail for the same rebrand window: catches | |
| # `_env_preferred("X", "X")` calls in the Python sidecar / | |
| # installer scripts, where a search-and-replace previously | |
| # collapsed the (preferred, legacy) pair into a single name and | |
| # silently killed the legacy fallback. Cheap grep-only check. | |
| - name: Check for collapsed _env_preferred(X, X) calls | |
| run: bash QSDM/scripts/check-no-collapsed-env-preferred.sh | |
| # Scripts live in QSDM/scripts; run from repo root so paths stay stable. | |
| - name: Build (no CGO) | |
| run: bash QSDM/scripts/go-build-no-cgo.sh /tmp/qsdm | |
| # Short tests cover API, monitoring (NVIDIA-lock/HMAC), dashboard, etc. (no GPU required). | |
| - name: Test (short, no CGO) | |
| run: bash QSDM/scripts/go-test-short-no-cgo.sh | |
| # Rust wasm_module + wazero validate_raw preflight (no CGO). | |
| wasm-preflight-smoke: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| targets: wasm32-unknown-unknown | |
| - name: Build wasm_module | |
| run: bash QSDM/scripts/build-wasm-module.sh | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version-file: QSDM/source/go.mod | |
| - name: Test generated release-cache maintenance guardrails | |
| shell: pwsh | |
| run: pwsh -NoProfile -File QSDM/scripts/test_generated_cache_maintenance.ps1 | |
| - name: Go test (wasm preflight integration) | |
| working-directory: QSDM/source | |
| env: | |
| CGO_ENABLED: "0" | |
| QSDM_METRICS_REGISTER_STRICT: "1" | |
| QSDM_WASM_PREFLIGHT_MODULE: ${{ github.workspace }}/QSDM/source/wasm_module/target/wasm32-unknown-unknown/release/wasm_module.wasm | |
| run: | | |
| unset CGO_CFLAGS CGO_LDFLAGS || true | |
| go test ./pkg/wasm/... -count=1 -timeout 10m -run TestTryPreflightP2PTransactionJSON_rustWasmModule | |
| wasm-preflight-smoke-windows: | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable | |
| with: | |
| targets: wasm32-unknown-unknown | |
| - name: Build wasm_module | |
| shell: bash | |
| run: bash QSDM/scripts/build-wasm-module.sh | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version-file: QSDM/source/go.mod | |
| - name: Go test (wasm preflight integration) | |
| shell: bash | |
| working-directory: QSDM/source | |
| env: | |
| CGO_ENABLED: "0" | |
| QSDM_METRICS_REGISTER_STRICT: "1" | |
| QSDM_WASM_PREFLIGHT_MODULE: ${{ github.workspace }}/QSDM/source/wasm_module/target/wasm32-unknown-unknown/release/wasm_module.wasm | |
| run: | | |
| unset CGO_CFLAGS CGO_LDFLAGS || true | |
| go test ./pkg/wasm/... -count=1 -timeout 10m -run TestTryPreflightP2PTransactionJSON_rustWasmModule | |
| govulncheck: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version-file: QSDM/source/go.mod | |
| # govulncheck does dataflow analysis on the ENTIRE module, which | |
| # means it has to compile every package including the CGO-gated | |
| # `pkg/crypto/dilithium.go` (imports `<oqs/oqs.h>`). Without | |
| # liboqs on the build path the job dies before govulncheck gets | |
| # a chance to report anything real: | |
| # | |
| # pkg/crypto/dilithium.go:6:10: fatal error: oqs/oqs.h: No such file or directory | |
| # | |
| # Mirror the exact setup test-cgo-transaction uses below (shared | |
| # cache key so we only build liboqs once per change to | |
| # rebuild_liboqs.sh across the whole workflow). | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential cmake git libssl-dev | |
| - name: Cache liboqs install | |
| id: liboqs-cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: QSDM/liboqs_install | |
| key: liboqs-install-${{ runner.os }}-${{ hashFiles('QSDM/scripts/rebuild_liboqs.sh') }} | |
| - name: Build liboqs | |
| if: steps.liboqs-cache.outputs.cache-hit != 'true' | |
| working-directory: QSDM | |
| run: chmod +x scripts/rebuild_liboqs.sh && ./scripts/rebuild_liboqs.sh | |
| - name: Run govulncheck (with allowlist filter) | |
| working-directory: QSDM/source | |
| env: | |
| CGO_ENABLED: "1" | |
| QSDM_METRICS_REGISTER_STRICT: "1" | |
| # Delegates to QSDM/scripts/govulncheck-filter.sh, which: | |
| # - runs `govulncheck -json ./...` | |
| # - fails ONLY when a finding is NOT on the explicit | |
| # unfixable-upstream allowlist (today: GO-2024-3218 in | |
| # go-libp2p-kad-dht, which has Fixed in: N/A) | |
| # - surfaces any NEW OSV id loudly so drift breaks CI | |
| # See the script header for the exact allowlist and rationale. | |
| run: | | |
| LIBOQS="${GITHUB_WORKSPACE}/QSDM/liboqs_install" | |
| if [ -d "$LIBOQS/lib64" ]; then L="$LIBOQS/lib64"; else L="$LIBOQS/lib"; fi | |
| export CGO_CFLAGS="-I${LIBOQS}/include" | |
| export CGO_LDFLAGS="-L${L} -loqs" | |
| export LD_LIBRARY_PATH="${L}:${LD_LIBRARY_PATH:-}" | |
| bash "${GITHUB_WORKSPACE}/QSDM/scripts/govulncheck-filter.sh" | |
| trivy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Trivy (filesystem — Go dependencies) | |
| uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 | |
| with: | |
| scan-type: fs | |
| scan-ref: QSDM/source | |
| severity: CRITICAL | |
| exit-code: "1" | |
| vuln-type: os,library | |
| - name: Trivy (filesystem — HIGH, non-blocking) | |
| uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 | |
| with: | |
| scan-type: fs | |
| scan-ref: QSDM/source | |
| severity: HIGH | |
| exit-code: "0" | |
| vuln-type: os,library | |
| - name: Trivy (misconfig — Dockerfile) | |
| uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 | |
| with: | |
| scan-type: config | |
| scan-ref: QSDM | |
| severity: CRITICAL,HIGH | |
| exit-code: "0" | |
| hide-progress: true | |
| # PoE / Dilithium: build liboqs (cached) and run transaction package tests with real signatures. | |
| test-cgo-transaction: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: QSDM | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version-file: QSDM/source/go.mod | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential cmake git libssl-dev | |
| - name: Cache liboqs install | |
| id: liboqs-cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: QSDM/liboqs_install | |
| key: liboqs-install-${{ runner.os }}-${{ hashFiles('QSDM/scripts/rebuild_liboqs.sh') }} | |
| - name: Build liboqs | |
| if: steps.liboqs-cache.outputs.cache-hit != 'true' | |
| run: chmod +x scripts/rebuild_liboqs.sh && ./scripts/rebuild_liboqs.sh | |
| - name: Test CGO packages (liboqs) | |
| working-directory: QSDM/source | |
| env: | |
| CGO_ENABLED: "1" | |
| QSDM_METRICS_REGISTER_STRICT: "1" | |
| run: | | |
| LIBOQS="${GITHUB_WORKSPACE}/QSDM/liboqs_install" | |
| if [ -d "$LIBOQS/lib64" ]; then L="$LIBOQS/lib64"; else L="$LIBOQS/lib"; fi | |
| export CGO_CFLAGS="-I${LIBOQS}/include" | |
| export CGO_LDFLAGS="-L${L} -loqs" | |
| export LD_LIBRARY_PATH="${L}:${LD_LIBRARY_PATH:-}" | |
| go test ./cmd/qsdm/transaction/... ./pkg/consensus/... ./pkg/crypto/... ./pkg/mesh3d/... ./cmd/migrate/... -count=1 -short -timeout 15m | |
| - name: Test API mesh companion (CGO + wallet) | |
| working-directory: QSDM/source | |
| env: | |
| CGO_ENABLED: "1" | |
| QSDM_METRICS_REGISTER_STRICT: "1" | |
| QSDM_PUBLISH_MESH_COMPANION: "1" | |
| run: | | |
| LIBOQS="${GITHUB_WORKSPACE}/QSDM/liboqs_install" | |
| if [ -d "$LIBOQS/lib64" ]; then L="$LIBOQS/lib64"; else L="$LIBOQS/lib"; fi | |
| export CGO_CFLAGS="-I${LIBOQS}/include" | |
| export CGO_LDFLAGS="-L${L} -loqs" | |
| export LD_LIBRARY_PATH="${L}:${LD_LIBRARY_PATH:-}" | |
| go test ./pkg/api/... -count=1 -short -timeout 10m -run TestSendTransaction_meshCompanionSecondBroadcast | |
| # Data race detector on hot pure-Go packages (slower than default job). Run via Actions → workflow_dispatch. | |
| race-hot-packages: | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 | |
| with: | |
| go-version-file: QSDM/source/go.mod | |
| - name: Install C toolchain (race detector needs CGO) | |
| run: sudo apt-get update && sudo apt-get install -y build-essential | |
| - name: Race tests (short, CGO) | |
| run: bash QSDM/scripts/race-hot-packages.sh | |
| # Full image: liboqs + CGO qsdm (matches production Dockerfile). Local builds need a running Docker engine. | |
| # Pushes to GHCR on default-branch push and on workflow_dispatch (not on pull_request). | |
| docker-image: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 | |
| # Fixed image name (avoids invalid OCI chars in repo names, e.g. '+'). | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 | |
| with: | |
| images: ghcr.io/${{ github.repository_owner }}/qsdm | |
| tags: | | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| type=sha,format=long | |
| - name: Log in to GitHub Container Registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build (and push when allowed) | |
| uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 | |
| with: | |
| context: QSDM | |
| file: QSDM/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event_name == 'workflow_dispatch' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max |