DR drill — exercise (orchestrated) #20
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: DR drill — exercise (orchestrated) | ||
| # Full scratch cycle (RFC-082 / #752 / #799): | ||
| # drill-infra-plan (fmt-check, validate, tofu plan) | ||
| # → drill-infra-apply (uploads encrypted state artifact; no git commit needed | ||
| # for the rest of this run) | ||
| # → drill-deploy → drill-restore-corpus → drill-e2e → drill-stack-playwright | ||
| # → ``drill-tfstate-bridge`` copies apply state to artifact ``drill-tfstate-for-teardown`` | ||
| # on this caller run (reusable-workflow artifacts are not always visible to | ||
| # later ``workflow_call`` jobs). ``drill-infra-destroy`` downloads that (or | ||
| # apply output as fallback), then ``tofu destroy``, hcloud sweep, Tailscale cleanup. | ||
| # → ``assert-post-conditions`` fails the run when any step red or Hetzner orphans remain. | ||
| # | ||
| # Schedule: Wednesdays 02:00 UTC (``github.event_name == schedule`` skips typed confirm). | ||
| # Sister cadence: compose backup-restore verify Sundays 04:00 UTC (#798). | ||
| # | ||
| # Typed gate (manual only): **DRILL_FULL_CYCLE** (alias **DRILL_EXERCISE**). Each reusable | ||
| # workflow keeps its own ``environment: drill`` rules where configured. | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| confirm: | ||
| # required:false so scheduled cron runs (no inputs) don't trip GHA | ||
| # workflow-load validation. The ``gate`` job below still enforces the | ||
| # typed value for ``workflow_dispatch`` events. | ||
| description: 'Type "DRILL_FULL_CYCLE" (or DRILL_EXERCISE) for plan→apply→app→always destroy' | ||
| required: false | ||
| default: '' | ||
| backup_tag: | ||
| description: "Restore: release tag (empty = newest compatible snapshot-prod-*; sibling snapshot.manifest.json required)" | ||
| required: false | ||
| default: '' | ||
| backup_repo: | ||
| description: "Restore: GitHub owner/repo for backup releases" | ||
| required: false | ||
| default: chipi/podcast_scraper-backup | ||
| override_image_sha: | ||
| description: "Deploy: image :sha-<7> (blank = this workflow run SHA short)" | ||
| required: false | ||
| default: '' | ||
| schedule: | ||
| - cron: '0 2 * * 3' # Wednesdays 02:00 UTC (#799) | ||
| permissions: | ||
| contents: read | ||
| actions: write | ||
| # drill-infra-plan (reusable) posts PR comments when permissions are evaluated for workflow_call. | ||
| pull-requests: write | ||
| concurrency: | ||
| group: drill-exercise | ||
| cancel-in-progress: false | ||
| jobs: | ||
| gate: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: "DR drill — scheduled run (skip typed confirm)" | ||
| if: github.event_name == 'schedule' | ||
| run: echo "Scheduled DR drill — DRILL_FULL_CYCLE implied (#799)." | ||
| - name: "DR drill — confirm DRILL_FULL_CYCLE" | ||
| if: github.event_name == 'workflow_dispatch' | ||
| run: | | ||
| C="${{ inputs.confirm }}" | ||
| if [ "$C" != "DRILL_FULL_CYCLE" ] && [ "$C" != "DRILL_EXERCISE" ]; then | ||
| echo "::error::Must type DRILL_FULL_CYCLE or DRILL_EXERCISE. Got: '$C'" | ||
| exit 1 | ||
| fi | ||
| plan: | ||
| needs: gate | ||
| uses: ./.github/workflows/drill-infra-plan.yml | ||
| secrets: inherit | ||
| apply: | ||
| needs: plan | ||
| uses: ./.github/workflows/drill-infra-apply.yml | ||
| secrets: inherit | ||
| with: | ||
| skip_confirm: true | ||
| drill-tfstate-bridge: | ||
| # Reusable-workflow artifacts are not always visible to a later ``workflow_call`` | ||
| # in the same parent run. Bridge copies apply output onto this caller run as | ||
| # ``drill-tfstate-for-teardown`` so ``drill-infra-destroy`` can download it. | ||
| needs: apply | ||
| if: always() | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: read | ||
| actions: write | ||
| steps: | ||
| - name: "DR drill — tfstate bridge — download apply artifact" | ||
| env: | ||
| GH_TOKEN: ${{ github.token }} | ||
| run: | | ||
| set -euo pipefail | ||
| name="terraform-state-after-apply-drill" | ||
| dest=".bridge-in" | ||
| repo="${{ github.repository }}" | ||
| rid="${{ github.run_id }}" | ||
| n="$(gh api "repos/${repo}/actions/runs/${rid}/artifacts" \ | ||
| --jq "[.artifacts[] | select(.name==\"${name}\")] | length")" | ||
| if [ "${n:-0}" -eq 0 ]; then | ||
| echo "::notice::No ${name} to bridge (apply did not upload or not finished)." | ||
| exit 0 | ||
| fi | ||
| mkdir -p "${dest}" | ||
| gh run download "${rid}" -R "${repo}" -n "${name}" -D "${dest}" | ||
| - name: "DR drill — tfstate bridge — republish encrypted state for teardown" | ||
| id: repub | ||
| run: | | ||
| set -euo pipefail | ||
| mkdir -p .bridge-out | ||
| FOUND=$(find .bridge-in -type f \ | ||
| \( -name 'terraform.tfstate.enc.drill' -o -name '*.enc.drill' \) \ | ||
| 2>/dev/null | head -1 || true) | ||
| if [ -n "$FOUND" ] && [ -f "$FOUND" ]; then | ||
| cp "$FOUND" .bridge-out/terraform.tfstate.enc.drill | ||
| echo "upload=yes" >> "$GITHUB_OUTPUT" | ||
| echo "Bridged drill encrypted state for teardown." | ||
| else | ||
| echo "upload=no" >> "$GITHUB_OUTPUT" | ||
| echo "No apply artifact to bridge (standalone destroy will use git state)." | ||
| fi | ||
| - name: "DR drill — tfstate bridge — upload teardown artifact" | ||
| if: steps.repub.outputs.upload == 'yes' | ||
| uses: actions/upload-artifact@v7 | ||
| with: | ||
| name: drill-tfstate-for-teardown | ||
| path: .bridge-out/terraform.tfstate.enc.drill | ||
| retention-days: 30 | ||
| deploy: | ||
|
Check failure on line 138 in .github/workflows/drill-exercise.yml
|
||
| needs: apply | ||
| uses: ./.github/workflows/drill-deploy.yml | ||
| secrets: inherit | ||
| with: | ||
| override_image_sha: ${{ inputs.override_image_sha }} | ||
| restore: | ||
| needs: deploy | ||
| uses: ./.github/workflows/drill-restore-corpus.yml | ||
| secrets: inherit | ||
| with: | ||
| backup_tag: ${{ inputs.backup_tag }} | ||
| backup_repo: ${{ inputs.backup_repo }} | ||
| skip_confirm: true | ||
| smoke: | ||
| needs: restore | ||
| uses: ./.github/workflows/drill-e2e.yml | ||
| secrets: inherit | ||
| with: | ||
| confirm: DRILL_SMOKE | ||
| stack-playwright: | ||
| needs: smoke | ||
| uses: ./.github/workflows/drill-stack-playwright.yml | ||
| secrets: inherit | ||
| with: | ||
| skip_confirm: true | ||
| finalize: | ||
| needs: [plan, apply, deploy, restore, smoke, stack-playwright] | ||
| if: always() | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: "DR drill — finalize — log cycle outcomes (always, before destroy)" | ||
| run: | | ||
| echo "plan=${{ needs.plan.result }} apply=${{ needs.apply.result }} deploy=${{ needs.deploy.result }} restore=${{ needs.restore.result }} smoke=${{ needs.smoke.result }} stack-playwright=${{ needs.stack-playwright.result }}" | ||
| echo "Proceeding to drill-infra-destroy (tofu destroy) regardless of outcomes above." | ||
| teardown: | ||
| needs: [finalize, drill-tfstate-bridge] | ||
| if: always() | ||
| uses: ./.github/workflows/drill-infra-destroy.yml | ||
| secrets: inherit | ||
| with: | ||
| skip_confirm: true | ||
| assert-post-conditions: | ||
| needs: [plan, apply, deploy, restore, smoke, stack-playwright, teardown] | ||
| if: always() | ||
| runs-on: ubuntu-latest | ||
| environment: drill | ||
| steps: | ||
| - name: "DR drill — assert cycle outcomes (fail on red)" | ||
| run: | | ||
| set -euo pipefail | ||
| fail=0 | ||
| check() { | ||
| local name="$1" | ||
| local result="$2" | ||
| case "$result" in | ||
| success) | ||
| echo "$name: success" | ||
| ;; | ||
| skipped) | ||
| echo "::error::$name: skipped (upstream failure or not run)" | ||
| fail=1 | ||
| ;; | ||
| failure) | ||
| echo "::error::$name: failure" | ||
| fail=1 | ||
| ;; | ||
| *) | ||
| echo "::error::$name: $result" | ||
| fail=1 | ||
| ;; | ||
| esac | ||
| } | ||
| check plan "${{ needs.plan.result }}" | ||
| check apply "${{ needs.apply.result }}" | ||
| check deploy "${{ needs.deploy.result }}" | ||
| check restore "${{ needs.restore.result }}" | ||
| check smoke "${{ needs.smoke.result }}" | ||
| check stack-playwright "${{ needs.stack-playwright.result }}" | ||
| check teardown "${{ needs.teardown.result }}" | ||
| if [ "$fail" -ne 0 ]; then | ||
| echo "::error::DR drill post-conditions failed — see job results above." | ||
| exit 1 | ||
| fi | ||
| echo "Post-condition evidence map (#799):" | ||
| echo " provision: apply=success (tofu apply + state artifact uploaded)" | ||
| echo " deploy: deploy=success (deploy.sh + drill-deploy :8080 health adapter)" | ||
| echo " restore: restore=success (snapshot-prod-* → /srv/podcast-scraper/corpus)" | ||
| echo " corpus readable + episodes>0: smoke=success (post_deploy_smoke EXPECT_POPULATED=1)" | ||
| echo " destroy clean: teardown=success + Hetzner orphan check below" | ||
| echo "All DR drill post-conditions green." | ||
| - name: "DR drill — checkout (orphan check script)" | ||
| uses: actions/checkout@v6 | ||
| with: | ||
| fetch-depth: 1 | ||
| - name: "DR drill — assert no Hetzner orphans after destroy" | ||
| env: | ||
| HCLOUD_TOKEN_DRILL: ${{ secrets.HCLOUD_TOKEN_DRILL }} | ||
| run: | | ||
| set -euo pipefail | ||
| bash scripts/ops/delete_drill_hetzner_orphans.sh --check-only | ||
| alert-on-failure: | ||
| needs: [assert-post-conditions] | ||
| if: failure() && github.event_name == 'schedule' | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: "DR drill — webhook alert (scheduled failure)" | ||
| env: | ||
| SMOKE_WEBHOOK_URL: ${{ secrets.SMOKE_WEBHOOK_URL }} | ||
| run: | | ||
| set -euo pipefail | ||
| if [ -z "${SMOKE_WEBHOOK_URL:-}" ]; then | ||
| echo "::notice::SMOKE_WEBHOOK_URL not set; skipping webhook alert." | ||
| exit 0 | ||
| fi | ||
| MSG="DR drill (#799): FAILED on scheduled run \`${GITHUB_SHA:0:7}\` — see drill-exercise workflow." | ||
| curl -fsS -X POST -H "Content-Type: application/json" \ | ||
| -d "{\"text\":\"${MSG}\"}" \ | ||
| "$SMOKE_WEBHOOK_URL" || echo "::warning::webhook ping failed (non-fatal)" | ||
