Post-Deploy Smoke Test #176
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Post-Deploy Smoke Test | |
| # Runs the production smoke test after a Vercel deploy completes for any push | |
| # to main that touched site-affecting files, plus once daily to catch external | |
| # drift between pushes. Push-triggered runs post a commit status next to the | |
| # Vercel deploy; push and scheduled runs both open (or update) a tracking issue | |
| # if anything fails. | |
| # | |
| # Trigger paths are intentionally broad - it's cheaper to run smoke unnecessarily | |
| # than to miss a real regression. Add new paths here if you introduce a new | |
| # build input that can affect rendered pages. | |
| # | |
| # Override the target URL for manual runs via workflow_dispatch (handy for | |
| # testing against a Vercel preview deployment before merging). | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' | |
| on: | |
| schedule: | |
| # Daily at 07:00 UTC, roughly 30 minutes after build-pages.yml finishes. | |
| - cron: '0 7 * * *' | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'data/**' | |
| - 'projects/**' | |
| - 'guide/**' | |
| - 'dev/**' | |
| - 'lists/**' | |
| - '*.html' | |
| - '*.xml' | |
| - 'llms.txt' | |
| - 'robots.txt' | |
| - 'index.html' | |
| - 'vercel.json' | |
| - 'lib/**' | |
| - 'api/**' | |
| - 'scripts/build-pages.js' | |
| - 'scripts/build-chunks.js' | |
| - 'scripts/smoke-test-prod.js' | |
| - '.github/workflows/post-deploy-smoke.yml' | |
| workflow_dispatch: | |
| inputs: | |
| base: | |
| description: 'Base URL to test (default https://hermesatlas.com)' | |
| required: false | |
| default: 'https://hermesatlas.com' | |
| sample: | |
| description: 'Number of project pages to sample' | |
| required: false | |
| default: '25' | |
| target_sha: | |
| description: 'Commit SHA whose Vercel deployment must be ready before smoke' | |
| required: false | |
| default: '' | |
| # Serialize - we don't want two smoke runs racing each other against the same | |
| # deploy. Cancel pending duplicate runs since only the latest deploy matters. | |
| concurrency: | |
| group: post-deploy-smoke | |
| cancel-in-progress: true | |
| jobs: | |
| smoke: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| issues: write | |
| statuses: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '24' | |
| - name: Install dependencies | |
| run: npm ci --no-audit --no-fund | |
| # Vercel posts a commit status with context="Vercel" once the deploy | |
| # completes. Poll for it. On workflow_dispatch we skip the wait because | |
| # the user-supplied base URL might already be live. | |
| - name: Wait for Vercel deploy to be ready | |
| if: github.event_name == 'push' || inputs.target_sha != '' | |
| uses: actions/github-script@v7 | |
| env: | |
| TARGET_SHA: ${{ inputs.target_sha }} | |
| with: | |
| script: | | |
| const sha = process.env.TARGET_SHA || context.sha; | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const MAX_WAIT_MS = 10 * 60 * 1000; // 10 minutes | |
| const INTERVAL_MS = 15 * 1000; | |
| const start = Date.now(); | |
| let lastState = null; | |
| core.info(`Polling Vercel deploy status for ${sha} (max 10min)`); | |
| while (Date.now() - start < MAX_WAIT_MS) { | |
| const { data } = await github.rest.repos.getCombinedStatusForRef({ | |
| owner, repo, ref: sha, | |
| }); | |
| const vercel = data.statuses.find(s => s.context === 'Vercel'); | |
| const state = vercel ? vercel.state : 'no-status-yet'; | |
| if (state !== lastState) { | |
| core.info(`Vercel status: ${state}`); | |
| lastState = state; | |
| } | |
| if (state === 'success') { | |
| core.info(`Vercel deploy ready in ${Math.round((Date.now()-start)/1000)}s`); | |
| return; | |
| } | |
| if (state === 'failure' || state === 'error') { | |
| core.setFailed(`Vercel deploy failed for ${sha} - skipping smoke test`); | |
| return; | |
| } | |
| await new Promise(r => setTimeout(r, INTERVAL_MS)); | |
| } | |
| core.setFailed(`Timed out waiting for Vercel deploy (10min)`); | |
| # A catalog deployment can invalidate the previous snapshot (for example, | |
| # after a removed repository is pruned). Seed a validated replacement | |
| # before checking the semantic `stale`/`complete` contract, rather than | |
| # waiting up to six hours for the scheduled refresh. | |
| - name: Seed stars snapshot before semantic smoke | |
| if: github.event_name == 'push' || inputs.target_sha != '' | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| CRON_SECRET: ${{ secrets.CRON_SECRET }} | |
| run: node scripts/push-stars-snapshot.js | |
| - name: Run smoke test | |
| id: smoke | |
| # Pass dispatch inputs via env, never interpolated straight into the | |
| # shell command (a crafted `base`/`sample` could otherwise inject shell). | |
| # | |
| # `set -o pipefail`: GitHub's default run shell is `bash -e` WITHOUT | |
| # pipefail, so `node ... | tee` otherwise reports tee's exit code and a | |
| # failing smoke run becomes a green check (bit the PR-preview gate for | |
| # weeks before being caught on 2026-07-07). | |
| env: | |
| SMOKE_BASE: ${{ inputs.base || 'https://hermesatlas.com' }} | |
| SMOKE_SAMPLE: ${{ inputs.sample || '25' }} | |
| run: | | |
| set -o pipefail | |
| node scripts/smoke-test-prod.js \ | |
| --base "$SMOKE_BASE" \ | |
| --sample "$SMOKE_SAMPLE" \ | |
| --continue \ | |
| | tee smoke-output.txt | |
| - name: Close smoke alert after recovery | |
| if: success() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = 'Smoke test failure on production'; | |
| const { data: issues } = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'workflow-issue', | |
| per_page: 100, | |
| }); | |
| const issue = issues.find(item => item.title === title); | |
| if (!issue) return; | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issue.number, | |
| body: `Recovered in ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}.`, | |
| }); | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issue.number, | |
| state: 'closed', | |
| state_reason: 'completed', | |
| }); | |
| - name: Post commit status (success) | |
| if: success() && github.event_name == 'push' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: context.sha, | |
| state: 'success', | |
| context: 'Smoke Test (production)', | |
| description: 'All production smoke checks passed', | |
| target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| }); | |
| - name: Post commit status (failure) and open / update tracking issue | |
| if: failure() && github.event_name != 'workflow_dispatch' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let smokeOutput = ''; | |
| try { | |
| smokeOutput = fs.readFileSync('smoke-output.txt', 'utf8'); | |
| } catch (e) { | |
| smokeOutput = '(smoke-output.txt not produced - check the job log)'; | |
| } | |
| // Commit status only makes sense for push-triggered deploy checks. | |
| if (context.eventName === 'push') { | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: context.sha, | |
| state: 'failure', | |
| context: 'Smoke Test (production)', | |
| description: 'Production smoke test failed - see Actions log', | |
| target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| }); | |
| } | |
| // Find or open the tracking issue. We keep one issue open at a time | |
| // so successive failures comment on the same thread instead of | |
| // spamming N separate issues. Uses the repo-standard workflow-issue | |
| // label + exact-title dedup (same pattern as build-pages et al.); | |
| // the old smoke-test-failure label was never created in the repo. | |
| const title = 'Smoke test failure on production'; | |
| const issues = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'workflow-issue', | |
| }); | |
| const match = issues.data.find(i => i.title === title); | |
| const body = [ | |
| context.eventName === 'schedule' | |
| ? 'Daily production smoke test failed.' | |
| : `Production smoke test failed on commit ${context.sha.slice(0,7)}.`, | |
| '', | |
| `**Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| '', | |
| '```', | |
| smokeOutput.slice(-3500), | |
| '```', | |
| ].join('\n'); | |
| if (match) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: match.number, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title, | |
| body, | |
| labels: ['workflow-issue'], | |
| }); | |
| } |