Drift Tests #128
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Drift Tests | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" # Daily 6am UTC | |
| pull_request: | |
| paths: | |
| - "src/agui-types.ts" | |
| - "src/__tests__/drift/agui-schema.drift.ts" | |
| workflow_dispatch: # Manual trigger | |
| permissions: | |
| contents: read | |
| jobs: | |
| agui-schema-drift: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: { persist-credentials: false } | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Clone ag-ui repo | |
| run: git clone --depth 1 https://github.com/ag-ui-protocol/ag-ui.git ../ag-ui | |
| - name: Run AG-UI schema drift test | |
| run: npx vitest run src/__tests__/drift/agui-schema.drift.ts --config vitest.config.drift.ts | |
| drift: | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| outputs: | |
| # Short, scannable Slack mrkdwn summary of which providers drifted and | |
| # what changed. Consumed by the `notify` job (which runs in a separate | |
| # workspace and so cannot read drift-report.json directly). | |
| summary: ${{ steps.summary.outputs.drift_summary }} | |
| # How many collector runs confirmed the drift (only set when drift | |
| # persisted across every retry). Lets the alert say "confirmed across N | |
| # runs". Empty on the common green/transient path. | |
| runs: ${{ steps.drift.outputs.drift_runs }} | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: { persist-credentials: false } | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version: 24 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| # Run the collector behind a "retry before alert" wrapper. A single | |
| # critical run can be a transient real-API hiccup (a streaming call | |
| # failing mid-flight), NOT a format change — so the wrapper re-runs the | |
| # collector and only reports critical drift (exit 2) if it PERSISTS | |
| # across every attempt. Any clean retry = transient = exit 0, no alert. | |
| # The common green path stays fast (no extra runs when the first is | |
| # clean). Exit codes mirror the collector contract so the steps below | |
| # are unchanged. | |
| - name: Run drift tests | |
| id: drift | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| run: | | |
| set +e | |
| npx tsx scripts/drift-retry.ts | |
| EXIT_CODE=$? | |
| set -e | |
| echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" | |
| if [ "$EXIT_CODE" -eq 2 ]; then | |
| : # critical drift persisted across retries, continue | |
| elif [ "$EXIT_CODE" -ne 0 ]; then | |
| echo "::error::Collector script crashed with exit code $EXIT_CODE" | |
| exit "$EXIT_CODE" | |
| fi | |
| - name: Upload drift report | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: drift-report | |
| path: drift-report.json | |
| if-no-files-found: warn | |
| retention-days: 30 | |
| # Distill drift-report.json into a short Slack summary and expose it as a | |
| # job output so the separate `notify` job can include it in the alert. | |
| # Runs before the failure step below so the summary is captured even when | |
| # critical drift is present. | |
| - name: Summarize drift for Slack | |
| id: summary | |
| if: always() | |
| run: npx tsx scripts/drift-slack-summary.ts | |
| - name: Fail if critical drift detected | |
| if: steps.drift.outputs.exit_code == '2' | |
| run: exit 1 | |
| notify: | |
| if: always() && github.event_name != 'pull_request' | |
| needs: [drift, agui-schema-drift] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| actions: read | |
| steps: | |
| - name: Check previous run status | |
| id: prev | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| PREV=$(gh run list --workflow="Drift Tests" --repo="${REPO}" --branch=main --limit=2 --json conclusion --jq '.[1].conclusion // "unknown"') | |
| echo "conclusion=$PREV" >> $GITHUB_OUTPUT | |
| - name: Notify Slack | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| PREV: ${{ steps.prev.outputs.conclusion }} | |
| DRIFT_RESULT: ${{ needs.drift.result }} | |
| AGUI_RESULT: ${{ needs.agui-schema-drift.result }} | |
| DRIFT_SUMMARY: ${{ needs.drift.outputs.summary }} | |
| DRIFT_RUNS: ${{ needs.drift.outputs.runs }} | |
| REPO: ${{ github.repository }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| if [ -z "$SLACK_WEBHOOK" ]; then echo "SLACK_WEBHOOK not set, skipping"; exit 0; fi | |
| HTTP_DRIFT=false | |
| AGUI_DRIFT=false | |
| INFRA_ERROR=false | |
| # Determine what happened in each job | |
| if [ "$DRIFT_RESULT" = "failure" ]; then | |
| HTTP_DRIFT=true | |
| elif [ "$DRIFT_RESULT" != "success" ] && [ "$DRIFT_RESULT" != "skipped" ]; then | |
| INFRA_ERROR=true | |
| fi | |
| if [ "$AGUI_RESULT" = "failure" ]; then | |
| AGUI_DRIFT=true | |
| elif [ "$AGUI_RESULT" != "success" ] && [ "$AGUI_RESULT" != "skipped" ]; then | |
| INFRA_ERROR=true | |
| fi | |
| RUN_URL="<https://github.com/${REPO}/actions/runs/${RUN_ID}|View run>" | |
| # Build the "what drifted" detail block from the drift job's summary | |
| # output. DRIFT_SUMMARY already contains real newlines (one bullet per | |
| # provider); we prepend a real newline so the detail sits on its own | |
| # lines under the headline. jq's --arg encodes these newlines into the | |
| # JSON payload correctly, so Slack renders real line breaks. | |
| # (No literal "\n" in any format() expression — that GitHub Actions | |
| # gotcha renders a visible backslash-n; here we use bash newlines.) | |
| NL=$'\n' | |
| DETAIL="" | |
| if [ -n "$DRIFT_SUMMARY" ]; then | |
| DETAIL="${NL}${DRIFT_SUMMARY}" | |
| fi | |
| # When HTTP API drift fired, it only alerts after the drift PERSISTED | |
| # across every retry of the collector. Note that confirmation so the | |
| # reader knows this was not a one-off transient blip. DRIFT_RUNS is | |
| # the count of runs that confirmed the drift (set by drift-retry.ts). | |
| CONFIRMED="" | |
| if [ "$DRIFT_RESULT" = "failure" ] && [ -n "$DRIFT_RUNS" ] && [ "$DRIFT_RUNS" -gt 1 ] 2>/dev/null; then | |
| CONFIRMED="${NL}_(confirmed across ${DRIFT_RUNS} runs)_" | |
| fi | |
| # Both types of drift | |
| if [ "$HTTP_DRIFT" = "true" ] && [ "$AGUI_DRIFT" = "true" ]; then | |
| EMOJI="🚨" | |
| MSG="*Drift detected* in aimock — HTTP API drift + AG-UI schema drift.${DETAIL}${CONFIRMED}${NL}${RUN_URL}" | |
| # HTTP API drift only | |
| elif [ "$HTTP_DRIFT" = "true" ]; then | |
| EMOJI="🚨" | |
| MSG="*HTTP API drift detected* in aimock — providers changed response formats.${DETAIL}${CONFIRMED}${NL}${RUN_URL}" | |
| # AG-UI schema drift only | |
| elif [ "$AGUI_DRIFT" = "true" ]; then | |
| EMOJI="🚨" | |
| MSG="*AG-UI schema drift detected* in aimock — canonical ag-ui types changed.${DETAIL}${NL}${RUN_URL}" | |
| # Infra failure — always notify | |
| elif [ "$INFRA_ERROR" = "true" ]; then | |
| EMOJI="❌" | |
| MSG="*Drift tests failed* (infra error). ${RUN_URL}" | |
| # Recovery: previous was bad, now good — notify once | |
| elif [ "$PREV" = "failure" ]; then | |
| EMOJI="✅" | |
| MSG="Drift tests passing again — all providers and AG-UI schema match." | |
| # Good → good — stay quiet | |
| else | |
| exit 0 | |
| fi | |
| PAYLOAD=$(jq -n --arg text "${EMOJI} ${MSG}" '{text: $text}') | |
| curl -sf -X POST "$SLACK_WEBHOOK" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$PAYLOAD" |