Cypher Health Check #606
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cypher Health Check | |
| on: | |
| push: | |
| branches: [main] | |
| schedule: | |
| - cron: '0 9 * * *' | |
| - cron: '0 12 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| ui-regression: | |
| if: github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check protected UI values | |
| run: bash scripts/check_ui_regression.sh | |
| deploy-verify: | |
| if: github.event_name == 'push' | |
| needs: [ui-regression] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Wait for backend rollout | |
| run: sleep 300 | |
| - name: Poll health endpoint | |
| run: | | |
| for i in $(seq 1 10); do | |
| echo "Attempt $i of 10..." | |
| RESPONSE=$(curl -s --max-time 30 "${{ secrets.GTD_API_URL }}/api/v1/health" || true) | |
| if [ -z "$RESPONSE" ]; then | |
| echo "Empty response. Waiting 30 seconds..." | |
| sleep 30 | |
| continue | |
| fi | |
| STATUS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('status','unknown'))" 2>/dev/null || echo "unknown") | |
| REASON=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('reason_code','unknown'))" 2>/dev/null || echo "unknown") | |
| STALE=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('stale_blocking_core_tables') or []))" 2>/dev/null || echo "1") | |
| LAST_RUN=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print((d.get('pipeline') or {}).get('last_run',{}).get('status','unknown'))" 2>/dev/null || echo "unknown") | |
| echo "Status=$STATUS reason=$REASON stale_core_tables=$STALE last_run=$LAST_RUN" | |
| if [ "$STATUS" = "ok" ] && [ "$STALE" = "0" ] && [ "$LAST_RUN" != "failed" ]; then | |
| echo "Core deploy verification passed." | |
| exit 0 | |
| fi | |
| echo "Waiting 30 seconds..." | |
| sleep 30 | |
| done | |
| echo "Deploy verify failed: core data health did not become ready." | |
| exit 1 | |
| - name: Send Telegram alert on failure | |
| if: failure() | |
| run: | | |
| MSG="🚨 Cypher CRITICAL - Deploy Failed"$'\n\n'"A code push to GitHub triggered a backend deploy, but the server is not responding after 10 minutes."$'\n\n'"WHAT THIS MEANS: The latest code change probably broke the Docker build or the server crashed on startup. The previous working version may still be running, or the server may be completely down."$'\n\n'"HOW TO FIX:"$'\n'"1. Connect to the Hetzner host and inspect container state and logs"$'\n'"2. Check for a failed image build or crash loop on the latest deploy"$'\n'"3. Read Docker/container logs - the error is usually near the bottom"$'\n'"4. If you see COPY failed or file not found - a file referenced in Dockerfile does not exist"$'\n'"5. If you see a Python error - the code has a bug"$'\n'"6. Open Claude Code and say Cypher deploy failed and paste the error from the logs" | |
| curl -s -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$(printf '%s' "$MSG" | python3 -c "import json,sys; print(json.dumps({'chat_id': '${{ secrets.TELEGRAM_CHAT_ID }}', 'text': sys.stdin.read()}))")" | |
| health-check: | |
| if: github.event_name != 'push' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check Cypher API health | |
| id: health | |
| run: | | |
| RESPONSE=$(curl -sf --max-time 30 "${{ secrets.GTD_API_URL }}/api/v1/health" 2>&1 || echo '{"status":"unreachable"}') | |
| echo "response=$RESPONSE" >> $GITHUB_OUTPUT | |
| echo "Raw response: $RESPONSE" | |
| STATUS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('status','unknown'))" 2>/dev/null || echo "unreachable") | |
| PIPELINE=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('pipeline',{}).get('status','unknown'))" 2>/dev/null || echo "unknown") | |
| AGE=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('data_age_hours','unknown'))" 2>/dev/null || echo "unknown") | |
| CARDS=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('cards',0))" 2>/dev/null || echo "0") | |
| INJ_SNAP=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print((d.get('core_table_freshness') or {}).get('injuries',{}).get('latest_date') or '')" 2>/dev/null || echo "") | |
| INJ_AGE=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print((d.get('core_table_freshness') or {}).get('injuries',{}).get('age_hours') or 'null')" 2>/dev/null || echo "null") | |
| INJ_STALE=$(echo "$RESPONSE" | python3 -c "import sys,json; d=json.load(sys.stdin); print('true' if (d.get('core_table_freshness') or {}).get('injuries',{}).get('stale') else 'false')" 2>/dev/null || echo "true") | |
| echo "status=$STATUS" >> $GITHUB_OUTPUT | |
| echo "pipeline=$PIPELINE" >> $GITHUB_OUTPUT | |
| echo "age=$AGE" >> $GITHUB_OUTPUT | |
| echo "cards=$CARDS" >> $GITHUB_OUTPUT | |
| echo "inj_snap=$INJ_SNAP" >> $GITHUB_OUTPUT | |
| echo "inj_age=$INJ_AGE" >> $GITHUB_OUTPUT | |
| echo "inj_stale=$INJ_STALE" >> $GITHUB_OUTPUT | |
| echo "Status: $STATUS | Pipeline: $PIPELINE | Data age: ${AGE}h | Cards: $CARDS" | |
| echo "Injuries: snapshot=$INJ_SNAP age=${INJ_AGE}h stale=$INJ_STALE" | |
| if [ "$STATUS" = "unreachable" ] || [ "$STATUS" = "error" ] || [ "$STATUS" = "degraded" ]; then | |
| echo "FAILING: status is $STATUS" | |
| exit 1 | |
| fi | |
| echo "Health check passed." | |
| - name: Injuries row-count guard | |
| id: injuries_guard | |
| run: | | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| BASE="${{ secrets.GTD_API_URL }}" | |
| echo "Triggering injuries refresh for $TODAY..." | |
| curl -s "${BASE}/api/v1/trigger?steps=injuries" || true | |
| for i in $(seq 1 12); do | |
| sleep 8 | |
| INJ_RESP=$(curl -s --max-time 30 "${BASE}/api/v1/injuries?sport=NBA" || echo '{}') | |
| SNAP=$(echo "$INJ_RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('snapshot_date') or '')" 2>/dev/null || echo "") | |
| TOTAL=$(echo "$INJ_RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('total_players') or 0)" 2>/dev/null || echo "0") | |
| echo "Attempt $i: snapshot=$SNAP total_players=$TOTAL" | |
| if [ "$SNAP" = "$TODAY" ] && [ "$TOTAL" -gt 0 ]; then | |
| echo "injuries_guard_passed: $TOTAL NBA injuries for $TODAY" | |
| echo "injuries_count=$TOTAL" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| done | |
| echo "FAIL: no injuries rows for today ($TODAY) after refresh" | |
| exit 1 | |
| - name: Send Telegram alert on failure | |
| if: failure() | |
| run: | | |
| STATUS="${{ steps.health.outputs.status }}" | |
| PIPELINE="${{ steps.health.outputs.pipeline }}" | |
| AGE="${{ steps.health.outputs.age }}" | |
| CARDS="${{ steps.health.outputs.cards }}" | |
| INJ_SNAP="${{ steps.health.outputs.inj_snap }}" | |
| INJ_AGE="${{ steps.health.outputs.inj_age }}" | |
| INJ_STALE="${{ steps.health.outputs.inj_stale }}" | |
| INJ_COUNT="${{ steps.injuries_guard.outputs.injuries_count }}" | |
| NOW=$(date -u '+%Y-%m-%d %H:%M UTC') | |
| TODAY=$(date -u '+%Y-%m-%d') | |
| if [ "$STATUS" = "unreachable" ]; then | |
| MSG="CRITICAL - Cypher Server Down - $NOW - The production backend is not responding. No data is being collected and the API is offline. Fix: Check Hetzner container logs and redeploy." | |
| elif [ "$INJ_STALE" = "true" ] || [ -z "$INJ_COUNT" ] || [ "$INJ_COUNT" = "0" ]; then | |
| MSG="WARNING - Cypher Injuries Data Missing - $NOW - Injuries ingestion failed or returned 0 rows for $TODAY. snapshot=$INJ_SNAP age=${INJ_AGE}h stale=$INJ_STALE - This means injury context is unavailable for card generation. Fix: Open Claude Code and say Cypher injuries ingestion is broken." | |
| elif [ "$STATUS" = "degraded" ]; then | |
| MSG="WARNING - Cypher Degraded - $NOW - STATUS: $STATUS | PIPELINE: $PIPELINE | DATA AGE: ${AGE}h | CARDS: $CARDS | INJURIES: snapshot=$INJ_SNAP age=${INJ_AGE}h - Fix: trigger manually at ${{ secrets.GTD_API_URL }}/api/v1/trigger or open Claude Code." | |
| else | |
| MSG="WARNING - Cypher Health Check Failed - $NOW - Status: $STATUS | Pipeline: $PIPELINE | Age: ${AGE}h | Cards: $CARDS | Injuries snapshot: $INJ_SNAP - Open Claude Code and paste this message." | |
| fi | |
| curl -s -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$(printf '%s' "$MSG" | python3 -c "import json,sys; print(json.dumps({'chat_id': '${{ secrets.TELEGRAM_CHAT_ID }}', 'text': sys.stdin.read()}))")" |