AI Eval #45
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI Eval | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| ai-eval: | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js 22 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| cache: 'npm' | |
| - name: Install dependencies | |
| run: npm install | |
| - name: Install Claude Code | |
| run: npm install -g @anthropic-ai/claude-code | |
| - name: Check Claude authentication | |
| id: claude-auth | |
| run: node scripts/check-claude-ai-eval-gate.js | |
| env: | |
| CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| - name: Run AI prompt evaluations | |
| if: steps.claude-auth.outputs.available == 'true' | |
| env: | |
| CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} | |
| run: npm run test:ai-eval | |
| - name: Upload AI eval responses | |
| if: always() && steps.claude-auth.outputs.available == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ai-eval-responses | |
| path: ai-evals/**/*.responses.md | |
| retention-days: 14 | |
| - name: Notify Slack on failure | |
| if: failure() && steps.claude-auth.outputs.available == 'true' | |
| uses: slackapi/slack-github-action@v2.1.0 | |
| with: | |
| method: chat.postMessage | |
| token: ${{ secrets.SLACK_BOT_TOKEN }} | |
| payload: | | |
| channel: "C0ASZRP7XRS" | |
| text: "🔴 AI Eval failed on `${{ github.ref_name }}` — <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" |