|
| 1 | +name: Promptfoo Redteam Scan |
| 2 | +on: |
| 3 | + workflow_dispatch: |
| 4 | + pull_request: |
| 5 | + push: |
| 6 | + branches: |
| 7 | + - main |
| 8 | + |
| 9 | +env: |
| 10 | + NODE_VERSION: '22' |
| 11 | + PROMPTFOO_HOST: ${{ vars.PROMPTFOO_HOST }} |
| 12 | + SCAN_TEMPLATE_ID: ${{ vars.SCAN_TEMPLATE_ID }} |
| 13 | + TARGET_ID: ${{ vars.TARGET_ID }} |
| 14 | + PROMPTFOO_API_KEY: ${{ secrets.PROMPTFOO_API_KEY }} |
| 15 | + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} |
| 16 | + PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo |
| 17 | + PORT: 4000 |
| 18 | + |
| 19 | +jobs: |
| 20 | + evaluate: |
| 21 | + runs-on: ubuntu-latest |
| 22 | + steps: |
| 23 | + - uses: actions/checkout@v4 |
| 24 | + |
| 25 | + - uses: actions/setup-node@v4 |
| 26 | + with: |
| 27 | + node-version: ${{ env.NODE_VERSION }} |
| 28 | + cache: 'npm' |
| 29 | + |
| 30 | + - name: Install project dependencies |
| 31 | + run: | |
| 32 | + npm install |
| 33 | +
|
| 34 | + - name: Start dev server |
| 35 | + run: | |
| 36 | + nohup npm run dev > /dev/null 2>&1 & |
| 37 | + sleep 10 |
| 38 | + echo "Dev server started in background" |
| 39 | +
|
| 40 | + - name: Install dependencies |
| 41 | + run: | |
| 42 | + npm install -g promptfoo@latest |
| 43 | +
|
| 44 | + - name: Cache promptfoo |
| 45 | + uses: actions/cache@v4 |
| 46 | + with: |
| 47 | + path: ~/.cache/promptfoo |
| 48 | + key: ${{ runner.os }}-promptfoo-${{ hashFiles('prompts/**') }} |
| 49 | + restore-keys: | |
| 50 | + ${{ runner.os }}-promptfoo- |
| 51 | +
|
| 52 | + - name: Promptfoo Login |
| 53 | + run: | |
| 54 | + promptfoo auth login --host "$PROMPTFOO_HOST" --api-key "$PROMPTFOO_API_KEY" |
| 55 | +
|
| 56 | + - name: Run redteam |
| 57 | + run: | |
| 58 | + promptfooredteam run -c "$SCAN_TEMPLATE_ID" -t "$TARGET_ID" \ |
| 59 | + -c promptfooconfig.yaml \ |
| 60 | + --share \ |
| 61 | + -o results.json \ |
| 62 | + -o report.html |
| 63 | +
|
| 64 | + - name: Check quality gate |
| 65 | + run: | |
| 66 | + FAILURES=$(jq '.results.stats.failures' results.json) |
| 67 | + EVAL_ID=$(jq '.evalId' results.json) |
| 68 | + EVAL_URL=$(jq -r '.shareableUrl' results.json) |
| 69 | + echo "Eval ID: $EVAL_ID" |
| 70 | + echo "URL: $EVAL_URL" |
| 71 | + if [ "$FAILURES" -gt 0 ]; then |
| 72 | + echo "❌ Eval failed with $FAILURES failures" |
| 73 | + exit 1 |
| 74 | + fi |
| 75 | + echo "✅ All tests passed!" |
| 76 | +
|
| 77 | + - name: Upload results |
| 78 | + if: always() |
| 79 | + uses: actions/upload-artifact@v4 |
| 80 | + with: |
| 81 | + name: eval-results |
| 82 | + path: | |
| 83 | + results.json |
| 84 | + report.html |
| 85 | +
|
0 commit comments