Skip to content

Commit b7c7da5

Browse files
author
James Hiester
committed
adding github action
1 parent ea14fae commit b7c7da5

File tree

1 file changed

+85
-0
lines changed

1 file changed

+85
-0
lines changed

.github/workflows/redteam-scan.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Promptfoo Redteam Scan
2+
on:
3+
workflow_dispatch:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
env:
10+
NODE_VERSION: '22'
11+
PROMPTFOO_HOST: ${{ vars.PROMPTFOO_HOST }}
12+
SCAN_TEMPLATE_ID: ${{ vars.SCAN_TEMPLATE_ID }}
13+
TARGET_ID: ${{ vars.TARGET_ID }}
14+
PROMPTFOO_API_KEY: ${{ secrets.PROMPTFOO_API_KEY }}
15+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
16+
PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo
17+
PORT: 4000
18+
19+
jobs:
20+
evaluate:
21+
runs-on: ubuntu-latest
22+
steps:
23+
- uses: actions/checkout@v4
24+
25+
- uses: actions/setup-node@v4
26+
with:
27+
node-version: ${{ env.NODE_VERSION }}
28+
cache: 'npm'
29+
30+
- name: Install project dependencies
31+
run: |
32+
npm install
33+
34+
- name: Start dev server
35+
run: |
36+
nohup npm run dev > /dev/null 2>&1 &
37+
sleep 10
38+
echo "Dev server started in background"
39+
40+
- name: Install dependencies
41+
run: |
42+
npm install -g promptfoo@latest
43+
44+
- name: Cache promptfoo
45+
uses: actions/cache@v4
46+
with:
47+
path: ~/.cache/promptfoo
48+
key: ${{ runner.os }}-promptfoo-${{ hashFiles('prompts/**') }}
49+
restore-keys: |
50+
${{ runner.os }}-promptfoo-
51+
52+
- name: Promptfoo Login
53+
run: |
54+
promptfoo auth login --host "$PROMPTFOO_HOST" --api-key "$PROMPTFOO_API_KEY"
55+
56+
- name: Run redteam
57+
run: |
58+
promptfooredteam run -c "$SCAN_TEMPLATE_ID" -t "$TARGET_ID" \
59+
-c promptfooconfig.yaml \
60+
--share \
61+
-o results.json \
62+
-o report.html
63+
64+
- name: Check quality gate
65+
run: |
66+
FAILURES=$(jq '.results.stats.failures' results.json)
67+
EVAL_ID=$(jq '.evalId' results.json)
68+
EVAL_URL=$(jq -r '.shareableUrl' results.json)
69+
echo "Eval ID: $EVAL_ID"
70+
echo "URL: $EVAL_URL"
71+
if [ "$FAILURES" -gt 0 ]; then
72+
echo "❌ Eval failed with $FAILURES failures"
73+
exit 1
74+
fi
75+
echo "✅ All tests passed!"
76+
77+
- name: Upload results
78+
if: always()
79+
uses: actions/upload-artifact@v4
80+
with:
81+
name: eval-results
82+
path: |
83+
results.json
84+
report.html
85+

0 commit comments

Comments
 (0)