Skip to content

Commit 225806e

Browse files
committed
feat: seed 996 runs across 3 models from hosted bench results
Import script converts JSON exports to seed SQL. claude-sonnet-4: 170 runs (96.5% pass rate) deepseek-chat: 251 runs (84.9%) gemini-2.5-flash: 575 runs (75.0%) Bump bench-cli to 0.5.7 (infra fixes).
1 parent 914cbc6 commit 225806e

3 files changed

Lines changed: 1164 additions & 647 deletions

File tree

demo/postgres/import-json.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/usr/bin/env python3
2+
"""Convert bench JSON exports to seed SQL."""
3+
import json
4+
import sys
5+
import os
6+
7+
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
8+
SCENARIOS_JSON = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/git/evidra-bench/runs/export/bench-scenarios.json")
9+
RUNS_JSON = sys.argv[2] if len(sys.argv) > 2 else os.path.expanduser("~/git/evidra-bench/runs/export/demo-runs.json")
10+
OUTPUT = os.path.join(SCRIPT_DIR, "seed-data.sql")
11+
12+
13+
def q(v):
14+
"""SQL-quote a value."""
15+
if v is None:
16+
return "NULL"
17+
s = str(v).replace("'", "''")
18+
return f"'{s}'"
19+
20+
21+
def n(v, default=0):
22+
"""Numeric value."""
23+
return str(v) if v is not None else str(default)
24+
25+
26+
def b(v):
27+
"""Boolean value."""
28+
return "true" if v else "false"
29+
30+
31+
with open(SCENARIOS_JSON) as f:
32+
scenarios = json.load(f)
33+
34+
with open(RUNS_JSON) as f:
35+
runs = json.load(f)
36+
37+
with open(OUTPUT, "w") as out:
38+
out.write("-- Auto-generated seed data from bench JSON exports.\n")
39+
out.write("-- Idempotent: ON CONFLICT DO NOTHING / DO UPDATE.\n")
40+
out.write("SET client_min_messages = warning;\n\n")
41+
42+
# Scenarios
43+
out.write(f"-- Scenarios ({len(scenarios)})\n")
44+
for s in scenarios:
45+
out.write(
46+
f"INSERT INTO bench_scenarios (id, category, title, track, level, tags, chaos, evidra_enabled) "
47+
f"VALUES ({q(s.get('id'))}, {q(s.get('category',''))}, {q(s.get('title',''))}, "
48+
f"{q(s.get('track',''))}, {q(s.get('level',''))}, {q(s.get('tags',''))}, "
49+
f"{b(s.get('chaos'))}, {b(s.get('evidra'))}) "
50+
f"ON CONFLICT (id) DO UPDATE SET category=EXCLUDED.category, title=EXCLUDED.title, "
51+
f"track=EXCLUDED.track, level=EXCLUDED.level, tags=EXCLUDED.tags, "
52+
f"chaos=EXCLUDED.chaos, evidra_enabled=EXCLUDED.evidra_enabled;\n"
53+
)
54+
55+
# Runs
56+
out.write(f"\n-- Runs ({len(runs)})\n")
57+
for r in runs:
58+
out.write(
59+
f"INSERT INTO bench_runs (id, tenant_id, scenario_id, model, provider, adapter, "
60+
f"evidence_mode, passed, duration_seconds, exit_code, turns, memory_window, "
61+
f"prompt_tokens, completion_tokens, estimated_cost_usd, checks_passed, checks_total, "
62+
f"checks_json, metadata_json, created_at, tool_server, tool_server_version, scenario_version) "
63+
f"VALUES ({q(r.get('id'))}, 'default', {q(r.get('scenario_id'))}, "
64+
f"{q(r.get('model'))}, {q(r.get('provider',''))}, {q(r.get('adapter','bench-cli'))}, "
65+
f"{q(r.get('evidence_mode','none'))}, {b(r.get('passed'))}, "
66+
f"{n(r.get('duration_seconds',0))}, {n(r.get('exit_code',0))}, "
67+
f"{n(r.get('turns',0))}, {n(r.get('memory_window',-1))}, "
68+
f"{n(r.get('prompt_tokens',0))}, {n(r.get('completion_tokens',0))}, "
69+
f"{n(r.get('estimated_cost_usd',0))}, {n(r.get('checks_passed',0))}, "
70+
f"{n(r.get('checks_total',0))}, {q(r.get('checks_json',''))}, "
71+
f"{q(r.get('metadata_json',''))}, {q(r.get('created_at','2026-03-29T00:00:00Z'))}, "
72+
f"{q(r.get('tool_server',''))}, {q(r.get('tool_server_version',''))}, "
73+
f"{q(r.get('scenario_version',''))}) ON CONFLICT (id) DO NOTHING;\n"
74+
)
75+
76+
lines = sum(1 for _ in open(OUTPUT))
77+
print(f"Written {OUTPUT} ({lines} lines)")
78+
print(f" Scenarios: {len(scenarios)}")
79+
print(f" Runs: {len(runs)}")
80+
81+
# Summary by model
82+
from collections import Counter
83+
model_counts = Counter(r['model'] for r in runs)
84+
model_passed = Counter(r['model'] for r in runs if r.get('passed'))
85+
for model in sorted(model_counts, key=model_counts.get, reverse=True):
86+
total = model_counts[model]
87+
passed = model_passed.get(model, 0)
88+
print(f" {model}: {total} runs, {passed} passed ({100*passed//total}%)")

0 commit comments

Comments
 (0)