|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Convert bench JSON exports to seed SQL.""" |
| 3 | +import json |
| 4 | +import sys |
| 5 | +import os |
| 6 | + |
| 7 | +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 8 | +SCENARIOS_JSON = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/git/evidra-bench/runs/export/bench-scenarios.json") |
| 9 | +RUNS_JSON = sys.argv[2] if len(sys.argv) > 2 else os.path.expanduser("~/git/evidra-bench/runs/export/demo-runs.json") |
| 10 | +OUTPUT = os.path.join(SCRIPT_DIR, "seed-data.sql") |
| 11 | + |
| 12 | + |
| 13 | +def q(v): |
| 14 | + """SQL-quote a value.""" |
| 15 | + if v is None: |
| 16 | + return "NULL" |
| 17 | + s = str(v).replace("'", "''") |
| 18 | + return f"'{s}'" |
| 19 | + |
| 20 | + |
| 21 | +def n(v, default=0): |
| 22 | + """Numeric value.""" |
| 23 | + return str(v) if v is not None else str(default) |
| 24 | + |
| 25 | + |
| 26 | +def b(v): |
| 27 | + """Boolean value.""" |
| 28 | + return "true" if v else "false" |
| 29 | + |
| 30 | + |
| 31 | +with open(SCENARIOS_JSON) as f: |
| 32 | + scenarios = json.load(f) |
| 33 | + |
| 34 | +with open(RUNS_JSON) as f: |
| 35 | + runs = json.load(f) |
| 36 | + |
| 37 | +with open(OUTPUT, "w") as out: |
| 38 | + out.write("-- Auto-generated seed data from bench JSON exports.\n") |
| 39 | + out.write("-- Idempotent: ON CONFLICT DO NOTHING / DO UPDATE.\n") |
| 40 | + out.write("SET client_min_messages = warning;\n\n") |
| 41 | + |
| 42 | + # Scenarios |
| 43 | + out.write(f"-- Scenarios ({len(scenarios)})\n") |
| 44 | + for s in scenarios: |
| 45 | + out.write( |
| 46 | + f"INSERT INTO bench_scenarios (id, category, title, track, level, tags, chaos, evidra_enabled) " |
| 47 | + f"VALUES ({q(s.get('id'))}, {q(s.get('category',''))}, {q(s.get('title',''))}, " |
| 48 | + f"{q(s.get('track',''))}, {q(s.get('level',''))}, {q(s.get('tags',''))}, " |
| 49 | + f"{b(s.get('chaos'))}, {b(s.get('evidra'))}) " |
| 50 | + f"ON CONFLICT (id) DO UPDATE SET category=EXCLUDED.category, title=EXCLUDED.title, " |
| 51 | + f"track=EXCLUDED.track, level=EXCLUDED.level, tags=EXCLUDED.tags, " |
| 52 | + f"chaos=EXCLUDED.chaos, evidra_enabled=EXCLUDED.evidra_enabled;\n" |
| 53 | + ) |
| 54 | + |
| 55 | + # Runs |
| 56 | + out.write(f"\n-- Runs ({len(runs)})\n") |
| 57 | + for r in runs: |
| 58 | + out.write( |
| 59 | + f"INSERT INTO bench_runs (id, tenant_id, scenario_id, model, provider, adapter, " |
| 60 | + f"evidence_mode, passed, duration_seconds, exit_code, turns, memory_window, " |
| 61 | + f"prompt_tokens, completion_tokens, estimated_cost_usd, checks_passed, checks_total, " |
| 62 | + f"checks_json, metadata_json, created_at, tool_server, tool_server_version, scenario_version) " |
| 63 | + f"VALUES ({q(r.get('id'))}, 'default', {q(r.get('scenario_id'))}, " |
| 64 | + f"{q(r.get('model'))}, {q(r.get('provider',''))}, {q(r.get('adapter','bench-cli'))}, " |
| 65 | + f"{q(r.get('evidence_mode','none'))}, {b(r.get('passed'))}, " |
| 66 | + f"{n(r.get('duration_seconds',0))}, {n(r.get('exit_code',0))}, " |
| 67 | + f"{n(r.get('turns',0))}, {n(r.get('memory_window',-1))}, " |
| 68 | + f"{n(r.get('prompt_tokens',0))}, {n(r.get('completion_tokens',0))}, " |
| 69 | + f"{n(r.get('estimated_cost_usd',0))}, {n(r.get('checks_passed',0))}, " |
| 70 | + f"{n(r.get('checks_total',0))}, {q(r.get('checks_json',''))}, " |
| 71 | + f"{q(r.get('metadata_json',''))}, {q(r.get('created_at','2026-03-29T00:00:00Z'))}, " |
| 72 | + f"{q(r.get('tool_server',''))}, {q(r.get('tool_server_version',''))}, " |
| 73 | + f"{q(r.get('scenario_version',''))}) ON CONFLICT (id) DO NOTHING;\n" |
| 74 | + ) |
| 75 | + |
| 76 | +lines = sum(1 for _ in open(OUTPUT)) |
| 77 | +print(f"Written {OUTPUT} ({lines} lines)") |
| 78 | +print(f" Scenarios: {len(scenarios)}") |
| 79 | +print(f" Runs: {len(runs)}") |
| 80 | + |
| 81 | +# Summary by model |
| 82 | +from collections import Counter |
| 83 | +model_counts = Counter(r['model'] for r in runs) |
| 84 | +model_passed = Counter(r['model'] for r in runs if r.get('passed')) |
| 85 | +for model in sorted(model_counts, key=model_counts.get, reverse=True): |
| 86 | + total = model_counts[model] |
| 87 | + passed = model_passed.get(model, 0) |
| 88 | + print(f" {model}: {total} runs, {passed} passed ({100*passed//total}%)") |
0 commit comments