Fix GitHub Actions for TwinBench v1 layout #16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| benchmark-ci: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python 3.10 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Compile evaluation scaffold | |
| run: python -m py_compile eval/*.py | |
| - name: Runner CLI smoke | |
| run: python eval/runner.py --help | |
| - name: Generate reference example artifact | |
| run: | | |
| python eval/runner.py \ | |
| --config benchmarks/configs/default.json \ | |
| --system-name "Reference Example System" \ | |
| --system-version "1.0.0" \ | |
| --observations benchmarks/fixtures/reference_observations.json \ | |
| --output /tmp/reference-example-v1.json | |
| - name: Validate TwinBench v1 result schema | |
| run: | | |
| python - <<'PY' | |
| import json | |
| from pathlib import Path | |
| p = Path("/tmp/reference-example-v1.json") | |
| if not p.exists(): | |
| raise SystemExit("reference example output not found") | |
| data = json.loads(p.read_text()) | |
| required = [ | |
| "benchmark_name", | |
| "benchmark_version", | |
| "benchmark_title", | |
| "benchmark_subtitle", | |
| "system_name", | |
| "system_version", | |
| "date_evaluated", | |
| "scenario_set", | |
| "scenarios", | |
| "metrics", | |
| "total_score", | |
| "scenario_coverage", | |
| "metric_coverage", | |
| "evaluator_notes", | |
| "caveats", | |
| ] | |
| missing = [k for k in required if k not in data] | |
| if missing: | |
| raise SystemExit(f"missing keys: {missing}") | |
| if data["benchmark_version"] != "1.0": | |
| raise SystemExit("benchmark_version must be 1.0") | |
| if len(data["scenarios"]) != 5: | |
| raise SystemExit("expected 5 scenarios") | |
| if sorted(data["metrics"].keys()) != ["CCC", "IC", "MR", "PG", "TC"]: | |
| raise SystemExit("unexpected metric keys") | |
| print("twinbench v1 artifact schema ok") | |
| PY | |
| - name: Check reference artifact reproducibility | |
| run: diff -u results/reference-example-v1.json /tmp/reference-example-v1.json |