Leaderboard Space: hosted HF board + refresh workflow + held-out path #70

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	workflow_dispatch:
	inputs:
	suite:
	description: Which live eval suite to run (manual dispatch only).
	type: choice
	options: [both, static, lift]
	default: both
	leaderboard:
	description: >-
	Leaderboard seeding (manual dispatch only): skip; build the board from the committed +
	fetched bundles; or seed (also live-record the second seed model, spending MODELS_PAT).
	type: choice
	options: [skip, build, seed]
	default: skip
	publish_space:
	description: >-
	After build/seed, push the rebuilt board to the static Hugging Face Space
	(needs the HF_TOKEN secret and the LEADERBOARD_SPACE_REPO variable).
	type: boolean
	default: false

	concurrency:
	group: ci-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	jobs:
	test:
	name: test (${{ matrix.os }}, py${{ matrix.python-version }})
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ['3.10', '3.11', '3.12']
	include:
	- os: windows-latest
	python-version: '3.12'
	env:
	UV_PYTHON: ${{ matrix.python-version }}
	# The suite mocks the embedder everywhere (FakeEmbedder / stubbed retriever), so no test loads
	# the real model. Enforce that: offline mode fails fast and loud if a test ever reaches for
	# HuggingFace, instead of silently downloading the model and flaking on a 429 rate limit.
	HF_HUB_OFFLINE: '1'
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups
	# `not gmat` keeps the dynamic dry-run tier out of the GMAT-free matrix; it runs in the gated
	# `dry-run` job below (setup-gmat + the [gmat] extra). The dry-run worker is omitted from the
	# coverage gate, so this `--cov` run stays GMAT-free and still meets the 90% floor.
	- run: uv run pytest -m "not eval_smoke and not gmat" --cov --cov-report=term-missing

	lint:
	name: lint
	runs-on: ubuntu-latest
	env:
	UV_PYTHON: '3.12'
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups
	- run: uv run ruff check
	- run: uv run ruff format --check

	typecheck:
	name: typecheck
	runs-on: ubuntu-latest
	env:
	UV_PYTHON: '3.12'
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups
	- run: uv run mypy

	eval-smoke:
	name: eval smoke (recorded, deterministic)
	runs-on: ubuntu-latest
	env:
	UV_PYTHON: '3.12'
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups
	# Replays recorded completions + judge verdicts. Zero model calls, zero quota.
	- run: uv run pytest -m eval_smoke

	gated-eval:
	name: gated eval (live, GitHub Models)
	# Live inference runs ONLY on manual dispatch (decision D7): per-PR CI is the deterministic
	# recorded replay above, so there is no per-PR inference budget to subset. This job is the
	# full-suite live run / fixture-refresh path; it needs a personal MODELS_PAT (the workflow
	# GITHUB_TOKEN is unreliable for GitHub Models inference). The `suite` dispatch input selects
	# which live eval(s) run, so this static suite can be refreshed without the close-the-loop run.
	if: >-
	github.event_name == 'workflow_dispatch'
	&& (inputs.suite == 'static' \|\| inputs.suite == 'both')
	runs-on: ubuntu-latest
	permissions:
	models: read
	contents: read
	env:
	UV_PYTHON: '3.12'
	MODELS_PAT: ${{ secrets.MODELS_PAT }}
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups
	- run: >
	uv run gmat-copilot eval --live
	--prompts tests/data/eval/prompts.json
	-m github:openai/gpt-4.1-mini
	--judge-model openai/gpt-4.1-mini
	--pace 4.5

	minimal-install:
	name: minimal install (no extras)
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6
	- uses: actions/setup-python@v6
	with:
	python-version: '3.12'
	- run: python -m pip install --upgrade pip
	- run: python -m pip install .
	- name: import cleanly with no extras
	run: python -c "import gmat_copilot; print(gmat_copilot.__version__)"
	- name: provider SDKs and the GMAT stack must not be pulled
	run: \|
	python - <<'PY'
	import importlib.util as u
	for mod in ("anthropic", "openai", "ollama", "gmat_run"):
	assert u.find_spec(mod) is None, f"{mod} leaked into the base install"
	print("base install is light: no provider SDK, no GMAT stack")
	PY

	extras-smoke:
	name: extras resolve and import
	runs-on: ubuntu-latest
	env:
	UV_PYTHON: '3.12'
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups --all-extras
	- run: uv run python -c "import anthropic, openai, ollama; import gmat_copilot; print('extras import OK')"

	dry-run:
	name: dry-run tier (setup-gmat, [gmat] extra)
	# The dynamic validation tier needs a real GMAT install, supplied by setup-gmat. Kept off the
	# base matrix (which is GMAT-free) and gated here; the dry-run worker is omitted from coverage,
	# so this job runs the gmat-marked tests without the coverage gate. These tests use no model
	# (the close-the-loop verification re-checks the recorded lift verdicts against a real GMAT), so
	# the job stays inference-free — the per-PR rate limit is never touched.
	runs-on: ubuntu-latest
	timeout-minutes: 30
	env:
	UV_PYTHON: '3.12'
	HF_HUB_OFFLINE: '1'
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- uses: astro-tools/setup-gmat@v0
	with:
	version: R2026a
	cache: true
	- run: uv sync --all-groups --extra gmat
	- run: uv run pytest -m gmat

	gated-lift-eval:
	name: gated close-the-loop eval (live, GitHub Models + setup-gmat)
	# The live close-the-loop eval drives a real model AND a real GMAT dry-run, so it runs ONLY on
	# manual dispatch (decision D7): per-PR CI replays the recorded lift (the eval-smoke job) and
	# re-checks the recorded verdicts against a real GMAT (the dry-run job), neither of which calls a
	# model. This job is the on-demand live run / fixture-refresh path; it needs a personal MODELS_PAT
	# and is paced to respect the free-tier daily budget. The `suite` dispatch input selects which
	# live eval(s) run, so the close-the-loop suite can be refreshed without the static run.
	if: >-
	github.event_name == 'workflow_dispatch'
	&& (inputs.suite == 'lift' \|\| inputs.suite == 'both')
	runs-on: ubuntu-latest
	timeout-minutes: 30
	permissions:
	models: read
	contents: read
	env:
	UV_PYTHON: '3.12'
	MODELS_PAT: ${{ secrets.MODELS_PAT }}
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- uses: astro-tools/setup-gmat@v0
	with:
	version: R2026a
	cache: true
	- run: uv sync --all-groups --extra gmat
	- run: >
	uv run gmat-copilot eval --lift
	--prompts tests/data/eval_lift/prompts.json
	-m github:openai/gpt-4.1-mini
	--judge-model openai/gpt-4.1-mini
	--budget 2
	--pace 4.5

	extension:
	name: vscode extension
	# Builds the editor surface (decision D15) and runs its activation smoke test headless. The
	# engine worker it drives is pure Python and covered by the test job above; here we only build
	# the TypeScript bundle, type-check, lint, run the activation test, and validate the .vsix.
	# Publishing to the Marketplace / Open VSX is a separate release concern.
	runs-on: ubuntu-latest
	timeout-minutes: 20
	defaults:
	run:
	working-directory: editors/vscode
	steps:
	- uses: actions/checkout@v6
	- uses: actions/setup-node@v6
	with:
	node-version: '22'
	cache: npm
	cache-dependency-path: editors/vscode/package-lock.json
	- run: npm ci
	- name: Lint, type-check, and run the activation smoke test (headless)
	run: xvfb-run -a npm test
	- name: Package the .vsix
	run: npx @vscode/vsce package --out gmat-copilot.vsix
	- uses: actions/upload-artifact@v7
	with:
	name: vscode-extension-vsix
	path: editors/vscode/gmat-copilot.vsix
	if-no-files-found: error

	leaderboard:
	name: leaderboard (build + verify, gated)
	# The per-model leaderboard engine (decision D16). Manual dispatch only: the held-out score uses
	# the LLM judge's frozen gold verdicts from a private store, and `seed` spends live GitHub Models
	# quota — neither belongs on the per-PR path. `build` assembles the board from the committed
	# public bundle plus any held-out fetched from the private store; `seed` additionally live-records
	# the second seed model. The board is uploaded as an artifact; rendering it on the static HF Space
	# is a separate surface. The committed board (leaderboard/leaderboard.json) is the gpt-4.1-mini
	# public anchor; this job refreshes it with held-out scores + additional seeds when configured.
	if: >-
	github.event_name == 'workflow_dispatch'
	&& (inputs.leaderboard == 'build' \|\| inputs.leaderboard == 'seed')
	runs-on: ubuntu-latest
	timeout-minutes: 60
	permissions:
	models: read
	contents: read
	env:
	UV_PYTHON: '3.12'
	# MODELS_PAT drives the `seed` live record (the workflow GITHUB_TOKEN is unreliable for GitHub
	# Models). HF_TOKEN + the HELDOUT_DATASET repo variable fetch the never-committed held-out from
	# its private HF Dataset (decision D16); until that dataset is authored (release cut), the fetch
	# is skipped and every held-out cell stays pending.
	MODELS_PAT: ${{ secrets.MODELS_PAT }}
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	HELDOUT_DATASET: ${{ vars.LEADERBOARD_HELDOUT_DATASET }}
	# The static HF Space the rebuilt board is published to (the project's one hosted artifact,
	# decision D16). Until the Space is created and this variable is set, the publish step is
	# skipped; the board still builds, verifies, and uploads as an artifact.
	SPACE_REPO: ${{ vars.LEADERBOARD_SPACE_REPO }}
	steps:
	- uses: actions/checkout@v6
	- uses: astral-sh/setup-uv@v7
	with:
	enable-cache: true
	cache-dependency-glob: uv.lock
	- run: uv sync --all-groups
	- name: Record the second seed's public bundle (live; paced for the free daily cap)
	if: inputs.leaderboard == 'seed'
	continue-on-error: true
	run: \|
	DIR=leaderboard/.cache/public/openai__gpt-4o-mini
	mkdir -p "$DIR"
	cp tests/data/eval/prompts.json "$DIR/prompts.json"
	uv run gmat-copilot eval --record "$DIR" \
	-m github:openai/gpt-4o-mini \
	--judge-model openai/gpt-4.1-mini \
	--pace 4.5
	# `eval --record` keys judge.json by the judge model; normalise it to the flat,
	# model-agnostic {prompt_id: verdicts} form the committed gold bundle uses, so the board's
	# run_recorded replay scores this generation model correctly (decisions D7, D11).
	uv run python -c 'import json, sys; p = sys.argv[1]; d = json.load(open(p, encoding="utf-8")); d = next(iter(d.values())) if d and all(isinstance(v, dict) for v in d.values()) else d; json.dump(d, open(p, "w", encoding="utf-8"), indent=2, sort_keys=True)' "$DIR/judge.json"
	- name: Fetch the private held-out bundles (if the dataset is configured)
	if: env.HF_TOKEN != '' && env.HELDOUT_DATASET != ''
	# The HF Hub Python API (stable across the huggingface-cli -> hf CLI rename). HF_TOKEN
	# authenticates the read of the private dataset; an empty dataset downloads to an empty dir
	# and leaves every held-out cell pending.
	run: \|
	uvx --from huggingface_hub python -c 'import os; from huggingface_hub import snapshot_download; snapshot_download(repo_id=os.environ["HELDOUT_DATASET"], repo_type="dataset", local_dir="leaderboard/.cache/heldout")'
	- name: Build the board
	run: \|
	uv run gmat-copilot leaderboard build \
	--out leaderboard/leaderboard.json \
	--held-out leaderboard/.cache/heldout
	- name: Verify the board reproduces offline and leaks no held-out gold
	run: uv run gmat-copilot leaderboard verify leaderboard/leaderboard.json
	- name: Publish the board to the static HF Space
	# The Space is the project's one hosted artifact (decision D16). The preceding `verify` step
	# is the firewall: it fails the job if the board is not aggregate-only, so a held-out gold can
	# never reach the Space. The Space payload is the committed static front end plus the freshly
	# built board, pushed via the HF Hub API; the published JSON carries aggregates only.
	if: inputs.publish_space && env.HF_TOKEN != '' && env.SPACE_REPO != ''
	# The HF Hub Python API (stable across the huggingface-cli -> hf CLI rename). The Space payload
	# is the committed static front end plus the freshly built board; the published JSON carries
	# aggregates only (the preceding verify step is the firewall).
	run: \|
	cp leaderboard/leaderboard.json leaderboard/space/leaderboard.json
	uvx --from huggingface_hub python -c 'import os, json; from huggingface_hub import upload_folder; v = json.load(open("leaderboard/leaderboard.json"))["eval_protocol_version"]; upload_folder(repo_id=os.environ["SPACE_REPO"], repo_type="space", folder_path="leaderboard/space", commit_message=f"Refresh leaderboard (eval protocol {v})")'
	- uses: actions/upload-artifact@v7
	with:
	name: leaderboard
	path: leaderboard/leaderboard.json
	if-no-files-found: error

Navigation Menu

Uh oh!

Leaderboard Space: hosted HF board + refresh workflow + held-out path #70

Workflow file

Leaderboard Space: hosted HF board + refresh workflow + held-out path #70

Uh oh!

Workflow file for this run