Merge pull request #1644 from kodustech/fix/kody-rules-file-scope-ext… #228
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Code-review evals (replay suite)" | |
| # Replacement for the live QA model benchmark. Runs the real review engine over | |
| # benchmark PR diffs via DETERMINISTIC TOOL REPLAY — same engine code, same | |
| # tier-0 models — so there is no deploy, no live reviews, and it runs in minutes | |
| # from the PR/branch itself instead of post-deploy on qa.web.kodus.io. | |
| # | |
| # - pull_request from a branch in this repo | |
| # → harness preflight + finder-recall gate on TWO | |
| # FAST models (gpt-5.4 + gemini-3-flash) + | |
| # kody-rules & anchoring (advisory). Fast and | |
| # reliable so it doesn't drag day-to-day PRs. | |
| # - pull_request from a fork → harness preflight only. GitHub does not | |
| # expose secrets to fork PRs. | |
| # - push to main (engine paths) → FULL matrix: every tier-0 model (per-model | |
| # comparison + per-provider coverage, incl. the | |
| # slow models that can't fit a PR timeout). | |
| # - workflow_dispatch → manual; pin one model via `only_model`. | |
| # | |
| # NOTE: the main matrix still uses the protected QA environment. The PR model | |
| # gate intentionally does NOT use that environment; it expects the same BYOK | |
| # names to exist as repo/org Actions secrets. | |
| # | |
| # Secrets reused from the QA environment (same names the benchmark uses): | |
| # BYOK_OPENAI_API_KEY, BYOK_ANTHROPIC_API_KEY, BYOK_GOOGLE_API_KEY, | |
| # BYOK_MOONSHOT_API_KEY, BYOK_ZHIPU_API_KEY | |
| # ANTHROPIC_API_KEY (Sonnet judge for recall/dedup scoring) | |
| # Least-privilege default for every job: these eval jobs only read the repo | |
| # and call model APIs — they never write back. Overrides go per-job if needed. | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| only_model: | |
| description: "Pin one tier-0 model id (e.g. gpt-5.4, claude-sonnet-4-6); empty = full matrix" | |
| required: false | |
| default: "" | |
| type: string | |
| push: | |
| branches: [main] | |
| paths: | |
| - "libs/code-review/**" | |
| - "libs/agent-harness/**" | |
| - "libs/ee/codeReview/**" | |
| - "libs/ee/kodyRules/**" | |
| - "libs/kodyRules/**" | |
| - "packages/kodus-flow/**" | |
| - "apps/web/src/features/ee/byok/_data/curated-models.json" | |
| - "evals/**" | |
| pull_request: | |
| paths: | |
| - "libs/code-review/**" | |
| - "libs/agent-harness/**" | |
| - "libs/ee/codeReview/**" | |
| - "libs/ee/kodyRules/**" | |
| - "libs/kodyRules/**" | |
| - "packages/kodus-flow/**" | |
| - "apps/web/src/features/ee/byok/_data/curated-models.json" | |
| - "evals/**" | |
| concurrency: | |
| group: code-review-evals-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| harness: | |
| if: github.event_name == 'pull_request' | |
| name: "eval harness (no secrets)" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| - name: Check eval harness wiring | |
| run: node evals/engine-gate.js --profile=harness | |
| # PR gate runs only FAST, reliable models — an engine regression shows on any | |
| # model, so two suffice: gpt-5.4 (OpenAI) + gemini-3-flash (Google), two | |
| # providers, ~10-15 min. kimi (Moonshot) and glm (z.ai) are deliberately NOT | |
| # here: their APIs throttle the many sequential tool-call round-trips (~15 | |
| # min/case), so 8 PRs blow the 45-min timeout. The full tier-0 matrix — | |
| # including the slow models and per-provider coverage — runs post-merge on | |
| # main (the `suite` job below), where duration doesn't block anyone. | |
| pr-finder-gate: | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false | |
| needs: harness | |
| name: "PR finder · ${{ matrix.model }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| model: | |
| - gpt-5.4 | |
| - gemini-3-flash-preview | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Check model secrets | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| BYOK_ANTHROPIC_API_KEY: ${{ secrets.BYOK_ANTHROPIC_API_KEY }} | |
| BYOK_GOOGLE_API_KEY: ${{ secrets.BYOK_GOOGLE_API_KEY }} | |
| BYOK_MOONSHOT_API_KEY: ${{ secrets.BYOK_MOONSHOT_API_KEY }} | |
| BYOK_ZHIPU_API_KEY: ${{ secrets.BYOK_ZHIPU_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: | | |
| missing=0 | |
| for name in \ | |
| BYOK_OPENAI_API_KEY \ | |
| BYOK_ANTHROPIC_API_KEY \ | |
| BYOK_GOOGLE_API_KEY \ | |
| BYOK_MOONSHOT_API_KEY \ | |
| BYOK_ZHIPU_API_KEY \ | |
| ANTHROPIC_API_KEY | |
| do | |
| if [ -z "${!name}" ]; then | |
| echo "::error::Missing required GitHub Actions secret: $name" | |
| missing=1 | |
| fi | |
| done | |
| exit "$missing" | |
| - name: Install deps | |
| run: pnpm install --frozen-lockfile | |
| - name: Create environment.ts (gitignored EE config) | |
| run: | | |
| cat <<'EOF' > libs/ee/configs/environment/environment.ts | |
| import { environment as devEnvironment } from './environment.dev'; | |
| export const environment = devEnvironment; | |
| EOF | |
| - name: Run finder recall | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| BYOK_ANTHROPIC_API_KEY: ${{ secrets.BYOK_ANTHROPIC_API_KEY }} | |
| BYOK_GOOGLE_API_KEY: ${{ secrets.BYOK_GOOGLE_API_KEY }} | |
| BYOK_MOONSHOT_API_KEY: ${{ secrets.BYOK_MOONSHOT_API_KEY }} | |
| BYOK_ZHIPU_API_KEY: ${{ secrets.BYOK_ZHIPU_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| # --gate: run mean vs the model's absolute floors (targets.json). | |
| # One retry before red: a floor breach re-runs the model once and | |
| # only fails if it breaches twice (recall_mean noise ≈ ±1 SEM/run; | |
| # floors sit at −2·SEM, so a double breach is a real regression, | |
| # not eval noise). Infra errors (exit 2) fail immediately, no retry. | |
| run: | | |
| run_gate() { node evals/investigation/run-recall.js --set=pr --gate --model=${{ matrix.model }}; } | |
| code=0; run_gate || code=$? | |
| # exit 2 = infra (quota/rate-limit/key/network) — environmental, | |
| # not a quality regression. Warn and pass, don't block merge. | |
| if [ "$code" -eq 2 ]; then echo "::warning::${{ matrix.model }}: infra failure (quota/rate-limit/key) — not measured, not gating."; exit 0; fi | |
| if [ "$code" -eq 0 ]; then exit 0; fi | |
| # exit 1 = floor breach — retry once to rule out eval noise. | |
| echo "::warning::Floor gate breached for ${{ matrix.model }} — retrying once." | |
| code=0; run_gate || code=$? | |
| if [ "$code" -eq 2 ]; then echo "::warning::${{ matrix.model }}: infra failure on retry — not gating."; exit 0; fi | |
| exit "$code" | |
| - name: Upload finder recall result | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: finder-recall-${{ matrix.model }} | |
| path: evals/investigation/results/finder-recall-${{ matrix.model }}.json | |
| if-no-files-found: ignore | |
| retention-days: 14 | |
| pr-kody-rules-gate: | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false | |
| needs: harness | |
| name: "PR kody-rules · ${{ matrix.model }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| model: | |
| - gpt-5.4 | |
| - gpt-5.4-mini | |
| - gemini-3-flash-preview | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Install deps | |
| run: pnpm install --frozen-lockfile | |
| - name: Create environment.ts (gitignored EE config) | |
| run: | | |
| cat <<'EOF' > libs/ee/configs/environment/environment.ts | |
| import { environment as devEnvironment } from './environment.dev'; | |
| export const environment = devEnvironment; | |
| EOF | |
| - name: Run kody-rules gate | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| BYOK_ANTHROPIC_API_KEY: ${{ secrets.BYOK_ANTHROPIC_API_KEY }} | |
| BYOK_GOOGLE_API_KEY: ${{ secrets.BYOK_GOOGLE_API_KEY }} | |
| BYOK_MOONSHOT_API_KEY: ${{ secrets.BYOK_MOONSHOT_API_KEY }} | |
| BYOK_ZHIPU_API_KEY: ${{ secrets.BYOK_ZHIPU_API_KEY }} | |
| # Tests the SHIPPED sharded path: behavioral-shipped.js drives the | |
| # real judgeKodyRulesSharded (its actual prompts) against real PR | |
| # diffs with a real model. This replaced real-agent.js, which drove | |
| # the old AGENTIC provider — obsolete since the #1449 rewrite, and | |
| # it now crashes (the sharded execute() needs a real prompt runner, | |
| # which that harness mocked). | |
| # | |
| # Advisory (no --gate) on this first landing: the sharded path is | |
| # structural (per-file sweep) so run-to-run variance is far lower | |
| # than the old agentic loop (which swung 72%→47%). gpt-5.4-mini | |
| # measured 100% (43/43) locally. Set --gate=<floor> once a few CI | |
| # runs confirm the number is stable per model. | |
| run: node evals/kody-rules/behavioral-shipped.js --dataset=github-cases --model=${{ matrix.model }} --temp=0 | |
| pr-kody-rules-atoms: | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false | |
| needs: harness | |
| name: "PR kody-rules atoms · gpt-5.4-mini" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Install deps | |
| run: pnpm install --frozen-lockfile | |
| - name: Create environment.ts (gitignored EE config) | |
| run: | | |
| cat <<'EOF' > libs/ee/configs/environment/environment.ts | |
| import { environment as devEnvironment } from './environment.dev'; | |
| export const environment = devEnvironment; | |
| EOF | |
| - name: Run atoms convention eval | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| # CONVENTION-rule coverage the mechanical github-cases can't give: | |
| # drives the REAL KodyRulesAgentProvider.execute() end-to-end | |
| # (atomic decomposition from the FROZEN rails-convention-atoms.json | |
| # fixture — no generation calls in CI — T0 detector sweep, shard | |
| # judge, mapAgentFindings) over the 27-file Rails analog PR with | |
| # 27 real-shaped rules, including the pull-request-scoped one. | |
| # | |
| # Advisory (no gate): the case has 19 ground-truth sites, so | |
| # single-run variance is ~±10pp — a hard floor would flake. | |
| # Reference band (gpt-5.4-mini, measured): 74-89% deliverable | |
| # (file-level + PR-scope). Gate once the case set grows enough to | |
| # damp the variance. | |
| run: pnpm run eval:kody-rules:atoms | |
| pr-anchoring-canary: | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false | |
| needs: harness | |
| name: "PR anchoring canary · gpt-5.4" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 35 | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Install deps | |
| run: pnpm install --frozen-lockfile | |
| - name: Create environment.ts (gitignored EE config) | |
| run: | | |
| cat <<'EOF' > libs/ee/configs/environment/environment.ts | |
| import { environment as devEnvironment } from './environment.dev'; | |
| export const environment = devEnvironment; | |
| EOF | |
| - name: Run anchoring canary | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| # Advisory (no --gate): the drop-rate ≤ 8% threshold is hardcoded | |
| # and uncalibrated to the current engine (gpt-5.4 measures ~33% on | |
| # 3 cases). Runs and reports, but doesn't block merge until the | |
| # threshold is set from a measured baseline. | |
| run: node evals/anchoring/anchor-eval.js --model=gpt-5.4 --limit=3 | |
| # PR-summary matrix: drives the REAL generateSummaryPR + updateSummarizationInPR | |
| # end-to-end and gates on the properties a prod incident violated (summary | |
| # generated, POSTED to the PR, correct model routing, replace/concatenate/ | |
| # complement composed right). Runs on THREE models so a per-model regression | |
| # can't hide behind a healthy one — the incident was one model degrading while | |
| # others worked. Each case is a single model call (not a tool loop), so kimi | |
| # and gemini are fast enough for a PR gate here (unlike the finder legs). | |
| # | |
| # GATING (--gate): the asserted properties are binary and word-independent | |
| # (does a non-empty summary come back and get posted, is it composed per the | |
| # client's behaviour, does the configured model actually route) — so a real | |
| # model can't make this flake. Infra errors (bad key/quota) exit 2 and fail | |
| # loudly; an all-cases-infra run is treated as "not measured", not a pass. | |
| pr-summary-gate: | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false | |
| needs: harness | |
| name: "PR pr-summary · ${{ matrix.model }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| model: | |
| - gpt-5.4-mini | |
| - kimi-k2.7-code | |
| - gemini-3-flash-preview | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Install deps | |
| run: pnpm install --frozen-lockfile | |
| - name: Create environment.ts (gitignored EE config) | |
| run: | | |
| cat <<'EOF' > libs/ee/configs/environment/environment.ts | |
| import { environment as devEnvironment } from './environment.dev'; | |
| export const environment = devEnvironment; | |
| EOF | |
| - name: Run pr-summary eval | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| BYOK_MOONSHOT_API_KEY: ${{ secrets.BYOK_MOONSHOT_API_KEY }} | |
| BYOK_GOOGLE_API_KEY: ${{ secrets.BYOK_GOOGLE_API_KEY }} | |
| # exit 2 = infra (absent/invalid key, quota, network) — environmental, | |
| # not a quality regression. Warn and pass, matching the finder gate, | |
| # so a bad org key or a depleted-credit provider doesn't block merges. | |
| # exit 1 (a real generate/post/route/compose regression) still gates. | |
| run: | | |
| code=0 | |
| node evals/pr-summary/run.js --model=${{ matrix.model }} --gate || code=$? | |
| if [ "$code" -eq 2 ]; then | |
| echo "::warning::${{ matrix.model }}: pr-summary infra (invalid/absent key, quota, network) — not measured, not gating." | |
| exit 0 | |
| fi | |
| exit "$code" | |
| # Single rollup status for branch protection: mark ONLY this check required. | |
| # The per-model jobs stay visible for debugging (expand to see which model | |
| # regressed), but the PR gates on one clean pass/fail. `always()` so it runs | |
| # even when an upstream job fails; skipped upstreams (fork PRs) are fine — | |
| # only a real failure/cancel trips it. | |
| pr-evals-gate: | |
| if: always() && github.event_name == 'pull_request' | |
| needs: [harness, pr-finder-gate, pr-kody-rules-gate, pr-anchoring-canary, pr-summary-gate] | |
| name: "PR evals gate" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - name: Require all eval jobs to pass | |
| env: | |
| R_HARNESS: ${{ needs.harness.result }} | |
| R_FINDER: ${{ needs.pr-finder-gate.result }} | |
| R_KODY: ${{ needs.pr-kody-rules-gate.result }} | |
| R_ANCHOR: ${{ needs.pr-anchoring-canary.result }} | |
| R_SUMMARY: ${{ needs.pr-summary-gate.result }} | |
| run: | | |
| echo "harness=$R_HARNESS finder=$R_FINDER kody=$R_KODY anchoring=$R_ANCHOR summary=$R_SUMMARY" | |
| failed=0 | |
| for r in "$R_HARNESS" "$R_FINDER" "$R_KODY" "$R_ANCHOR" "$R_SUMMARY"; do | |
| if [ "$r" = "failure" ] || [ "$r" = "cancelled" ]; then | |
| failed=1 | |
| fi | |
| done | |
| if [ "$failed" -eq 1 ]; then | |
| echo "::error::One or more eval jobs did not pass — see the per-model checks above." | |
| exit 1 | |
| fi | |
| echo "All eval jobs passed (or were skipped)." | |
| # Model matrix: the pinned model (dispatch) or the full tier-0 set. | |
| prepare: | |
| if: github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| models: ${{ steps.pick.outputs.models }} | |
| steps: | |
| - id: pick | |
| run: | | |
| if [ -n "${{ github.event.inputs.only_model }}" ]; then | |
| echo "models=[\"${{ github.event.inputs.only_model }}\"]" >> "$GITHUB_OUTPUT" | |
| else | |
| echo 'models=["gpt-5.4","claude-sonnet-4-6","gemini-3.1-pro-preview-customtools","gemini-3-flash-preview","kimi-k2.7-code","glm-5.2"]' >> "$GITHUB_OUTPUT" | |
| fi | |
| suite: | |
| if: github.event_name != 'pull_request' | |
| needs: prepare | |
| name: "suite · ${{ matrix.model }}" | |
| runs-on: ubuntu-latest | |
| # Secrets live in the QA environment (same as the benchmark). | |
| environment: QA | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| model: ${{ fromJSON(needs.prepare.outputs.models) }} | |
| steps: | |
| - uses: actions/checkout@v6.0.2 | |
| - uses: pnpm/action-setup@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Install deps | |
| run: pnpm install --frozen-lockfile | |
| - name: Create environment.ts (gitignored EE config) | |
| run: | | |
| cat <<'EOF' > libs/ee/configs/environment/environment.ts | |
| import { environment as devEnvironment } from './environment.dev'; | |
| export const environment = devEnvironment; | |
| EOF | |
| - name: Run eval suite | |
| # engine-gate profile=ci runs the replay suite with --enforce. | |
| # Infra errors and enabled quality gates fail this job. | |
| env: | |
| BYOK_OPENAI_API_KEY: ${{ secrets.BYOK_OPENAI_API_KEY }} | |
| BYOK_ANTHROPIC_API_KEY: ${{ secrets.BYOK_ANTHROPIC_API_KEY }} | |
| BYOK_GOOGLE_API_KEY: ${{ secrets.BYOK_GOOGLE_API_KEY }} | |
| BYOK_MOONSHOT_API_KEY: ${{ secrets.BYOK_MOONSHOT_API_KEY }} | |
| BYOK_ZHIPU_API_KEY: ${{ secrets.BYOK_ZHIPU_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| run: node evals/engine-gate.js --profile=ci --model=${{ matrix.model }} --prs=5 --runs=3 |