Fix missing model bot issues #449
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fix missing model bot issues | |
| on: | |
| schedule: | |
| - cron: "0 10 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| issue_number: | |
| description: "Specific issue number to fix" | |
| required: false | |
| type: string | |
| debug_enabled: | |
| description: "Reserved for parity with other workflows" | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: fix-missing-model-bot-issues | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| fix: | |
| runs-on: ubuntu-latest | |
| env: | |
| # Route Claude Code through the Braintrust gateway. The gateway | |
| # authenticates with a Braintrust API key (BRAINTRUST_AGENT_SPEND_API_KEY, | |
| # passed as anthropic_api_key below) and calls the provider on our behalf. | |
| # https://www.braintrust.dev/docs/deploy/gateway | |
| ANTHROPIC_BASE_URL: https://gateway.braintrust.dev | |
| # Attribute gateway spend to the automations-spend-control project. | |
| ANTHROPIC_CUSTOM_HEADERS: "x-bt-project-name: automations-spend-control" | |
| BLOCKED_LABEL: autofix-blocked | |
| BATCH_DIR: fix-missing-model-bot-issues | |
| steps: | |
| - name: Select issues | |
| id: issues | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| env: | |
| MANUAL_ISSUE_NUMBER: ${{ inputs.issue_number || '' }} | |
| BATCH_DIR: ${{ runner.temp }}/${{ env.BATCH_DIR }} | |
| with: | |
| script: | | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const owner = context.repo.owner; | |
| const repo = context.repo.repo; | |
| const manualIssueNumber = (process.env.MANUAL_ISSUE_NUMBER || "").trim(); | |
| const blockedLabel = process.env.BLOCKED_LABEL; | |
| const botIssuePattern = /^\[(?:BOT ISSUE|Bot Issue)\]/i; | |
| const batchDir = process.env.BATCH_DIR; | |
| fs.mkdirSync(batchDir, { recursive: true }); | |
| async function getIssue(issueNumber) { | |
| const { data } = await github.rest.issues.get({ | |
| owner, | |
| repo, | |
| issue_number: issueNumber, | |
| }); | |
| if (data.pull_request) { | |
| throw new Error(`#${issueNumber} is a pull request, not an issue`); | |
| } | |
| return data; | |
| } | |
| async function hasOpenPullRequest(branchName) { | |
| const { data } = await github.rest.pulls.list({ | |
| owner, | |
| repo, | |
| state: "open", | |
| head: `${owner}:${branchName}`, | |
| per_page: 1, | |
| }); | |
| return data.length > 0; | |
| } | |
| async function getIssuesInOpenBatchPullRequests() { | |
| const pulls = await github.paginate(github.rest.pulls.list, { | |
| owner, | |
| repo, | |
| state: "open", | |
| per_page: 100, | |
| }); | |
| const issueNumbers = new Set(); | |
| for (const pull of pulls) { | |
| if (!pull.head?.ref?.startsWith("chore/autofix-bot-issues-")) { | |
| continue; | |
| } | |
| const body = pull.body || ""; | |
| for (const match of body.matchAll(/\b(?:Closes|Fixes|Resolves)\s+#(\d+)\b/gi)) { | |
| issueNumbers.add(Number(match[1])); | |
| } | |
| } | |
| return issueNumbers; | |
| } | |
| let candidates = []; | |
| if (manualIssueNumber.length > 0) { | |
| const issueNumber = Number.parseInt(manualIssueNumber, 10); | |
| if (!Number.isInteger(issueNumber)) { | |
| throw new Error(`Invalid issue number: ${manualIssueNumber}`); | |
| } | |
| candidates = [await getIssue(issueNumber)]; | |
| } else { | |
| const issues = await github.paginate(github.rest.issues.listForRepo, { | |
| owner, | |
| repo, | |
| state: "open", | |
| per_page: 100, | |
| }); | |
| candidates = issues | |
| .filter((issue) => !issue.pull_request && botIssuePattern.test(issue.title)) | |
| .sort( | |
| (a, b) => | |
| new Date(a.created_at).getTime() - new Date(b.created_at).getTime(), | |
| ); | |
| } | |
| const selectedIssues = []; | |
| const issuesInOpenBatchPullRequests = | |
| manualIssueNumber.length === 0 | |
| ? await getIssuesInOpenBatchPullRequests() | |
| : new Set(); | |
| for (const issue of candidates) { | |
| if (!botIssuePattern.test(issue.title)) { | |
| continue; | |
| } | |
| const branchName = `chore/autofix-issue-${issue.number}`; | |
| if ( | |
| manualIssueNumber.length === 0 && | |
| ((await hasOpenPullRequest(branchName)) || | |
| issuesInOpenBatchPullRequests.has(issue.number) || | |
| issue.labels?.some((label) => | |
| typeof label === "string" | |
| ? label === blockedLabel | |
| : label?.name === blockedLabel, | |
| )) | |
| ) { | |
| continue; | |
| } | |
| selectedIssues.push({ | |
| number: issue.number, | |
| title: issue.title, | |
| body: issue.body ?? "", | |
| url: issue.html_url, | |
| }); | |
| } | |
| const today = new Date().toISOString().slice(0, 10); | |
| const branchName = | |
| manualIssueNumber.length > 0 | |
| ? `chore/autofix-bot-issues-manual-${manualIssueNumber}` | |
| : `chore/autofix-bot-issues-${today}`; | |
| const manifest = { | |
| branch: branchName, | |
| issues: selectedIssues, | |
| }; | |
| fs.writeFileSync( | |
| path.join(batchDir, "issues.json"), | |
| JSON.stringify(manifest, null, 2) + "\n", | |
| ); | |
| for (const issue of selectedIssues) { | |
| fs.writeFileSync( | |
| path.join(batchDir, `issue-${issue.number}.body.md`), | |
| issue.body, | |
| ); | |
| } | |
| if (selectedIssues.length === 0) { | |
| core.setOutput("found", "false"); | |
| await core.summary | |
| .addHeading("Fix missing model bot issues") | |
| .addRaw("No eligible bot issues found.") | |
| .write(); | |
| return; | |
| } | |
| core.setOutput("found", "true"); | |
| core.setOutput("count", String(selectedIssues.length)); | |
| core.setOutput("branch", branchName); | |
| - name: Checkout repository | |
| if: steps.issues.outputs.found == 'true' | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| if: steps.issues.outputs.found == 'true' | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: 24 | |
| registry-url: "https://registry.npmjs.org" | |
| - name: Setup pnpm | |
| if: steps.issues.outputs.found == 'true' | |
| uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| with: | |
| version: 10.33.0 | |
| - name: Get pnpm store directory | |
| if: steps.issues.outputs.found == 'true' | |
| shell: bash | |
| run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV | |
| - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| if: steps.issues.outputs.found == 'true' | |
| with: | |
| path: ${{ env.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Install dependencies | |
| if: steps.issues.outputs.found == 'true' | |
| run: pnpm install --frozen-lockfile | |
| - name: Apply deterministic fixes | |
| if: steps.issues.outputs.found == 'true' | |
| id: fix_initial | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| BATCH_DIR="$RUNNER_TEMP/$BATCH_DIR" | |
| mkdir -p "$BATCH_DIR/results" | |
| : > "$BATCH_DIR/unsupported.txt" | |
| while IFS= read -r encoded_issue; do | |
| issue_json=$(printf '%s' "$encoded_issue" | base64 --decode) | |
| issue_number=$(jq -r '.number' <<<"$issue_json") | |
| issue_title=$(jq -r '.title' <<<"$issue_json") | |
| body_file="$BATCH_DIR/issue-${issue_number}.body.md" | |
| result_file="$BATCH_DIR/results/issue-${issue_number}.initial.json" | |
| pnpm exec tsx packages/proxy/scripts/fix_bot_issue.ts resolve-issue \ | |
| --title "$issue_title" \ | |
| --body-file "$body_file" \ | |
| --result-path "$result_file" \ | |
| --write | |
| if [ "$(jq -r '.action' "$result_file")" = "unsupported" ]; then | |
| echo "$issue_number" >> "$BATCH_DIR/unsupported.txt" | |
| fi | |
| done < <(jq -r '.issues[] | @base64' "$BATCH_DIR/issues.json") | |
| unsupported_count=$(wc -l < "$BATCH_DIR/unsupported.txt" | tr -d ' ') | |
| echo "unsupported_count=$unsupported_count" >> "$GITHUB_OUTPUT" | |
| - name: Verify proposed changes and document sync_models deviations | |
| if: steps.fix_initial.outputs.unsupported_count != '0' | |
| timeout-minutes: 30 | |
| uses: anthropics/claude-code-action@fbda2eb1bdc90d319b8d853f5deb53bca199a7c1 # v1.0.140 | |
| with: | |
| anthropic_api_key: ${{ secrets.BRAINTRUST_AGENT_SPEND_API_KEY }} | |
| github_token: ${{ github.token }} | |
| display_report: "true" | |
| claude_args: | | |
| --model claude-sonnet-5 | |
| --max-turns 80 | |
| --allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,Edit,Write" | |
| --disallowedTools "Bash,MultiEdit,Replace,NotebookEditCell,mcp__github__create_issue,mcp__github__create_issue_comment,mcp__github__update_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files" | |
| prompt: | | |
| # Goal | |
| The deterministic resolver could not apply one or more bot issues, either due to missing metadata or an unresolvable field. Your job is to improve each unsupported issue body independently so the resolver can retry it as part of one daily batch PR. | |
| This is a verification and documentation step, not just gap-filling. Keep each issue scoped to the exact models it already names. | |
| # Inputs | |
| - Batch manifest: `${{ runner.temp }}/${{ env.BATCH_DIR }}/issues.json` | |
| - Unsupported issue numbers: `${{ runner.temp }}/${{ env.BATCH_DIR }}/unsupported.txt` | |
| - Initial result files: `${{ runner.temp }}/${{ env.BATCH_DIR }}/results/issue-<number>.initial.json` | |
| - Original issue bodies: `${{ runner.temp }}/${{ env.BATCH_DIR }}/issue-<number>.body.md` | |
| - Output files: write enriched bodies to `${{ runner.temp }}/${{ env.BATCH_DIR }}/issue-<number>.body.enriched.md` | |
| # Local files to read first | |
| - `packages/proxy/scripts/fix_bot_issue.ts` — read this to understand exactly what fields the resolver requires and what caused it to bail | |
| - `packages/proxy/schema/model_list.json` — use neighboring entries as the style reference for a complete spec | |
| - `packages/proxy/schema/index.ts` | |
| - `packages/proxy/schema/models.ts` | |
| # Process | |
| 1. Read the unsupported issue numbers file, then process each listed issue independently. | |
| 2. For each issue, read its initial result file and original issue body. Use the resolver message to identify exactly which field(s) caused the bail. Focus your research on those fields first. | |
| 3. Fetch the official source URLs already listed in the issue first. Search additional official provider docs or APIs only if the listed sources don't provide the missing field. | |
| 4. For each model in the issue, focus on the field(s) that caused the bail (identified in step 2). Fill in additional fields only when the official source you are already reading readily provides them — do not open extra sources or spend turns exhaustively re-verifying fields that are unlikely to be wrong. Fields to consider, bail field(s) first: | |
| - `format`, `flavor`, `displayName` | |
| - `parent` when official sources explicitly document a relationship to a stable base alias and that parent model id exists in `model_list.json` — patterns include: dated snapshot → stable alias (e.g. `claude-3-5-sonnet-20241022` → `claude-3-5-sonnet-latest`), `@NNN`/`-vN` versioned snapshots, location-scoped prefixes (`global.`, `us.`, `eu.`, `apac.`), and provider-documented tier variants (e.g. `:free`, `:nitro` on Together AI) | |
| - `available_providers` and provider mapping | |
| - `max_input_tokens`, `max_output_tokens`, and other token/context limits | |
| - pricing fields (input, output, cache — use on-demand public pricing only) | |
| - `deprecation`/`retirement` status and dates | |
| - capability flags (`reasoning`, multimodal, etc.) when they map cleanly to the local schema | |
| - `locations` and supported regions when required | |
| - `supported_regions` for any model whose `available_providers` includes `"vertex"` — see the dedicated section below | |
| 5. **Vertex `supported_regions`**: For every model with `"vertex"` in `available_providers`, populate `supported_regions` in the model spec. The resolver does not scrape the Vertex docs page, so this field must come from you. | |
| - Source of truth: `https://docs.cloud.google.com/gemini-enterprise-agent-platform/resources/locations`. Look for the `Google models` section and find the row whose `<code>` cell matches the model id (e.g. `gemini-2.5-pro`), then collect every region code (e.g. `us-central1`, `europe-west1`) whose cell is marked Supported. | |
| - Always include `"global"` first if Google lists the model as available on the global endpoint, followed by per-region codes sorted alphabetically. Match the ordering used by existing vertex entries in `model_list.json`. | |
| - If the model id is not listed on the Google docs page (e.g. preview-only or recently announced), check the model's product page or release notes for explicit region availability. If you still cannot find an official region list, omit `supported_regions` entirely and call this out in the `## Verification` section — do not invent regions. | |
| - Do not copy `supported_regions` from sync_models; sync_models does not track Vertex regions accurately. | |
| 6. **Cross-check against sync_models (only the fields you added or changed)**: For just the numeric field(s) you set or corrected in this fix — not the whole spec — compare against the sync_models value for the same model. Look the model up in the sync_models catalog (`https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/litellm/model_prices_and_context_window_backup.json`) for those specific fields; do not scan or diff the entire file. In the `## Verification` section, for every changed field that differs from sync_models, you must: | |
| - State the sync_models value explicitly | |
| - State the official source URL that justifies the deviation | |
| - Briefly explain why the official source should be preferred (e.g., sync_models lags, collapses aliases, or has an incorrect mapping) | |
| - If you cannot cite an official source that justifies a deviation, use the sync_models value or omit the field instead of overriding silently | |
| 7. Re-verify provider/model relationships: only keep providers where the model appears on the provider's public shared inference surface (serverless, general catalog). Ignore bring-your-own-model flows, custom deployments, template galleries, and self-hosted guides. For Baseten, ignore custom-deployment-only evidence. | |
| 8. Match the style of neighboring entries in `model_list.json` and `index.ts` for a complete, up-to-date spec. | |
| 9. Do not add models not already named in the issue. Do not broaden scope to other providers or models. | |
| 10. If a multi-model issue is only partially valid, do not shrink it to a verified subset — write the original body unchanged and explain in `## Verification` which models failed and why, so the resolver continues to bail rather than applying a partial fix. | |
| 11. Do not guess any field. If a value is not published or not applicable, omit it and note it in the verification section. | |
| 12. **Out of scope: computer-vision-only models.** The catalog only tracks text/chat LLMs (including multimodal LLMs that accept images as input alongside text). Do not enrich, add, or research models whose sole purpose is computer vision — image generation (e.g. Imagen, DALL·E, Stable Diffusion, Flux), image editing/inpainting, image embeddings/classification, OCR, object detection, segmentation, video generation (e.g. Veo, Sora, Runway), or speech-only models. If the issue names a CV-only model, do not write a verified spec for it — leave the original body unchanged and note in `## Verification` that the model is out of scope so the resolver bails to `unsupported`. | |
| # Output | |
| **If you can verify enough metadata for the resolver to proceed:** | |
| Write the full enriched issue body to that issue's `.body.enriched.md` output file. Include: | |
| - A `## Verification` section with: | |
| - Each official source URL and which fields it verified | |
| - For each field that deviates from sync_models: the sync_models value, the proposed value, the source URL justifying the deviation, and why it is preferred | |
| - Fields that were not published or not applicable | |
| - A single `<!-- fix-bot-issue-metadata -->` JSON block with a fully populated `model_specs` map | |
| **If you cannot verify enough to improve the issue safely:** | |
| Write the original issue body unchanged to that issue's `.body.enriched.md` output file. | |
| The output file must contain the full issue body text, not just the JSON block. | |
| # Display name examples | |
| **Good** (model id → `displayName`): | |
| - `"mistral-small-latest"` → `"Mistral Small"` — stable alias, no date suffix, no version number | |
| - `"mistral-large-2512"` → `"Mistral Large 3 (2512)"` — version "3" confirmed in Mistral's official release docs; YYMM kept as-is in parentheses | |
| - `"claude-3-5-sonnet-20241022"` → `"Claude 3.5 Sonnet (2024-10-22)"` — ISO date in parentheses, matches official Anthropic naming exactly | |
| - `"gpt-4o-2024-11-20"` → `"GPT-4o (2024-11-20)"` — ISO date in parentheses, provider spells it "GPT-4o" | |
| **Bad** (what NOT to do): | |
| - `"mistral-small-2603"` → `"Mistral Small 4 (2603)"` ❌ — "4" is a generation number not found in the model id or official Mistral docs for this snapshot; if official docs do not confirm the generation label, omit it and write `"Mistral Small (2603)"` instead | |
| - `"mistral-small-2603"` → `"Mistral Small (March 2026)"` ❌ — never expand a YYMM code into prose month/year; keep the original numeric form in parentheses | |
| - `"claude-3-5-sonnet-20241022"` → `"Claude 3.5 Sonnet October 2024"` ❌ — date must be in parentheses, not free prose | |
| # Constraints | |
| - Do not edit repository source files. | |
| - Do not create or update GitHub issues, comments, or pull requests. | |
| - Do not use third-party aggregators as the source of truth. | |
| - Do not invent values. | |
| - name: Re-apply enriched fixes | |
| if: steps.fix_initial.outputs.unsupported_count != '0' | |
| run: | | |
| set -euo pipefail | |
| BATCH_DIR="$RUNNER_TEMP/$BATCH_DIR" | |
| while IFS= read -r issue_number; do | |
| [ -n "$issue_number" ] || continue | |
| issue_title=$(jq -r --arg number "$issue_number" '.issues[] | select((.number | tostring) == $number) | .title' "$BATCH_DIR/issues.json") | |
| original_body="$BATCH_DIR/issue-${issue_number}.body.md" | |
| enriched_body="$BATCH_DIR/issue-${issue_number}.body.enriched.md" | |
| result_file="$BATCH_DIR/results/issue-${issue_number}.retry.json" | |
| if [ ! -f "$enriched_body" ]; then | |
| cp "$original_body" "$enriched_body" | |
| fi | |
| pnpm exec tsx packages/proxy/scripts/fix_bot_issue.ts resolve-issue \ | |
| --title "$issue_title" \ | |
| --body-file "$enriched_body" \ | |
| --result-path "$result_file" \ | |
| --write | |
| done < "$BATCH_DIR/unsupported.txt" | |
| - name: Read final fix results | |
| id: result | |
| if: steps.issues.outputs.found == 'true' | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| env: | |
| BATCH_DIR: ${{ runner.temp }}/${{ env.BATCH_DIR }} | |
| with: | |
| script: | | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const batchDir = process.env.BATCH_DIR; | |
| const manifest = JSON.parse( | |
| fs.readFileSync(path.join(batchDir, "issues.json"), "utf8"), | |
| ); | |
| const escapeCell = (value) => | |
| String(value ?? "n/a").replace(/\|/g, "\\|").replace(/\n/g, "<br>"); | |
| const codeList = (items) => | |
| items && items.length > 0 | |
| ? items.map((item) => `\`${item}\``).join("<br>") | |
| : "None"; | |
| const sourceLinks = (items) => | |
| items && items.length > 0 | |
| ? items.map((item, index) => `[${index + 1}](${item})`).join("<br>") | |
| : "None"; | |
| const readFinalResult = (issueNumber) => { | |
| const retryPath = path.join( | |
| batchDir, | |
| "results", | |
| `issue-${issueNumber}.retry.json`, | |
| ); | |
| const initialPath = path.join( | |
| batchDir, | |
| "results", | |
| `issue-${issueNumber}.initial.json`, | |
| ); | |
| return JSON.parse( | |
| fs.readFileSync(fs.existsSync(retryPath) ? retryPath : initialPath, "utf8"), | |
| ); | |
| }; | |
| const buildVerificationSection = (issueNumber) => { | |
| const enrichedPath = path.join( | |
| batchDir, | |
| `issue-${issueNumber}.body.enriched.md`, | |
| ); | |
| if (!fs.existsSync(enrichedPath)) { | |
| return "_No LLM verification step ran; model metadata was already complete in the issue._"; | |
| } | |
| const body = fs.readFileSync(enrichedPath, "utf8"); | |
| const match = body.match( | |
| /^## Verification[\s\S]*?(?=\n## (?!Verification)|\n<!-- fix-bot-issue-metadata -->|$)/m, | |
| ); | |
| return match ? match[0].trim() : "_No Verification section found in enriched issue body._"; | |
| }; | |
| const verificationTable = (rows) => | |
| rows && rows.length > 0 | |
| ? [ | |
| "| Model | Display name | Parent | Providers | Format | Flavor | Token limits | Pricing | Lifecycle |", | |
| "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", | |
| ...rows.map((row) => | |
| `| ${[ | |
| row.model, | |
| row.display_name ?? "", | |
| row.parent ?? "", | |
| row.providers, | |
| row.format, | |
| row.flavor, | |
| row.token_limits, | |
| row.pricing, | |
| row.lifecycle, | |
| ] | |
| .map((value) => escapeCell(value)) | |
| .join(" | ")} |`, | |
| ), | |
| ].join("\n") | |
| : "No per-model verification rows were generated."; | |
| const litellmTable = (rows) => | |
| rows && rows.length > 0 | |
| ? [ | |
| "| Model | Field | Proposed update | sync_models | sync_models source models |", | |
| "| --- | --- | --- | --- | --- |", | |
| ...rows.map((row) => | |
| `| ${[ | |
| row.model, | |
| row.field, | |
| row.proposed_value, | |
| row.sync_models_value, | |
| row.sync_models_models, | |
| ] | |
| .map((value) => escapeCell(value)) | |
| .join(" | ")} |`, | |
| ), | |
| ].join("\n") | |
| : "No sync_models discrepancies were noted."; | |
| const changed = []; | |
| const alreadyResolved = []; | |
| const unsupported = []; | |
| const allAddedModels = []; | |
| const allUpdatedModels = []; | |
| for (const issue of manifest.issues) { | |
| const result = readFinalResult(issue.number); | |
| const entry = { ...issue, result }; | |
| if (result.action === "changed") { | |
| changed.push(entry); | |
| allAddedModels.push(...(result.added_models || [])); | |
| allUpdatedModels.push(...(result.updated_models || [])); | |
| } else if ( | |
| result.action === "already_present" || | |
| result.action === "deprecated" | |
| ) { | |
| alreadyResolved.push(entry); | |
| } else { | |
| unsupported.push(entry); | |
| } | |
| } | |
| const unique = (items) => Array.from(new Set(items)); | |
| const bodyLines = [ | |
| "Automated daily batch of model catalog updates from bot issues.", | |
| "", | |
| "## Included issues", | |
| "", | |
| ...changed.map((entry) => `- Closes #${entry.number}: ${entry.title}`), | |
| "", | |
| "## Summary", | |
| "", | |
| "| Issue | Provider | Primary model | Changed models | Added models | Updated models | Verification sources |", | |
| "| --- | --- | --- | --- | --- | --- | --- |", | |
| ...changed.map(({ number, result }) => | |
| `| #${number} | ${escapeCell(result.provider || "n/a")} | ${escapeCell(result.model || "n/a")} | ${codeList(result.changed_models || [])} | ${codeList(result.added_models || [])} | ${codeList(result.updated_models || [])} | ${sourceLinks(result.source_urls || [])} |`, | |
| ), | |
| "", | |
| "## Verified metadata", | |
| "", | |
| ...changed.flatMap(({ number, title, result }) => [ | |
| `### #${number}: ${title}`, | |
| "", | |
| verificationTable(result.verification_rows || []), | |
| "", | |
| "### Verification notes", | |
| "", | |
| buildVerificationSection(number), | |
| "", | |
| "### sync_models vs proposed update", | |
| "", | |
| result.litellm_comparison_summary || | |
| "sync_models cross-check was not needed for this result.", | |
| "", | |
| litellmTable(result.litellm_comparison_rows || []), | |
| "", | |
| ]), | |
| ]; | |
| if (changed.length === 0) { | |
| bodyLines.push("No issue produced catalog changes in this batch."); | |
| } | |
| fs.writeFileSync(path.join(batchDir, "pull-request-body.md"), bodyLines.join("\n")); | |
| fs.writeFileSync( | |
| path.join(batchDir, "final-results.json"), | |
| JSON.stringify({ changed, alreadyResolved, unsupported }, null, 2) + "\n", | |
| ); | |
| core.setOutput("action", changed.length > 0 ? "changed" : "unchanged"); | |
| core.setOutput("changed_count", String(changed.length)); | |
| core.setOutput("already_resolved_count", String(alreadyResolved.length)); | |
| core.setOutput("unsupported_count", String(unsupported.length)); | |
| core.setOutput("added_models", unique(allAddedModels).join(", ")); | |
| core.setOutput("updated_models", unique(allUpdatedModels).join(", ")); | |
| # Always regenerate AvailableEndpointTypes from model_list.json before | |
| # opening the PR. Models added by the Claude fallback (rather than the | |
| # deterministic resolve-issue path) don't call updateAvailableEndpointTypes, | |
| # so without this their index.ts provider mapping is missing. Running it | |
| # here (node_modules is installed; no PR-branch re-checkout yet) means the | |
| # mapping lands in the first PR commit regardless of the Codex path. | |
| - name: Canonicalize model list | |
| if: steps.result.outputs.action == 'changed' | |
| run: pnpm exec tsx packages/proxy/scripts/sync_models.ts normalize-local-models --write | |
| - name: Check expected files only | |
| id: changes | |
| if: steps.result.outputs.action == 'changed' | |
| run: | | |
| set -euo pipefail | |
| if [ -z "$(git status --short)" ]; then | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| unexpected=$(git status --porcelain | awk '{print $2}' | grep -vE '^(packages/proxy/schema/model_list\.json|packages/proxy/schema/index\.ts)$' || true) | |
| if [ -n "$unexpected" ]; then | |
| echo "Unexpected files modified:" | |
| echo "$unexpected" | |
| exit 1 | |
| fi | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| - name: Build proxy package | |
| if: steps.result.outputs.action == 'changed' && steps.changes.outputs.changed == 'true' | |
| run: pnpm --filter @braintrust/proxy run build | |
| - name: Validate model schema | |
| if: steps.result.outputs.action == 'changed' && steps.changes.outputs.changed == 'true' | |
| run: pnpm exec vitest run packages/proxy/schema/models.test.ts packages/proxy/scripts/fix_bot_issue.test.ts packages/proxy/scripts/sync_models.test.ts | |
| - name: Create PR | |
| id: pr | |
| if: steps.result.outputs.action == 'changed' && steps.changes.outputs.changed == 'true' | |
| uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7.0.11 | |
| with: | |
| token: ${{ github.token }} | |
| base: main | |
| branch: ${{ steps.issues.outputs.branch }} | |
| commit-message: "chore: update model catalog from bot issues" | |
| title: "chore: update model catalog from bot issues" | |
| body-path: ${{ runner.temp }}/${{ env.BATCH_DIR }}/pull-request-body.md | |
| reviewers: | | |
| knjiang | |
| cpinn | |
| erin2722 | |
| CLowbrow | |
| aswink | |
| labels: auto-sync | |
| signoff: false | |
| - name: Request Codex review | |
| id: codex_request | |
| if: steps.pr.outputs.pull-request-number != '' | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| env: | |
| PULL_REQUEST_NUMBER: ${{ steps.pr.outputs.pull-request-number }} | |
| with: | |
| script: | | |
| const requestedAt = new Date().toISOString(); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: Number(process.env.PULL_REQUEST_NUMBER), | |
| body: "<!-- codex-model-sync-review-request -->\n@codex review", | |
| }); | |
| core.setOutput("requested_at", requestedAt); | |
| - name: Update bot issues | |
| if: steps.issues.outputs.found == 'true' | |
| uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 | |
| env: | |
| BATCH_DIR: ${{ runner.temp }}/${{ env.BATCH_DIR }} | |
| BLOCKED_LABEL: ${{ env.BLOCKED_LABEL }} | |
| PULL_REQUEST_URL: ${{ steps.pr.outputs.pull-request-url }} | |
| with: | |
| script: | | |
| const fs = require("fs"); | |
| const path = require("path"); | |
| const finalResults = JSON.parse( | |
| fs.readFileSync( | |
| path.join(process.env.BATCH_DIR, "final-results.json"), | |
| "utf8", | |
| ), | |
| ); | |
| async function removeBlockedLabel(issueNumber) { | |
| try { | |
| await github.rest.issues.removeLabel({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issueNumber, | |
| name: process.env.BLOCKED_LABEL, | |
| }); | |
| } catch (error) { | |
| if (error.status !== 404) { | |
| throw error; | |
| } | |
| } | |
| } | |
| async function ensureLabel() { | |
| try { | |
| await github.rest.issues.getLabel({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: process.env.BLOCKED_LABEL, | |
| }); | |
| } catch (error) { | |
| if (error.status !== 404) { | |
| throw error; | |
| } | |
| await github.rest.issues.createLabel({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: process.env.BLOCKED_LABEL, | |
| color: "BFDADC", | |
| description: | |
| "Bot issue is blocked until metadata or source information is updated.", | |
| }); | |
| } | |
| } | |
| if (process.env.PULL_REQUEST_URL) { | |
| for (const entry of finalResults.changed) { | |
| await removeBlockedLabel(entry.number); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: entry.number, | |
| body: `${entry.result.message}\n\n${entry.result.verification_summary || ""}\n\nIncluded in the daily batch PR: ${process.env.PULL_REQUEST_URL}`, | |
| }); | |
| } | |
| } | |
| for (const entry of finalResults.alreadyResolved) { | |
| await removeBlockedLabel(entry.number); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: entry.number, | |
| body: `${entry.result.message}\n\n${entry.result.verification_summary || ""}`, | |
| }); | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: entry.number, | |
| state: "closed", | |
| }); | |
| } | |
| if (finalResults.unsupported.length > 0) { | |
| await ensureLabel(); | |
| } | |
| for (const entry of finalResults.unsupported) { | |
| await github.rest.issues.addLabels({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: entry.number, | |
| labels: [process.env.BLOCKED_LABEL], | |
| }); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: entry.number, | |
| body: entry.result.comment_body || entry.result.message, | |
| }); | |
| } |