Skip to content

Fix missing model bot issues #166

Fix missing model bot issues

Fix missing model bot issues #166

name: Fix missing model bot issues
on:
schedule:
- cron: "0 */2 * * *"
workflow_dispatch:
inputs:
issue_number:
description: "Specific issue number to fix"
required: false
type: string
debug_enabled:
description: "Reserved for parity with other workflows"
required: false
default: false
type: boolean
issues:
types: [opened, reopened, edited]
concurrency:
group: fix-missing-model-bot-issues
cancel-in-progress: false
permissions:
contents: write
pull-requests: write
issues: write
jobs:
fix:
runs-on: ubuntu-latest
env:
BLOCKED_LABEL: autofix-blocked
steps:
- name: Select issue
id: issue
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
env:
MANUAL_ISSUE_NUMBER: ${{ inputs.issue_number || '' }}
with:
script: |
const owner = context.repo.owner;
const repo = context.repo.repo;
const manualIssueNumber = (process.env.MANUAL_ISSUE_NUMBER || "").trim();
const blockedLabel = process.env.BLOCKED_LABEL;
const botIssuePattern = /^\[(?:BOT ISSUE|Bot Issue)\]/i;
async function getIssue(issueNumber) {
const { data } = await github.rest.issues.get({
owner,
repo,
issue_number: issueNumber,
});
if (data.pull_request) {
throw new Error(`#${issueNumber} is a pull request, not an issue`);
}
return data;
}
async function hasOpenPullRequest(branchName) {
const { data } = await github.rest.pulls.list({
owner,
repo,
state: "open",
head: `${owner}:${branchName}`,
per_page: 1,
});
return data.length > 0;
}
let candidates = [];
if (context.eventName === "issues") {
candidates = [context.payload.issue];
} else if (manualIssueNumber.length > 0) {
const issueNumber = Number.parseInt(manualIssueNumber, 10);
if (!Number.isInteger(issueNumber)) {
throw new Error(`Invalid issue number: ${manualIssueNumber}`);
}
candidates = [await getIssue(issueNumber)];
} else {
const issues = await github.paginate(github.rest.issues.listForRepo, {
owner,
repo,
state: "open",
per_page: 100,
});
candidates = issues
.filter((issue) => !issue.pull_request && botIssuePattern.test(issue.title))
.sort(
(a, b) =>
new Date(a.created_at).getTime() - new Date(b.created_at).getTime(),
);
}
let selected = null;
for (const issue of candidates) {
if (!botIssuePattern.test(issue.title)) {
continue;
}
const branchName = `chore/autofix-issue-${issue.number}`;
if (
context.eventName === "schedule" &&
manualIssueNumber.length === 0 &&
((await hasOpenPullRequest(branchName)) ||
issue.labels?.some((label) =>
typeof label === "string"
? label === blockedLabel
: label?.name === blockedLabel,
))
) {
continue;
}
selected = { issue, branchName };
break;
}
if (!selected) {
core.setOutput("found", "false");
await core.summary
.addHeading("Fix missing model bot issues")
.addRaw("No eligible bot issues found.")
.write();
return;
}
core.setOutput("found", "true");
core.setOutput("number", String(selected.issue.number));
core.setOutput("title", selected.issue.title);
core.setOutput("body", selected.issue.body ?? "");
core.setOutput("url", selected.issue.html_url);
core.setOutput("branch", selected.branchName);
- name: Checkout repository
if: steps.issue.outputs.found == 'true'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Set up Node.js
if: steps.issue.outputs.found == 'true'
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: 24
registry-url: "https://registry.npmjs.org"
- name: Setup pnpm
if: steps.issue.outputs.found == 'true'
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
with:
version: 10.33.0
- name: Get pnpm store directory
if: steps.issue.outputs.found == 'true'
shell: bash
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
if: steps.issue.outputs.found == 'true'
with:
path: ${{ env.STORE_PATH }}
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-store-
- name: Install dependencies
if: steps.issue.outputs.found == 'true'
run: pnpm install --frozen-lockfile
- name: Write issue body to file
if: steps.issue.outputs.found == 'true'
env:
ISSUE_BODY: ${{ steps.issue.outputs.body }}
run: |
set -euo pipefail
printf '%s' "$ISSUE_BODY" > "$RUNNER_TEMP/fix-missing-model-bot-issue-body.md"
- name: Apply fix from issue body
if: steps.issue.outputs.found == 'true'
id: fix_initial
env:
ISSUE_TITLE: ${{ steps.issue.outputs.title }}
run: |
set -euo pipefail
pnpm dlx tsx packages/proxy/scripts/fix_bot_issue.ts resolve-issue \
--title "$ISSUE_TITLE" \
--body-file "$RUNNER_TEMP/fix-missing-model-bot-issue-body.md" \
--result-path "$RUNNER_TEMP/fix-missing-model-bot-issue-result.initial.json" \
--write
- name: Read initial fix result
id: result_initial
if: steps.issue.outputs.found == 'true'
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
env:
RESULT_PATH: ${{ runner.temp }}/fix-missing-model-bot-issue-result.initial.json
with:
script: |
const fs = require("fs");
const resultPath = process.env.RESULT_PATH;
const result = JSON.parse(fs.readFileSync(resultPath, "utf8"));
core.setOutput("action", result.action);
core.setOutput("message", result.message);
core.setOutput("provider", result.provider || "");
core.setOutput("model", result.model || "");
core.setOutput("pr_title", result.pr_title || "");
core.setOutput("changed_models", (result.changed_models || []).join(", "));
core.setOutput("added_models", (result.added_models || []).join(", "));
core.setOutput("updated_models", (result.updated_models || []).join(", "));
core.setOutput("comment_body", result.comment_body || result.message);
core.setOutput("source_urls", (result.source_urls || []).join(", "));
core.setOutput("verification_summary", result.verification_summary || "");
- name: Verify proposed changes and document sync_models deviations
if: steps.result_initial.outputs.action == 'unsupported'
uses: anthropics/claude-code-action@df37d2f0760a4b5683a6e617c9325bc1a36443f6 # v1.0.75
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ github.token }}
show_full_output: "true"
display_report: "true"
timeout_minutes: "15"
claude_args: |
--model claude-opus-4-6
--max-turns 80
--allowedTools "Read,Glob,Grep,LS,WebSearch,WebFetch,Edit,Write"
--disallowedTools "Bash,MultiEdit,Replace,NotebookEditCell,mcp__github__create_issue,mcp__github__create_issue_comment,mcp__github__update_issue,mcp__github__create_pr,mcp__github__create_or_update_file,mcp__github__delete_file,mcp__github_file_ops__commit_files,mcp__github_file_ops__delete_files"
prompt: |
# Goal
The deterministic resolver could not apply a bot issue — either missing metadata or an unresolvable field. Your job is to:
1. Verify the proposed changes against official provider sources
2. Cross-check every numeric field against the sync_models (LiteLLM) reference catalog and explicitly document any deviation with a source URL and reason
3. Produce an updated issue body with fully verified metadata so the resolver can retry and open a PR
This is a verification and documentation step, not just gap-filling.
# Inputs
- Issue title: `${{ steps.issue.outputs.title }}`
- Issue URL: `${{ steps.issue.outputs.url }}`
- Original issue body: `${{ runner.temp }}/fix-missing-model-bot-issue-body.md`
- Output file (write enriched body here): `${{ runner.temp }}/fix-missing-model-bot-issue-body.enriched.md`
- Why the resolver failed: `${{ steps.result_initial.outputs.message }}`
# Local files to read first
- `packages/proxy/scripts/fix_bot_issue.ts` — read this to understand exactly what fields the resolver requires and what caused it to bail
- `packages/proxy/schema/model_list.json` — use neighboring entries as the style reference for a complete spec
- `packages/proxy/schema/index.ts`
- `packages/proxy/schema/models.ts`
# Process
1. Read `fix_bot_issue.ts` and the resolver failure message to identify exactly which field(s) caused the bail. Focus your research on those fields first.
2. Read the original issue body and note the official source URLs already listed.
3. Fetch those URLs first. Search additional official provider docs or APIs only if the listed sources don't provide the missing field.
4. For each model in the issue, build the most complete verified spec you can — not just the missing field. Verify all fields that official sources support:
- `format`, `flavor`, `displayName`
- `parent` when official sources explicitly document a relationship to a stable base alias and that parent model id exists in `model_list.json` — patterns include: dated snapshot → stable alias (e.g. `claude-3-5-sonnet-20241022` → `claude-3-5-sonnet-latest`), `@NNN`/`-vN` versioned snapshots, location-scoped prefixes (`global.`, `us.`, `eu.`, `apac.`), and provider-documented tier variants (e.g. `:free`, `:nitro` on Together AI)
- `available_providers` and provider mapping
- `max_input_tokens`, `max_output_tokens`, and other token/context limits
- pricing fields (input, output, cache — use on-demand public pricing only)
- `deprecation`/`retirement` status and dates
- capability flags (`reasoning`, multimodal, etc.) when they map cleanly to the local schema
- `locations` and supported regions when required
5. **Cross-check against sync_models**: Fetch the sync_models reference catalog at `https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/litellm/model_prices_and_context_window_backup.json`. For each numeric field in your proposed spec (token limits, pricing), compare it against the sync_models value for the same model. In the `## Verification` section, for every field that differs from sync_models, you must:
- State the sync_models value explicitly
- State the official source URL that justifies the deviation
- Briefly explain why the official source should be preferred (e.g., sync_models lags, collapses aliases, or has an incorrect mapping)
- If you cannot cite an official source that justifies a deviation, use the sync_models value or omit the field instead of overriding silently
6. Re-verify provider/model relationships: only keep providers where the model appears on the provider's public shared inference surface (serverless, general catalog). Ignore bring-your-own-model flows, custom deployments, template galleries, and self-hosted guides. For Baseten, ignore custom-deployment-only evidence.
7. Match the style of neighboring entries in `model_list.json` and `index.ts` for a complete, up-to-date spec.
8. Do not add models not already named in the issue. Do not broaden scope to other providers or models.
9. If a multi-model issue is only partially valid, do not shrink it to a verified subset — write the original body unchanged and explain in `## Verification` which models failed and why, so the resolver continues to bail rather than applying a partial fix.
10. Do not guess any field. If a value is not published or not applicable, omit it and note it in the verification section.
# Output
**If you can verify enough metadata for the resolver to proceed:**
Write the full enriched issue body to the output file. Include:
- A `## Verification` section with:
- Each official source URL and which fields it verified
- For each field that deviates from sync_models: the sync_models value, the proposed value, the source URL justifying the deviation, and why it is preferred
- Fields that were not published or not applicable
- A single `<!-- fix-bot-issue-metadata -->` JSON block with a fully populated `model_specs` map
**If you cannot verify enough to improve the issue safely:**
Write the original issue body unchanged to the output file.
The output file must contain the full issue body text, not just the JSON block.
# Display name examples
**Good** (model id → `displayName`):
- `"mistral-small-latest"` → `"Mistral Small"` — stable alias, no date suffix, no version number
- `"mistral-large-2512"` → `"Mistral Large 3 (2512)"` — version "3" confirmed in Mistral's official release docs; YYMM kept as-is in parentheses
- `"claude-3-5-sonnet-20241022"` → `"Claude 3.5 Sonnet (2024-10-22)"` — ISO date in parentheses, matches official Anthropic naming exactly
- `"gpt-4o-2024-11-20"` → `"GPT-4o (2024-11-20)"` — ISO date in parentheses, provider spells it "GPT-4o"
**Bad** (what NOT to do):
- `"mistral-small-2603"` → `"Mistral Small 4 (2603)"` ❌ — "4" is a generation number not found in the model id or official Mistral docs for this snapshot; if official docs do not confirm the generation label, omit it and write `"Mistral Small (2603)"` instead
- `"mistral-small-2603"` → `"Mistral Small (March 2026)"` ❌ — never expand a YYMM code into prose month/year; keep the original numeric form in parentheses
- `"claude-3-5-sonnet-20241022"` → `"Claude 3.5 Sonnet October 2024"` ❌ — date must be in parentheses, not free prose
# Constraints
- Do not edit repository source files.
- Do not create or update GitHub issues, comments, or pull requests.
- Do not use third-party aggregators as the source of truth.
- Do not invent values.
- name: Ensure enriched body file exists
if: steps.result_initial.outputs.action == 'unsupported'
run: |
set -euo pipefail
if [ ! -f "$RUNNER_TEMP/fix-missing-model-bot-issue-body.enriched.md" ]; then
cp \
"$RUNNER_TEMP/fix-missing-model-bot-issue-body.md" \
"$RUNNER_TEMP/fix-missing-model-bot-issue-body.enriched.md"
fi
- name: Re-apply fix from enriched issue body
if: steps.result_initial.outputs.action == 'unsupported'
id: fix_retry
env:
ISSUE_TITLE: ${{ steps.issue.outputs.title }}
run: |
set -euo pipefail
pnpm dlx tsx packages/proxy/scripts/fix_bot_issue.ts resolve-issue \
--title "$ISSUE_TITLE" \
--body-file "$RUNNER_TEMP/fix-missing-model-bot-issue-body.enriched.md" \
--result-path "$RUNNER_TEMP/fix-missing-model-bot-issue-result.retry.json" \
--write
- name: Extract verification section from enriched issue body
id: verification
if: steps.issue.outputs.found == 'true'
shell: bash
run: |
set -euo pipefail
enriched="$RUNNER_TEMP/fix-missing-model-bot-issue-body.enriched.md"
if [ ! -f "$enriched" ]; then
{
echo "section<<VEOF"
echo "_No LLM verification step ran — model metadata was already complete in the issue._"
echo "VEOF"
} >> "$GITHUB_OUTPUT"
exit 0
fi
section=$(awk '
/^## Verification/{found=1}
found && /^## / && !/^## Verification/{exit}
found && /<!-- fix-bot-issue-metadata -->/{exit}
found{print}
' "$enriched")
if [ -z "$section" ]; then
section="_No Verification section found in enriched issue body._"
fi
{
echo "section<<VEOF"
printf '%s\n' "$section"
echo "VEOF"
} >> "$GITHUB_OUTPUT"
- name: Read final fix result
id: result
if: steps.issue.outputs.found == 'true'
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
env:
INITIAL_RESULT_PATH: ${{ runner.temp }}/fix-missing-model-bot-issue-result.initial.json
RETRY_RESULT_PATH: ${{ runner.temp }}/fix-missing-model-bot-issue-result.retry.json
with:
script: |
const fs = require("fs");
const escapeCell = (value) =>
String(value ?? "n/a").replace(/\|/g, "\\|").replace(/\n/g, "<br>");
const codeList = (items) =>
items && items.length > 0
? items.map((item) => `\`${item}\``).join("<br>")
: "None";
const sourceLinks = (items) =>
items && items.length > 0
? items.map((item, index) => `[${index + 1}](${item})`).join("<br>")
: "None";
const resultPath = fs.existsSync(process.env.RETRY_RESULT_PATH)
? process.env.RETRY_RESULT_PATH
: process.env.INITIAL_RESULT_PATH;
const result = JSON.parse(fs.readFileSync(resultPath, "utf8"));
const summaryTable = [
"| Field | Value |",
"| --- | --- |",
`| Provider | ${escapeCell(result.provider || "n/a")} |`,
`| Primary model | ${escapeCell(result.model || "n/a")} |`,
`| Changed models | ${codeList(result.changed_models || [])} |`,
`| Added models | ${codeList(result.added_models || [])} |`,
`| Updated models | ${codeList(result.updated_models || [])} |`,
`| Verification sources | ${sourceLinks(result.source_urls || [])} |`,
].join("\n");
const verificationRows = result.verification_rows || [];
const verificationTable =
verificationRows.length > 0
? [
"| Model | Display name | Parent | Providers | Format | Flavor | Token limits | Pricing | Lifecycle |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
...verificationRows.map((row) =>
[
row.model,
row.display_name ?? "",
row.parent ?? "",
row.providers,
row.format,
row.flavor,
row.token_limits,
row.pricing,
row.lifecycle,
]
.map((value) => escapeCell(value))
.join(" | "),
).map((row) => `| ${row} |`),
].join("\n")
: "No per-model verification rows were generated.";
const liteLLMRows = result.litellm_comparison_rows || [];
const liteLLMTable =
liteLLMRows.length > 0
? [
"| Model | Field | Proposed update | sync_models | sync_models source models |",
"| --- | --- | --- | --- | --- |",
...liteLLMRows.map((row) =>
[
row.model,
row.field,
row.proposed_value,
row.sync_models_value,
row.sync_models_models,
]
.map((value) => escapeCell(value))
.join(" | "),
).map((row) => `| ${row} |`),
].join("\n")
: "No sync_models discrepancies were noted.";
core.setOutput("action", result.action);
core.setOutput("message", result.message);
core.setOutput("provider", result.provider || "");
core.setOutput("model", result.model || "");
core.setOutput("pr_title", result.pr_title || "");
core.setOutput("changed_models", (result.changed_models || []).join(", "));
core.setOutput("added_models", (result.added_models || []).join(", "));
core.setOutput("updated_models", (result.updated_models || []).join(", "));
core.setOutput("comment_body", result.comment_body || result.message);
core.setOutput("source_urls", (result.source_urls || []).join(", "));
core.setOutput("verification_summary", result.verification_summary || "");
core.setOutput("pr_summary_table", summaryTable);
core.setOutput("pr_verification_table", verificationTable);
core.setOutput(
"litellm_comparison_summary",
result.litellm_comparison_summary ||
"sync_models cross-check was not needed for this result.",
);
core.setOutput("pr_litellm_table", liteLLMTable);
- name: Check expected files only
id: changes
if: steps.result.outputs.action == 'changed'
run: |
set -euo pipefail
if [ -z "$(git status --short)" ]; then
echo "changed=false" >> "$GITHUB_OUTPUT"
exit 0
fi
unexpected=$(git status --porcelain | awk '{print $2}' | grep -vE '^(packages/proxy/schema/model_list\.json|packages/proxy/schema/index\.ts)$' || true)
if [ -n "$unexpected" ]; then
echo "Unexpected files modified:"
echo "$unexpected"
exit 1
fi
echo "changed=true" >> "$GITHUB_OUTPUT"
- name: Build proxy package
if: steps.result.outputs.action == 'changed' && steps.changes.outputs.changed == 'true'
run: pnpm --filter @braintrust/proxy run build
- name: Validate model schema
if: steps.result.outputs.action == 'changed' && steps.changes.outputs.changed == 'true'
run: pnpm exec vitest run packages/proxy/schema/models.test.ts packages/proxy/scripts/fix_bot_issue.test.ts packages/proxy/scripts/sync_models.test.ts
- name: Create PR
id: pr
if: steps.result.outputs.action == 'changed' && steps.changes.outputs.changed == 'true'
uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7.0.11
with:
token: ${{ github.token }}
base: main
branch: ${{ steps.issue.outputs.branch }}
commit-message: "${{ steps.result.outputs.pr_title }}"
title: "${{ steps.result.outputs.pr_title }}"
body: |
${{ steps.result.outputs.pr_title }}
Closes #${{ steps.issue.outputs.number }}
Source issue: ${{ steps.issue.outputs.url }}
**Summary**
${{ steps.result.outputs.pr_summary_table }}
**Verified metadata**
${{ steps.result.outputs.pr_verification_table }}
**Verification notes**
${{ steps.verification.outputs.section }}
**sync_models vs proposed update**
${{ steps.result.outputs.litellm_comparison_summary }}
${{ steps.result.outputs.pr_litellm_table }}
reviewers: |
knjiang
cpinn
erin2722
CLowbrow
aswink
labels: auto-sync
signoff: false
- name: Comment on fix PR
if: steps.pr.outputs.pull-request-url != ''
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
env:
BLOCKED_LABEL: ${{ env.BLOCKED_LABEL }}
ISSUE_NUMBER: ${{ steps.issue.outputs.number }}
PULL_REQUEST_URL: ${{ steps.pr.outputs.pull-request-url }}
RESULT_MESSAGE: ${{ steps.result.outputs.message }}
VERIFICATION_SUMMARY: ${{ steps.result.outputs.verification_summary }}
with:
script: |
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
name: process.env.BLOCKED_LABEL,
});
} catch (error) {
if (error.status !== 404) {
throw error;
}
}
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
body: `${process.env.RESULT_MESSAGE}\n\n${process.env.VERIFICATION_SUMMARY}\n\nOpened ${process.env.PULL_REQUEST_URL} to apply the catalog update.`,
});
- name: Close already resolved issue
if: steps.result.outputs.action == 'already_present' || steps.result.outputs.action == 'deprecated'
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
env:
BLOCKED_LABEL: ${{ env.BLOCKED_LABEL }}
ISSUE_NUMBER: ${{ steps.issue.outputs.number }}
RESULT_MESSAGE: ${{ steps.result.outputs.message }}
VERIFICATION_SUMMARY: ${{ steps.result.outputs.verification_summary }}
with:
script: |
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
name: process.env.BLOCKED_LABEL,
});
} catch (error) {
if (error.status !== 404) {
throw error;
}
}
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
body: `${process.env.RESULT_MESSAGE}\n\n${process.env.VERIFICATION_SUMMARY}`,
});
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
state: "closed",
});
- name: Comment on unsupported issue
if: steps.result.outputs.action == 'unsupported'
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
env:
BLOCKED_LABEL: ${{ env.BLOCKED_LABEL }}
ISSUE_NUMBER: ${{ steps.issue.outputs.number }}
RESULT_MESSAGE: ${{ steps.result.outputs.comment_body }}
with:
script: |
async function ensureLabel() {
try {
await github.rest.issues.getLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: process.env.BLOCKED_LABEL,
});
} catch (error) {
if (error.status !== 404) {
throw error;
}
await github.rest.issues.createLabel({
owner: context.repo.owner,
repo: context.repo.repo,
name: process.env.BLOCKED_LABEL,
color: "BFDADC",
description:
"Bot issue is blocked until metadata or source information is updated.",
});
}
}
await ensureLabel();
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
labels: [process.env.BLOCKED_LABEL],
});
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: Number(process.env.ISSUE_NUMBER),
body: process.env.RESULT_MESSAGE,
});