Skip to content

Commit c005779

Browse files
srcKodnicobailon
andauthored
fix: detect context-overflow errors separately from retryable failures (#1312)
* fix: detect context-overflow errors separately from retryable failures Context-overflow errors ("maximum context length", "too many tokens", "context_length_exceeded", etc.) mean the input is too large for the model's window. Retrying the same input on another model cannot succeed, yet today these errors can match generic retryable patterns and burn fallback attempts on a guaranteed failure. Add isContextOverflow() as a dedicated, non-retryable classifier so callers can stop the fallback loop early and surface a clear "input too large" error. Tool-failure prefixed errors are excluded, matching isRetryableModelFailure's convention. * docs: credit context overflow contribution --------- Co-authored-by: Nico Bailon <nico.bailon@gmail.com>
1 parent 1f3999b commit c005779

3 files changed

Lines changed: 56 additions & 0 deletions

File tree

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
profile examples (#1295).
1414

1515
### Fixed
16+
- Add a separate classifier for model context-overflow errors. Thanks to [@srcKod](https://github.com/srcKod) for #1312.
1617
- Quote only confidently identified leading Windows executable paths in acceptance verification commands. Thanks to [@srcKod](https://github.com/srcKod) for #1294.
1718

1819
### Changed

src/runs/shared/model-fallback.ts

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,34 @@ export function isRetryableModelFailure(error: string | undefined): boolean {
424424
return RETRYABLE_MODEL_FAILURE_PATTERNS.some((pattern) => pattern.test(error));
425425
}
426426

427+
/**
428+
* Context-overflow signals. These are deliberately NOT part of
429+
* {@link RETRYABLE_MODEL_FAILURE_PATTERNS}: an overflow means the input was too
430+
* large for the model's context window, so retrying the same input on another
431+
* model (or the same model again) cannot succeed. Callers should treat overflow
432+
* as a terminal, non-retryable failure and surface a clear "input too large"
433+
* error instead of burning fallback attempts on a guaranteed failure.
434+
*/
435+
const CONTEXT_OVERFLOW_PATTERNS = [
436+
/context(?: length| window| limit)? (?:exceed|overflow|too long)/i,
437+
/maximum context length/i,
438+
/too many tokens/i,
439+
/token limit/i,
440+
/context_length_exceeded/i,
441+
/length_required/i,
442+
/maximum.*tokens/i,
443+
/prompt.*too long/i,
444+
/input.*too long/i,
445+
/exceeded.*context/i,
446+
/context.*overflow/i,
447+
];
448+
449+
export function isContextOverflow(error: string | undefined): boolean {
450+
if (!error) return false;
451+
if (TOOL_FAILURE_PREFIX.test(error.trim())) return false;
452+
return CONTEXT_OVERFLOW_PATTERNS.some((pattern) => pattern.test(error));
453+
}
454+
427455
export function formatModelAttemptNote(attempt: ModelAttemptSummary, nextModel?: string): string {
428456
const failure = attempt.error?.trim() || `exit ${attempt.exitCode ?? 1}`;
429457
return nextModel

test/unit/model-fallback.test.ts

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { describe, it } from "node:test";
33
import {
44
buildModelCandidates,
55
fuzzyResolveModel,
6+
isContextOverflow,
67
isRetryableModelFailure,
78
normalizeModelSegment,
89
resolveEffectiveSubagentModel,
@@ -529,3 +530,29 @@ describe("resolveSubagentModelOverride scope enforcement", () => {
529530
);
530531
});
531532
});
533+
534+
describe("isContextOverflow", () => {
535+
it("detects common context-overflow error shapes", () => {
536+
assert.equal(isContextOverflow("This model's maximum context length is 8192 tokens"), true);
537+
assert.equal(isContextOverflow("context length exceeded for the requested prompt"), true);
538+
assert.equal(isContextOverflow("too many tokens in the request"), true);
539+
assert.equal(isContextOverflow("context_length_exceeded"), true);
540+
assert.equal(isContextOverflow("prompt is too long for this model"), true);
541+
assert.equal(isContextOverflow("input too long: 40000 tokens"), true);
542+
});
543+
544+
it("does not flag unrelated retryable failures as overflow", () => {
545+
assert.equal(isContextOverflow("rate limit exceeded for provider"), false);
546+
assert.equal(isContextOverflow("503 service unavailable"), false);
547+
assert.equal(isContextOverflow("connection refused"), false);
548+
});
549+
550+
it("does not flag tool failures as overflow", () => {
551+
assert.equal(isContextOverflow("bash failed (exit 1): context length exceeded in output"), false);
552+
});
553+
554+
it("returns false for empty or undefined input", () => {
555+
assert.equal(isContextOverflow(undefined), false);
556+
assert.equal(isContextOverflow(""), false);
557+
});
558+
});

0 commit comments

Comments
 (0)