Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion core/config/yaml/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,28 @@ import {
ModelConfig,
} from "@continuedev/config-yaml";

import { ContinueConfig, ILLMLogger, LLMOptions } from "../..";
import { ContinueConfig, ILLMLogger, LLMOptions, ToolOverride } from "../..";
import { BaseLLM } from "../../llm";
import { LLMClasses } from "../../llm/llms";

const AUTODETECT = "AUTODETECT";

/**
* Converts YAML record format { toolName: {...} } to array format [{ name: toolName, ...}]
* for use with applyToolOverrides
*/
function convertToolPromptOverridesToArray(
record: Record<string, Omit<ToolOverride, "name">> | undefined,
): ToolOverride[] | undefined {
if (!record) {
return undefined;
}
return Object.entries(record).map(([name, override]) => ({
name,
...override,
}));
}

function getModelClass(
model: ModelConfig,
): (typeof LLMClasses)[number] | undefined {
Expand Down Expand Up @@ -67,6 +83,9 @@ async function modelConfigToBaseLLM({
baseAgentSystemMessage: model.chatOptions?.baseAgentSystemMessage,
basePlanSystemMessage: model.chatOptions?.basePlanSystemMessage,
baseChatSystemMessage: model.chatOptions?.baseSystemMessage,
toolPromptOverrides: convertToolPromptOverridesToArray(
model.chatOptions?.toolPromptOverrides,
),
capabilities: {
tools: model.capabilities?.includes("tool_use"),
uploadImage: model.capabilities?.includes("image_input"),
Expand Down
27 changes: 27 additions & 0 deletions core/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,9 @@ export interface LLMOptions {

sourceFile?: string;
isFromAutoDetect?: boolean;

/** Tool prompt overrides for this model */
toolPromptOverrides?: ToolOverride[];
}

type RequireAtLeastOne<T, Keys extends keyof T = keyof T> = Pick<
Expand Down Expand Up @@ -1139,6 +1142,30 @@ export interface Tool {
) => ToolPolicy;
}

/**
* Configuration for overriding built-in tool prompts.
* Allows customization of tool descriptions and behavior per model.
*/
export interface ToolOverride {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally use Partial<Pick<Tool, ....>> or similar for this type

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cheers, thanks for having a look. I missed that detail about JSON deprecation. No problem at all. I'll put the JSON implementation on a separate branch on my fork in case there is any use for it, and bring in your chatOptions suggestion to replace the PR code. I'll get right on getting it back through the CI.

/** Tool name to override (matches function.name, e.g., "read_file", "run_terminal_command") */
name: string;
/** Override the tool's description shown to the LLM */
description?: string;
/** Override the display title shown in UI */
displayTitle?: string;
/** Override the action phrases */
wouldLikeTo?: string;
isCurrently?: string;
hasAlready?: string;
/** Override system message description for non-native tool calling */
systemMessageDescription?: {
prefix?: string;
exampleArgs?: Array<[string, string | number]>;
};
/** Set to true to disable this tool */
disabled?: boolean;
}

interface ToolChoice {
type: "function";
function: {
Expand Down
31 changes: 29 additions & 2 deletions core/llm/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
RequestOptions,
TabAutocompleteOptions,
TemplateType,
ToolOverride,
Usage,
} from "../index.js";
import { isLemonadeInstalled } from "../util/lemonadeHelper.js";
Expand Down Expand Up @@ -65,6 +66,8 @@ import {
toCompleteBody,
toFimBody,
} from "./openaiTypeConverters.js";
import { applyToolOverrides } from "../tools/applyToolOverrides.js";

export class LLMError extends Error {
constructor(
message: string,
Expand Down Expand Up @@ -196,6 +199,9 @@ export abstract class BaseLLM implements ILLM {

isFromAutoDetect?: boolean;

/** Tool prompt overrides for this model */
toolPromptOverrides?: ToolOverride[];

lastRequestId: string | undefined;

private _llmOptions: LLMOptions;
Expand Down Expand Up @@ -303,6 +309,7 @@ export abstract class BaseLLM implements ILLM {
this.autocompleteOptions = options.autocompleteOptions;
this.sourceFile = options.sourceFile;
this.isFromAutoDetect = options.isFromAutoDetect;
this.toolPromptOverrides = options.toolPromptOverrides;
}

get contextLength() {
Expand Down Expand Up @@ -1111,8 +1118,28 @@ export abstract class BaseLLM implements ILLM {
messageOptions?: MessageOption,
): AsyncGenerator<ChatMessage, PromptLog> {
this.lastRequestId = undefined;

// Apply per-model tool prompt overrides if configured
let effectiveTools = options.tools;
if (this.toolPromptOverrides?.length && options.tools?.length) {
const { tools: overriddenTools, errors } = applyToolOverrides(
options.tools,
this.toolPromptOverrides,
);
effectiveTools = overriddenTools;
// Log any warnings for unknown tool names
for (const error of errors) {
if (!error.fatal) {
console.warn(`Tool override warning: ${error.message}`);
}
}
}

// Use effectiveTools for the rest of this method
const optionsWithOverrides = { ...options, tools: effectiveTools };

let { completionOptions, logEnabled } =
this._parseCompletionOptions(options);
this._parseCompletionOptions(optionsWithOverrides);
const interaction = logEnabled
? this.logger?.createInteractionLog()
: undefined;
Expand All @@ -1130,7 +1157,7 @@ export abstract class BaseLLM implements ILLM {
knownContextLength: this._contextLength,
maxTokens: completionOptions.maxTokens ?? DEFAULT_MAX_TOKENS,
supportsImages: this.supportsImages(),
tools: options.tools,
tools: optionsWithOverrides.tools,
});

messages = compiledChatMessages;
Expand Down
69 changes: 69 additions & 0 deletions core/tools/applyToolOverrides.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import { ConfigValidationError } from "@continuedev/config-yaml";
import { Tool, ToolOverride } from "..";

export interface ApplyToolOverridesResult {
tools: Tool[];
errors: ConfigValidationError[];
}

/**
* Applies tool overrides from config to the list of tools.
* Overrides can modify tool descriptions, display titles, action phrases,
* system message descriptions, or disable tools entirely.
*/
export function applyToolOverrides(
tools: Tool[],
overrides: ToolOverride[] | undefined,
): ApplyToolOverridesResult {
if (!overrides?.length) {
return { tools, errors: [] };
}

const errors: ConfigValidationError[] = [];
const toolsByName = new Map(tools.map((t) => [t.function.name, t]));

for (const override of overrides) {
const tool = toolsByName.get(override.name);

if (!tool) {
errors.push({
fatal: false,
message: `Tool override "${override.name}" does not match any known tool. Available tools: ${Array.from(toolsByName.keys()).join(", ")}`,
});
continue;
}

if (override.disabled) {
toolsByName.delete(override.name);
continue;
}

const updatedTool: Tool = {
...tool,
function: {
...tool.function,
description: override.description ?? tool.function.description,
},
displayTitle: override.displayTitle ?? tool.displayTitle,
wouldLikeTo: override.wouldLikeTo ?? tool.wouldLikeTo,
isCurrently: override.isCurrently ?? tool.isCurrently,
hasAlready: override.hasAlready ?? tool.hasAlready,
};

if (override.systemMessageDescription) {
updatedTool.systemMessageDescription = {
prefix:
override.systemMessageDescription.prefix ??
tool.systemMessageDescription?.prefix ??
"",
exampleArgs:
override.systemMessageDescription.exampleArgs ??
tool.systemMessageDescription?.exampleArgs,
};
}

toolsByName.set(override.name, updatedTool);
}

return { tools: Array.from(toolsByName.values()), errors };
}
Loading
Loading