Skip to content

Commit 5835a5a

Browse files
committed
fix: cache tool call names in streaming to handle providers with incomplete chunks
- Added toolCallNameCache map in combineStreamingChatResponse to store tool call names by ID - Modified updateToolCall to accept nameCache parameter and restore missing names from cache - Fixed streaming callback errors when providers omit function names in subsequent chunks (e.g., GPT-4.1 via OpenRouter) - Added unit tests for name caching logic covering first chunk caching and subsequent chunk restoration - Added integration tests for different streaming formats (GPT-4 style multi-chunk, Gemini style single-chunk, parallel tool calls) This ensures streaming tool calls work correctly with all provider formats, including those that send function names only in the first chunk.
1 parent 649d2f5 commit 5835a5a

2 files changed

Lines changed: 453 additions & 5 deletions

File tree

llms/openai/internal/openaiclient/chat.go

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -691,8 +691,9 @@ func combineStreamingChatResponse(
691691
defer streaming.CallWithDone(ctx, payload.StreamingFunc) //nolint:errcheck
692692

693693
var (
694-
response ChatCompletionResponse
695-
splitters []reasoning.ChunkContentSplitter
694+
response ChatCompletionResponse
695+
splitters []reasoning.ChunkContentSplitter
696+
toolCallNameCache = make(map[string]string) // Cache tool call names by ID for streaming
696697
)
697698

698699
for streamResponse := range responseChan {
@@ -748,7 +749,7 @@ func combineStreamingChatResponse(
748749
}
749750

750751
for _, toolCall := range choice.Delta.ToolCalls {
751-
updateToolCall(&responseChoice.Message, toolCall)
752+
updateToolCall(&responseChoice.Message, toolCall, toolCallNameCache)
752753

753754
toolCall := streaming.NewToolCall(toolCall.ID, toolCall.Function.Name, toolCall.Function.Arguments)
754755
if err := streaming.CallWithToolCall(ctx, payload.StreamingFunc, toolCall); err != nil {
@@ -805,8 +806,8 @@ func updateFunctionCall(message *ChatMessage, functionCall *FunctionCall) {
805806
}
806807
}
807808

808-
func updateToolCall(message *ChatMessage, delta *StreamedToolCall) {
809-
if delta == nil {
809+
func updateToolCall(message *ChatMessage, delta *StreamedToolCall, nameCache map[string]string) {
810+
if delta == nil || nameCache == nil {
810811
return
811812
}
812813

@@ -837,16 +838,28 @@ func updateToolCall(message *ChatMessage, delta *StreamedToolCall) {
837838
toolCall.Type = delta.Type
838839
toolCall.Function.Name = delta.Function.Name
839840
toolCall.Function.Arguments = delta.Function.Arguments
841+
842+
// Cache the tool call name by ID for subsequent chunks
843+
nameCache[delta.ID] = delta.Function.Name
840844
}
841845

842846
// For next delta chunks, append arguments to the current tool call
843847
if delta.ID == "" {
848+
// Standard case: no ID in subsequent chunks (most providers)
844849
toolCall.Function.Arguments += delta.Function.Arguments
845850

846851
// Complete the tool call fields with stored values from the current tool call
847852
delta.Function.Name = toolCall.Function.Name
848853
delta.ID = toolCall.ID
849854
delta.Type = toolCall.Type
855+
} else if delta.Function.Name == "" {
856+
// If ID is present but name is missing (some providers don't send name in subsequent chunks),
857+
// restore the name from cache
858+
if cachedName, ok := nameCache[delta.ID]; ok {
859+
delta.Function.Name = cachedName
860+
toolCall.Function.Arguments += delta.Function.Arguments
861+
delta.Type = toolCall.Type
862+
}
850863
}
851864
}
852865

0 commit comments

Comments
 (0)