diff --git a/.changeset/thick-turtles-deny.md b/.changeset/thick-turtles-deny.md new file mode 100644 index 000000000..3c761ad2c --- /dev/null +++ b/.changeset/thick-turtles-deny.md @@ -0,0 +1,7 @@ +--- +"create-llama": patch +"@llamaindex/server": patch +"@create-llama/llama-index-server": patch +--- + +feat: bump chat-ui with inline artifact diff --git a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts index 733faf1e0..95f960fda 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/code_generator/src/app/workflow.ts @@ -1,4 +1,4 @@ -import { extractLastArtifact } from "@llamaindex/server"; +import { artifactEvent, extractLastArtifact } from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { @@ -52,19 +52,6 @@ const synthesizeAnswerEvent = workflowEvent(); const uiEvent = workflowEvent(); -const artifactEvent = workflowEvent<{ - type: "artifact"; - data: { - type: "code"; - created_at: number; - data: { - language: string; - file_name: string; - code: string; - }; - }; -}>(); - export function workflowFactory(reqBody: any) { const llm = Settings.llm; diff --git a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts index 70ba93cb9..ca2316578 100644 --- a/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts +++ b/packages/create-llama/templates/components/use-cases/typescript/document_generator/src/app/workflow.ts @@ -1,4 +1,4 @@ -import { extractLastArtifact } from "@llamaindex/server"; +import { artifactEvent, extractLastArtifact } from "@llamaindex/server"; import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; import { @@ -55,19 +55,6 @@ const synthesizeAnswerEvent = workflowEvent<{ const uiEvent = workflowEvent(); -const artifactEvent = workflowEvent<{ - type: "artifact"; - data: { - type: "document"; - created_at: number; - data: { - title: string; - content: string; - type: "markdown" | "html"; - }; - }; -}>(); - export function workflowFactory(reqBody: any) { const llm = Settings.llm; diff --git a/packages/server/examples/code-gen/README.md b/packages/server/examples/code-gen/README.md new file mode 100644 index 000000000..8ac84b5d3 --- /dev/null +++ b/packages/server/examples/code-gen/README.md @@ -0,0 +1,22 @@ +This example demonstrates how to use the code generation workflow. + +```ts +new LlamaIndexServer({ + workflow: workflowFactory, + uiConfig: { + starterQuestions: [ + "Generate a calculator app", + "Create a simple todo list app", + ], + componentsDir: "components", + }, + port: 3000, +}).start(); +``` + +Export OpenAI API key and start the server in dev mode. + +```bash +export OPENAI_API_KEY= +npx nodemon --exec tsx index.ts +``` diff --git a/packages/server/examples/code-gen/components/ui_event.jsx b/packages/server/examples/code-gen/components/ui_event.jsx new file mode 100644 index 000000000..9b2db6683 --- /dev/null +++ b/packages/server/examples/code-gen/components/ui_event.jsx @@ -0,0 +1,132 @@ +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Progress } from "@/components/ui/progress"; +import { Skeleton } from "@/components/ui/skeleton"; +import { cn } from "@/lib/utils"; +import { Markdown } from "@llamaindex/chat-ui/widgets"; +import { ListChecks, Loader2, Wand2 } from "lucide-react"; +import { useEffect, useState } from "react"; + +const STAGE_META = { + plan: { + icon: ListChecks, + badgeText: "Step 1/2: Planning", + gradient: "from-blue-100 via-blue-50 to-white", + progress: 33, + iconBg: "bg-blue-100 text-blue-600", + badge: "bg-blue-100 text-blue-700", + }, + generate: { + icon: Wand2, + badgeText: "Step 2/2: Generating", + gradient: "from-violet-100 via-violet-50 to-white", + progress: 66, + iconBg: "bg-violet-100 text-violet-600", + badge: "bg-violet-100 text-violet-700", + }, +}; + +function ArtifactWorkflowCard({ event }) { + const [visible, setVisible] = useState(event?.state !== "completed"); + const [fade, setFade] = useState(false); + + useEffect(() => { + if (event?.state === "completed") { + setVisible(false); + } else { + setVisible(true); + setFade(false); + } + }, [event?.state]); + + if (!event || !visible) return null; + + const { state, requirement } = event; + const meta = STAGE_META[state]; + + if (!meta) return null; + + return ( +
+ + +
+ +
+ + + {meta.badgeText} + + + + + {state === "plan" && ( +
+ +
+ Analyzing your request... +
+ +
+ )} + {state === "generate" && ( +
+
+ + + Working on the requirement: + +
+
+ {requirement ? ( + + ) : ( + + No requirements available yet. + + )} +
+
+ )} + +
+ +
+ +
+ ); +} + +export default function Component({ events }) { + const aggregateEvents = () => { + if (!events || events.length === 0) return null; + return events[events.length - 1]; + }; + + const event = aggregateEvents(); + + return ; +} diff --git a/packages/server/examples/code-gen/index.ts b/packages/server/examples/code-gen/index.ts new file mode 100644 index 000000000..6cbb9227e --- /dev/null +++ b/packages/server/examples/code-gen/index.ts @@ -0,0 +1,20 @@ +import { OpenAI } from "@llamaindex/openai"; +import { LlamaIndexServer } from "@llamaindex/server"; +import { Settings } from "llamaindex"; +import { workflowFactory } from "./src/app/workflow"; + +Settings.llm = new OpenAI({ + model: "gpt-4o-mini", +}); + +new LlamaIndexServer({ + workflow: workflowFactory, + uiConfig: { + starterQuestions: [ + "Generate a calculator app", + "Create a simple todo list app", + ], + componentsDir: "components", + }, + port: 3000, +}).start(); diff --git a/packages/server/examples/code-gen/src/app/workflow.ts b/packages/server/examples/code-gen/src/app/workflow.ts new file mode 100644 index 000000000..e16af7bfc --- /dev/null +++ b/packages/server/examples/code-gen/src/app/workflow.ts @@ -0,0 +1,337 @@ +import { artifactEvent, extractLastArtifact } from "@llamaindex/server"; +import { ChatMemoryBuffer, MessageContent, Settings } from "llamaindex"; + +import { + agentStreamEvent, + createStatefulMiddleware, + createWorkflow, + startAgentEvent, + stopAgentEvent, + workflowEvent, +} from "@llamaindex/workflow"; + +import { z } from "zod"; + +export const RequirementSchema = z.object({ + next_step: z.enum(["answering", "coding"]), + language: z.string().nullable().optional(), + file_name: z.string().nullable().optional(), + requirement: z.string(), +}); + +export type Requirement = z.infer; + +export const UIEventSchema = z.object({ + type: z.literal("ui_event"), + data: z.object({ + state: z + .enum(["plan", "generate", "completed"]) + .describe( + "The current state of the workflow: 'plan', 'generate', or 'completed'.", + ), + requirement: z + .string() + .optional() + .describe( + "An optional requirement creating or updating a code, if applicable.", + ), + }), +}); + +export type UIEvent = z.infer; +const planEvent = workflowEvent<{ + userInput: MessageContent; + context?: string | undefined; +}>(); + +const generateArtifactEvent = workflowEvent<{ + requirement: Requirement; +}>(); + +const synthesizeAnswerEvent = workflowEvent(); + +const uiEvent = workflowEvent(); + +export function workflowFactory(reqBody: unknown) { + const llm = Settings.llm; + + const { withState, getContext } = createStatefulMiddleware(() => { + return { + memory: new ChatMemoryBuffer({ llm }), + lastArtifact: extractLastArtifact(reqBody), + }; + }); + const workflow = withState(createWorkflow()); + + workflow.handle([startAgentEvent], async ({ data }) => { + const { userInput, chatHistory = [] } = data; + // Prepare chat history + const { state } = getContext(); + // Put user input to the memory + if (!userInput) { + throw new Error("Missing user input to start the workflow"); + } + state.memory.set(chatHistory); + state.memory.put({ role: "user", content: userInput }); + + return planEvent.with({ + userInput: userInput, + context: state.lastArtifact + ? JSON.stringify(state.lastArtifact) + : undefined, + }); + }); + + workflow.handle([planEvent], async ({ data: planData }) => { + const { sendEvent } = getContext(); + const { state } = getContext(); + sendEvent( + uiEvent.with({ + type: "ui_event", + data: { + state: "plan", + }, + }), + ); + const user_msg = planData.userInput; + const context = planData.context + ? `## The context is: \n${planData.context}\n` + : ""; + const prompt = ` +You are a product analyst responsible for analyzing the user's request and providing the next step for code or document generation. +You are helping user with their code artifact. To update the code, you need to plan a coding step. + +Follow these instructions: +1. Carefully analyze the conversation history and the user's request to determine what has been done and what the next step should be. +2. The next step must be one of the following two options: + - "coding": To make the changes to the current code. + - "answering": If you don't need to update the current code or need clarification from the user. +Important: Avoid telling the user to update the code themselves, you are the one who will update the code (by planning a coding step). +3. If the next step is "coding", you may specify the language ("typescript" or "python") and file_name if known, otherwise set them to null. +4. The requirement must be provided clearly what is the user request and what need to be done for the next step in details + as precise and specific as possible, don't be stingy with in the requirement. +5. If the next step is "answering", set language and file_name to null, and the requirement should describe what to answer or explain to the user. +6. Be concise; only return the requirements for the next step. +7. The requirements must be in the following format: + \`\`\`json + { + "next_step": "answering" | "coding", + "language": "typescript" | "python" | null, + "file_name": string | null, + "requirement": string + } + \`\`\` + +## Example 1: +User request: Create a calculator app. +You should return: +\`\`\`json +{ + "next_step": "coding", + "language": "typescript", + "file_name": "calculator.tsx", + "requirement": "Generate code for a calculator app that has a simple UI with a display and button layout. The display should show the current input and the result. The buttons should include basic operators, numbers, clear, and equals. The calculation should work correctly." +} +\`\`\` + +## Example 2: +User request: Explain how the game loop works. +Context: You have already generated the code for a snake game. +You should return: +\`\`\`json +{ + "next_step": "answering", + "language": null, + "file_name": null, + "requirement": "The user is asking about the game loop. Explain how the game loop works." +} +\`\`\` + +${context} + +Now, plan the user's next step for this request: +${user_msg} +`; + + const response = await llm.complete({ + prompt, + }); + // parse the response to Requirement + // 1. use regex to find the json block + const jsonBlock = response.text.match(/```json\s*([\s\S]*?)\s*```/); + if (!jsonBlock) { + throw new Error("No JSON block found in the response."); + } + const requirement = RequirementSchema.parse(JSON.parse(jsonBlock[1])); + state.memory.put({ + role: "assistant", + content: `The plan for next step: \n${response.text}`, + }); + + if (requirement.next_step === "coding") { + return generateArtifactEvent.with({ + requirement, + }); + } else { + return synthesizeAnswerEvent.with({}); + } + }); + + workflow.handle([generateArtifactEvent], async ({ data: planData }) => { + const { sendEvent } = getContext(); + const { state } = getContext(); + + sendEvent( + uiEvent.with({ + type: "ui_event", + data: { + state: "generate", + requirement: planData.requirement.requirement, + }, + }), + ); + + const previousArtifact = state.lastArtifact + ? JSON.stringify(state.lastArtifact) + : "There is no previous artifact"; + const requirementText = planData.requirement.requirement; + + const prompt = ` + You are a skilled developer who can help user with coding. + You are given a task to generate or update a code for a given requirement. + + ## Follow these instructions: + **1. Carefully read the user's requirements.** + If any details are ambiguous or missing, make reasonable assumptions and clearly reflect those in your output. + If the previous code is provided: + + Carefully analyze the code with the request to make the right changes. + + Avoid making a lot of changes from the previous code if the request is not to write the code from scratch again. + **2. For code requests:** + - If the user does not specify a framework or language, default to a React component using the Next.js framework. + - For Next.js, use Shadcn UI components, Typescript, @types/node, @types/react, @types/react-dom, PostCSS, and TailwindCSS. + The import pattern should be: + \`\`\`typescript + import { ComponentName } from "@/components/ui/component-name" + import { Markdown } from "@llamaindex/chat-ui" + import { cn } from "@/lib/utils" + \`\`\` + - Ensure the code is idiomatic, production-ready, and includes necessary imports. + - Only generate code relevant to the user's request—do not add extra boilerplate. + **3. Don't be verbose on response** + - No other text or comments only return the code which wrapped by \`\`\`language\`\`\` block. + - If the user's request is to update the code, only return the updated code. + **4. Only the following languages are allowed: "typescript", "python".** + **5. If there is no code to update, return the reason without any code block.** + + ## Example: + \`\`\`typescript + import React from "react"; + import { Button } from "@/components/ui/button"; + import { cn } from "@/lib/utils"; + + export default function MyComponent() { + return ( +
+ +
+ ); + } + \`\`\` + + The previous code is: + {previousArtifact} + + Now, i have to generate the code for the following requirement: + {requirement} + ` + .replace("{previousArtifact}", previousArtifact) + .replace("{requirement}", requirementText); + + const response = await llm.complete({ + prompt, + }); + + // Extract the code from the response + const codeMatch = response.text.match(/```(\w+)([\s\S]*)```/); + if (!codeMatch) { + return synthesizeAnswerEvent.with({}); + } + + const code = codeMatch[2].trim(); + + // Put the generated code to the memory + state.memory.put({ + role: "assistant", + content: `Updated the code: \n${response.text}`, + }); + + // To show the Canvas panel for the artifact + sendEvent( + artifactEvent.with({ + type: "artifact", + data: { + type: "code", + created_at: Date.now(), + data: { + language: planData.requirement.language || "", + file_name: planData.requirement.file_name || "", + code, + }, + }, + }), + ); + + return synthesizeAnswerEvent.with({}); + }); + + workflow.handle([synthesizeAnswerEvent], async () => { + const { sendEvent } = getContext(); + const { state } = getContext(); + + const chatHistory = await state.memory.getMessages(); + const messages = [ + ...chatHistory, + { + role: "system" as const, + content: ` + You are a helpful assistant who is responsible for explaining the work to the user. + Based on the conversation history, provide an answer to the user's question. + The user has access to the code so avoid mentioning the whole code again in your response. + `, + }, + ]; + + const responseStream = await llm.chat({ + messages, + stream: true, + }); + + sendEvent( + uiEvent.with({ + type: "ui_event", + data: { + state: "completed", + }, + }), + ); + + let response = ""; + for await (const chunk of responseStream) { + response += chunk.delta; + sendEvent( + agentStreamEvent.with({ + delta: chunk.delta, + response: "", + currentAgentName: "assistant", + raw: chunk, + }), + ); + } + + return stopAgentEvent.with({ + result: response, + }); + }); + + return workflow; +} diff --git a/packages/server/next/app/components/ui/chat/chat-message-content.tsx b/packages/server/next/app/components/ui/chat/chat-message-content.tsx index 3ba488ed7..caf132633 100644 --- a/packages/server/next/app/components/ui/chat/chat-message-content.tsx +++ b/packages/server/next/app/components/ui/chat/chat-message-content.tsx @@ -19,7 +19,6 @@ export function ChatMessageContent({ - diff --git a/packages/server/package.json b/packages/server/package.json index bbe896723..29fac104f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -65,7 +65,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.4.9", + "@llamaindex/chat-ui": "0.5.2", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-alert-dialog": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.3", diff --git a/packages/server/project-config/package.json b/packages/server/project-config/package.json index f5c5bb2e6..ae5083908 100644 --- a/packages/server/project-config/package.json +++ b/packages/server/project-config/package.json @@ -41,7 +41,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.4.9", + "@llamaindex/chat-ui": "0.5.2", "@llamaindex/env": "~0.1.30", "@llamaindex/openai": "~0.4.0", "@llamaindex/readers": "~3.1.4", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index e08c57892..b0d3f9abe 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -2,4 +2,5 @@ export * from "./server"; export * from "./types"; export * from "./utils/events"; export { generateEventComponent } from "./utils/gen-ui"; +export * from "./utils/inline"; export * from "./utils/prompts"; diff --git a/packages/server/src/utils/events.ts b/packages/server/src/utils/events.ts index c20751a2b..86b31d52d 100644 --- a/packages/server/src/utils/events.ts +++ b/packages/server/src/utils/events.ts @@ -1,8 +1,13 @@ import { randomUUID } from "@llamaindex/env"; import { workflowEvent } from "@llamaindex/workflow"; -import type { Message } from "ai"; -import { MetadataMode, type Metadata, type NodeWithScore } from "llamaindex"; +import { + MetadataMode, + type ChatMessage, + type Metadata, + type NodeWithScore, +} from "llamaindex"; import { z } from "zod"; +import { getInlineAnnotations } from "./inline"; // Events that appended to stream as annotations export type SourceEventNode = { @@ -148,24 +153,22 @@ export const artifactAnnotationSchema = z.object({ data: artifactSchema, }); -export function extractAllArtifacts(messages: Message[]): Artifact[] { - const allArtifacts: Artifact[] = []; - - for (const message of messages) { - const artifacts = - message.annotations - ?.filter( - ( - annotation, - ): annotation is z.infer => - artifactAnnotationSchema.safeParse(annotation).success, - ) - .map((annotation) => annotation.data as Artifact) ?? []; - - allArtifacts.push(...artifacts); - } +export function extractArtifactsFromMessage(message: ChatMessage): Artifact[] { + const inlineAnnotations = getInlineAnnotations(message); + const artifacts = inlineAnnotations.filter( + (annotation): annotation is z.infer => { + return artifactAnnotationSchema.safeParse(annotation).success; + }, + ); + return artifacts.map((artifact) => artifact.data); +} - return allArtifacts; +export function extractArtifactsFromAllMessages( + messages: ChatMessage[], +): Artifact[] { + return messages + .flatMap((message) => extractArtifactsFromMessage(message)) + .sort((a, b) => a.created_at - b.created_at); } export function extractLastArtifact( @@ -187,10 +190,10 @@ export function extractLastArtifact( requestBody: unknown, type?: ArtifactType, ): CodeArtifact | DocumentArtifact | Artifact | undefined { - const { messages } = (requestBody as { messages?: Message[] }) ?? {}; + const { messages } = (requestBody as { messages?: ChatMessage[] }) ?? {}; if (!messages) return undefined; - const artifacts = extractAllArtifacts(messages); + const artifacts = extractArtifactsFromAllMessages(messages); if (!artifacts.length) return undefined; if (type) { diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 8b5184b00..3e69d4652 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -1,6 +1,7 @@ export * from "./events"; export * from "./file"; export * from "./gen-ui"; +export * from "./inline"; export * from "./prompts"; export * from "./request"; export * from "./stream"; diff --git a/packages/server/src/utils/inline.ts b/packages/server/src/utils/inline.ts new file mode 100644 index 000000000..db4d596b2 --- /dev/null +++ b/packages/server/src/utils/inline.ts @@ -0,0 +1,90 @@ +import { agentStreamEvent, type WorkflowEventData } from "@llamaindex/workflow"; +import { type ChatMessage } from "llamaindex"; +import { z } from "zod"; + +const INLINE_ANNOTATION_KEY = "annotation"; // the language key to detect inline annotation code in markdown + +export const AnnotationSchema = z.object({ + type: z.string(), + data: z.any(), +}); + +export type Annotation = z.infer; + +export function getInlineAnnotations(message: ChatMessage): Annotation[] { + const markdownContent = getMessageMarkdownContent(message); + + const inlineAnnotations: Annotation[] = []; + + // Regex to match annotation code blocks + // Matches ```annotation followed by content until closing ``` + const annotationRegex = new RegExp( + `\`\`\`${INLINE_ANNOTATION_KEY}\\s*\\n([\\s\\S]*?)\\n\`\`\``, + "g", + ); + + let match; + while ((match = annotationRegex.exec(markdownContent)) !== null) { + const jsonContent = match[1]?.trim(); + + if (!jsonContent) { + continue; + } + + try { + // Parse the JSON content + const parsed = JSON.parse(jsonContent); + + // Validate against the annotation schema + const validated = AnnotationSchema.parse(parsed); + + // Extract the artifact data + inlineAnnotations.push(validated); + } catch (error) { + // Skip invalid annotations - they might be malformed JSON or invalid schema + console.warn("Failed to parse annotation:", error); + } + } + + return inlineAnnotations; +} + +/** + * To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. + * The language key is `annotation` and the code block is wrapped in backticks. + * + * \`\`\`annotation + * \{ + * "type": "artifact", + * "data": \{...\} + * \} + * \`\`\` + */ +export function toInlineAnnotation(item: unknown) { + return `\n\`\`\`${INLINE_ANNOTATION_KEY}\n${JSON.stringify(item)}\n\`\`\`\n`; +} + +export function toInlineAnnotationEvent(event: WorkflowEventData) { + return agentStreamEvent.with({ + delta: toInlineAnnotation(event.data), + response: "", + currentAgentName: "assistant", + raw: event.data, + }); +} + +function getMessageMarkdownContent(message: ChatMessage): string { + let markdownContent = ""; + + if (typeof message.content === "string") { + markdownContent = message.content; + } else { + message.content.forEach((item) => { + if (item.type === "text") { + markdownContent += item.text; + } + }); + } + + return markdownContent; +} diff --git a/packages/server/src/utils/workflow.ts b/packages/server/src/utils/workflow.ts index 566c8f330..4771efe15 100644 --- a/packages/server/src/utils/workflow.ts +++ b/packages/server/src/utils/workflow.ts @@ -15,12 +15,14 @@ import { type NodeWithScore, } from "llamaindex"; import { + artifactEvent, sourceEvent, toAgentRunEvent, toSourceEvent, type SourceEventNode, } from "./events"; import { downloadFile } from "./file"; +import { toInlineAnnotationEvent } from "./inline"; export async function runWorkflow( workflow: Workflow, @@ -74,6 +76,10 @@ function processWorkflowStream( transformedEvent = toSourceEvent(sourceNodes); } } + // Handle artifact events, transform to agentStreamEvent + else if (artifactEvent.include(event)) { + transformedEvent = toInlineAnnotationEvent(event); + } // Post-process for llama-cloud files if (sourceEvent.include(transformedEvent)) { const sourceNodesForDownload = transformedEvent.data.data.nodes; // These are SourceEventNode[] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index baa92997d..75525440f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,8 +181,8 @@ importers: specifier: ^5.0.1 version: 5.0.1(react-hook-form@7.56.1(react@19.1.0)) '@llamaindex/chat-ui': - specifier: 0.4.9 - version: 0.4.9(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + specifier: 0.5.2 + version: 0.5.2(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@llamaindex/env': specifier: ~0.1.30 version: 0.1.30 @@ -1186,8 +1186,8 @@ packages: zod: optional: true - '@llamaindex/chat-ui@0.4.9': - resolution: {integrity: sha512-KEdydC+aJ22VK/TltxIHlMWbWLfh6I0YkyVd1D/CS3FRfLt8l9jfQ/YjY10MiEd8oc1fFfk6ek/FhVWe9Szstg==} + '@llamaindex/chat-ui@0.5.2': + resolution: {integrity: sha512-zIOaf5cuhDtllrcDEDaGNRq2bon8STUYoex4n+zzowR6eirRRYPYoUpRBNBK7fZi4BJhBPmqTS/tkXzjw908Gw==} peerDependencies: react: ^18.2.0 || ^19.0.0 || ^19.0.0-rc @@ -2454,6 +2454,9 @@ packages: '@types/mdast@3.0.15': resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + '@types/mdurl@2.0.0': resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} @@ -4621,6 +4624,9 @@ packages: mdast-util-from-markdown@1.3.1: resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} + mdast-util-from-markdown@2.0.2: + resolution: {integrity: sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==} + mdast-util-gfm-autolink-literal@1.0.3: resolution: {integrity: sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==} @@ -4645,15 +4651,24 @@ packages: mdast-util-phrasing@3.0.1: resolution: {integrity: sha512-WmI1gTXUBJo4/ZmSk79Wcb2HcjPJBzM1nlI/OUWA8yk2X9ik3ffNbBGsU+09BFmXaL1IBb9fiuvq6/KMiNycSg==} + mdast-util-phrasing@4.1.0: + resolution: {integrity: sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==} + mdast-util-to-hast@12.3.0: resolution: {integrity: sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw==} mdast-util-to-markdown@1.5.0: resolution: {integrity: sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==} + mdast-util-to-markdown@2.1.2: + resolution: {integrity: sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==} + mdast-util-to-string@3.2.0: resolution: {integrity: sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==} + mdast-util-to-string@4.0.0: + resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + mdurl@2.0.0: resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} @@ -4686,6 +4701,9 @@ packages: micromark-core-commonmark@1.1.0: resolution: {integrity: sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw==} + micromark-core-commonmark@2.0.3: + resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} + micromark-extension-gfm-autolink-literal@1.0.5: resolution: {integrity: sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg==} @@ -4713,63 +4731,123 @@ packages: micromark-factory-destination@1.1.0: resolution: {integrity: sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg==} + micromark-factory-destination@2.0.1: + resolution: {integrity: sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==} + micromark-factory-label@1.1.0: resolution: {integrity: sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w==} + micromark-factory-label@2.0.1: + resolution: {integrity: sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==} + micromark-factory-space@1.1.0: resolution: {integrity: sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==} + micromark-factory-space@2.0.1: + resolution: {integrity: sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==} + micromark-factory-title@1.1.0: resolution: {integrity: sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ==} + micromark-factory-title@2.0.1: + resolution: {integrity: sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==} + micromark-factory-whitespace@1.1.0: resolution: {integrity: sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ==} + micromark-factory-whitespace@2.0.1: + resolution: {integrity: sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==} + micromark-util-character@1.2.0: resolution: {integrity: sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==} + micromark-util-character@2.1.1: + resolution: {integrity: sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==} + micromark-util-chunked@1.1.0: resolution: {integrity: sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ==} + micromark-util-chunked@2.0.1: + resolution: {integrity: sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==} + micromark-util-classify-character@1.1.0: resolution: {integrity: sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw==} + micromark-util-classify-character@2.0.1: + resolution: {integrity: sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==} + micromark-util-combine-extensions@1.1.0: resolution: {integrity: sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA==} + micromark-util-combine-extensions@2.0.1: + resolution: {integrity: sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==} + micromark-util-decode-numeric-character-reference@1.1.0: resolution: {integrity: sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw==} + micromark-util-decode-numeric-character-reference@2.0.2: + resolution: {integrity: sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==} + micromark-util-decode-string@1.1.0: resolution: {integrity: sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ==} + micromark-util-decode-string@2.0.1: + resolution: {integrity: sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==} + micromark-util-encode@1.1.0: resolution: {integrity: sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw==} + micromark-util-encode@2.0.1: + resolution: {integrity: sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==} + micromark-util-html-tag-name@1.2.0: resolution: {integrity: sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q==} + micromark-util-html-tag-name@2.0.1: + resolution: {integrity: sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==} + micromark-util-normalize-identifier@1.1.0: resolution: {integrity: sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q==} + micromark-util-normalize-identifier@2.0.1: + resolution: {integrity: sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==} + micromark-util-resolve-all@1.1.0: resolution: {integrity: sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA==} + micromark-util-resolve-all@2.0.1: + resolution: {integrity: sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==} + micromark-util-sanitize-uri@1.2.0: resolution: {integrity: sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A==} + micromark-util-sanitize-uri@2.0.1: + resolution: {integrity: sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==} + micromark-util-subtokenize@1.1.0: resolution: {integrity: sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A==} + micromark-util-subtokenize@2.1.0: + resolution: {integrity: sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==} + micromark-util-symbol@1.1.0: resolution: {integrity: sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==} + micromark-util-symbol@2.0.1: + resolution: {integrity: sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==} + micromark-util-types@1.1.0: resolution: {integrity: sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==} + micromark-util-types@2.0.2: + resolution: {integrity: sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==} + micromark@3.2.0: resolution: {integrity: sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA==} + micromark@4.0.2: + resolution: {integrity: sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==} + micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -5634,14 +5712,17 @@ packages: remark-parse@10.0.2: resolution: {integrity: sha512-3ydxgHa/ZQzG8LvC7jTXccARYDcRld3VfcgIIFs7bI6vbRSxJJmzgLEIIoYKyrfhaY+ujuWaf/PJiMZXoiCXgw==} + remark-parse@11.0.0: + resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} + remark-rehype@10.1.0: resolution: {integrity: sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==} - remark-stringify@10.0.3: - resolution: {integrity: sha512-koyOzCMYoUHudypbj4XpnAKFbkddRMYZHwghnxd7ue5210WzGw6kOBwauJTRUMq16jsovXx8dYNvSSWP89kZ3A==} + remark-stringify@11.0.0: + resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} - remark@14.0.3: - resolution: {integrity: sha512-bfmJW1dmR2LvaMJuAnE88pZP9DktIFYXazkTfOIKZzi3Knk9lT0roItIA24ydOucI3bV/g/tXBA6hzqq3FV9Ew==} + remark@15.0.1: + resolution: {integrity: sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==} require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} @@ -6265,6 +6346,9 @@ packages: unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} + unified@11.0.5: + resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} + unist-util-find-after@5.0.0: resolution: {integrity: sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==} @@ -7367,7 +7451,7 @@ snapshots: p-retry: 6.2.1 zod: 3.25.13 - '@llamaindex/chat-ui@0.4.9(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': + '@llamaindex/chat-ui@0.5.2(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.0)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.10)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.36.7)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@codemirror/lang-css': 6.3.1 '@codemirror/lang-html': 6.4.9 @@ -7401,11 +7485,13 @@ snapshots: react: 19.1.0 react-markdown: 8.0.7(@types/react@19.1.2)(react@19.1.0) rehype-katex: 7.0.1 - remark: 14.0.3 + remark: 15.0.1 remark-code-import: 1.2.0 remark-gfm: 3.0.1 remark-math: 5.1.1 + remark-parse: 11.0.0 tailwind-merge: 2.6.0 + unist-util-visit: 5.0.0 vaul: 0.9.9(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) transitivePeerDependencies: - '@babel/runtime' @@ -8713,6 +8799,10 @@ snapshots: dependencies: '@types/unist': 2.0.11 + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + '@types/mdurl@2.0.0': {} '@types/ms@2.1.0': {} @@ -11247,6 +11337,23 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-from-markdown@2.0.2: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + decode-named-character-reference: 1.1.0 + devlop: 1.1.0 + mdast-util-to-string: 4.0.0 + micromark: 4.0.2 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-decode-string: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + unist-util-stringify-position: 4.0.0 + transitivePeerDependencies: + - supports-color + mdast-util-gfm-autolink-literal@1.0.3: dependencies: '@types/mdast': 3.0.15 @@ -11302,6 +11409,11 @@ snapshots: '@types/mdast': 3.0.15 unist-util-is: 5.2.1 + mdast-util-phrasing@4.1.0: + dependencies: + '@types/mdast': 4.0.4 + unist-util-is: 6.0.0 + mdast-util-to-hast@12.3.0: dependencies: '@types/hast': 2.3.10 @@ -11324,10 +11436,26 @@ snapshots: unist-util-visit: 4.1.2 zwitch: 2.0.4 + mdast-util-to-markdown@2.1.2: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + longest-streak: 3.1.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-string: 4.0.0 + micromark-util-classify-character: 2.0.1 + micromark-util-decode-string: 2.0.1 + unist-util-visit: 5.0.0 + zwitch: 2.0.4 + mdast-util-to-string@3.2.0: dependencies: '@types/mdast': 3.0.15 + mdast-util-to-string@4.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdurl@2.0.0: {} media-typer@1.1.0: {} @@ -11363,6 +11491,25 @@ snapshots: micromark-util-types: 1.1.0 uvu: 0.5.6 + micromark-core-commonmark@2.0.3: + dependencies: + decode-named-character-reference: 1.1.0 + devlop: 1.1.0 + micromark-factory-destination: 2.0.1 + micromark-factory-label: 2.0.1 + micromark-factory-space: 2.0.1 + micromark-factory-title: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-classify-character: 2.0.1 + micromark-util-html-tag-name: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-subtokenize: 2.1.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-extension-gfm-autolink-literal@1.0.5: dependencies: micromark-util-character: 1.2.0 @@ -11437,6 +11584,12 @@ snapshots: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-factory-destination@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-label@1.1.0: dependencies: micromark-util-character: 1.2.0 @@ -11444,11 +11597,23 @@ snapshots: micromark-util-types: 1.1.0 uvu: 0.5.6 + micromark-factory-label@2.0.1: + dependencies: + devlop: 1.1.0 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-space@1.1.0: dependencies: micromark-util-character: 1.2.0 micromark-util-types: 1.1.0 + micromark-factory-space@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-types: 2.0.2 + micromark-factory-title@1.1.0: dependencies: micromark-factory-space: 1.1.0 @@ -11456,6 +11621,13 @@ snapshots: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-factory-title@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-factory-whitespace@1.1.0: dependencies: micromark-factory-space: 1.1.0 @@ -11463,30 +11635,61 @@ snapshots: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-factory-whitespace@2.0.1: + dependencies: + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-character@1.2.0: dependencies: micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-util-character@2.1.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-chunked@1.1.0: dependencies: micromark-util-symbol: 1.1.0 + micromark-util-chunked@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-classify-character@1.1.0: dependencies: micromark-util-character: 1.2.0 micromark-util-symbol: 1.1.0 micromark-util-types: 1.1.0 + micromark-util-classify-character@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-combine-extensions@1.1.0: dependencies: micromark-util-chunked: 1.1.0 micromark-util-types: 1.1.0 + micromark-util-combine-extensions@2.0.1: + dependencies: + micromark-util-chunked: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-decode-numeric-character-reference@1.1.0: dependencies: micromark-util-symbol: 1.1.0 + micromark-util-decode-numeric-character-reference@2.0.2: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-decode-string@1.1.0: dependencies: decode-named-character-reference: 1.1.0 @@ -11494,24 +11697,49 @@ snapshots: micromark-util-decode-numeric-character-reference: 1.1.0 micromark-util-symbol: 1.1.0 + micromark-util-decode-string@2.0.1: + dependencies: + decode-named-character-reference: 1.1.0 + micromark-util-character: 2.1.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-symbol: 2.0.1 + micromark-util-encode@1.1.0: {} + micromark-util-encode@2.0.1: {} + micromark-util-html-tag-name@1.2.0: {} + micromark-util-html-tag-name@2.0.1: {} + micromark-util-normalize-identifier@1.1.0: dependencies: micromark-util-symbol: 1.1.0 + micromark-util-normalize-identifier@2.0.1: + dependencies: + micromark-util-symbol: 2.0.1 + micromark-util-resolve-all@1.1.0: dependencies: micromark-util-types: 1.1.0 + micromark-util-resolve-all@2.0.1: + dependencies: + micromark-util-types: 2.0.2 + micromark-util-sanitize-uri@1.2.0: dependencies: micromark-util-character: 1.2.0 micromark-util-encode: 1.1.0 micromark-util-symbol: 1.1.0 + micromark-util-sanitize-uri@2.0.1: + dependencies: + micromark-util-character: 2.1.1 + micromark-util-encode: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-subtokenize@1.1.0: dependencies: micromark-util-chunked: 1.1.0 @@ -11519,10 +11747,21 @@ snapshots: micromark-util-types: 1.1.0 uvu: 0.5.6 + micromark-util-subtokenize@2.1.0: + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-util-symbol@1.1.0: {} + micromark-util-symbol@2.0.1: {} + micromark-util-types@1.1.0: {} + micromark-util-types@2.0.2: {} + micromark@3.2.0: dependencies: '@types/debug': 4.1.12 @@ -11545,6 +11784,28 @@ snapshots: transitivePeerDependencies: - supports-color + micromark@4.0.2: + dependencies: + '@types/debug': 4.1.12 + debug: 4.4.0(supports-color@5.5.0) + decode-named-character-reference: 1.1.0 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.3 + micromark-factory-space: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-chunked: 2.0.1 + micromark-util-combine-extensions: 2.0.1 + micromark-util-decode-numeric-character-reference: 2.0.2 + micromark-util-encode: 2.0.1 + micromark-util-normalize-identifier: 2.0.1 + micromark-util-resolve-all: 2.0.1 + micromark-util-sanitize-uri: 2.0.1 + micromark-util-subtokenize: 2.1.0 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + transitivePeerDependencies: + - supports-color + micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -12401,6 +12662,15 @@ snapshots: transitivePeerDependencies: - supports-color + remark-parse@11.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.2 + micromark-util-types: 2.0.2 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + remark-rehype@10.1.0: dependencies: '@types/hast': 2.3.10 @@ -12408,18 +12678,18 @@ snapshots: mdast-util-to-hast: 12.3.0 unified: 10.1.2 - remark-stringify@10.0.3: + remark-stringify@11.0.0: dependencies: - '@types/mdast': 3.0.15 - mdast-util-to-markdown: 1.5.0 - unified: 10.1.2 + '@types/mdast': 4.0.4 + mdast-util-to-markdown: 2.1.2 + unified: 11.0.5 - remark@14.0.3: + remark@15.0.1: dependencies: - '@types/mdast': 3.0.15 - remark-parse: 10.0.2 - remark-stringify: 10.0.3 - unified: 10.1.2 + '@types/mdast': 4.0.4 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.5 transitivePeerDependencies: - supports-color @@ -13142,6 +13412,16 @@ snapshots: trough: 2.2.0 vfile: 5.3.7 + unified@11.0.5: + dependencies: + '@types/unist': 3.0.3 + bail: 2.0.2 + devlop: 1.1.0 + extend: 3.0.2 + is-plain-obj: 4.1.0 + trough: 2.2.0 + vfile: 6.0.3 + unist-util-find-after@5.0.0: dependencies: '@types/unist': 3.0.3 diff --git a/python/llama-index-server/llama_index/server/api/callbacks/__init__.py b/python/llama-index-server/llama_index/server/api/callbacks/__init__.py index 196e61b29..bd4275887 100644 --- a/python/llama-index-server/llama_index/server/api/callbacks/__init__.py +++ b/python/llama-index-server/llama_index/server/api/callbacks/__init__.py @@ -1,4 +1,7 @@ from llama_index.server.api.callbacks.agent_call_tool import AgentCallTool +from llama_index.server.api.callbacks.artifact_transform import ( + InlineAnnotationTransformer, +) from llama_index.server.api.callbacks.base import EventCallback from llama_index.server.api.callbacks.llamacloud import LlamaCloudFileDownload from llama_index.server.api.callbacks.source_nodes import SourceNodesFromToolCall @@ -12,4 +15,5 @@ "SuggestNextQuestions", "LlamaCloudFileDownload", "AgentCallTool", + "InlineAnnotationTransformer", ] diff --git a/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py new file mode 100644 index 000000000..ccd0197ba --- /dev/null +++ b/python/llama-index-server/llama_index/server/api/callbacks/artifact_transform.py @@ -0,0 +1,24 @@ +import logging +from typing import Any + +from llama_index.server.api.callbacks.base import EventCallback +from llama_index.server.models.artifacts import ArtifactEvent +from llama_index.server.utils.inline import to_inline_annotation_event + +logger = logging.getLogger("uvicorn") + + +class InlineAnnotationTransformer(EventCallback): + """ + Transforms an event to AgentStream with inline annotation format. + """ + + async def run(self, event: Any) -> Any: + # handle for ArtifactEvent specifically as it's only supported by inline annotation + if isinstance(event, ArtifactEvent): + return to_inline_annotation_event(event) + return event + + @classmethod + def from_default(cls, *args: Any, **kwargs: Any) -> "InlineAnnotationTransformer": + return cls() diff --git a/python/llama-index-server/llama_index/server/api/routers/chat.py b/python/llama-index-server/llama_index/server/api/routers/chat.py index 18970860a..8ccf207bd 100644 --- a/python/llama-index-server/llama_index/server/api/routers/chat.py +++ b/python/llama-index-server/llama_index/server/api/routers/chat.py @@ -18,6 +18,7 @@ from llama_index.server.api.callbacks import ( AgentCallTool, EventCallback, + InlineAnnotationTransformer, LlamaCloudFileDownload, SourceNodesFromToolCall, SuggestNextQuestions, @@ -72,6 +73,7 @@ async def chat( callbacks: list[EventCallback] = [ AgentCallTool(), + InlineAnnotationTransformer(), SourceNodesFromToolCall(), LlamaCloudFileDownload(background_tasks), ] diff --git a/python/llama-index-server/llama_index/server/models/artifacts.py b/python/llama-index-server/llama_index/server/models/artifacts.py index 431b12e8b..ecc146928 100644 --- a/python/llama-index-server/llama_index/server/models/artifacts.py +++ b/python/llama-index-server/llama_index/server/models/artifacts.py @@ -5,6 +5,7 @@ from llama_index.core.workflow.events import Event from llama_index.server.models.chat import ChatAPIMessage from pydantic import BaseModel +from llama_index.server.utils.inline import get_inline_annotations logger = logging.getLogger(__name__) @@ -33,10 +34,9 @@ class Artifact(BaseModel): @classmethod def from_message(cls, message: ChatAPIMessage) -> Optional["Artifact"]: - if not message.annotations or not isinstance(message.annotations, list): - return None + inline_annotations = get_inline_annotations(message) - for annotation in message.annotations: + for annotation in inline_annotations: if isinstance(annotation, dict) and annotation.get("type") == "artifact": try: artifact = cls.model_validate(annotation.get("data")) diff --git a/python/llama-index-server/llama_index/server/utils/inline.py b/python/llama-index-server/llama_index/server/utils/inline.py new file mode 100644 index 000000000..daa97ce45 --- /dev/null +++ b/python/llama-index-server/llama_index/server/utils/inline.py @@ -0,0 +1,81 @@ +import json +import re +from typing import Any, List + +from pydantic import ValidationError + +from llama_index.core.workflow.events import Event +from llama_index.server.models.chat import ChatAPIMessage +from llama_index.core.agent.workflow.workflow_events import AgentStream + +INLINE_ANNOTATION_KEY = ( + "annotation" # the language key to detect inline annotation code in markdown +) + + +def get_inline_annotations(message: ChatAPIMessage) -> List[Any]: + """Extract inline annotations from a chat message.""" + markdown_content = message.content + + inline_annotations: List[Any] = [] + + # Regex to match annotation code blocks + # Matches ```annotation followed by content until closing ``` + annotation_regex = re.compile( + rf"```{re.escape(INLINE_ANNOTATION_KEY)}\s*\n([\s\S]*?)\n```", re.MULTILINE + ) + + for match in annotation_regex.finditer(markdown_content): + json_content = match.group(1).strip() if match.group(1) else None + + if not json_content: + continue + + try: + # Parse the JSON content + parsed = json.loads(json_content) + + # Check for required fields in the parsed annotation + if ( + not isinstance(parsed, dict) + or "type" not in parsed + or "data" not in parsed + ): + continue + + # Extract the annotation data + inline_annotations.append(parsed) + except (json.JSONDecodeError, ValidationError) as error: + # Skip invalid annotations - they might be malformed JSON or invalid schema + print(f"Failed to parse annotation: {error}") + + return inline_annotations + + +def to_inline_annotation(item: dict) -> str: + """ + To append inline annotations to the stream, we need to wrap the annotation in a code block with the language key. + The language key is `annotation` and the code block is wrapped in backticks. + + ```annotation + { + "type": "artifact", + "data": {...} + } + ``` + """ + return f"\n```{INLINE_ANNOTATION_KEY}\n{json.dumps(item)}\n```\n" + + +def to_inline_annotation_event(event: Event) -> AgentStream: + """ + Convert an event to an AgentStream with inline annotation format. + """ + event_dict = event.model_dump() + return AgentStream( + delta=to_inline_annotation(event_dict), + response="", + current_agent_name="assistant", + tool_calls=[], + raw=event_dict, + )