From 16479ea1bc6eb8813b0c258fa050f730219d7420 Mon Sep 17 00:00:00 2001 From: Andrew Mikofalvy Date: Mon, 20 Oct 2025 12:37:10 -0700 Subject: [PATCH] PRD-5111: attempt to fix file generation --- agents-cli/src/codegen/plan-builder.ts | 97 +-- agents-cli/src/codegen/plan-to-specs.ts | 163 +++++ .../pull.batch-generator-with-tools.ts | 604 ++++++++++++++++++ agents-cli/src/commands/pull.json-diff.ts | 294 +++++++++ agents-cli/src/commands/pull.llm-generate.ts | 2 +- agents-cli/src/commands/pull.ts | 66 +- .../src/commands/pull.validation-tools.ts | 266 ++++++++ agents-cli/src/index.ts | 1 + .../agents/data-workshop-agent.ts | 23 - .../weather-project/agents/weather-agent.ts | 69 -- .../data-components/weather-forecast.ts | 17 - .../environments/development.env.ts | 6 - .../weather-project/environments/index.ts | 6 - examples/weather-project/index.ts | 24 +- .../weather-project/tools/forecast-weather.ts | 7 - .../weather-project/tools/geocode-address.ts | 7 - examples/weather-project/weather-project.json | 429 +++++++++++++ 17 files changed, 1893 insertions(+), 188 deletions(-) create mode 100644 agents-cli/src/codegen/plan-to-specs.ts create mode 100644 agents-cli/src/commands/pull.batch-generator-with-tools.ts create mode 100644 agents-cli/src/commands/pull.json-diff.ts create mode 100644 agents-cli/src/commands/pull.validation-tools.ts delete mode 100644 examples/weather-project/agents/data-workshop-agent.ts delete mode 100644 examples/weather-project/agents/weather-agent.ts delete mode 100644 examples/weather-project/data-components/weather-forecast.ts delete mode 100644 examples/weather-project/environments/development.env.ts delete mode 100644 examples/weather-project/environments/index.ts delete mode 100644 examples/weather-project/tools/forecast-weather.ts delete mode 100644 examples/weather-project/tools/geocode-address.ts create mode 100644 examples/weather-project/weather-project.json diff --git a/agents-cli/src/codegen/plan-builder.ts b/agents-cli/src/codegen/plan-builder.ts index 32bd0f615..e27e9de93 100644 --- a/agents-cli/src/codegen/plan-builder.ts +++ b/agents-cli/src/codegen/plan-builder.ts @@ -171,38 +171,51 @@ ${fileNames} CRITICAL RULES: -1. TOOL TYPES - VERY IMPORTANT: +1. AGENT AND SUBAGENT STRUCTURE - MOST IMPORTANT: + - **Top-level Agents** (entityType: "agent"): Create ONE file per top-level agent in agents/ directory + - **SubAgents** (entityType: "subAgent"): NEVER create separate files - ALWAYS include as entities in their parent agent's file + - Each agent file contains: + * All subAgents that belong to that agent (as entities in the same file) + * The top-level agent itself (as an entity) + - EXAMPLE: If "weather-agent" has subAgents "forecast-agent" and "geocode-agent", create ONE file "agents/weather-agent.ts" with THREE entities: + * Entity 1: forecast-agent (entityType: "subAgent") + * Entity 2: geocode-agent (entityType: "subAgent") + * Entity 3: weather-agent (entityType: "agent") + - WRONG: Creating separate files "agents/forecast-agent.ts" and "agents/geocode-agent.ts" + - The filename is based on the top-level agent's ID, NOT the subAgent IDs + +2. TOOL TYPES - VERY IMPORTANT: - **Function Tools** (type: "function"): ALWAYS define INLINE within agent files using "inlineContent" array - **MCP Tools** (type: "mcp"): Create separate files in tools/ directory - VALID FILE TYPES: Only use these exact types: "agent", "tool", "dataComponent", "artifactComponent", "statusComponent", "environment", "index" - NEVER create file type "functionTool" - function tools go in "inlineContent" of agent files -2. STATUS COMPONENTS - VERY IMPORTANT: +3. STATUS COMPONENTS - VERY IMPORTANT: - **Status Components**: ALWAYS create separate files in status-components/ directory - Status components are found in agent.statusUpdates.statusComponents array - Each status component should get its own file - Agents must import status components from status-components/ directory - Status components are NEVER inlined in agent files -3. ENVIRONMENT FILES - VERY IMPORTANT: +4. ENVIRONMENT FILES - VERY IMPORTANT: - **When credential references exist**: Create environment files in environments/ directory - **Environment Structure**: Create ONLY "${targetEnvironment}.env.ts" file for target environment (credentials are embedded in this file) - **Environment Index**: Create "environments/index.ts" that imports environment files and exports envSettings using createEnvironmentSettings - **NO separate credential files**: Credentials are defined INSIDE the environment files, not as separate files - **Environment entities**: Use type "environment" for both environment files and index file -4. File Structure: +5. File Structure: - If patterns show "toolsLocation": "inline", ALL tools should be in "inlineContent" of agent files - If patterns show "toolsLocation": "separate", MCP tools get separate files, function tools still inline - Follow the detected file naming convention (kebab-case, camelCase, or snake_case) -4. Variable Names: +6. Variable Names: - MUST use the exact variable names from the mappings above - If ID "weather" is used by both agent and subAgent, they will have different variable names - Do NOT generate new variable names - use what's provided -5. File Placement: - - agents/ directory: Agent files (with function tools in "inlineContent") +7. File Placement: + - agents/ directory: Agent files (with subAgents and function tools inline) - tools/ directory: MCP tool files only - data-components/ directory: Data component files - artifact-components/ directory: Artifact component files @@ -210,13 +223,13 @@ CRITICAL RULES: - environments/ directory: Environment/credential files - index.ts: Main project file -6. File Paths (CRITICAL): +8. File Paths (CRITICAL): - Paths MUST be relative to the project root directory - DO NOT include the project name in the path - CORRECT: "agents/weather-agent.ts", "tools/inkeep-facts.ts", "status-components/tool-summary.ts" - WRONG: "my-project/agents/weather-agent.ts", "project-name/tools/inkeep-facts.ts" -7. Dependencies: +9. Dependencies: - Each file should list which variables it needs to import from other files - Imports should use relative paths - Respect detected import style (named vs default) @@ -225,47 +238,52 @@ OUTPUT FORMAT (JSON): { "files": [ { - "path": "agents/weather-agent.ts", + "path": "agents/weather-basic.ts", "type": "agent", "entities": [ { - "id": "weather", - "variableName": "weatherSubAgent", + "id": "get-coordinates-agent", + "variableName": "getCoordinatesAgent", "entityType": "subAgent", - "exportName": "weatherSubAgent" + "exportName": "getCoordinatesAgent" }, { - "id": "weather", - "variableName": "weatherAgent", + "id": "weather-forecaster", + "variableName": "weatherForecaster", + "entityType": "subAgent", + "exportName": "weatherForecaster" + }, + { + "id": "weather-assistant", + "variableName": "weatherAssistant", + "entityType": "subAgent", + "exportName": "weatherAssistant" + }, + { + "id": "weather-basic", + "variableName": "weatherBasic", "entityType": "agent", - "exportName": "weatherAgent" + "exportName": "weatherBasic" } ], "dependencies": [ { - "variableName": "weatherApi", - "fromPath": "../tools/weather-api", + "variableName": "weatherMcp", + "fromPath": "../tools/weather-mcp", "entityType": "tool" } ], - "inlineContent": [ - { - "id": "get-forecast", - "variableName": "getForecast", - "entityType": "tool", - "exportName": "getForecast" - } - ] + "inlineContent": [] }, { - "path": "tools/weather-api.ts", + "path": "tools/weather-mcp.ts", "type": "tool", "entities": [ { - "id": "weather-api", - "variableName": "weatherApi", + "id": "weather-mcp", + "variableName": "weatherMcp", "entityType": "tool", - "exportName": "weatherApi" + "exportName": "weatherMcp" } ], "dependencies": [], @@ -324,22 +342,27 @@ OUTPUT FORMAT (JSON): "type": "index", "entities": [ { - "id": "my-weather-project", - "variableName": "myWeatherProject", + "id": "weather-project", + "variableName": "weatherProject", "entityType": "project", - "exportName": "myWeatherProject" + "exportName": "weatherProject" } ], "dependencies": [ { - "variableName": "weatherAgent", - "fromPath": "./agents/weather-agent", + "variableName": "weatherBasic", + "fromPath": "./agents/weather-basic", "entityType": "agent" }, { - "variableName": "weatherApi", - "fromPath": "./tools/weather-api", + "variableName": "weatherMcp", + "fromPath": "./tools/weather-mcp", "entityType": "tool" + }, + { + "variableName": "temperatureData", + "fromPath": "./data-components/temperature-data", + "entityType": "dataComponent" } ] } diff --git a/agents-cli/src/codegen/plan-to-specs.ts b/agents-cli/src/codegen/plan-to-specs.ts new file mode 100644 index 000000000..9f7689e43 --- /dev/null +++ b/agents-cli/src/codegen/plan-to-specs.ts @@ -0,0 +1,163 @@ +import { join } from 'node:path'; +import type { FullProjectDefinition } from '@inkeep/agents-core'; +import type { FileSpec } from '../commands/pull.batch-generator-with-tools'; +import type { GenerationPlan } from './types'; + +/** + * Build file specs from a generation plan + * Converts the plan-based structure to the file spec structure needed by batch generator + */ +export async function buildFileSpecsFromPlan( + plan: GenerationPlan, + projectData: FullProjectDefinition, + dirs: { + projectRoot: string; + agentsDir: string; + toolsDir: string; + dataComponentsDir: string; + artifactComponentsDir: string; + statusComponentsDir: string; + environmentsDir: string; + } +): Promise { + const fileSpecs: FileSpec[] = []; + + // Build filename and variable name mappings from the plan's flat file array + const toolFilenames = new Map(); + const componentFilenames = new Map(); + const toolVariableNames = new Map(); + const componentVariableNames = new Map(); + + // Collect filenames and variable names from plan files + for (const fileInfo of plan.files) { + for (const entity of fileInfo.entities) { + const fileName = fileInfo.path.split('/').pop()?.replace('.ts', '') || ''; + const variableName = entity.variableName || entity.exportName || entity.id; + + if (entity.entityType === 'tool') { + toolFilenames.set(entity.id, fileName); + toolVariableNames.set(entity.id, variableName); + } else if ( + entity.entityType === 'dataComponent' || + entity.entityType === 'artifactComponent' || + entity.entityType === 'statusComponent' + ) { + componentFilenames.set(entity.id, fileName); + componentVariableNames.set(entity.id, variableName); + } + } + } + + // Process each file in the plan + for (const fileInfo of plan.files) { + const fullPath = join(dirs.projectRoot, fileInfo.path); + + // Handle different file types + if (fileInfo.type === 'index') { + fileSpecs.push({ + type: 'index', + id: projectData.id, + data: projectData, + outputPath: fullPath, + toolFilenames, + componentFilenames, + toolVariableNames, + componentVariableNames, + }); + } else if (fileInfo.type === 'agent') { + // Find the agent entity in this file + const agentEntity = fileInfo.entities.find((e) => e.entityType === 'agent'); + if (agentEntity) { + const agentData = projectData.agents?.[agentEntity.id]; + if (agentData) { + fileSpecs.push({ + type: 'agent', + id: agentEntity.id, + data: agentData, + outputPath: fullPath, + toolFilenames, + componentFilenames, + toolVariableNames, + componentVariableNames, + }); + } + } + } else if (fileInfo.type === 'tool') { + // Find the tool entity in this file + const toolEntity = fileInfo.entities.find((e) => e.entityType === 'tool'); + if (toolEntity) { + const toolData = projectData.tools?.[toolEntity.id]; + if (toolData) { + const variableName = toolEntity.variableName || toolEntity.exportName || toolEntity.id; + fileSpecs.push({ + type: 'tool', + id: toolEntity.id, + data: toolData, + outputPath: fullPath, + variableName, + }); + } + } + } else if (fileInfo.type === 'dataComponent') { + // Find the data component entity in this file + const componentEntity = fileInfo.entities.find((e) => e.entityType === 'dataComponent'); + if (componentEntity) { + const componentData = projectData.dataComponents?.[componentEntity.id]; + if (componentData) { + const variableName = componentEntity.variableName || componentEntity.exportName || componentEntity.id; + fileSpecs.push({ + type: 'data_component', + id: componentEntity.id, + data: componentData, + outputPath: fullPath, + variableName, + }); + } + } + } else if (fileInfo.type === 'artifactComponent') { + // Find the artifact component entity in this file + const componentEntity = fileInfo.entities.find((e) => e.entityType === 'artifactComponent'); + if (componentEntity) { + const componentData = projectData.artifactComponents?.[componentEntity.id]; + if (componentData) { + const variableName = componentEntity.variableName || componentEntity.exportName || componentEntity.id; + fileSpecs.push({ + type: 'artifact_component', + id: componentEntity.id, + data: componentData, + outputPath: fullPath, + variableName, + }); + } + } + } else if (fileInfo.type === 'statusComponent') { + // Find the status component entity in this file + const componentEntity = fileInfo.entities.find((e) => e.entityType === 'statusComponent'); + if (componentEntity) { + // Status components come from agent definitions + // Find the component data from the agent's statusUpdates + for (const agent of Object.values(projectData.agents || {})) { + const agentObj = agent as any; + const statusComponents = agentObj.statusUpdates?.statusComponents || []; + + for (const statusComponent of statusComponents) { + if (statusComponent.type === componentEntity.id) { + const variableName = componentEntity.variableName || componentEntity.exportName || componentEntity.id; + fileSpecs.push({ + type: 'status_component', + id: componentEntity.id, + data: statusComponent, + outputPath: fullPath, + variableName, + }); + break; + } + } + } + } + } + // Skip environment files - they're handled by generateEnvironmentFiles + } + + return fileSpecs; +} diff --git a/agents-cli/src/commands/pull.batch-generator-with-tools.ts b/agents-cli/src/commands/pull.batch-generator-with-tools.ts new file mode 100644 index 000000000..9fa300c78 --- /dev/null +++ b/agents-cli/src/commands/pull.batch-generator-with-tools.ts @@ -0,0 +1,604 @@ +import { writeFileSync } from 'node:fs'; +import type { FullProjectDefinition, ModelSettings } from '@inkeep/agents-core'; +import { generateText, stepCountIs } from 'ai'; +import { isLangfuseConfigured } from '../instrumentation'; +import { + cleanGeneratedCode, + createModel, + getTypeDefinitions, + IMPORT_INSTRUCTIONS, + NAMING_CONVENTION_RULES, + PROJECT_JSON_EXAMPLE, +} from './pull.llm-generate'; +import { + calculateTokenSavings, + createPlaceholders, + restorePlaceholders, +} from './pull.placeholder-system'; +import type { ValidationContext } from './pull.validation-tools'; +import { getValidationTools } from './pull.validation-tools'; + +/** + * Specification for a file to generate + */ +export interface FileSpec { + type: 'index' | 'agent' | 'tool' | 'data_component' | 'artifact_component' | 'status_component'; + id: string; + data: any; + outputPath: string; + variableName?: string; // Variable name to use for export (for individual entity files) + toolFilenames?: Map; + componentFilenames?: Map; + toolVariableNames?: Map; + componentVariableNames?: Map; +} + +/** + * Options for batch generation with validation + */ +export interface BatchGenerationOptions { + fileSpecs: FileSpec[]; + modelSettings: ModelSettings; + originalProjectDefinition: FullProjectDefinition; + projectId: string; + projectRoot: string; + tenantId: string; + apiUrl: string; + maxAttempts?: number; + debug?: boolean; + reasoningConfig?: Record; +} + +/** + * Result from batch generation + */ +export interface BatchGenerationResult { + success: boolean; + attemptCount: number; + validationPassed: boolean; + filesGenerated: number; + errors: string[]; + warnings: string[]; +} + +/** + * Generate all files in a batch with iterative validation + * Uses LLM function calling to allow self-correction + */ +export async function generateAllFilesWithValidation( + options: BatchGenerationOptions +): Promise { + const { + fileSpecs, + modelSettings, + originalProjectDefinition, + projectId, + projectRoot, + tenantId, + apiUrl, + maxAttempts = 3, + debug = false, + reasoningConfig, + } = options; + + if (fileSpecs.length === 0) { + return { + success: true, + attemptCount: 0, + validationPassed: true, + filesGenerated: 0, + errors: [], + warnings: [], + }; + } + + if (debug) { + console.log(`\n[DEBUG] === Starting batch generation with validation ===`); + console.log(`[DEBUG] Files to generate: ${fileSpecs.length}`); + console.log(`[DEBUG] Max attempts: ${maxAttempts}`); + console.log(`[DEBUG] Model: ${modelSettings.model || 'default'}`); + } + + const model = createModel(modelSettings); + + // Create placeholders for all file data to reduce token usage + const fileDataWithPlaceholders = fileSpecs.map((spec) => { + const { processedData, replacements } = createPlaceholders(spec.data, { fileType: spec.type }); + return { + spec, + processedData, + replacements, + }; + }); + + // Merge all placeholder replacements + const allReplacements: Record = {}; + for (const { replacements } of fileDataWithPlaceholders) { + Object.assign(allReplacements, replacements); + } + + if (debug) { + const totalReplacements = Object.keys(allReplacements).length; + console.log(`[DEBUG] Created ${totalReplacements} placeholders across all files`); + + // Calculate total token savings + let originalSize = 0; + let processedSize = 0; + for (const { spec, processedData } of fileDataWithPlaceholders) { + originalSize += JSON.stringify(spec.data).length; + processedSize += JSON.stringify(processedData).length; + } + const savings = calculateTokenSavings( + { size: originalSize } as any, + { size: processedSize } as any + ); + console.log( + `[DEBUG] Total token savings: ${savings.savings} characters (${savings.savingsPercentage.toFixed(1)}%)` + ); + } + + // Build the initial prompt + const prompt = buildBatchGenerationPrompt(fileDataWithPlaceholders); + + if (debug) { + console.log(`[DEBUG] Initial prompt size: ${prompt.length} characters`); + } + + // Track generated files across attempts + const generatedFiles = new Map(); + let lastValidationErrors: string[] = []; + let validationPassed = false; + let attemptCount = 0; + + // Create validation context + const validationContext: ValidationContext = { + originalProjectDefinition, + projectId, + generatedFiles, + placeholderReplacements: allReplacements, + projectRoot, + tenantId, + apiUrl, + debug, + }; + + // Get validation tools + const tools = getValidationTools(validationContext); + + // Iterative generation loop + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + attemptCount = attempt; + + if (debug) { + console.log(`\n[DEBUG] === Attempt ${attempt} of ${maxAttempts} ===`); + } + + try { + // Build the prompt for this attempt + const attemptPrompt = + attempt === 1 + ? prompt + : buildRetryPrompt(prompt, lastValidationErrors, attempt, maxAttempts); + + if (debug && attempt > 1) { + console.log(`[DEBUG] Retry prompt includes ${lastValidationErrors.length} previous errors`); + } + + // Generate with tools enabled + const startTime = Date.now(); + + const result = await generateText({ + model, + prompt: attemptPrompt, + temperature: 0.1, + stopWhen: stepCountIs(5), // Allow multiple tool calls + maxOutputTokens: 32000, + abortSignal: AbortSignal.timeout(600000), // 10 minute timeout + tools, + ...reasoningConfig, + // Enable Langfuse telemetry if configured + ...(isLangfuseConfigured() && { + experimental_telemetry: { + isEnabled: true, + metadata: { + batchGeneration: true, + withValidation: true, + attempt, + maxAttempts, + fileCount: fileSpecs.length, + promptSize: attemptPrompt.length, + }, + }, + }), + }); + + const duration = Date.now() - startTime; + + if (debug) { + console.log(`[DEBUG] LLM response received in ${duration}ms`); + console.log(`[DEBUG] Response text length: ${result.text.length} characters`); + console.log(`[DEBUG] Tool calls made: ${result.steps.length - 1}`); // -1 for initial generation + } + + // Parse the generated files from the response + const parsedFiles = parseMultiFileResponse(result.text, fileSpecs); + + if (debug) { + console.log(`[DEBUG] Parsed ${parsedFiles.length} files from response`); + } + + // Update generated files map + // Only clear if we successfully parsed files, to preserve previous attempts + if (parsedFiles.length > 0) { + generatedFiles.clear(); + for (const { path, content } of parsedFiles) { + generatedFiles.set(path, content); + } + } + + // Check if validation tool was called + const validationStep = result.steps.find((step) => + step.toolCalls?.some((call: any) => call.toolName === 'validate_generated_code') + ); + + if (validationStep) { + if (debug) { + console.log('[DEBUG] LLM called validation tool during generation'); + } + + // Get the validation result from tool results + const validationToolResult = result.steps + .flatMap((step) => step.toolResults || []) + .find((toolResult: any) => toolResult.toolName === 'validate_generated_code'); + + if (validationToolResult) { + // The result might be in .result or directly on the tool result + const toolOutput = validationToolResult.result || validationToolResult; + + if (debug) { + console.log(`[DEBUG] Validation tool output:`, JSON.stringify(toolOutput, null, 2)); + } + + if (toolOutput && toolOutput.success) { + if (debug) { + console.log('[DEBUG] Validation passed!'); + } + validationPassed = true; + break; + } + + if (toolOutput && toolOutput.success === false) { + if (debug) { + console.log('[DEBUG] Validation failed, preparing for retry'); + } + lastValidationErrors = toolOutput.errors || []; + + // If this is the last attempt, break + if (attempt === maxAttempts) { + if (debug) { + console.log('[DEBUG] Max attempts reached, using last generation'); + } + break; + } + } + } + } else { + if (debug) { + console.log('[DEBUG] LLM did not call validation tool, assuming success'); + } + // If LLM didn't call validation, we assume success + validationPassed = true; + break; + } + } catch (error) { + if (debug) { + console.error(`[DEBUG] Error during attempt ${attempt}:`, error); + } + + // Store the error for the next attempt or final result + lastValidationErrors = [error instanceof Error ? error.message : String(error)]; + + // If this is the last attempt, break and use whatever files we have + if (attempt === maxAttempts) { + if (debug) { + console.log('[DEBUG] Max attempts reached after error, using best available generation'); + } + break; + } + } + } + + // Check if we have any files to write + if (generatedFiles.size === 0) { + return { + success: false, + attemptCount, + validationPassed: false, + filesGenerated: 0, + errors: ['No files were successfully generated after all attempts'], + warnings: lastValidationErrors.length > 0 ? ['Last errors:', ...lastValidationErrors] : [], + }; + } + + // Write all generated files to disk with placeholders restored + if (debug) { + console.log(`\n[DEBUG] Writing ${generatedFiles.size} files to disk`); + } + + for (const [path, content] of generatedFiles.entries()) { + // Restore placeholders + const restoredContent = restorePlaceholders(content, allReplacements); + const cleanedContent = cleanGeneratedCode(restoredContent); + + writeFileSync(path, cleanedContent, 'utf-8'); + + if (debug) { + console.log(`[DEBUG] Wrote file: ${path}`); + } + } + + // Prepare result + const warnings: string[] = []; + if (!validationPassed) { + warnings.push( + `Validation did not pass after ${attemptCount} attempts. Using best effort generation.` + ); + if (lastValidationErrors.length > 0) { + warnings.push('Last validation errors:'); + warnings.push(...lastValidationErrors); + } + } + + if (debug) { + console.log(`[DEBUG] === Batch generation completed ===`); + console.log(`[DEBUG] Attempts: ${attemptCount}`); + console.log(`[DEBUG] Validation passed: ${validationPassed}`); + console.log(`[DEBUG] Files generated: ${generatedFiles.size}`); + } + + return { + success: true, + attemptCount, + validationPassed, + filesGenerated: generatedFiles.size, + errors: [], + warnings, + }; +} + +/** + * Build the initial batch generation prompt + */ +function buildBatchGenerationPrompt( + fileDataWithPlaceholders: Array<{ + spec: FileSpec; + processedData: any; + replacements: Record; + }> +): string { + const typeDefinitions = getTypeDefinitions(); + const sharedInstructions = ` +${NAMING_CONVENTION_RULES} + +${IMPORT_INSTRUCTIONS} +`; + + // Build individual file prompts + const filePrompts = fileDataWithPlaceholders.map(({ spec, processedData }, index) => { + let fileSpecificInstructions = ''; + + switch (spec.type) { + case 'index': { + // Build variable name mappings info + let variableNamesInfo = ''; + if (spec.toolVariableNames && spec.toolVariableNames.size > 0) { + variableNamesInfo += '\nTOOL VARIABLE NAMES (use these for imports):\n'; + for (const [id, variableName] of spec.toolVariableNames.entries()) { + variableNamesInfo += `- Tool ID "${id}" should be imported as: ${variableName}\n`; + } + } + if (spec.componentVariableNames && spec.componentVariableNames.size > 0) { + variableNamesInfo += '\nCOMPONENT VARIABLE NAMES (use these for imports):\n'; + for (const [id, variableName] of spec.componentVariableNames.entries()) { + variableNamesInfo += `- Component ID "${id}" should be imported as: ${variableName}\n`; + } + } + + fileSpecificInstructions = ` +REQUIREMENTS FOR INDEX FILE: +1. Import the project function from '@inkeep/agents-sdk' +2. The project object should include all required properties +3. Export the project instance +4. CRITICAL: Use the variable names specified below for imports, NOT the tool/component IDs +${variableNamesInfo} + +EXAMPLE: +${PROJECT_JSON_EXAMPLE} +`; + break; + } + + case 'agent': { + // Build variable name mappings info for agents + let agentVariableNamesInfo = ''; + if (spec.toolVariableNames && spec.toolVariableNames.size > 0) { + agentVariableNamesInfo += '\nTOOL VARIABLE NAMES (use these for imports):\n'; + for (const [id, variableName] of spec.toolVariableNames.entries()) { + agentVariableNamesInfo += `- Tool ID "${id}" should be imported as: ${variableName}\n`; + } + } + if (spec.componentVariableNames && spec.componentVariableNames.size > 0) { + agentVariableNamesInfo += '\nCOMPONENT VARIABLE NAMES (use these for imports):\n'; + for (const [id, variableName] of spec.componentVariableNames.entries()) { + agentVariableNamesInfo += `- Component ID "${id}" should be imported as: ${variableName}\n`; + } + } + + fileSpecificInstructions = ` +REQUIREMENTS FOR AGENT FILE: +1. Import { agent, subAgent } from '@inkeep/agents-sdk' +2. Import tools and components from their respective files +3. Use proper TypeScript types and Zod schemas +4. Use template literals for all string values +5. Define contextConfig using helper functions if needed +6. CRITICAL: Use the variable names specified below for imports, NOT the tool/component IDs +${agentVariableNamesInfo} +`; + break; + } + + case 'tool': + fileSpecificInstructions = ` +REQUIREMENTS FOR TOOL FILE: +1. Import mcpTool from '@inkeep/agents-sdk' +2. Include serverUrl property +3. Add credential if credentialReferenceId exists +4. CRITICAL: Export this tool with the variable name: ${spec.variableName || spec.id} + Example: export const ${spec.variableName || spec.id} = mcpTool({ ... }); +`; + break; + + case 'data_component': + fileSpecificInstructions = ` +REQUIREMENTS FOR DATA COMPONENT: +1. Import dataComponent from '@inkeep/agents-sdk' +2. Import z from 'zod' +3. Define clean Zod schemas +4. CRITICAL: Export this component with the variable name: ${spec.variableName || spec.id} + Example: export const ${spec.variableName || spec.id} = dataComponent({ ... }); +`; + break; + + case 'artifact_component': + fileSpecificInstructions = ` +REQUIREMENTS FOR ARTIFACT COMPONENT: +1. Import artifactComponent from '@inkeep/agents-sdk' +2. Import z from 'zod' and preview from '@inkeep/agents-core' +3. Use preview() for fields shown in previews +4. CRITICAL: Export this component with the variable name: ${spec.variableName || spec.id} + Example: export const ${spec.variableName || spec.id} = artifactComponent({ ... }); +`; + break; + + case 'status_component': + fileSpecificInstructions = ` +REQUIREMENTS FOR STATUS COMPONENT: +1. Import statusComponent from '@inkeep/agents-sdk' +2. Import z from 'zod' +3. Use 'type' field as identifier +4. CRITICAL: Export this component with the variable name: ${spec.variableName || spec.id} + Example: export const ${spec.variableName || spec.id} = statusComponent({ ... }); +`; + break; + } + + return ` +--- FILE ${index + 1} OF ${fileDataWithPlaceholders.length}: ${spec.outputPath} --- +FILE TYPE: ${spec.type} +FILE ID: ${spec.id} + +DATA FOR THIS FILE: +${JSON.stringify(processedData, null, 2)} + +${fileSpecificInstructions} + +Generate ONLY the TypeScript code for this file. +--- END FILE ${index + 1} --- +`; + }); + + return `You are generating multiple TypeScript files for an Inkeep project. + +${typeDefinitions} + +${sharedInstructions} + +CRITICAL INSTRUCTIONS: +1. Generate ${fileDataWithPlaceholders.length} separate TypeScript files +2. Each file MUST be wrapped with separator markers: --- FILE: --- and --- END FILE: --- +3. Include ONLY raw TypeScript code between markers (no markdown) +4. After generating all files, call the validate_generated_code tool to verify correctness +5. If validation fails, regenerate the problematic files and validate again +6. Continue until validation passes or you've made reasonable attempts + +FILE SPECIFICATIONS: +${filePrompts.join('\n\n')} + +OUTPUT FORMAT: +--- FILE: /path/to/file.ts --- +[TypeScript code here] +--- END FILE: /path/to/file.ts --- + +Now generate all files and validate them using the validate_generated_code tool.`; +} + +/** + * Build a retry prompt with previous validation errors + */ +function buildRetryPrompt( + originalPrompt: string, + validationErrors: string[], + attempt: number, + maxAttempts: number +): string { + return `${originalPrompt} + +RETRY CONTEXT - ATTEMPT ${attempt} OF ${maxAttempts}: +Previous generation had validation errors. Please fix these issues: + +${validationErrors.join('\n\n')} + +IMPORTANT: +- Focus on the specific errors mentioned above +- Ensure all IDs, names, and configurations match exactly +- Double-check the structure of the generated code +- Call validate_generated_code after regenerating to verify + +Now regenerate the files with these fixes applied.`; +} + +/** + * Parse multi-file response from LLM + */ +function parseMultiFileResponse( + response: string, + fileSpecs: FileSpec[] +): Array<{ path: string; content: string }> { + const results: Array<{ path: string; content: string }> = []; + + for (const spec of fileSpecs) { + const startMarker = `--- FILE: ${spec.outputPath} ---`; + const endMarker = `--- END FILE: ${spec.outputPath} ---`; + + const startIndex = response.indexOf(startMarker); + const endIndex = response.indexOf(endMarker); + + if (startIndex === -1 || endIndex === -1) { + // Try alternate format without full path + const altStartMarker = `--- FILE ${fileSpecs.indexOf(spec) + 1}`; + const altEndMarker = `--- END FILE ${fileSpecs.indexOf(spec) + 1}`; + + const altStartIndex = response.indexOf(altStartMarker); + const altEndIndex = response.indexOf(altEndMarker); + + if (altStartIndex !== -1 && altEndIndex !== -1) { + const content = response + .substring(altStartIndex + altStartMarker.length, altEndIndex) + .trim(); + results.push({ path: spec.outputPath, content }); + continue; + } + + throw new Error(`Failed to find file markers for ${spec.outputPath}`); + } + + const content = response.substring(startIndex + startMarker.length, endIndex).trim(); + + results.push({ + path: spec.outputPath, + content, + }); + } + + return results; +} diff --git a/agents-cli/src/commands/pull.json-diff.ts b/agents-cli/src/commands/pull.json-diff.ts new file mode 100644 index 000000000..d5a9ff9ce --- /dev/null +++ b/agents-cli/src/commands/pull.json-diff.ts @@ -0,0 +1,294 @@ +import type { FullProjectDefinition } from '@inkeep/agents-core'; + +/** + * Represents a difference found between two objects + */ +export interface JsonDifference { + path: string; + type: 'missing' | 'extra' | 'mismatch' | 'type_mismatch'; + expected?: any; + actual?: any; + message: string; +} + +/** + * Compare two project definitions and return detailed differences + * @param original - The original project definition from the API + * @param regenerated - The regenerated project definition from TypeScript + * @returns Array of differences found + */ +export function compareProjectDefinitions( + original: FullProjectDefinition, + regenerated: FullProjectDefinition +): JsonDifference[] { + const differences: JsonDifference[] = []; + + // Compare at the root level + compareObjects(original, regenerated, '', differences); + + return differences; +} + +/** + * Recursively compare two objects and collect differences + */ +function compareObjects( + expected: any, + actual: any, + path: string, + differences: JsonDifference[] +): void { + // Handle null/undefined cases + if (expected === null || expected === undefined) { + if (actual !== null && actual !== undefined) { + differences.push({ + path, + type: 'mismatch', + expected, + actual, + message: `Expected ${expected} but got ${actual}`, + }); + } + return; + } + + if (actual === null || actual === undefined) { + differences.push({ + path, + type: 'missing', + expected, + actual, + message: `Missing value at path ${path}`, + }); + return; + } + + // Handle type mismatches + const expectedType = getType(expected); + const actualType = getType(actual); + + if (expectedType !== actualType) { + differences.push({ + path, + type: 'type_mismatch', + expected: expectedType, + actual: actualType, + message: `Type mismatch at ${path}: expected ${expectedType} but got ${actualType}`, + }); + return; + } + + // Handle arrays + if (Array.isArray(expected)) { + compareArrays(expected, actual, path, differences); + return; + } + + // Handle objects + if (expectedType === 'object') { + // Check for missing keys in actual + for (const key of Object.keys(expected)) { + const newPath = path ? `${path}.${key}` : key; + + // Skip certain fields that are expected to be different + if (shouldSkipField(key)) { + continue; + } + + if (!(key in actual)) { + differences.push({ + path: newPath, + type: 'missing', + expected: expected[key], + message: `Missing key "${key}" at path ${path || 'root'}`, + }); + } else { + compareObjects(expected[key], actual[key], newPath, differences); + } + } + + // Check for extra keys in actual + for (const key of Object.keys(actual)) { + if (shouldSkipField(key)) { + continue; + } + + if (!(key in expected)) { + const newPath = path ? `${path}.${key}` : key; + differences.push({ + path: newPath, + type: 'extra', + actual: actual[key], + message: `Extra key "${key}" at path ${path || 'root'}`, + }); + } + } + + return; + } + + // Handle primitives + if (expected !== actual) { + differences.push({ + path, + type: 'mismatch', + expected, + actual, + message: `Value mismatch at ${path}: expected ${JSON.stringify(expected)} but got ${JSON.stringify(actual)}`, + }); + } +} + +/** + * Compare two arrays and collect differences + */ +function compareArrays( + expected: any[], + actual: any[], + path: string, + differences: JsonDifference[] +): void { + // Check length + if (expected.length !== actual.length) { + differences.push({ + path, + type: 'mismatch', + expected: `array of length ${expected.length}`, + actual: `array of length ${actual.length}`, + message: `Array length mismatch at ${path}: expected ${expected.length} items but got ${actual.length}`, + }); + } + + // Compare elements + const minLength = Math.min(expected.length, actual.length); + for (let i = 0; i < minLength; i++) { + compareObjects(expected[i], actual[i], `${path}[${i}]`, differences); + } + + // Report missing elements if actual is shorter + if (actual.length < expected.length) { + for (let i = actual.length; i < expected.length; i++) { + differences.push({ + path: `${path}[${i}]`, + type: 'missing', + expected: expected[i], + message: `Missing array element at ${path}[${i}]`, + }); + } + } + + // Report extra elements if actual is longer + if (actual.length > expected.length) { + for (let i = expected.length; i < actual.length; i++) { + differences.push({ + path: `${path}[${i}]`, + type: 'extra', + actual: actual[i], + message: `Extra array element at ${path}[${i}]`, + }); + } + } +} + +/** + * Get the type of a value for comparison + */ +function getType(value: any): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (Array.isArray(value)) return 'array'; + return typeof value; +} + +/** + * Check if a field should be skipped during comparison + * These fields are expected to differ between original and regenerated + */ +function shouldSkipField(key: string): boolean { + // Skip timestamp fields as they will naturally differ + if (key === 'createdAt' || key === 'updatedAt') { + return true; + } + + // Skip internal metadata fields + if (key.startsWith('_')) { + return true; + } + + return false; +} + +/** + * Format differences into a human-readable report + */ +export function formatDifferencesReport(differences: JsonDifference[]): string { + if (differences.length === 0) { + return 'āœ… No differences found - validation passed!'; + } + + const lines: string[] = [ + `āŒ Found ${differences.length} difference(s):`, + '', + ]; + + // Group by type + const byType = { + missing: differences.filter((d) => d.type === 'missing'), + extra: differences.filter((d) => d.type === 'extra'), + mismatch: differences.filter((d) => d.type === 'mismatch'), + type_mismatch: differences.filter((d) => d.type === 'type_mismatch'), + }; + + if (byType.missing.length > 0) { + lines.push('Missing fields:'); + for (const diff of byType.missing) { + lines.push(` - ${diff.path}`); + lines.push(` Expected: ${formatValue(diff.expected)}`); + } + lines.push(''); + } + + if (byType.extra.length > 0) { + lines.push('Extra fields:'); + for (const diff of byType.extra) { + lines.push(` - ${diff.path}`); + lines.push(` Got: ${formatValue(diff.actual)}`); + } + lines.push(''); + } + + if (byType.type_mismatch.length > 0) { + lines.push('Type mismatches:'); + for (const diff of byType.type_mismatch) { + lines.push(` - ${diff.path}`); + lines.push(` Expected type: ${diff.expected}, Got type: ${diff.actual}`); + } + lines.push(''); + } + + if (byType.mismatch.length > 0) { + lines.push('Value mismatches:'); + for (const diff of byType.mismatch) { + lines.push(` - ${diff.path}`); + lines.push(` Expected: ${formatValue(diff.expected)}`); + lines.push(` Got: ${formatValue(diff.actual)}`); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Format a value for display in the report + */ +function formatValue(value: any): string { + if (value === null) return 'null'; + if (value === undefined) return 'undefined'; + if (typeof value === 'string') return `"${value}"`; + if (typeof value === 'object') { + const str = JSON.stringify(value); + return str.length > 100 ? `${str.substring(0, 100)}...` : str; + } + return String(value); +} diff --git a/agents-cli/src/commands/pull.llm-generate.ts b/agents-cli/src/commands/pull.llm-generate.ts index 969bbfe39..6ae507e6c 100644 --- a/agents-cli/src/commands/pull.llm-generate.ts +++ b/agents-cli/src/commands/pull.llm-generate.ts @@ -165,7 +165,7 @@ export function createModel(config: ModelSettings): any { } } -const PROJECT_JSON_EXAMPLE = ` +export const PROJECT_JSON_EXAMPLE = ` ---START OF PROJECT JSON EXAMPLE--- { "id": "my-project", diff --git a/agents-cli/src/commands/pull.ts b/agents-cli/src/commands/pull.ts index a9022f346..00e1f5d6d 100644 --- a/agents-cli/src/commands/pull.ts +++ b/agents-cli/src/commands/pull.ts @@ -26,6 +26,7 @@ export interface PullOptions { env?: string; json?: boolean; debug?: boolean; + validate?: boolean; // Enable iterative validation with LLM function calling } interface VerificationResult { @@ -955,17 +956,64 @@ export async function pullProjectCommand(options: PullOptions): Promise { // Step 4: Generate files from plan using unified generator s.start('Generating project files with LLM...'); - const { generateFilesFromPlan } = await import('../codegen/unified-generator'); + + // Use iterative validation if requested (PRD-5111) + const useIterativeValidation = options.validate || process.env.INKEEP_ITERATIVE_VALIDATION === 'true'; const generationStart = Date.now(); - await generateFilesFromPlan( - plan, - projectData, - dirs, - modelSettings, - options.debug || false, - reasoningConfig // Pass reasoning config for enhanced code generation - ); + + if (useIterativeValidation) { + if (options.debug) { + console.log(chalk.gray('\nšŸ“ Using iterative validation with LLM function calling')); + } + + // Import the new batch generator with validation + const { generateAllFilesWithValidation } = await import('./pull.batch-generator-with-tools'); + const { buildFileSpecsFromPlan } = await import('../codegen/plan-to-specs'); + + // Build file specs from the plan + const fileSpecs = await buildFileSpecsFromPlan(plan, projectData, dirs); + + // Generate with validation + const result = await generateAllFilesWithValidation({ + fileSpecs, + modelSettings, + originalProjectDefinition: projectData, + projectId: finalConfig.projectId, + projectRoot: dirs.projectRoot, + tenantId: finalConfig.tenantId, + apiUrl: finalConfig.agentsManageApiUrl, + maxAttempts: 3, + debug: options.debug || false, + reasoningConfig, + }); + + if (options.debug) { + console.log(chalk.gray(`\nšŸ“ Validation result:`)); + console.log(chalk.gray(` ā€¢ Attempts: ${result.attemptCount}`)); + console.log(chalk.gray(` ā€¢ Validation passed: ${result.validationPassed}`)); + console.log(chalk.gray(` ā€¢ Files generated: ${result.filesGenerated}`)); + } + + if (!result.validationPassed) { + console.log(chalk.yellow('\nāš ļø Validation warnings:')); + for (const warning of result.warnings) { + console.log(chalk.gray(` ā€¢ ${warning}`)); + } + } + } else { + // Use the current plan-based generation + const { generateFilesFromPlan } = await import('../codegen/unified-generator'); + await generateFilesFromPlan( + plan, + projectData, + dirs, + modelSettings, + options.debug || false, + reasoningConfig // Pass reasoning config for enhanced code generation + ); + } + const generationDuration = Date.now() - generationStart; s.stop('Project files generated'); diff --git a/agents-cli/src/commands/pull.validation-tools.ts b/agents-cli/src/commands/pull.validation-tools.ts new file mode 100644 index 000000000..5da6817c2 --- /dev/null +++ b/agents-cli/src/commands/pull.validation-tools.ts @@ -0,0 +1,266 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { FullProjectDefinition } from '@inkeep/agents-core'; +import { tool, type CoreTool } from 'ai'; +import { z } from 'zod'; +import { compareProjectDefinitions, formatDifferencesReport } from './pull.json-diff'; +import { restorePlaceholders } from './pull.placeholder-system'; + +/** + * Context needed for validation tools + */ +export interface ValidationContext { + originalProjectDefinition: FullProjectDefinition; + projectId: string; + generatedFiles: Map; // path -> content (may contain placeholders) + placeholderReplacements: Record; // placeholder -> original value + tenantId: string; + apiUrl: string; + projectRoot: string; // project root directory to make paths relative + debug?: boolean; +} + +/** + * Result from validation + */ +export interface ValidationResult { + success: boolean; + errors: string[]; + warnings: string[]; + differencesCount: number; + regeneratedDefinition?: FullProjectDefinition; +} + +/** + * Create validation tool for the LLM to use + * This tool validates generated TypeScript code by: + * 1. Writing files to a temp directory + * 2. Loading the project using dynamic import + * 3. Calling project.getFullDefinition() + * 4. Comparing the result with the original API response + */ +export function createValidationTool(context: ValidationContext): CoreTool { + return tool({ + description: `Validates that the generated TypeScript code correctly represents the project definition. +This tool: +1. Writes all generated files to a temporary directory +2. Loads the project TypeScript code +3. Converts it back to JSON using project.getFullDefinition() +4. Compares the regenerated JSON with the original API response +5. Returns any differences found (missing fields, incorrect values, etc.) + +Call this tool after generating code to verify it's correct. If there are errors, regenerate the problematic files.`, + inputSchema: z.object({ + reason: z + .string() + .describe('Why you want to validate (e.g., "checking if initial generation is correct")'), + }), + execute: async ({ reason }) => { + if (context.debug) { + console.log(`\n[DEBUG] Validation tool called: ${reason}`); + } + + try { + const result = await validateGeneratedFiles(context); + + if (context.debug) { + console.log(`[DEBUG] Validation result: ${result.success ? 'PASS' : 'FAIL'}`); + console.log(`[DEBUG] Differences found: ${result.differencesCount}`); + } + + // Return a formatted response for the LLM + if (result.success) { + return { + success: true, + message: 'āœ… Validation passed! Generated code correctly represents the project definition.', + }; + } + + return { + success: false, + message: `āŒ Validation failed with ${result.differencesCount} differences.`, + errors: result.errors, + warnings: result.warnings, + details: + 'Review the errors above and regenerate the affected files. Focus on ensuring all IDs, names, and configurations match exactly.', + }; + } catch (error) { + if (context.debug) { + console.error('[DEBUG] Validation error:', error); + } + + return { + success: false, + message: 'āŒ Validation failed with an error', + error: error instanceof Error ? error.message : String(error), + details: + 'An error occurred during validation. This usually means the generated code has syntax errors or cannot be loaded.', + }; + } + }, + }); +} + +/** + * Create lint tool (placeholder for future implementation) + */ +export function createLintTool(): CoreTool { + return tool({ + description: + 'Runs linting checks on generated code (not yet implemented). Reserved for future use.', + inputSchema: z.object({ + reason: z.string().describe('Why you want to lint'), + }), + execute: async ({ reason }) => { + return { + success: true, + message: 'Linting is not yet implemented', + note: 'This tool is reserved for future use', + }; + }, + }); +} + +/** + * Validate generated files by loading them and comparing with original definition + */ +async function validateGeneratedFiles(context: ValidationContext): Promise { + // Create a temporary directory for validation + const tempDir = join(tmpdir(), `inkeep-validation-${Date.now()}`); + mkdirSync(tempDir, { recursive: true }); + + try { + // Write all files to temp directory with placeholders restored + for (const [absolutePath, content] of context.generatedFiles.entries()) { + // Convert absolute path to relative by removing project root + const relativePath = absolutePath.startsWith(context.projectRoot) + ? absolutePath.slice(context.projectRoot.length).replace(/^\//, '') + : absolutePath; + + const fullPath = join(tempDir, relativePath); + + // Ensure directory exists + const dir = join(fullPath, '..'); + mkdirSync(dir, { recursive: true }); + + // Restore placeholders in the content + const restoredContent = restorePlaceholders(content, context.placeholderReplacements); + + // Write the file + writeFileSync(fullPath, restoredContent, 'utf-8'); + + if (context.debug) { + console.log(`[DEBUG] Wrote validation file: ${fullPath}`); + } + } + + // Now try to load the project and get its definition + const indexPath = join(tempDir, 'index.ts'); + + if (context.debug) { + console.log(`[DEBUG] Loading project from: ${indexPath}`); + } + + // Dynamically import the project file using tsx loader + // Note: This requires tsx to be installed as a dependency + const { Project } = await import('@inkeep/agents-sdk'); + + // Use tsx to load and execute the TypeScript file + // We need to use a worker or subprocess to load TypeScript in a clean environment + // For now, we'll use a simpler approach with require and tsx + const { register } = await import('tsx/esm/api'); + const unregister = register(); + + let projectModule: any; + try { + // Import the generated index file + projectModule = await import(`${indexPath}?t=${Date.now()}`); + } finally { + unregister(); + } + + // Find the exported project instance + let projectInstance: any = null; + for (const exportName of Object.keys(projectModule)) { + const exported = projectModule[exportName]; + if (exported && typeof exported === 'object' && exported.__type === 'project') { + projectInstance = exported; + break; + } + } + + if (!projectInstance) { + return { + success: false, + errors: ['No project instance found in generated index.ts. Expected an exported project() call.'], + warnings: [], + differencesCount: 1, + }; + } + + // Set config on the project + projectInstance.setConfig(context.tenantId, context.apiUrl); + + // Get the full definition from the regenerated code + const regeneratedDefinition = await projectInstance.getFullDefinition(); + + if (context.debug) { + console.log('[DEBUG] Successfully regenerated project definition from TypeScript'); + } + + // Compare the regenerated definition with the original + const differences = compareProjectDefinitions( + context.originalProjectDefinition, + regeneratedDefinition + ); + + if (differences.length === 0) { + return { + success: true, + errors: [], + warnings: [], + differencesCount: 0, + regeneratedDefinition, + }; + } + + // Format the differences into error messages + const report = formatDifferencesReport(differences); + + return { + success: false, + errors: [report], + warnings: [], + differencesCount: differences.length, + regeneratedDefinition, + }; + } catch (error) { + if (context.debug) { + console.error('[DEBUG] Error during validation:', error); + } + + // Return a detailed error + const errorMessage = + error instanceof Error + ? `${error.message}\n\nStack trace:\n${error.stack}` + : String(error); + + return { + success: false, + errors: [`Failed to load or validate generated code: ${errorMessage}`], + warnings: [], + differencesCount: 1, + }; + } +} + +/** + * Get all validation tools + */ +export function getValidationTools(context: ValidationContext): Record { + return { + validate_generated_code: createValidationTool(context), + lint_code: createLintTool(), + }; +} diff --git a/agents-cli/src/index.ts b/agents-cli/src/index.ts index d156344a3..4c405dbb5 100644 --- a/agents-cli/src/index.ts +++ b/agents-cli/src/index.ts @@ -105,6 +105,7 @@ program ) .option('--json', 'Generate project data JSON file instead of updating files') .option('--debug', 'Enable debug logging for LLM generation') + .option('--validate', 'Enable iterative validation with LLM function calling (max 3 attempts)') .action(async (options) => { await pullProjectCommand(options); }); diff --git a/examples/weather-project/agents/data-workshop-agent.ts b/examples/weather-project/agents/data-workshop-agent.ts deleted file mode 100644 index 634f77584..000000000 --- a/examples/weather-project/agents/data-workshop-agent.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { agent, subAgent } from '@inkeep/agents-sdk'; - -export const dataWorkshopAgent = agent({ - id: 'data-workshop-agent', - name: `Data Workshop Agent`, - description: `A versatile data workshop agent that provides various utility functions including text analysis, calculations, data formatting, and more.`, - defaultSubAgent: subAgent({ - id: 'data-workshop-sub-agent', - name: `data-workshop-agent`, - description: `A versatile data workshop agent that provides various utility functions including text analysis, calculations, data formatting, and more.`, - prompt: `You are a helpful data workshop assistant with access to various utility tools. You can help users with: - -- Text analysis and processing -- Mathematical calculations (BMI, age, etc.) -- Data formatting and conversion -- Password generation and security -- QR code generation -- Currency conversion -- Entertainment (jokes and quotes) - -Always use the appropriate tools to provide accurate results and be helpful in explaining what each tool does.` - }) -}); \ No newline at end of file diff --git a/examples/weather-project/agents/weather-agent.ts b/examples/weather-project/agents/weather-agent.ts deleted file mode 100644 index a255aedae..000000000 --- a/examples/weather-project/agents/weather-agent.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { agent, subAgent } from '@inkeep/agents-sdk'; -import { fdxgfv9HL7SXlfynPx8hf } from '../tools/geocode-address'; -import { fUI2riwrBVJ6MepT8rjx0 } from '../tools/forecast-weather'; -import { weatherForecast } from '../data-components/weather-forecast'; - -export const weatherAgent = agent({ - id: 'weather-agent', - name: `Weather agent`, - defaultSubAgent: subAgent({ - id: 'weather-assistant', - name: `Weather assistant`, - description: `Main weather assistant that coordinates between geocoding and forecasting services to provide comprehensive weather information. This assistant handles user queries and delegates tasks to specialized sub-agents as needed.`, - prompt: `You are a helpful weather assistant that provides comprehensive weather information -for any location worldwide. You coordinate with specialized agents to: - -1. Convert location names/addresses to coordinates (via geocoder) -2. Retrieve detailed weather forecasts (via weather forecaster) -3. Present weather information in a clear, user-friendly format - -When users ask about weather: -- If they provide a location name or address, delegate to the geocoder first -- Once you have coordinates, delegate to the weather forecaster -- Present the final weather information in an organized, easy-to-understand format -- Include relevant details like temperature, conditions, precipitation, wind, etc. -- Provide helpful context and recommendations when appropriate - -You have access to weather forecast data components that can enhance your responses -with structured weather information.`, - canDelegateTo: () => [ - subAgent({ - id: 'weather-forecaster', - name: `Weather forecaster`, - description: `Specialized agent for retrieving detailed weather forecasts and current conditions. This agent focuses on providing accurate, up-to-date weather information using geographic coordinates.`, - prompt: `You are a weather forecasting specialist that provides detailed weather information -including current conditions, forecasts, and weather-related insights. - -You work with precise geographic coordinates to deliver: -- Current weather conditions -- Short-term and long-term forecasts -- Temperature, humidity, wind, and precipitation data -- Weather alerts and advisories -- Seasonal and climate information - -Always provide clear, actionable weather information that helps users plan their activities.`, - canUse: () => [fUI2riwrBVJ6MepT8rjx0] - }), - subAgent({ - id: 'geocoder-agent', - name: `Geocoder agent`, - description: `Specialized agent for converting addresses and location names into geographic coordinates. This agent handles all location-related queries and provides accurate latitude/longitude data for weather lookups.`, - prompt: `You are a geocoding specialist that converts addresses, place names, and location descriptions -into precise geographic coordinates. You help users find the exact location they're asking about -and provide the coordinates needed for weather forecasting. - -When users provide: -- Street addresses -- City names -- Landmarks -- Postal codes -- General location descriptions - -You should use your geocoding tools to find the most accurate coordinates and provide clear -information about the location found.`, - canUse: () => [fdxgfv9HL7SXlfynPx8hf] - }) - ], - dataComponents: () => [weatherForecast] - }) -}); \ No newline at end of file diff --git a/examples/weather-project/data-components/weather-forecast.ts b/examples/weather-project/data-components/weather-forecast.ts deleted file mode 100644 index ae20a0cb4..000000000 --- a/examples/weather-project/data-components/weather-forecast.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { dataComponent } from '@inkeep/agents-sdk'; -import { z } from 'zod'; - -export const weatherForecast = dataComponent({ - id: 'weather-forecast', - name: `WeatherForecast`, - description: `A hourly forecast for the weather at a given location`, - props: z.object({ - forecast: z.array( - z.object({ - time: z.string().describe(`The time of current item E.g. 12PM, 1PM`), - temperature: z.number().describe(`The temperature at given time in Farenheit`), - code: z.number().describe(`Weather code at given time`) - }) - ).describe(`The hourly forecast for the weather at a given location`) - }) -}); \ No newline at end of file diff --git a/examples/weather-project/environments/development.env.ts b/examples/weather-project/environments/development.env.ts deleted file mode 100644 index b4f4acb38..000000000 --- a/examples/weather-project/environments/development.env.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { registerEnvironmentSettings } from '@inkeep/agents-sdk'; - -export const development = registerEnvironmentSettings({ - credentials: { - } -}); diff --git a/examples/weather-project/environments/index.ts b/examples/weather-project/environments/index.ts deleted file mode 100644 index 2d37419f8..000000000 --- a/examples/weather-project/environments/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { createEnvironmentSettings } from '@inkeep/agents-sdk'; -import { development } from './development.env'; - -export const envSettings = createEnvironmentSettings({ - development, -}); diff --git a/examples/weather-project/index.ts b/examples/weather-project/index.ts index b1c1fbd04..e03ab2e3e 100644 --- a/examples/weather-project/index.ts +++ b/examples/weather-project/index.ts @@ -1,10 +1,22 @@ import { project } from '@inkeep/agents-sdk'; +import { weatherAdvanced } from './agents/weather-advanced'; +import { weatherBasic } from './agents/weather-basic'; +import { weatherIntermediate } from './agents/weather-intermediate'; -export const myWeatherProject = project({ - id: 'my-weather-project', - name: 'Weather Project', - description: 'Project containing sample agent framework using ', +export const weatherProject = project({ + id: `weather-project`, + name: `Weather Project`, + description: `Weather project template`, models: { - base: { model: 'openai/gpt-4o-mini' } - } + base: { + model: `anthropic/claude-sonnet-4-5-20250929` + }, + structuredOutput: { + model: `anthropic/claude-sonnet-4-5-20250929` + }, + summarizer: { + model: `anthropic/claude-sonnet-4-5-20250929` + } + }, + agents: () => [weatherBasic, weatherAdvanced, weatherIntermediate] }); \ No newline at end of file diff --git a/examples/weather-project/tools/forecast-weather.ts b/examples/weather-project/tools/forecast-weather.ts deleted file mode 100644 index ca260c8fb..000000000 --- a/examples/weather-project/tools/forecast-weather.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { mcpTool } from '@inkeep/agents-sdk'; - -export const fUI2riwrBVJ6MepT8rjx0 = mcpTool({ - id: 'fUI2riwrBVJ6MepT8rjx0', - name: `Forecast weather`, - serverUrl: `https://weather-forecast-mcp.vercel.app/mcp` -}); \ No newline at end of file diff --git a/examples/weather-project/tools/geocode-address.ts b/examples/weather-project/tools/geocode-address.ts deleted file mode 100644 index 371af5a08..000000000 --- a/examples/weather-project/tools/geocode-address.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { mcpTool } from '@inkeep/agents-sdk'; - -export const fdxgfv9HL7SXlfynPx8hf = mcpTool({ - id: 'fdxgfv9HL7SXlfynPx8hf', - name: `Geocode address`, - serverUrl: `https://geocoder-mcp.vercel.app/mcp` -}); \ No newline at end of file diff --git a/examples/weather-project/weather-project.json b/examples/weather-project/weather-project.json new file mode 100644 index 000000000..cb15bd584 --- /dev/null +++ b/examples/weather-project/weather-project.json @@ -0,0 +1,429 @@ +{ + "id": "weather-project", + "name": "Weather Project", + "description": "Weather project template", + "models": { + "base": { + "model": "anthropic/claude-sonnet-4-5-20250929" + }, + "structuredOutput": { + "model": "anthropic/claude-sonnet-4-5-20250929" + }, + "summarizer": { + "model": "anthropic/claude-sonnet-4-5-20250929" + } + }, + "agents": { + "weather-basic": { + "id": "weather-basic", + "name": "Weather basic", + "description": "Asks for the weather forecast for the given location", + "defaultSubAgentId": "weather-assistant", + "subAgents": { + "get-coordinates-agent": { + "id": "get-coordinates-agent", + "name": "Coordinates agent", + "description": "Responsible for converting location or address into coordinates", + "prompt": "You are a helpful assistant responsible for converting location or address into coordinates using your coordinate converter tool", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [], + "dataComponents": [], + "artifactComponents": [], + "canUse": [ + { + "agentToolRelationId": "jBGWN-yg0Ux873bLleUOe", + "toolId": "weather-mcp", + "toolSelection": [ + "get_coordinates" + ], + "headers": null + } + ] + }, + "weather-assistant": { + "id": "weather-assistant", + "name": "Weather assistant", + "description": "Responsible for routing between the coordinates agent and weather forecast agent", + "prompt": "You are a helpful assistant. When the user asks about the weather in a given location, first ask the coordinates agent for the coordinates, and then pass those coordinates to the weather forecast agent to get the weather forecast. If the user does not ask about weather related questions, politely decline to answer and redirect the user to a weather related question.", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [ + "weather-forecaster", + "get-coordinates-agent" + ], + "dataComponents": [], + "artifactComponents": [], + "canUse": [] + }, + "weather-forecaster": { + "id": "weather-forecaster", + "name": "Weather forecaster", + "description": "This agent is responsible for taking in coordinates and returning the forecast for the weather at that location", + "prompt": "You are a helpful assistant responsible for taking in coordinates and returning the forecast for that location using your forecasting tool", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [], + "dataComponents": [], + "artifactComponents": [], + "canUse": [ + { + "agentToolRelationId": "B9FHQiR1fWbu1RMDfIV_u", + "toolId": "weather-mcp", + "toolSelection": [ + "get_weather_forecast" + ], + "headers": null + } + ] + } + }, + "createdAt": "2025-10-14T20:37:28.442Z", + "updatedAt": "2025-10-14T20:37:28.442Z", + "tools": { + "weather-mcp": { + "id": "weather-mcp", + "name": "Weather", + "description": null, + "config": { + "type": "mcp", + "mcp": { + "server": { + "url": "https://weather-mcp-hazel.vercel.app/mcp" + } + } + }, + "credentialReferenceId": null, + "imageUrl": "https://cdn.iconscout.com/icon/free/png-256/free-ios-weather-icon-svg-download-png-461610.png?f=webp" + } + }, + "functionTools": {} + }, + "weather-advanced": { + "id": "weather-advanced", + "name": "Weather advanced", + "description": "Asks for the weather forecast for the given location with time context and rich UI rendering", + "defaultSubAgentId": "weather-assistant-advanced", + "subAgents": { + "coordinates-agent-advanced": { + "id": "coordinates-agent-advanced", + "name": "Coordinates agent", + "description": "Responsible for converting location or address into coordinates", + "prompt": "You are a helpful assistant responsible for converting location or address into coordinates using your coordinate converter tool", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [], + "dataComponents": [], + "artifactComponents": [], + "canUse": [ + { + "agentToolRelationId": "pow8XkQdrlUE1tGOfBJ7V", + "toolId": "weather-mcp", + "toolSelection": [ + "get_coordinates" + ], + "headers": null + } + ] + }, + "weather-assistant-advanced": { + "id": "weather-assistant-advanced", + "name": "Weather assistant", + "description": "Responsible for routing between the coordinates agent and using the weather forecast tool", + "prompt": "You are a helpful assistant. The time is {{time}} in the timezone {{requestContext.tz}}. When the user asks about the weather in a given location, first ask the coordinates agent for the coordinates, \n and then pass those coordinates to the weather forecast agent to get the weather forecast. Be sure to pass todays date to the weather forecaster.", + "models": { + "base": { + "model": "anthropic/claude-sonnet-4-20250514" + }, + "structuredOutput": { + "model": "anthropic/claude-sonnet-4-20250514" + }, + "summarizer": { + "model": "anthropic/claude-3-5-haiku-20241022" + } + }, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [ + "coordinates-agent-advanced" + ], + "dataComponents": [ + "temperature-data" + ], + "artifactComponents": [], + "canUse": [ + { + "agentToolRelationId": "2tP9MpBkqPdiBMPX9Vhb3", + "toolId": "weather-mcp", + "toolSelection": [ + "get_weather_forecast_for_date_range" + ], + "headers": null + } + ] + } + }, + "createdAt": "2025-10-14T20:37:28.442Z", + "updatedAt": "2025-10-14T20:37:28.457Z", + "contextConfig": { + "id": "i1b42sq626v93ggi60vn0", + "headersSchema": { + "type": "object", + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "additionalProperties": false + }, + "contextVariables": { + "time": { + "id": "time-info", + "name": "Time Information", + "trigger": "invocation", + "fetchConfig": { + "url": "https://world-time-api3.p.rapidapi.com/timezone/{{{{headers.tz}}}}", + "method": "GET", + "headers": { + "x-rapidapi-key": "590c52974dmsh0da44377420ef4bp1c64ebjsnf8d55149e28d" + } + }, + "responseSchema": { + "type": "object", + "properties": { + "datetime": { + "type": "string" + }, + "timezone": { + "type": "string" + } + }, + "required": [ + "datetime" + ], + "additionalProperties": false + }, + "defaultValue": "Unable to fetch time information" + } + } + }, + "tools": { + "weather-mcp": { + "id": "weather-mcp", + "name": "Weather", + "description": null, + "config": { + "type": "mcp", + "mcp": { + "server": { + "url": "https://weather-mcp-hazel.vercel.app/mcp" + } + } + }, + "credentialReferenceId": null, + "imageUrl": "https://cdn.iconscout.com/icon/free/png-256/free-ios-weather-icon-svg-download-png-461610.png?f=webp" + } + }, + "functionTools": {} + }, + "weather-intermediate": { + "id": "weather-intermediate", + "name": "Weather intermediate", + "description": "Asks for the weather forecast for the given location with time context", + "defaultSubAgentId": "weather-assistant", + "subAgents": { + "coordinates-agent-intermediate": { + "id": "coordinates-agent-intermediate", + "name": "Coordinates agent", + "description": "Responsible for converting location or address into coordinates", + "prompt": "You are a helpful assistant responsible for converting location or address into coordinates using your coordinate converter tool", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [], + "dataComponents": [], + "artifactComponents": [], + "canUse": [ + { + "agentToolRelationId": "HyKPRtDjShZRKzxxe2yD5", + "toolId": "weather-mcp", + "toolSelection": [ + "get_coordinates" + ], + "headers": null + } + ] + }, + "weather-assistant": { + "id": "weather-assistant", + "name": "Weather assistant", + "description": "Responsible for routing between the coordinates agent and weather forecast agent", + "prompt": "You are a helpful assistant. The time is {{time}} in the timezone {{requestContext.tz}}. When the user asks about the weather in a given location, first ask the coordinates agent for the coordinates, and then pass those coordinates to the weather forecast agent to get the weather forecast. Be sure to pass todays date to the weather forecaster.", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [ + "coordinates-agent-intermediate", + "weather-forecaster-intermediate" + ], + "dataComponents": [], + "artifactComponents": [], + "canUse": [] + }, + "weather-forecaster-intermediate": { + "id": "weather-forecaster-intermediate", + "name": "Weather forecaster", + "description": "This agent is responsible for taking in coordinates and returning the forecast for the weather at that location", + "prompt": "You are a helpful assistant responsible for taking in coordinates and returning the forecast for that location using your forecasting tool. Pass in todays date as the start date if the user does not specify a date and 7 days from today as the end date.", + "models": null, + "stopWhen": null, + "canTransferTo": [], + "canDelegateTo": [], + "dataComponents": [], + "artifactComponents": [], + "canUse": [ + { + "agentToolRelationId": "LIs-UindTGjAKFWxVixVF", + "toolId": "weather-mcp", + "toolSelection": [ + "get_weather_forecast_for_date_range" + ], + "headers": null + } + ] + } + }, + "createdAt": "2025-10-14T20:37:28.442Z", + "updatedAt": "2025-10-14T20:37:28.457Z", + "contextConfig": { + "id": "ttnzm2ntvamq269l0j1x8", + "headersSchema": { + "type": "object", + "properties": { + "tz": { + "type": "string" + } + }, + "required": [ + "tz" + ], + "additionalProperties": false + }, + "contextVariables": { + "time": { + "id": "time-info", + "name": "Time Information", + "trigger": "invocation", + "fetchConfig": { + "url": "https://world-time-api3.p.rapidapi.com/timezone/{{{{headers.tz}}}}", + "method": "GET", + "headers": { + "x-rapidapi-key": "590c52974dmsh0da44377420ef4bp1c64ebjsnf8d55149e28d" + } + }, + "responseSchema": { + "type": "object", + "properties": { + "datetime": { + "type": "string" + }, + "timezone": { + "type": "string" + } + }, + "required": [ + "datetime" + ], + "additionalProperties": false + }, + "defaultValue": "Unable to fetch time information" + } + } + }, + "tools": { + "weather-mcp": { + "id": "weather-mcp", + "name": "Weather", + "description": null, + "config": { + "type": "mcp", + "mcp": { + "server": { + "url": "https://weather-mcp-hazel.vercel.app/mcp" + } + } + }, + "credentialReferenceId": null, + "imageUrl": "https://cdn.iconscout.com/icon/free/png-256/free-ios-weather-icon-svg-download-png-461610.png?f=webp" + } + }, + "functionTools": {} + } + }, + "tools": { + "weather-mcp": { + "id": "weather-mcp", + "name": "Weather", + "config": { + "type": "mcp", + "mcp": { + "server": { + "url": "https://weather-mcp-hazel.vercel.app/mcp" + } + } + }, + "imageUrl": "https://cdn.iconscout.com/icon/free/png-256/free-ios-weather-icon-svg-download-png-461610.png?f=webp" + } + }, + "dataComponents": { + "temperature-data": { + "id": "temperature-data", + "name": "Temperature data", + "description": "Temperature data", + "props": { + "type": "object", + "properties": { + "temperature_data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string", + "description": "The date of the temperature data" + }, + "temperature": { + "type": "number", + "description": "The temperature in degrees Fahrenheit" + }, + "weather_code": { + "type": "number", + "description": "The weather code" + } + }, + "required": [ + "date", + "temperature", + "weather_code" + ] + } + } + }, + "required": [ + "temperature_data" + ] + } + } + }, + "artifactComponents": {}, + "credentialReferences": {}, + "createdAt": "2025-10-14T20:37:28.421Z", + "updatedAt": "2025-10-14T20:37:28.421Z" +} \ No newline at end of file