diff --git a/cli/README.md b/cli/README.md index e4a2e199..891b3869 100644 --- a/cli/README.md +++ b/cli/README.md @@ -278,6 +278,9 @@ ralphy --parallel --sandbox - Retryable rate-limit or quota errors are detected and deferred for later retry - Local changes are stashed before the merge phase and restored after - Agents should not modify PRD files, `.ralphy/progress.txt`, `.ralphy-worktrees`, or `.ralphy-sandboxes` +- Parallel batches can be conflict-aware (file-overlap graph coloring) to reduce merge collisions +- Worktree cleanup is tracked and recovered across crashes; stale worktrees are cleaned on next run +- Merge phase uses a global lock to avoid concurrent merge corruption from multiple ralphy processes ## Options diff --git a/cli/src/cli/commands/run.ts b/cli/src/cli/commands/run.ts index e9fc95b7..6482f168 100644 --- a/cli/src/cli/commands/run.ts +++ b/cli/src/cli/commands/run.ts @@ -40,19 +40,19 @@ export async function runLoop(options: RuntimeOptions): Promise { if (!existsSync(options.prdFile)) { logError(`${options.prdFile} not found in current directory`); logInfo(`Create a ${options.prdFile} file with tasks`); - process.exit(1); + throw new Error(`PRD source not found: ${options.prdFile}`); } } else if (options.prdSource === "markdown-folder") { if (!existsSync(options.prdFile)) { logError(`PRD folder ${options.prdFile} not found`); logInfo(`Create a ${options.prdFile}/ folder with markdown files containing tasks`); - process.exit(1); + throw new Error(`PRD folder not found: ${options.prdFile}`); } } if (options.prdSource === "github" && !options.githubRepo) { logError("GitHub repository not specified. Use --github owner/repo"); - process.exit(1); + throw new Error("GitHub repository not specified"); } // Check engine availability @@ -61,7 +61,7 @@ export async function runLoop(options: RuntimeOptions): Promise { if (!available) { logError(`${engine.name} CLI not found. Make sure '${engine.cliCommand}' is in your PATH.`); - process.exit(1); + throw new Error(`${engine.name} CLI not available`); } // Create task source with caching for better performance @@ -91,7 +91,7 @@ export async function runLoop(options: RuntimeOptions): Promise { logError("Cannot run in parallel/branch mode: repository has no commits yet."); logInfo("Please make an initial commit first:"); logInfo(' git add . && git commit -m "Initial commit"'); - process.exit(1); + throw new Error("Repository has no commits yet"); } } @@ -195,6 +195,6 @@ export async function runLoop(options: RuntimeOptions): Promise { } if (result.tasksFailed > 0) { - process.exit(1); + throw new Error(`${result.tasksFailed} task(s) failed`); } } diff --git a/cli/src/cli/commands/task.ts b/cli/src/cli/commands/task.ts index 312977e7..f34d633c 100644 --- a/cli/src/cli/commands/task.ts +++ b/cli/src/cli/commands/task.ts @@ -28,7 +28,7 @@ export async function runTask(task: string, options: RuntimeOptions): Promise dir !== ".git"); + } + return DEFAULT_SYMLINK_DIRS; +} + +function resolveSafeRelativePath(baseDir: string, candidatePath: string): string | null { + if (!candidatePath || isAbsolute(candidatePath)) { + return null; + } + + const normalized = normalize(candidatePath); + const resolved = resolve(baseDir, normalized); + const rel = relative(baseDir, resolved); + + if (rel === "" || rel === ".") { + return normalized; + } + + if (rel.startsWith(`..${sep}`) || rel === "..") { + return null; + } + + if (isAbsolute(rel)) { + return null; + } + + return rel; +} + +/** + * Common logic to run an agent in a specific directory + */ +async function runAgent(targetDir: string, options: AgentRunnerOptions): Promise { + const { + engine, + prdFile, + skipTests, + skipLint, + browserEnabled, + engineArgs, + maxRetries, + retryDelay, + debug, + originalDir, + task, + } = options; + + // If planning model is provided, first determine which files are needed + const filesToCopy = options.filesToCopy; + if (options.planningModel && (!filesToCopy || filesToCopy.length === 0)) { + // Signal planning phase + StaticAgentDisplay.getInstance()?.setAgentStatus( + options.agentNum, + task.title, + "working", + "planning", + "planning", + ); + + // Create planning progress callback + const onPlanningProgress = (event: PlanningProgressEvent) => { + let stepText = event.message; + + if (event.status === "started") { + stepText = "Planning started - analyzing task..."; + } else if (event.status === "thinking" && event.message) { + stepText = `Thinking: ${event.message}`; + } else if (event.status === "completed") { + stepText = `Planning complete! Identified ${event.metadata?.fileCount || 0} files`; + } else if (!event.message) { + stepText = event.status; + } + + if (options.onProgress && stepText) { + options.onProgress(stepText); + } + }; + + const planningResult = await planTaskFiles( + engine, + task, + originalDir, + options.planningModel, // Use planningModel as modelOverride + undefined, // maxReplans + options.planningModel, + undefined, // fullTasksContext + debug, + onPlanningProgress, // onProgress - NOW WITH CALLBACK! + options.debugOpenCode, + options.logThoughts, + options.engineArgs, + ); + + if (planningResult.error) { + logDebug(`Agent ${options.agentNum}: Planning failed: ${planningResult.error}`); + // Signal failure if planning was required but failed + StaticAgentDisplay.getInstance()?.setAgentStatus(options.agentNum, task.title, "failed"); + return { + success: false, + response: "", + inputTokens: 0, + outputTokens: 0, + error: `Planning failed: ${planningResult.error}`, + }; + } + + if ( + planningResult.files.length > 0 || + (planningResult.plan && planningResult.plan.length > 0) + ) { + logDebug( + `Agent ${options.agentNum}: Planning phase identified ${planningResult.files.length} files with ${planningResult.plan?.length || 0} steps`, + ); + // Store planning results for execution phase + options.planningAnalysis = planningResult.analysis; + options.planningSteps = planningResult.plan; + // Copy these files to the target directory if they aren't there already + await copyPlannedFilesIsolated(originalDir, targetDir, planningResult.files); + logDebug( + `Agent ${options.agentNum}: Pre-copied ${planningResult.files.length} files based on plan`, + ); + + // Add explicit transition feedback + const display = StaticAgentDisplay.getInstance(); + if (display) { + display.setAgentStatus(options.agentNum, task.title, "working", "execution", "main"); + display.updateAgent( + options.agentNum, + `Starting execution with ${planningResult.files.length} planned files...`, + ); + display.clearAgentSteps(options.agentNum); + } + } else { + logDebug(`Agent ${options.agentNum}: Planning returned no useful files or steps.`); + // Optional: Fallback or warning + + // Still transition to working phase even without planning results + const display = StaticAgentDisplay.getInstance(); + if (display) { + display.setAgentStatus(options.agentNum, task.title, "working", "execution", "main"); + display.updateAgent(options.agentNum, "Starting execution without planning..."); + display.clearAgentSteps(options.agentNum); + } + } + } + + // Check if we should use orchestrator pattern for test model + const useOrchestrator = Boolean( + options.testModel && + shouldUseOrchestrator(task.title || "", task.description || "", options.testModel), + ); + + // Build execution prompt (with orchestrator instructions if enabled) + const prompt = buildExecutionPrompt({ + task: task.title, + progressFile: PROGRESS_FILE, + prdFile, + skipTests, + skipLint, + browserEnabled, + allowCommit: false, + planningAnalysis: options.planningAnalysis, + planningSteps: options.planningSteps, + enableOrchestrator: useOrchestrator, + }); + + if (useOrchestrator) { + logDebug( + `Agent ${options.agentNum}: Using orchestrator pattern with test model ${options.testModel}`, + ); + + // Status update + if (!options.planningModel) { + const display = StaticAgentDisplay.getInstance(); + if (display) { + display.setAgentStatus(options.agentNum, task.title, "working", "execution", "main"); + display.clearAgentSteps(options.agentNum); + } + } + + // Execute with orchestrator + const orchestratorResult = await executeWithOrchestrator( + prompt, + { + mainEngine: engine, + testEngine: engine, // Same engine, different model + mainModel: options.modelOverride, + testModel: options.testModel, + workDir: targetDir, + maxIterations: 5, + debug, + agentNum: options.agentNum, + }, + options.onProgress, + ); + + const result: AIResult = orchestratorResult.success + ? { + success: true, + response: orchestratorResult.response, + inputTokens: 0, + outputTokens: 0, + } + : { + success: false, + response: orchestratorResult.response, + inputTokens: 0, + outputTokens: 0, + error: orchestratorResult.error, + }; + + // Update final status in UI + if (result.success) { + StaticAgentDisplay.getInstance()?.setAgentStatus(options.agentNum, task.title, "completed"); + } else { + StaticAgentDisplay.getInstance()?.setAgentStatus(options.agentNum, task.title, "failed"); + } + + return result; + } + + // Determine if this is a test-related task and select appropriate model and phase + const isTestTask = /test|testing|tests?|spec|coverage/i.test(task.title || task.id); + const effectiveModel = + isTestTask && options.testModel ? options.testModel : options.modelOverride; + + // Set phase to testing for test tasks + if (isTestTask) { + StaticAgentDisplay.getInstance()?.setAgentStatus( + options.agentNum, + task.title, + "working", + "testing", + options.testModel || "test", + ); + } + + // Status is already set during planning → execution transition + // Only update if planning was skipped (no planningModel) + if (!options.planningModel) { + const display = StaticAgentDisplay.getInstance(); + if (display) { + display.setAgentStatus(options.agentNum, task.title, "working", "execution", "main"); + display.clearAgentSteps(options.agentNum); + } + } + + // Execute with retry + const engineOptions = { + ...(effectiveModel && { modelOverride: effectiveModel }), + ...(engineArgs && engineArgs.length > 0 && { engineArgs }), + ...(options.env && { env: options.env }), + ...(options.debugOpenCode && { debugOpenCode: options.debugOpenCode }), + // Default to true for autonomous operation - only disable if explicitly set to false + allowOpenCodeSandboxAccess: options.allowOpenCodeSandboxAccess !== false, + ...(options.logThoughts !== undefined && { logThoughts: options.logThoughts }), + ...(options.dryRun && { dryRun: options.dryRun }), + }; + + const result = await withRetry( + async () => { + let res: AIResult; + if (options.onProgress && engine.executeStreaming) { + res = await engine.executeStreaming(prompt, targetDir, options.onProgress, engineOptions); + } else { + res = await engine.execute(prompt, targetDir, engineOptions); + } + + if (debug) { + logDebug(`Agent ${options.agentNum}: Full AI Response:`, res.response); + if (res.error) logDebug(`Agent ${options.agentNum}: Full AI Error:`, res.error); + } + if (!res.success && res.error && isRetryableError(res.error)) { + throw new Error(res.error); + } + return res; + }, + { maxRetries, retryDelay }, + ); + + // Update final status in UI + if (result.success) { + StaticAgentDisplay.getInstance()?.setAgentStatus(options.agentNum, task.title, "completed"); + } else { + StaticAgentDisplay.getInstance()?.setAgentStatus(options.agentNum, task.title, "failed"); + } + + return result; +} + +/** + * Run a single agent in a lightweight sandbox + */ +export async function runAgentInSandbox( + sandboxBase: string, + options: AgentRunnerOptions, +): Promise { + const { + agentNum, + originalDir, + prdSource, + prdFile, + prdIsFolder, + task, + filesToCopy, + noGitParallel, + } = options; + // Use cryptographically secure random for sandbox directory naming + const uniqueSuffix = randomBytes(4).toString("hex"); + const sandboxDir = join( + sandboxBase, + `${SANDBOX_DIR_PREFIX}${agentNum}-${Date.now()}-${uniqueSuffix}`, + ); + + try { + mkdirSync(sandboxDir, { recursive: true }); + + // If selective isolation is requested (filesToCopy provided) + if (filesToCopy && Array.isArray(filesToCopy) && filesToCopy.length > 0) { + // Copy skill folders and symlink shared resources + copySkillFolders(originalDir, sandboxDir); + symlinkSharedResources(originalDir, sandboxDir, getFilteredSymlinkDirs(!!noGitParallel)); + + // Copy planned files into sandbox + await copyPlannedFilesIsolated(originalDir, sandboxDir, filesToCopy); + logDebug( + `Agent ${agentNum}: Copied ${filesToCopy.length} planned files for selective isolation`, + ); + } else if (options.useSemanticChunking !== false) { + // Use semantic chunking to determine relevant files + try { + const taskDescription = `${task.title || ""} ${task.description || ""}`; + const relevantFiles = await getRelevantFilesForTask(originalDir, taskDescription, { + maxFiles: 30, + minRelevance: 0.15, + }); + + if (relevantFiles.length > 0) { + // Copy skill folders and symlink shared resources + copySkillFolders(originalDir, sandboxDir); + symlinkSharedResources(originalDir, sandboxDir, getFilteredSymlinkDirs(!!noGitParallel)); + + // Copy relevant files into sandbox + await copyPlannedFilesIsolated(originalDir, sandboxDir, relevantFiles); + logDebug(`Agent ${agentNum}: Semantic chunking selected ${relevantFiles.length} files`); + + // Continue with selective isolation using relevant files + const beforeSnapshot = createSelectiveSnapshot(sandboxDir, relevantFiles); + + // Ensure .ralphy/ exists + const ralphyDir = join(sandboxDir, RALPHY_DIR); + if (!existsSync(ralphyDir)) mkdirSync(ralphyDir, { recursive: true }); + + // Copy PRD resources + copyPrdResources(originalDir, sandboxDir, prdSource, prdFile, prdIsFolder); + + // Run agent + const result = await runAgent(sandboxDir, options); + + // Snapshot after execution and discover new files + const afterSnapshot = createSelectiveSnapshot(sandboxDir, relevantFiles); + const fullDirSnapshot = createDirectorySnapshot(sandboxDir); + + for (const [relPath, snap] of fullDirSnapshot) { + if (!afterSnapshot.has(relPath) && !relevantFiles.includes(relPath)) { + if (!shouldIgnoreFile(relPath, ["node_modules/**", ".git/**", ".ralphy/**"])) { + afterSnapshot.set(relPath, snap); + } + } + } + + const { modified, added } = compareSnapshots(beforeSnapshot, afterSnapshot); + const allChanges = [...modified, ...added].filter( + (file) => !isInRalphyDir(file) && normalize(file) !== normalize(prdFile), + ); + + if (allChanges.length > 0) { + try { + await copyBackPlannedFilesParallel(originalDir, sandboxDir, allChanges); + logDebug(`Agent ${agentNum}: Copied back ${allChanges.length} modified/new files`); + } catch (copyErr) { + logWarn(`Agent ${agentNum}: Failed to copy back files: ${copyErr}`); + } + } + + // Release locks if they were held + try { + releaseLocksForFiles(relevantFiles, originalDir); + } catch (lockErr) { + logDebug(`Agent ${agentNum}: Failed to release locks: ${lockErr}`); + } + + return { + task, + agentNum, + worktreeDir: sandboxDir, + branchName: "", + result, + usedSandbox: true, + }; + } + } catch (error) { + logDebug( + `Agent ${agentNum}: Semantic chunking failed, falling back to full sandbox: ${error}`, + ); + } + } + + // Traditional full isolation mode + const sandboxResult = await createSandbox({ + originalDir, + sandboxDir, + agentNum, + symlinkDirs: getFilteredSymlinkDirs(!!noGitParallel), + }); + + logDebug( + `Agent ${agentNum}: Created full sandbox (${sandboxResult.symlinksCreated} symlinks, ${sandboxResult.filesCopied} copies)`, + ); + + // Copy PRD resources + copyPrdResources(originalDir, sandboxDir, prdSource, prdFile, prdIsFolder); + + // Ensure .ralphy/ exists + const ralphyDir = join(sandboxDir, RALPHY_DIR); + if (!existsSync(ralphyDir)) mkdirSync(ralphyDir, { recursive: true }); + + const result = await runAgent(sandboxDir, options); + + return { + task, + agentNum, + worktreeDir: sandboxDir, + branchName: "", + result, + usedSandbox: true, + }; + } catch (error) { + const errorMsg = standardizeError(error).message; + if (filesToCopy) releaseLocksForFiles(filesToCopy, originalDir); + + // Enhanced error logging for engine execution issues + if (errorMsg.includes("exitCode")) { + const engineName = options.engine.name; + logDebug( + `Agent ${options.agentNum}: Engine execution error - possibly ${engineName} CLI/API issue: ${errorMsg}`, + ); + logDebug(`Check ${engineName} CLI availability: '${options.engine.cliCommand} --help'`); + } + + return { + task, + agentNum, + worktreeDir: sandboxDir, + branchName: "", + result: null, + error: errorMsg, + usedSandbox: true, + }; + } +} + +/** + * Run a single agent in a git worktree + */ +export async function runAgentInWorktree( + worktreeBase: string, + baseBranch: string, + options: AgentRunnerOptions, +): Promise { + const { agentNum, originalDir, prdSource, prdFile, prdIsFolder, task } = options; + let worktreeDir = ""; + let branchName = ""; + + try { + // Create worktree + const worktree = await createAgentWorktree( + task.title, + agentNum, + baseBranch, + worktreeBase, + originalDir, + ); + worktreeDir = worktree.worktreeDir; + branchName = worktree.branchName; + options.onWorktreeCreated?.(worktreeDir, branchName); + + logDebug(`Agent ${agentNum}: Created worktree at ${worktreeDir}`); + + // Copy PRD file or folder to worktree + copyPrdResources(originalDir, worktreeDir, prdSource, prdFile, prdIsFolder); + + // Ensure .ralphy/ exists in worktree + const ralphyDir = join(worktreeDir, RALPHY_DIR); + if (!existsSync(ralphyDir)) { + mkdirSync(ralphyDir, { recursive: true }); + } + + const result = await runAgent(worktreeDir, options); + + return { + task, + agentNum, + worktreeDir, + branchName, + result, + usedSandbox: false, + }; + } catch (error) { + const errorMsg = standardizeError(error).message; + return { + task, + agentNum, + worktreeDir, + branchName, + result: null, + error: errorMsg, + usedSandbox: false, + }; + } +} + +function copyPrdResources( + originalDir: string, + targetDir: string, + prdSource: string, + prdFile: string, + prdIsFolder: boolean, +) { + const safePrdPath = resolveSafeRelativePath(originalDir, prdFile); + if (!safePrdPath) { + throw new Error(`Invalid PRD path outside project: ${prdFile}`); + } + + if ( + prdSource === "markdown" || + prdSource === "yaml" || + prdSource === "json" || + prdSource === "csv" + ) { + const srcPath = join(originalDir, safePrdPath); + const destPath = join(targetDir, safePrdPath); + if (existsSync(srcPath)) { + mkdirSync(dirname(destPath), { recursive: true }); + copyFileSync(srcPath, destPath); + } + } else if (prdSource === "markdown-folder" && prdIsFolder) { + const srcPath = join(originalDir, safePrdPath); + const destPath = join(targetDir, safePrdPath); + if (existsSync(srcPath)) { + mkdirSync(dirname(destPath), { recursive: true }); + cpSync(srcPath, destPath, { recursive: true }); + } + } +} diff --git a/cli/src/execution/conflict-resolution.ts b/cli/src/execution/conflict-resolution.ts index c465610c..66a7cf9f 100644 --- a/cli/src/execution/conflict-resolution.ts +++ b/cli/src/execution/conflict-resolution.ts @@ -1,6 +1,7 @@ import type { AIEngine } from "../engines/types.ts"; import { completeMerge, getConflictedFiles } from "../git/merge.ts"; import { logDebug, logError, logInfo } from "../ui/logger.ts"; +import { standardizeError } from "../utils/errors.ts"; /** * Build a prompt for AI-assisted conflict resolution @@ -78,7 +79,7 @@ export async function resolveConflictsWithAI( logError(`AI conflict resolution failed: ${result.error || "Unknown error"}`); return false; } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); + const errorMsg = standardizeError(error).message; logError(`AI conflict resolution error: ${errorMsg}`); return false; } diff --git a/cli/src/execution/deferred.ts b/cli/src/execution/deferred.ts index 56dffeaf..64c5d51e 100644 --- a/cli/src/execution/deferred.ts +++ b/cli/src/execution/deferred.ts @@ -76,3 +76,31 @@ export function clearDeferredTask( writeState(workDir, state); } } + +/** + * Get the current deferral count for a task + */ +export function getDeferredCount( + type: TaskSourceType, + task: Task, + workDir: string, + prdFile?: string, +): number { + const state = readState(workDir); + const key = buildKey(type, task, prdFile); + return state.tasks[key]?.count ?? 0; +} + +/** + * Check if a task has exceeded the maximum number of deferrals + */ +export function hasExceededMaxDeferrals( + type: TaskSourceType, + task: Task, + workDir: string, + maxRetries: number, + prdFile?: string, +): boolean { + const count = getDeferredCount(type, task, workDir, prdFile); + return count >= maxRetries; +} diff --git a/cli/src/execution/file-utils.ts b/cli/src/execution/file-utils.ts new file mode 100644 index 00000000..86b22c48 --- /dev/null +++ b/cli/src/execution/file-utils.ts @@ -0,0 +1,432 @@ +import { createHash } from "node:crypto"; +import { + copyFileSync, + createReadStream, + existsSync, + lstatSync, + mkdirSync, + type ReadStream, + readdirSync, + readFileSync, + statSync, +} from "node:fs"; +import { join, sep } from "node:path"; +import { DEFAULT_RECURSION_DEPTH, MAX_FILE_SIZE_FOR_HASH } from "../config/constants.ts"; +import { logDebug } from "../ui/logger.ts"; + +export interface FileSnapshot { + path: string; + size: number; + mtime: number; + hash?: string; +} + +/** + * Enhanced ignore patterns for sandbox copying + */ +export const DEFAULT_IGNORE_PATTERNS = [ + // Dependencies and build artifacts + "node_modules/**", + ".pnpm-store/**", + ".yarn/**", + "bower_components/**", + "dist/**", + "build/**", + "out/**", + ".next/**", + ".nuxt/**", + ".cache/**", + ".tmp/**", + + // Version control + ".git/**", + ".svn/**", + ".hg/**", + + // IDE and editor files + ".vscode/**", + ".idea/**", + "*.swp", + "*.swo", + "*~", + + // OS files + "Thumbs.db", + ".DS_Store", + "Desktop.ini", + + // Logs and temp + "*.log", + "*.tmp", + "temp/**", + "tmp/**", + "*.pid", + + // Test coverage + "coverage/**", + ".coverage/**", + ".nyc_output/**", + + // Environment and secrets + ".env*", + "*.key", + "*.pem", + "*.p12", + "*.pfx", +]; + +export function copyIfExists(src: string, dest: string): void { + if (!existsSync(src)) return; + mkdirSync(join(dest, ".."), { recursive: true }); + try { + copyFileSync(src, dest); + } catch { + // Ignore copy errors + } +} + +export function shouldIgnoreFile(filePath: string, ignorePatterns: string[]): boolean { + const normalizedPath = filePath.replace(/\\/g, "/"); + + for (const pattern of ignorePatterns) { + if (matchesGlob(normalizedPath, pattern)) { + return true; + } + } + + return false; +} + +function matchesGlob(filePath: string, pattern: string): boolean { + // Handle ** patterns properly + const regexPattern = globToRegex(pattern); + return regexPattern.test(filePath); +} + +/** + * Maximum glob pattern length to prevent ReDoS attacks + */ +const MAX_GLOB_PATTERN_LENGTH = 1000; + +function globToRegex(pattern: string): RegExp { + const safePattern = + pattern.length > MAX_GLOB_PATTERN_LENGTH + ? pattern.slice(0, MAX_GLOB_PATTERN_LENGTH) + : pattern; + + // Limit pattern length to prevent ReDoS attacks + if (safePattern.length < pattern.length) { + logDebug(`Glob pattern too long (${pattern.length} > ${MAX_GLOB_PATTERN_LENGTH}), truncating`); + } + + // Escape special regex characters except * and ? + // Use a bounded approach to prevent catastrophic backtracking + let regex = safePattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*\*/g, "\0DOUBLESTAR\0") // Temporarily mark ** + .replace(/\*/g, "[^/]*") // Single * matches anything except / + .replace(/\?/g, "[^/]"); // ? matches single char except / + + // Handle ** (match any number of directories) using non-capturing group + // The (?:.*/)? pattern is bounded - it won't cause catastrophic backtracking + regex = regex.replace(/\0DOUBLESTAR\0/g, "(?:.*/)?"); + + // Handle directory separators + regex = regex.replace(/\//g, "[/\\\\]"); + + // Anchor to start + regex = `^${regex}`; + + // Match at end if pattern doesn't end with /** + if (!safePattern.endsWith("/**")) { + regex += "$"; + } + + return new RegExp(regex, "i"); +} + +export function createFileSnapshot(filePath: string, maxSizeForHash = MAX_FILE_SIZE_FOR_HASH): FileSnapshot | null { + if (!existsSync(filePath)) return null; + + try { + const stat = statSync(filePath); + const snapshot: FileSnapshot = { + path: filePath, + size: stat.size, + mtime: stat.mtime.getTime(), + }; + + if (stat.size <= maxSizeForHash) { + try { + const content = readFileSync(filePath, "utf-8"); + snapshot.hash = createHash("sha256").update(content).digest("hex").slice(0, 16); + } catch { + // Skip hash for unreadable files + } + } + + return snapshot; + } catch (error) { + // Log snapshot failures for debugging + logDebug(`Failed to create snapshot for ${filePath}: ${error}`); + return null; + } +} + +/** + * Async streaming hash for large files + */ +export async function hashFileStreaming(filePath: string): Promise { + return new Promise((resolve) => { + const hash = createHash("sha256"); + let stream: ReadStream | null = null; + + try { + stream = createReadStream(filePath); + } catch (err) { + logDebug(`Failed to create read stream for ${filePath}: ${err}`); + return resolve(null); + } + + const handleError = (error: Error | unknown) => { + if (stream) stream.destroy(); + logDebug(`Streaming hash failed for ${filePath}: ${error}`); + resolve(null); + }; + + stream.on("data", (chunk: Buffer) => { + try { + hash.update(chunk); + } catch (err) { + handleError(err); + } + }); + + stream.on("end", () => { + resolve(hash.digest("hex").slice(0, 16)); + }); + + stream.on("error", handleError); + }); +} + +export function createDirectorySnapshot( + dir: string, + maxSizeForHash = MAX_FILE_SIZE_FOR_HASH, + maxDepth = 50, + currentDepth = 0, +): Map { + const snapshot = new Map(); + + if (!existsSync(dir)) return snapshot; + + // Prevent stack overflow on very deep structures + if (currentDepth > maxDepth) { + logDebug(`Directory depth limit reached (${maxDepth}) at: ${dir}`); + return snapshot; + } + + // Iterative DFS to avoid recursion overhead and improve cache locality + const stack: Array<{ path: string; relPath: string }> = [{ path: dir, relPath: "" }]; + + while (stack.length > 0) { + const item = stack.pop(); + if (!item) break; // Safety check + const { path: currentPath, relPath } = item; + + try { + const entries = readdirSync(currentPath, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = join(currentPath, entry.name); + const entryRelPath = relPath ? join(relPath, entry.name) : entry.name; + + if (entry.isDirectory()) { + // Check current depth before pushing + const pathDepth = entryRelPath.split(sep).length; + if (pathDepth < maxDepth - currentDepth) { + stack.push({ path: entryPath, relPath: entryRelPath }); + } else { + logDebug(`Skipping deep directory: ${entryRelPath} (depth ${pathDepth})`); + } + } else { + const fileSnapshot = createFileSnapshot(entryPath, maxSizeForHash); + if (fileSnapshot) { + snapshot.set(entryRelPath, fileSnapshot); + } + } + } + } catch (error) { + // Log error but continue processing other directories + logDebug(`Failed to read directory ${currentPath}: ${error}`); + } + } + + return snapshot; +} + +/** + * Create a selective snapshot of only specific files + */ +export function createSelectiveSnapshot( + baseDir: string, + files: string[], + maxSizeForHash = MAX_FILE_SIZE_FOR_HASH, +): Map { + const snapshot = new Map(); + + for (const relPath of files) { + const fullPath = join(baseDir, relPath); + const fileSnapshot = createFileSnapshot(fullPath, maxSizeForHash); + if (fileSnapshot) { + snapshot.set(relPath, fileSnapshot); + } + } + + return snapshot; +} + +export function compareSnapshots( + before: Map, + after: Map, +): { modified: string[]; added: string[]; deleted: string[] } { + const modified: string[] = []; + const added: string[] = []; + const deleted: string[] = []; + + for (const [relPath, beforeSnap] of before) { + const afterSnap = after.get(relPath); + + if (!afterSnap) { + deleted.push(relPath); + } else { + const contentChanged = + beforeSnap.hash && afterSnap.hash + ? beforeSnap.hash !== afterSnap.hash + : beforeSnap.mtime !== afterSnap.mtime || beforeSnap.size !== afterSnap.size; + + if (contentChanged) { + modified.push(relPath); + } + } + } + + for (const [relPath] of after) { + if (!before.has(relPath)) { + added.push(relPath); + } + } + + return { modified, added, deleted }; +} + +export function collectFilesRecursively( + dir: string, + root: string, + maxDepth = DEFAULT_RECURSION_DEPTH, + currentDepth = 0, +): string[] { + if (!existsSync(dir) || maxDepth < currentDepth) return []; + + const files: string[] = []; + const entries = readdirSync(dir); + const dirs: string[] = []; + + for (const entry of entries) { + const fullPath = join(dir, entry); + const relPath = join(root, entry); + + try { + const stat = lstatSync(fullPath); + + if (stat.isSymbolicLink()) { + // Handle symlinks to prevent infinite loops + logDebug(`Skipping symlink during file collection: ${relPath}`); + } else if (stat.isDirectory()) { + dirs.push(fullPath); + } + if (stat.isFile()) { + files.push(relPath); + } + } catch (err) { + logDebug(`Failed to stat ${fullPath}: ${err}`); + } + } + + for (const d of dirs) { + files.push(...collectFilesRecursively(d, root, maxDepth, currentDepth + 1)); + } + + return files; +} + +/** + * Enhanced file modification detection for parallel agents + */ +export function getModifiedFiles(sandboxDir: string, originalDir: string, symlinkDirs: string[] = []): string[] { + const modified: string[] = []; + + function scanDir(relPath: string) { + const sandboxPath = join(sandboxDir, relPath); + const originalPath = join(originalDir, relPath); + + if (!existsSync(sandboxPath)) return; + + const stat = lstatSync(sandboxPath); + + // Skip symlinks + if (stat.isSymbolicLink()) return; + + // Skip known symlink directories if any + if (relPath) { + const topLevel = relPath.split(sep)[0]; + if (symlinkDirs.includes(topLevel)) return; + } + + if (stat.isDirectory()) { + try { + const items = readdirSync(sandboxPath); + for (const item of items) { + scanDir(relPath ? join(relPath, item) : item); + } + } catch (err) { + logDebug(`Failed to scan dir ${relPath}: ${err}`); + } + } else if (stat.isFile()) { + let isModified = false; + + if (!existsSync(originalPath)) { + isModified = true; + } else { + try { + const originalStat = statSync(originalPath); + + // Check mtime and size + const mtimeDifferent = stat.mtimeMs !== originalStat.mtimeMs; + const sizeDifferent = stat.size !== originalStat.size; + + if (mtimeDifferent || sizeDifferent) { + isModified = true; + } + } catch { + isModified = true; + } + } + + if (isModified) { + modified.push(relPath); + } + } + } + + // Start scanning from root + try { + const items = readdirSync(sandboxDir); + for (const item of items) { + scanDir(item); + } + } catch (err) { + logDebug(`Failed to read sandbox dir: ${err}`); + } + + return modified; +} diff --git a/cli/src/execution/graph-coloring.ts b/cli/src/execution/graph-coloring.ts new file mode 100644 index 00000000..3fd3d341 --- /dev/null +++ b/cli/src/execution/graph-coloring.ts @@ -0,0 +1,136 @@ +import type { Task } from "../tasks/types.ts"; + +export interface PlannedTask { + task: Task; + files: string[]; + color?: number; +} + +export interface ConflictGraph { + nodes: Map; +} + +export function buildConflictGraph(tasks: PlannedTask[]): ConflictGraph { + const graph: ConflictGraph = { nodes: new Map() }; + const fileToTasks = new Map(); + + // Build file-to-task mapping: O(n × m) + for (const task of tasks) { + for (const file of task.files) { + const tasksForFile = fileToTasks.get(file) || []; + tasksForFile.push(task.task.id); + fileToTasks.set(file, tasksForFile); + } + } + + // Build conflicts from file mapping: O(n × m) + for (const task of tasks) { + const conflicts = new Set(); + for (const file of task.files) { + const tasksForFile = fileToTasks.get(file) || []; + for (const taskId of tasksForFile) { + if (taskId !== task.task.id) { + conflicts.add(taskId); + } + } + } + graph.nodes.set(task.task.id, Array.from(conflicts)); + } + + return graph; +} + +/** + * DSatur graph coloring algorithm + * Prioritizes nodes by saturation (number of different colors used by neighbors) + */ +export function colorGraph(tasks: PlannedTask[], graph: ConflictGraph): Map { + const colors = new Map(); + if (tasks.length === 0) return colors; + + const saturation = new Map>(); + const uncolored = new Set(); + + for (const task of tasks) { + saturation.set(task.task.id, new Set()); + uncolored.add(task.task.id); + } + + while (uncolored.size > 0) { + let maxSatNode: string | null = null; + let maxSat = -1; + let maxDegree = -1; + + for (const nodeId of uncolored) { + const sat = saturation.get(nodeId)?.size || 0; + const degree = graph.nodes.get(nodeId)?.length || 0; + + if (sat > maxSat || (sat === maxSat && degree > maxDegree)) { + maxSat = sat; + maxDegree = degree; + maxSatNode = nodeId; + } + } + + if (!maxSatNode) break; + + const usedColors = saturation.get(maxSatNode) || new Set(); + let color = 0; + while (usedColors.has(color)) { + color++; + } + + colors.set(maxSatNode, color); + uncolored.delete(maxSatNode); + + const neighbors = graph.nodes.get(maxSatNode) || []; + for (const neighborId of neighbors) { + if (uncolored.has(neighborId)) { + saturation.get(neighborId)?.add(color); + } + } + } + + return colors; +} + +export function batchByColor( + tasks: PlannedTask[], + colors: Map, + maxParallel: number, +): Map { + const batches = new Map(); + const safeMaxParallel = Number.isFinite(maxParallel) && maxParallel > 0 ? Math.floor(maxParallel) : 1; + + for (const task of tasks) { + const color = colors.get(task.task.id) || 0; + task.color = color; + + let batch = batches.get(color); + if (!batch) { + batch = []; + batches.set(color, batch); + } + batch.push(task); + } + + // If any batch exceeds maxParallel, split it into sub-batches + const finalBatches = new Map(); + let nextBatchId = 0; + + // Sort colors to maintain deterministic order + const sortedColors = Array.from(batches.keys()).sort((a, b) => a - b); + + for (const color of sortedColors) { + const batch = batches.get(color) || []; + if (batch.length <= safeMaxParallel) { + finalBatches.set(nextBatchId++, batch); + } else { + for (let i = 0; i < batch.length; i += safeMaxParallel) { + finalBatches.set(nextBatchId++, batch.slice(i, i + safeMaxParallel)); + } + } + } + + return finalBatches; +} diff --git a/cli/src/execution/hash-store.ts b/cli/src/execution/hash-store.ts new file mode 100644 index 00000000..e46c65a6 --- /dev/null +++ b/cli/src/execution/hash-store.ts @@ -0,0 +1,834 @@ +/** + * Hash Storage System for Ralphy CLI + * + * Manages file hashes per task to avoid copying unchanged files and reduce token usage. + * Stores file content, metadata, and hash references in .ralphy-hashes// + */ + +import { createHash } from "node:crypto"; +import { + createReadStream, + createWriteStream, + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { join, resolve } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { createGunzip, createGzip } from "node:zlib"; +import { + ENABLE_HASH_STORE, + HASH_STORE_DIR, + HASH_STORE_MAX_AGE_MS, + MAX_FILE_SIZE_FOR_HASH, +} from "../config/constants.ts"; +import { logDebug, logError, logWarn } from "../ui/logger.ts"; + +const COMPRESSION_TIMEOUT_MS = 30000; // 30 second timeout for compression/decompression + +function validateTaskId(taskId: string): string { + const trimmed = taskId.trim(); + if (!trimmed) { + throw new HashStoreError("Task ID cannot be empty"); + } + if (!/^[a-zA-Z0-9_-]+$/.test(trimmed)) { + throw new HashStoreError("Task ID contains invalid characters"); + } + return trimmed; +} + +/** + * Wrap a promise with a timeout + */ +async function withTimeout( + promise: Promise, + timeoutMs: number, + operation: string, +): Promise { + let timeoutId: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`${operation} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }); + + try { + return await Promise.race([promise, timeoutPromise]); + } finally { + if (timeoutId) { + clearTimeout(timeoutId); + } + } +} + +// ============================================================================ +// Types +// ============================================================================ + +/** + * Metadata for a stored file hash + */ +export interface HashMetadata { + /** Original file path (relative to project root) */ + originalPath: string; + /** SHA256 hash of file content */ + hash: string; + /** File size in bytes */ + size: number; + /** Last modified timestamp */ + mtime: number; + /** Content MIME type (if detectable) */ + mimeType?: string; + /** Whether content is compressed */ + compressed: boolean; + /** Original content size before compression */ + originalSize: number; + /** Timestamp when hash was stored */ + storedAt: number; + /** Task ID that owns this hash */ + taskId: string; +} + +/** + * Hash reference file structure + */ +export interface HashReference { + /** Hash value */ + hash: string; + /** Path to the hash file (relative to hash store root) */ + hashPath: string; + /** Metadata path (relative to hash store root) */ + metadataPath: string; +} + +/** + * Task hash index - maps files to their hash references + */ +export interface TaskHashIndex { + /** Task ID */ + taskId: string; + /** Mapping of file paths to hash references */ + files: Record; + /** Created timestamp */ + createdAt: number; + /** Last updated timestamp */ + updatedAt: number; +} + +/** + * Options for adding a file to the hash store + */ +export interface AddFileOptions { + /** Whether to compress the content */ + compress?: boolean; + /** Minimum size threshold for compression (default: 1KB) */ + compressionThreshold?: number; +} + +/** + * Result of adding a file to the hash store + */ +export interface AddFileResult { + /** Whether the operation succeeded */ + success: boolean; + /** The hash value (if successful) */ + hash?: string; + /** Error message (if failed) */ + error?: string; + /** Whether the hash was already in the store */ + alreadyExists?: boolean; +} + +/** + * Result of retrieving a file from the hash store + */ +export interface GetFileResult { + /** Whether the operation succeeded */ + success: boolean; + /** File content buffer (if successful) */ + content?: Buffer; + /** File metadata (if successful) */ + metadata?: HashMetadata; + /** Error message (if failed) */ + error?: string; +} + +// ============================================================================ +// Error Classes +// ============================================================================ + +/** + * Base error for hash store operations + */ +export class HashStoreError extends Error { + constructor( + message: string, + public readonly cause?: Error, + ) { + super(message); + this.name = "HashStoreError"; + } +} + +/** + * Error when hash store is disabled + */ +export class HashStoreDisabledError extends HashStoreError { + constructor() { + super("Hash store is disabled"); + this.name = "HashStoreDisabledError"; + } +} + +/** + * Error when file is not found in hash store + */ +export class HashNotFoundError extends HashStoreError { + constructor(hash: string) { + super(`Hash not found: ${hash}`); + this.name = "HashNotFoundError"; + } +} + +/** + * Error when file reference is not found + */ +export class FileReferenceNotFoundError extends HashStoreError { + constructor(filePath: string) { + super(`File reference not found: ${filePath}`); + this.name = "FileReferenceNotFoundError"; + } +} + +// ============================================================================ +// Hash Store Implementation +// ============================================================================ + +/** + * FileHashStore - Manages file hashes per task + * + * This class provides: + * - SHA256 hash generation for files + * - Compressed storage of file content + * - File reference tracking + * - Automatic cleanup after task completion + */ +export class FileHashStore { + /** Task identifier */ + private readonly taskId: string; + /** Base directory for hash storage */ + private readonly baseDir: string; + /** Task-specific directory */ + private readonly taskDir: string; + /** Path to the hash index file */ + private readonly indexPath: string; + /** In-memory hash index */ + private index: TaskHashIndex; + /** Whether the store is initialized */ + private initialized = false; + /** Whether the store has been cleaned up */ + private cleanedUp = false; + + /** + * Create a new FileHashStore instance + * + * @param taskId - Unique task identifier + * @param projectRoot - Project root directory (defaults to current working directory) + */ + constructor( + taskId: string, + private readonly projectRoot: string = process.cwd(), + ) { + this.taskId = validateTaskId(taskId); + this.baseDir = resolve(projectRoot, HASH_STORE_DIR); + this.taskDir = join(this.baseDir, this.taskId); + this.indexPath = join(this.taskDir, ".ralphy-hashes-ref.json"); + this.index = { + taskId: this.taskId, + files: {}, + createdAt: Date.now(), + updatedAt: Date.now(), + }; + } + + /** + * Initialize the hash store + * Creates directories and loads existing index if present + */ + async initialize(): Promise { + if (!ENABLE_HASH_STORE) { + logDebug("[HashStore] Hash store is disabled, skipping initialization"); + return; + } + + if (this.initialized) { + return; + } + + try { + // Create task directory + mkdirSync(this.taskDir, { recursive: true }); + + // Create content directory for hashes + mkdirSync(join(this.taskDir, "content"), { recursive: true }); + + // Load existing index if present + if (existsSync(this.indexPath)) { + try { + const data = readFileSync(this.indexPath, "utf-8"); + // SECURITY: Validate JSON before parsing to prevent prototype pollution + if (data.match(/"(__proto__|constructor|prototype)"\s*:/)) { + throw new Error( + "Hash index file contains potentially malicious prototype pollution keys", + ); + } + try { + this.index = JSON.parse(data) as TaskHashIndex; + } catch { + throw new Error("Failed to parse hash index: invalid JSON"); + } + logDebug( + `[HashStore] Loaded existing index for task ${this.taskId} with ${Object.keys(this.index.files).length} files`, + ); + } catch (error) { + logWarn(`[HashStore] Failed to load existing index, creating new one: ${error}`); + this.index = { + taskId: this.taskId, + files: {}, + createdAt: Date.now(), + updatedAt: Date.now(), + }; + } + } else { + logDebug(`[HashStore] Created new index for task ${this.taskId}`); + } + + this.initialized = true; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + logError(`[HashStore] Initialization failed: ${err.message}`); + throw new HashStoreError("Failed to initialize hash store", err); + } + } + + /** + * Generate SHA256 hash for a file + * + * @param filePath - Path to the file + * @returns The SHA256 hash string + */ + async generateHash(filePath: string): Promise { + const absolutePath = resolve(this.projectRoot, filePath); + const hash = createHash("sha256"); + + // For small files, read entirely into memory + const stats = statSync(absolutePath); + if (stats.size <= MAX_FILE_SIZE_FOR_HASH) { + const content = readFileSync(absolutePath); + hash.update(content); + } else { + // Stream large files to avoid memory issues + const stream = createReadStream(absolutePath); + for await (const chunk of stream) { + hash.update(chunk as Buffer); + } + } + + return hash.digest("hex"); + } + + /** + * Add a file to the hash store + * + * @param filePath - Path to the file (relative to project root) + * @param options - Options for adding the file + * @returns Result of the operation + */ + async addFile(filePath: string, options: AddFileOptions = {}): Promise { + if (!ENABLE_HASH_STORE) { + return { success: false, error: "Hash store is disabled" }; + } + + if (this.cleanedUp) { + return { success: false, error: "Hash store has been cleaned up" }; + } + + await this.initialize(); + + const { + compress = true, + compressionThreshold = 1024, // 1KB + } = options; + + try { + const absolutePath = resolve(this.projectRoot, filePath); + + // Check if file exists + if (!existsSync(absolutePath)) { + return { success: false, error: `File not found: ${filePath}` }; + } + + // Get file stats + const stats = statSync(absolutePath); + + // Generate hash + const hash = await this.generateHash(filePath); + + const compressedHashPath = join("content", `${hash}.gz`); + const uncompressedHashPath = join("content", hash); + const absoluteCompressedPath = join(this.taskDir, compressedHashPath); + const absoluteUncompressedPath = join(this.taskDir, uncompressedHashPath); + + const hasCompressed = existsSync(absoluteCompressedPath); + const hasUncompressed = existsSync(absoluteUncompressedPath); + const alreadyExists = hasCompressed || hasUncompressed; + const shouldCompress = compress && stats.size >= compressionThreshold; + + // Store content if not already present + if (!alreadyExists) { + if (shouldCompress) { + // Compress and store + await this.storeCompressed(absolutePath, absoluteCompressedPath); + } else { + // Store uncompressed + await this.storeUncompressed(absolutePath, absoluteUncompressedPath); + } + + // Store metadata + const metadata: HashMetadata = { + originalPath: filePath, + hash, + size: stats.size, + mtime: stats.mtime.getTime(), + compressed: shouldCompress, + originalSize: stats.size, + storedAt: Date.now(), + taskId: this.taskId, + }; + + const metadataPath = join(this.taskDir, "content", `${hash}.json`); + writeFileSync(metadataPath, JSON.stringify(metadata, null, 2)); + } + + // Update index + const storedHashPath = alreadyExists + ? hasCompressed + ? compressedHashPath + : uncompressedHashPath + : shouldCompress + ? compressedHashPath + : uncompressedHashPath; + + this.index.files[filePath] = { + hash, + hashPath: storedHashPath, + metadataPath: join("content", `${hash}.json`), + }; + this.index.updatedAt = Date.now(); + + // Save index + this.saveIndex(); + + logDebug( + `[HashStore] Added file ${filePath} with hash ${hash.slice(0, 16)}... (${alreadyExists ? "deduplicated" : "new"})`, + ); + + return { + success: true, + hash, + alreadyExists, + }; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + logError(`[HashStore] Failed to add file ${filePath}: ${err.message}`); + return { success: false, error: err.message }; + } + } + + /** + * Store file with compression + */ + private async storeCompressed(sourcePath: string, destPath: string): Promise { + const source = createReadStream(sourcePath); + const gzip = createGzip({ level: 6 }); + const dest = createWriteStream(destPath); + + await withTimeout(pipeline(source, gzip, dest), COMPRESSION_TIMEOUT_MS, "File compression"); + } + + /** + * Store file without compression + */ + private async storeUncompressed(sourcePath: string, destPath: string): Promise { + const source = createReadStream(sourcePath); + const dest = createWriteStream(destPath); + + await withTimeout(pipeline(source, dest), COMPRESSION_TIMEOUT_MS, "File copy"); + } + + /** + * Get a file from the hash store + * + * @param filePath - Path to the file (relative to project root) + * @returns Result with content and metadata + */ + async getFile(filePath: string): Promise { + if (!ENABLE_HASH_STORE) { + return { success: false, error: "Hash store is disabled" }; + } + + await this.initialize(); + + try { + const reference = this.index.files[filePath]; + if (!reference) { + return { success: false, error: `File not in hash store: ${filePath}` }; + } + + // Load metadata + const metadataPath = join(this.taskDir, reference.metadataPath); + if (!existsSync(metadataPath)) { + return { success: false, error: `Metadata not found for: ${filePath}` }; + } + + let metadata: HashMetadata; + try { + metadata = JSON.parse(readFileSync(metadataPath, "utf-8")) as HashMetadata; + } catch { + return { success: false, error: `Failed to parse metadata for: ${filePath}` }; + } + + // Load content + const hashPath = join(this.taskDir, reference.hashPath); + if (!existsSync(hashPath)) { + return { success: false, error: `Content not found for: ${filePath}` }; + } + + let content: Buffer; + if (metadata.compressed) { + content = await this.loadCompressed(hashPath); + } else { + content = readFileSync(hashPath); + } + + logDebug(`[HashStore] Retrieved file ${filePath} (${content.length} bytes)`); + + return { + success: true, + content, + metadata, + }; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + logError(`[HashStore] Failed to get file ${filePath}: ${err.message}`); + return { success: false, error: err.message }; + } + } + + /** + * Load compressed content + */ + private async loadCompressed(filePath: string): Promise { + const chunks: Buffer[] = []; + const source = createReadStream(filePath); + const gunzip = createGunzip(); + + await withTimeout( + pipeline(source, gunzip, async function collectChunks(stream: AsyncIterable) { + for await (const chunk of stream) { + chunks.push(chunk as Buffer); + } + }), + COMPRESSION_TIMEOUT_MS, + "File decompression", + ); + + return Buffer.concat(chunks); + } + + /** + * Check if a file is in the hash store + * + * @param filePath - Path to the file (relative to project root) + * @returns True if the file is stored + */ + async hasFile(filePath: string): Promise { + if (!ENABLE_HASH_STORE || this.cleanedUp) { + return false; + } + + await this.initialize(); + + return filePath in this.index.files; + } + + /** + * Get the hash for a file + * + * @param filePath - Path to the file (relative to project root) + * @returns The hash string, or null if not found + */ + async getHash(filePath: string): Promise { + if (!ENABLE_HASH_STORE || this.cleanedUp) { + return null; + } + + await this.initialize(); + + const reference = this.index.files[filePath]; + return reference?.hash ?? null; + } + + /** + * Compare a file's current hash with the stored hash + * + * @param filePath - Path to the file (relative to project root) + * @returns True if the file has changed (or not in store), false if unchanged + */ + async hasChanged(filePath: string): Promise { + if (!ENABLE_HASH_STORE || this.cleanedUp) { + return true; // Assume changed if store is unavailable + } + + const storedHash = await this.getHash(filePath); + if (!storedHash) { + return true; // Not in store, treat as changed + } + + try { + const currentHash = await this.generateHash(filePath); + return currentHash !== storedHash; + } catch { + return true; // Error reading file, treat as changed + } + } + + /** + * Get all files in the hash store for this task + * + * @returns Array of file paths + */ + async getStoredFiles(): Promise { + if (!ENABLE_HASH_STORE || this.cleanedUp) { + return []; + } + + await this.initialize(); + + return Object.keys(this.index.files); + } + + /** + * Get statistics about the hash store + */ + async getStats(): Promise<{ + totalFiles: number; + totalSize: number; + compressedSize: number; + deduplicationRatio: number; + }> { + if (!ENABLE_HASH_STORE || this.cleanedUp) { + return { + totalFiles: 0, + totalSize: 0, + compressedSize: 0, + deduplicationRatio: 0, + }; + } + + await this.initialize(); + + let totalSize = 0; + let compressedSize = 0; + const uniqueHashes = new Set(); + + for (const [_filePath, reference] of Object.entries(this.index.files)) { + uniqueHashes.add(reference.hash); + + const metadataPath = join(this.taskDir, reference.metadataPath); + if (existsSync(metadataPath)) { + try { + const metadata: HashMetadata = JSON.parse( + readFileSync(metadataPath, "utf-8"), + ) as HashMetadata; + totalSize += metadata.originalSize; + } catch { + // Skip files with corrupted metadata + } + } + + const hashPath = join(this.taskDir, reference.hashPath); + if (existsSync(hashPath)) { + const stats = statSync(hashPath); + compressedSize += stats.size; + } + } + + const totalFiles = Object.keys(this.index.files).length; + const deduplicationRatio = totalFiles > 0 ? (totalFiles - uniqueHashes.size) / totalFiles : 0; + + return { + totalFiles, + totalSize, + compressedSize, + deduplicationRatio, + }; + } + + /** + * Save the index to disk + */ + private saveIndex(): void { + try { + writeFileSync(this.indexPath, JSON.stringify(this.index, null, 2)); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + logWarn(`[HashStore] Failed to save index: ${err.message}`); + } + } + + /** + * Clean up the hash store for this task + * Removes all stored files and the task directory + */ + async cleanup(): Promise { + if (this.cleanedUp) { + return; + } + + if (!ENABLE_HASH_STORE) { + this.cleanedUp = true; + return; + } + + try { + if (existsSync(this.taskDir)) { + // Get stats before cleanup for logging + const stats = await this.getStats(); + + rmSync(this.taskDir, { recursive: true, force: true }); + + logDebug( + `[HashStore] Cleaned up task ${this.taskId} (${stats.totalFiles} files, ${(stats.compressedSize / 1024 / 1024).toFixed(2)} MB)`, + ); + } + + this.cleanedUp = true; + this.initialized = false; + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + logError(`[HashStore] Cleanup failed: ${err.message}`); + // Don't throw - cleanup failures shouldn't break execution + } + } + + /** + * Clean up old hash stores across all tasks + * Should be called periodically to remove stale data + * + * @param maxAgeMs - Maximum age in milliseconds (defaults to HASH_STORE_MAX_AGE_MS) + */ + static async cleanupOldStores( + projectRoot: string = process.cwd(), + maxAgeMs: number = HASH_STORE_MAX_AGE_MS, + ): Promise { + if (!ENABLE_HASH_STORE) { + return 0; + } + + const baseDir = resolve(projectRoot, HASH_STORE_DIR); + + if (!existsSync(baseDir)) { + return 0; + } + + let cleanedCount = 0; + const now = Date.now(); + + try { + const entries = readdirSync(baseDir, { withFileTypes: true }); + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + + const taskDir = join(baseDir, entry.name); + const indexPath = join(taskDir, ".ralphy-hashes-ref.json"); + + try { + let shouldClean = false; + + if (existsSync(indexPath)) { + const data = readFileSync(indexPath, "utf-8"); + try { + const index: TaskHashIndex = JSON.parse(data) as TaskHashIndex; + shouldClean = now - index.updatedAt > maxAgeMs; + } catch { + // If index is corrupted, clean it up + shouldClean = true; + } + } else { + // No index file, check directory modification time + const stats = statSync(taskDir); + shouldClean = now - stats.mtime.getTime() > maxAgeMs; + } + + if (shouldClean) { + rmSync(taskDir, { recursive: true, force: true }); + cleanedCount++; + logDebug(`[HashStore] Cleaned up old store: ${entry.name}`); + } + } catch (error) { + // Log but continue cleaning other stores + logWarn(`[HashStore] Failed to check/clean ${entry.name}: ${error}`); + } + } + } catch (error) { + logWarn(`[HashStore] Failed to cleanup old stores: ${error}`); + } + + if (cleanedCount > 0) { + logDebug(`[HashStore] Cleaned up ${cleanedCount} old hash stores`); + } + + return cleanedCount; + } +} + +// ============================================================================ +// Utility Functions +// ============================================================================ + +/** + * Create a hash store for a task + * Convenience function for creating and initializing a hash store + * + * @param taskId - Task identifier + * @param projectRoot - Project root directory + * @returns Initialized FileHashStore instance + */ +export async function createHashStore( + taskId: string, + projectRoot?: string, +): Promise { + const store = new FileHashStore(taskId, projectRoot); + await store.initialize(); + return store; +} + +/** + * Check if hash store is enabled + */ +export function isHashStoreEnabled(): boolean { + return ENABLE_HASH_STORE; +} + +/** + * Get the path to the hash store directory + */ +export function getHashStorePath(projectRoot: string = process.cwd()): string { + return resolve(projectRoot, HASH_STORE_DIR); +} diff --git a/cli/src/execution/index.ts b/cli/src/execution/index.ts index 89d770b8..5f3f8970 100644 --- a/cli/src/execution/index.ts +++ b/cli/src/execution/index.ts @@ -1,4 +1,4 @@ +export * from "./parallel.ts"; export * from "./prompt.ts"; export * from "./retry.ts"; export * from "./sequential.ts"; -export * from "./parallel.ts"; diff --git a/cli/src/execution/locking.ts b/cli/src/execution/locking.ts new file mode 100644 index 00000000..2916502b --- /dev/null +++ b/cli/src/execution/locking.ts @@ -0,0 +1,397 @@ +import { createHash, randomBytes } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join, normalize, resolve } from "node:path"; +import process from "node:process"; +import { + LOCK_CLEANUP_INTERVAL_MS, + LOCK_DIR, + LOCK_MAX_LOCKS, + LOCK_TIMEOUT_MS, +} from "../config/constants.ts"; +import { logDebug, logWarn } from "../ui/logger.ts"; +import { registerCleanup } from "../utils/cleanup.ts"; + +interface LockInfo { + timestamp: number; + timeout: number; + owner: string; // Track lock owner + refreshCount: number; +} + +// Unified lock structure for better performance +const locks = new Map(); +const lockOwner = `${process.pid.toString()}-${Date.now()}`; +const sleepBuffer = new SharedArrayBuffer(4); +const sleepArray = new Int32Array(sleepBuffer); +function sleepBlocking(ms: number): void { + if (ms <= 0) return; + + if (typeof Bun !== "undefined" && Bun.sleepSync) { + Bun.sleepSync(ms); + return; + } + + try { + // Node runtime fallback. If unavailable in current runtime/thread, skip blocking delay. + Atomics.wait(sleepArray, 0, 0, ms); + } catch { + // No-op fallback. + } +} + +function refreshLock(normalizedPath: string, workDir: string): void { + const lockInfo = locks.get(normalizedPath); + if (!lockInfo) return; + + lockInfo.timestamp = Date.now(); + lockInfo.refreshCount++; + + // Update lock file on disk + const lockFile = getLockFilePath(normalizedPath, workDir); + try { + writeFileSync(lockFile, JSON.stringify(lockInfo)); + } catch (err) { + logDebug(`Failed to refresh lock ${normalizedPath}: ${err}`); + } +} + +// Define global state interface for type safety +declare global { + interface RalphyGlobalState { + _lockState?: { + _lastLockCleanup?: number; + }; + verboseMode?: boolean; + } +} + +// Register for global cleanup +registerCleanup(() => { + locks.clear(); +}); + +function getLockFilePath(normalizedPath: string, workDir: string): string { + const hash = createHash("sha256").update(normalizedPath).digest("hex"); + const lockDir = join(workDir, LOCK_DIR); + return join(lockDir, `${hash}.lock`); +} + +function ensureLockDir(workDir: string): void { + const lockDir = join(workDir, LOCK_DIR); + try { + mkdirSync(lockDir, { recursive: true }); + } catch (err) { + // Directory may already exist, that's OK + if ((err as NodeJS.ErrnoException).code !== "EEXIST") { + throw err; + } + } +} + +function cleanupStaleLockFiles(workDir: string): void { + const lockDir = join(workDir, LOCK_DIR); + if (!existsSync(lockDir)) return; + + const files = readdirSync(lockDir); + const now = Date.now(); + + for (const file of files) { + if (!file.endsWith(".lock")) continue; + const filePath = join(lockDir, file); + try { + const content = readFileSync(filePath, "utf8"); + const lockInfo: LockInfo = JSON.parse(content); + if (now - lockInfo.timestamp >= lockInfo.timeout) { + try { + unlinkSync(filePath); + } catch { + // Best-effort cleanup: lock may be removed by another process. + } + } + } catch { + try { + unlinkSync(filePath); + } catch { + // Best-effort cleanup: lock may be removed by another process. + } + } + } +} + +export function normalizePathForLocking(filePath: string, workDir: string): string { + // Resolve to absolute path first + const absolutePath = resolve(workDir, filePath); + + // Normalize path separators and resolve .. etc. + const normalized = normalize(absolutePath); + + // On Windows, convert to lowercase for case-insensitive comparison + if (process.platform === "win32") { + return normalized.toLowerCase(); + } + + return normalized; +} + +export function isInRalphyDir(filePath: string): boolean { + return filePath.includes(".ralphy") || filePath.includes(".ralphy-worktrees"); +} + +function getGlobalLockState(): NonNullable { + if (!(globalThis as RalphyGlobalState)._lockState) { + (globalThis as RalphyGlobalState)._lockState = { _lastLockCleanup: 0 }; + } + // biome-ignore lint/style/noNonNullAssertion: guaranteed to be set above + return (globalThis as RalphyGlobalState)._lockState!; +} + +export function acquireFileLock( + filePath: string, + workDir: string, + maxRetries = 5, + allowReentrant = false, +): boolean { + const normalizedPath = normalizePathForLocking(filePath, workDir); + const now = Date.now(); + + // CRITICAL FIX: Check in-memory lock FIRST before any file operations + // This handles re-entrant locks without file I/O + const existing = locks.get(normalizedPath); + if (existing && now - existing.timestamp < existing.timeout) { + if (existing.owner === lockOwner && allowReentrant) { + refreshLock(normalizedPath, workDir); + return true; + } + return false; // Someone else owns it + } + + ensureLockDir(workDir); + const lockState = getGlobalLockState(); + const lastCleanupTime = lockState._lastLockCleanup || 0; + + if (now - lastCleanupTime > LOCK_CLEANUP_INTERVAL_MS) { + cleanupStaleLocks(); + cleanupStaleLockFiles(workDir); + lockState._lastLockCleanup = now; + } + + const lockFile = getLockFilePath(normalizedPath, workDir); + + // Atomic lock acquisition using writeFileSync with exclusive flag + // This is the ONLY source of truth - in-memory cache is updated AFTER file succeeds + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const lockInfo = { + timestamp: Date.now(), + timeout: LOCK_TIMEOUT_MS, + owner: lockOwner, + refreshCount: 0, + }; + + // CRITICAL: Use writeFileSync with 'wx' flag for atomic creation + // This is the race condition prevention - only one process can succeed + writeFileSync(lockFile, JSON.stringify(lockInfo), { flag: "wx" }); + + // ONLY update in-memory cache AFTER successful file write + // This ensures file is the source of truth + locks.set(normalizedPath, lockInfo); + + return true; + } catch (_error) { + const currentTime = Date.now(); + + // Check if we should retry based on lock file state + if (existsSync(lockFile)) { + try { + const content = readFileSync(lockFile, "utf8"); + + // Handle empty or corrupt lock file + if (!content || content.trim().length === 0) { + logDebug(`Lock file ${lockFile} is empty, removing`); + unlinkSync(lockFile); + continue; + } + + let fileLockInfo: unknown; + try { + fileLockInfo = JSON.parse(content); + } catch (parseError) { + logDebug(`Failed to parse lock file ${lockFile}: ${parseError}`); + unlinkSync(lockFile); + continue; + } + + // Validate lock info and check if stale + if ( + fileLockInfo && + typeof fileLockInfo === "object" && + "timestamp" in fileLockInfo && + typeof fileLockInfo.timestamp === "number" && + "timeout" in fileLockInfo && + typeof fileLockInfo.timeout === "number" + ) { + // Check if lock is stale + if (currentTime - fileLockInfo.timestamp >= fileLockInfo.timeout) { + logDebug(`Removing stale lock file ${lockFile}`); + unlinkSync(lockFile); + continue; // Retry after removing stale lock + } + + // Lock is valid and held by someone else + logDebug(`Lock file ${lockFile} is held by another process`); + + // Check if it's our own lock (file exists but memory doesn't have it) + // Use type assertion for owner/refreshCount which may not be in older lock files + const typedLockInfo = fileLockInfo as LockInfo; + if (typedLockInfo.owner === lockOwner && allowReentrant) { + logDebug(`Reclaiming our own lock ${lockFile}`); + // Reclaim the lock in memory + locks.set(normalizedPath, { + timestamp: typedLockInfo.timestamp, + timeout: typedLockInfo.timeout, + owner: typedLockInfo.owner, + refreshCount: typedLockInfo.refreshCount || 0, + }); + refreshLock(normalizedPath, workDir); + return true; + } + } + } catch (readError) { + logDebug(`Error reading lock file ${lockFile}: ${readError}`); + try { + unlinkSync(lockFile); + } catch (unlinkError) { + logDebug(`Failed to remove lock file ${lockFile}: ${unlinkError}`); + } + } + } + + // Exponential backoff with jitter - use non-blocking approach + if (attempt < maxRetries) { + const baseDelay = 2 ** attempt * 100; // 100, 200, 400, 800, 1600ms + // Use cryptographically secure random for jitter (not Math.random()) + const jitter = Number.parseInt(randomBytes(2).toString("hex"), 16) % 50; // 0-50ms jitter + const delay = Math.min(baseDelay + jitter, 5000); // Max 5 seconds + + logDebug( + `Lock acquisition attempt ${attempt}/${maxRetries} failed, retrying in ${Math.round(delay)}ms`, + ); + sleepBlocking(delay); + } + } + } + logDebug(`Failed to acquire lock after ${maxRetries} attempts: ${normalizedPath}`); + return false; +} + +export function releaseFileLock(filePath: string, workDir: string): void { + const normalizedPath = normalizePathForLocking(filePath, workDir); + locks.delete(normalizedPath); + + // Remove persistent lock file + const lockFile = getLockFilePath(normalizedPath, workDir); + if (existsSync(lockFile)) { + try { + unlinkSync(lockFile); + } catch (err) { + logDebug(`Failed to delete lock file ${lockFile}: ${err}`); + } + } +} + +export function acquireLocksForFiles(files: string[], workDir: string): boolean { + // Remove duplicates by normalizing paths first + const fileMap = new Map(); + + for (const file of files) { + const normalizedPath = normalizePathForLocking(file, workDir); + if (!fileMap.has(normalizedPath)) { + fileMap.set(normalizedPath, file); + } + } + + const uniqueFiles = Array.from(fileMap.values()); + const acquiredThisAttempt: string[] = []; + + try { + for (const file of uniqueFiles) { + if (acquireFileLock(file, workDir)) { + acquiredThisAttempt.push(file); + } else { + // Rollback: release only locks acquired in THIS attempt + for (const acquiredFile of acquiredThisAttempt) { + releaseFileLock(acquiredFile, workDir); + } + return false; + } + } + return true; + } catch (err) { + // Rollback on error + for (const acquiredFile of acquiredThisAttempt) { + releaseFileLock(acquiredFile, workDir); + } + throw err; + } +} + +export function releaseLocksForFiles(files: string[], workDir: string): void { + for (const file of files) { + releaseFileLock(file, workDir); + } +} + +export function clearAllLocks(): void { + locks.clear(); +} + +export function getActiveLocks(): string[] { + return Array.from(locks.keys()); +} + +export function cleanupStaleLocks(): void { + const now = Date.now(); + const locksToEvict: string[] = []; + + // Remove expired locks first + for (const [path, lockInfo] of locks.entries()) { + if (now - lockInfo.timestamp > lockInfo.timeout) { + locksToEvict.push(path); + } + } + + // Notify before eviction + for (const path of locksToEvict) { + const lockInfo = locks.get(path); + if (lockInfo && lockInfo.owner !== lockOwner) { + logDebug(`Evicting lock owned by ${lockInfo.owner}: ${path}`); + } + locks.delete(path); + } + + // If still too many, remove oldest but check ownership + if (locks.size > LOCK_MAX_LOCKS) { + logWarn( + `Lock registry size (${locks.size}) exceeded ${LOCK_MAX_LOCKS}. Evicting oldest non-own locks.`, + ); + + const sorted = Array.from(locks.entries()).sort((a, b) => a[1].timestamp - b[1].timestamp); + + // Keep all locks owned by this process, evict oldest of others first + const others = sorted.filter(([_path, info]) => info.owner !== lockOwner); + const overflow = locks.size - LOCK_MAX_LOCKS; + const toEvictOthers = others.slice(0, Math.max(overflow, 0)); + + for (const [path] of toEvictOthers) { + logDebug(`Evicting lock from other process: ${path}`); + locks.delete(path); + } + } +} diff --git a/cli/src/execution/orchestrator.ts b/cli/src/execution/orchestrator.ts new file mode 100644 index 00000000..43f5507f --- /dev/null +++ b/cli/src/execution/orchestrator.ts @@ -0,0 +1,333 @@ +/** + * Simplified Orchestrator for Test Model Integration + * + * Automatically runs tests after main model completes, no special markers needed. + * Test model analyzes results and suggests fixes if tests fail. + */ + +import type { AIEngine, AIResult } from "../engines/types.ts"; +import { logDebug, logError, logWarn } from "../ui/logger.ts"; +import { StaticAgentDisplay } from "../ui/static-agent-display.ts"; +import { + canMakeConnectionAttempt, + circuitBreaker, + sleep, + waitForConnectionRestore, +} from "./retry.ts"; + +const MAX_CONTEXT_CHARS = 12000; + +function truncateContext(mainOutput: string): string { + if (mainOutput.length <= MAX_CONTEXT_CHARS) { + return mainOutput; + } + + const omitted = mainOutput.length - MAX_CONTEXT_CHARS; + return `${mainOutput.slice(0, MAX_CONTEXT_CHARS)}\n\n[...output truncated, ${omitted} chars omitted...]`; +} + +export interface OrchestratorOptions { + mainEngine: AIEngine; + testEngine?: AIEngine; + mainModel?: string; + testModel?: string; + workDir: string; + maxIterations?: number; + debug?: boolean; + /** Agent number for display updates */ + agentNum?: number; +} + +export interface OrchestratorResult { + success: boolean; + response: string; + iterations: number; + mainModelCalls: number; + testModelCalls: number; + error?: string; +} + +async function executeWithRetry( + engine: AIEngine, + prompt: string, + workDir: string, + options: { modelOverride?: string }, + maxRetries = 3, +): Promise { + let lastError: string | undefined; + + const circuitCheck = canMakeConnectionAttempt(); + if (!circuitCheck.allowed) { + logError(`Circuit breaker preventing execution: ${circuitCheck.reason}`); + const restored = await waitForConnectionRestore(60000); + if (!restored) { + return { + success: false, + response: "", + inputTokens: 0, + outputTokens: 0, + error: circuitCheck.reason || "Connection circuit open - too many failures", + }; + } + } + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + const attemptCheck = canMakeConnectionAttempt(); + if (!attemptCheck.allowed) { + logError(`Circuit breaker preventing retry: ${attemptCheck.reason}`); + return { + success: false, + response: "", + inputTokens: 0, + outputTokens: 0, + error: attemptCheck.reason || "Connection circuit open - stopping retries", + }; + } + + const result = await engine.execute(prompt, workDir, options); + + if (result.success) { + circuitBreaker.recordSuccess(); + return result; + } + + lastError = result.error; + + const isConnectionError = + /connection|network|timeout|unable to connect|internet connection|econnrefused|econnreset|socket hang up|dns|ENOTFOUND/i.test( + result.error || "", + ); + + if (isConnectionError) { + circuitBreaker.recordFailure(new Error(result.error || "Connection error")); + + if (attempt < maxRetries) { + const delayMs = Math.min(2000 * 2 ** (attempt - 1), 30000); + logWarn( + `Connection error on attempt ${attempt}/${maxRetries}. Retrying in ${delayMs}ms...`, + ); + await sleep(delayMs); + + const postFailureCheck = canMakeConnectionAttempt(); + if (!postFailureCheck.allowed) { + logError(`Circuit opened after ${attempt} attempts: ${postFailureCheck.reason}`); + return { + success: false, + response: "", + inputTokens: 0, + outputTokens: 0, + error: postFailureCheck.reason || `Connection failed after ${attempt} attempts`, + }; + } + } else { + break; + } + } else if (attempt >= maxRetries) { + break; + } else { + return result; + } + } + + return { + success: false, + response: "", + inputTokens: 0, + outputTokens: 0, + error: lastError || "All retry attempts failed", + }; +} + +function buildTestPrompt(mainOutput: string, _workDir: string): string { + return `You are a test runner. Your job is to verify that the implementation is correct by RUNNING the actual tests. + +## Previous Implementation Work + +${truncateContext(mainOutput)} + +## Your Task + +1. First, identify what test framework is being used (jest, pytest, npm test, cargo test, etc.) +2. Run the tests using the appropriate command +3. Report the results clearly: + - How many tests passed/failed + - Any error messages + - Specific files that failed + +## Commands to try (in order): +- npm test +- npm run test +- yarn test +- pnpm test +- pytest +- python -m pytest +- cargo test +- go test +- make test + +## Output Format + +Report your findings in this format: + +TEST RESULTS: +- Framework: +- Command: +- Passed: +- Failed: +- Status: PASS / FAIL / PARTIAL + +DETAILS: +`; +} + +function buildFixPrompt(originalPrompt: string, mainOutput: string, testResults: string): string { + return `${originalPrompt} + +## Your Previous Implementation + +${truncateContext(mainOutput)} + +## Test Results + +${testResults} + +## Instructions + +The tests have revealed issues. Please: +1. Fix the problems identified in the test results +2. Run tests again to verify fixes +3. Provide the corrected implementation`; +} + +/** + * Execute with orchestrator pattern - automatically runs tests after main model + */ +export async function executeWithOrchestrator( + prompt: string, + options: OrchestratorOptions, + onProgress?: (step: string) => void, +): Promise { + const { mainEngine, testEngine, mainModel, testModel, workDir, debug = false } = options; + + const reportProgress = (message: string) => { + if (debug) logDebug(`[Orchestrator] ${message}`); + onProgress?.(message); + }; + + reportProgress("Starting execution with test feedback"); + + // Step 1: Run main model to implement the task + reportProgress("Running main model..."); + const mainResult = await executeWithRetry(mainEngine, prompt, workDir, { + modelOverride: mainModel, + }); + + if (!mainResult.success) { + return { + success: false, + response: mainResult.response, + iterations: 1, + mainModelCalls: 1, + testModelCalls: 0, + error: `Main model failed: ${mainResult.error}`, + }; + } + + const mainOutput = mainResult.response || ""; + reportProgress("Main model complete, running tests..."); + + // Update display to show test model is running + const display = StaticAgentDisplay.getInstance(); + if (display && options.agentNum) { + display.setAgentStatus(options.agentNum, "", "working", "testing", testModel || "test"); + } + + // Step 2: Run test model to verify the work + reportProgress(`Sending to test model (${testModel || "default"})...`); + const testPrompt = buildTestPrompt(mainOutput, workDir); + const testEngineToUse = testEngine || mainEngine; + reportProgress("Test prompt ready, executing test model..."); + const testResult = await executeWithRetry(testEngineToUse, testPrompt, workDir, { + modelOverride: testModel, + }); + + const testOutput = testResult.success + ? testResult.response || "Tests completed" + : `Test execution failed: ${testResult.error}`; + + reportProgress(`Test model complete. Response length: ${testOutput.length} chars`); + reportProgress(`Test output preview: ${testOutput.slice(0, 100)}...`); + + // Check if tests indicate failures that need fixing + const hasFailures = + /test (fail|error|broken)|failed|failing|✗|❌|assertion|exception/i.test(testOutput) && + !/0 fail|no fail|all pass|✓|✔|passed/i.test(testOutput); + + if (!hasFailures) { + // Tests passed or no issues found + return { + success: true, + response: `${mainOutput}\n\n---\n\nTest Results:\n${testOutput}`, + iterations: 1, + mainModelCalls: 1, + testModelCalls: 1, + }; + } + + // Step 3: Tests failed - run main model again with fix instructions + reportProgress("Issues found, requesting fixes..."); + const fixPrompt = buildFixPrompt(prompt, mainOutput, testOutput); + const fixResult = await executeWithRetry(mainEngine, fixPrompt, workDir, { + modelOverride: mainModel, + }); + + if (!fixResult.success) { + return { + success: false, + response: `${mainOutput}\n\n---\n\nTest Results:\n${testOutput}`, + iterations: 2, + mainModelCalls: 2, + testModelCalls: 1, + error: `Failed to fix issues: ${fixResult.error}`, + }; + } + + return { + success: true, + response: `${fixResult.response}\n\n---\n\nOriginal Test Results:\n${testOutput}`, + iterations: 2, + mainModelCalls: 2, + testModelCalls: 1, + }; +} + +/** + * Check if orchestrator pattern should be used for this task + */ +export function shouldUseOrchestrator( + taskTitle: string, + taskDescription: string, + testModel?: string, +): boolean { + if (!testModel) return false; + + const combined = `${taskTitle} ${taskDescription}`.toLowerCase(); + + // Use orchestrator for tasks that likely need testing + const testKeywords = [ + "test", + "spec", + "jest", + "vitest", + "mocha", + "cypress", + "playwright", + "implement", + "create", + "build", + "fix", + "debug", + ]; + + return testKeywords.some((kw) => combined.includes(kw)); +} diff --git a/cli/src/execution/parallel-no-git.ts b/cli/src/execution/parallel-no-git.ts new file mode 100644 index 00000000..ef466191 --- /dev/null +++ b/cli/src/execution/parallel-no-git.ts @@ -0,0 +1,368 @@ +import { logTaskProgress } from "../config/writer.ts"; +import type { Task } from "../tasks/types.ts"; +import { logDebug, logError, logInfo, logSuccess, logWarn } from "../ui/logger.ts"; +import { notifyTaskComplete, notifyTaskFailed } from "../ui/notify.ts"; +import { StaticAgentDisplay } from "../ui/static-agent-display.ts"; +import { runAgentInSandbox } from "./agent-runner.ts"; +import { clearDeferredTask, recordDeferredTask } from "./deferred.ts"; +import { isRetryableError } from "./retry.ts"; +import type { AgentRunnerOptions } from "./runner-types.ts"; +import { cleanupSandbox, getSandboxBase } from "./sandbox.ts"; +import type { ExecutionOptions, ExecutionResult } from "./sequential.ts"; +import { type StateFormat, TaskState, TaskStateManager, detectStateFormat } from "./task-state.ts"; + +/** + * Run tasks in parallel using sandboxes only (no git worktrees) + * + * This is a simplified version of parallel.ts that: + * - Always uses sandboxes (never git worktrees) + * - Skips merge phase entirely + * - Uses static display to show agents without constant refreshing + * - Shows 5 static rows per agent with formatted AI output + * + * @param options - Execution options including maxParallel, taskSource, etc. + * @returns Execution result with completed/failed task counts + */ +export async function runParallelNoGit( + options: ExecutionOptions & { + maxParallel: number; + prdSource: string; + prdFile: string; + prdIsFolder?: boolean; + taskStateManager?: TaskStateManager; + }, +): Promise { + const { + engine, + taskSource, + workDir, + skipTests, + skipLint, + dryRun, + maxIterations, + maxRetries, + retryDelay, + maxParallel, + prdSource, + prdFile, + prdIsFolder = false, + browserEnabled, + modelOverride, + planningModel, + testModel, + engineArgs, + debug, + debugOpenCode, + allowOpenCodeSandboxAccess, + logThoughts, + taskStateManager: externalTaskStateManager, + } = options; + + const result: ExecutionResult = { + tasksCompleted: 0, + tasksFailed: 0, + totalInputTokens: 0, + totalOutputTokens: 0, + }; + + // Initialize task state manager if not provided + let taskStateManager: TaskStateManager; + if (externalTaskStateManager) { + taskStateManager = externalTaskStateManager; + } else { + // Detect format from prdFile extension + const format: StateFormat = detectStateFormat(prdFile); + + taskStateManager = new TaskStateManager( + workDir, + taskSource.type, + prdFile || "tasks.yaml", + format, + ); + + // Get all tasks and initialize state manager + const allTasks = await taskSource.getAllTasks(); + await taskStateManager.initialize(allTasks); + } + + // Use lightweight sandbox mode + logInfo("Using lightweight sandbox mode (no git worktrees)"); + + const sandboxBase = getSandboxBase(workDir); + + // Global agent counter to ensure unique numbering across batches + let globalAgentNum = 0; + const getNextAgentNum = () => ++globalAgentNum; + + // Track processed tasks in dry-run mode (since we don't modify the source file) + const dryRunProcessedIds = new Set(); + + // Static agent display - shows agents without constant refreshing + const staticAgentDisplay = StaticAgentDisplay.getInstance() || new StaticAgentDisplay(); + staticAgentDisplay.startDisplay(); + + // Process tasks in batches + let iteration = 0; + + try { + while (true) { + // Check iteration limit + if (maxIterations > 0 && iteration >= maxIterations) { + logInfo(`Reached max iterations (${maxIterations})`); + break; + } + + // Get pending tasks from state manager + const pendingTasks = taskStateManager.getTasksByState(TaskState.PENDING); + + if (pendingTasks.length === 0) { + logSuccess("All tasks completed!"); + break; + } + + // Get all tasks from source to find full task objects + const allSourceTasks = await taskSource.getAllTasks(); + + // Map pending state entries to full task objects + let tasks: Task[] = pendingTasks + .map((pt) => allSourceTasks.find((t) => t.id === pt.id)) + .filter((t): t is Task => t !== undefined); + + // Filter out already processed tasks in dry-run mode + if (dryRun) { + tasks = tasks.filter((t) => !dryRunProcessedIds.has(t.id)); + } + + if (tasks.length === 0) { + logSuccess("All tasks completed!"); + break; + } + + // Filter out tasks that have exceeded max attempts + const filteredTasks: Task[] = []; + for (const task of tasks) { + if (taskStateManager.hasExceededMaxAttempts(task.id, maxRetries)) { + logWarn(`Task "${task.title}" has exceeded max attempts (${maxRetries}), skipping...`); + await taskStateManager.transitionState(task.id, TaskState.SKIPPED); + await taskSource.markComplete(task.id); + result.tasksFailed++; + notifyTaskFailed(task.title, "Exceeded maximum retry attempts"); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } else { + filteredTasks.push(task); + } + } + + if (filteredTasks.length === 0) { + // All tasks in this batch were skipped due to max attempts + continue; + } + + // Limit to maxParallel + const batch = filteredTasks.slice(0, maxParallel); + iteration++; + + logInfo(`Batch ${iteration}: ${batch.length} tasks in parallel`); + + if (dryRun && !debugOpenCode) { + logInfo("(dry run) Skipping batch"); + // Track processed tasks to avoid infinite loop + for (const task of batch) { + dryRunProcessedIds.add(task.id); + } + continue; + } + + // Claim tasks for execution before starting and assign stable agent numbers + const claimedTasks: Array<{ task: Task; agentNum: number }> = []; + for (const task of batch) { + const claimed = await taskStateManager.claimTaskForExecution(task.id); + if (claimed) { + const agentNum = getNextAgentNum(); + const initialPhase = planningModel ? "planning" : "execution"; + const initialModel = planningModel ? "planning" : "main"; + staticAgentDisplay.setAgentStatus( + agentNum, + task.title, + "working", + initialPhase, + initialModel, + ); + claimedTasks.push({ task, agentNum }); + } else { + logDebug(`Task "${task.title}" is already being executed, skipping...`); + } + } + + if (claimedTasks.length === 0) { + // No tasks could be claimed, continue to next batch + continue; + } + + // Parallel execution with progress callback + const promises = claimedTasks.map(({ task, agentNum }) => { + const agentOptions: AgentRunnerOptions = { + engine, + task, + agentNum, + originalDir: workDir, + prdSource, + prdFile, + prdIsFolder, + maxRetries, + retryDelay, + skipTests, + skipLint, + browserEnabled, + modelOverride, + planningModel, + testModel, + engineArgs, + env: options.env, + debug, + debugOpenCode, + allowOpenCodeSandboxAccess, + logThoughts, + onProgress: (step) => { + // Detect OpenCode JSON and parse it properly + if ( + step.includes('"type":"tool_use"') || + step.includes('"type":"step_') || + step.includes('"type":"text"') + ) { + staticAgentDisplay.updateAgentFromOpenCode(agentNum, step); + } else { + staticAgentDisplay.updateAgent(agentNum, step); + } + }, + dryRun, + noGitParallel: true, + }; + + return runAgentInSandbox(sandboxBase, agentOptions); + }); + + const results = await Promise.allSettled(promises); + + // Process results + let sawRetryableFailure = false; + + for (let i = 0; i < results.length; i++) { + const res = results[i]; + const claimedTask = claimedTasks[i]; + const task = claimedTask.task; + + if (res.status === "rejected") { + const error = res.reason; + const retryableFailure = isRetryableError(error); + if (retryableFailure) { + sawRetryableFailure = true; + const deferrals = recordDeferredTask(taskSource.type, task, workDir, prdFile); + if (deferrals >= maxRetries) { + logError(`Task "${task.title}" failed after ${deferrals} deferrals: ${error}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, String(error)); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, String(error)); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + staticAgentDisplay.agentComplete(claimedTask.agentNum); + } else { + logWarn(`Task "${task.title}" deferred (${deferrals}/${maxRetries}): ${error}`); + await taskStateManager.transitionState(task.id, TaskState.DEFERRED, String(error)); + result.tasksFailed++; + } + } else { + logError(`Task "${task.title}" failed: ${error}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, String(error)); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, String(error)); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + staticAgentDisplay.agentComplete(claimedTask.agentNum); + } + continue; + } + + const agentResult = res.value; + const { agentNum, worktreeDir, result: aiResult, error: failureReason } = agentResult; + + staticAgentDisplay.agentComplete(agentNum); + + if (failureReason) { + const retryable = isRetryableError(failureReason); + if (retryable) { + const deferrals = recordDeferredTask(taskSource.type, task, workDir, prdFile); + sawRetryableFailure = true; + if (deferrals >= maxRetries) { + logError( + `Task "${task.title}" failed after ${deferrals} deferrals: ${failureReason}`, + ); + await taskStateManager.transitionState(task.id, TaskState.FAILED, failureReason); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, failureReason); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } else { + logWarn( + `Task "${task.title}" deferred (${deferrals}/${maxRetries}): ${failureReason}`, + ); + await taskStateManager.transitionState(task.id, TaskState.DEFERRED, failureReason); + } + } else { + logError(`Task "${task.title}" failed: ${failureReason}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, failureReason); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, failureReason); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } + } else if (aiResult?.success) { + logSuccess(`Task "${task.title}" completed`); + result.totalInputTokens += aiResult.inputTokens; + result.totalOutputTokens += aiResult.outputTokens; + + await taskStateManager.transitionState(task.id, TaskState.COMPLETED); + await taskSource.markComplete(task.id); + logTaskProgress(task.title, "completed", workDir); + result.tasksCompleted++; + notifyTaskComplete(task.title); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } else { + // Logic failure (success=false) but no exception thrown (e.g. Planning Failed) + const errorMsg = aiResult?.error || "Unknown logic failure"; + logError(`Task "${task.title}" failed (logic): ${errorMsg}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, errorMsg); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, errorMsg); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } + + // Cleanup sandbox + if (worktreeDir) { + try { + await cleanupSandbox(worktreeDir); + logDebug(`Cleaned up sandbox: ${worktreeDir}`); + } catch (cleanupErr) { + logWarn(`Failed to cleanup sandbox ${worktreeDir}: ${cleanupErr}`); + } + } + } + + if (sawRetryableFailure) { + logWarn("Stopping early due to retryable errors. Try again later."); + break; + } + } + } finally { + // Stop static display + staticAgentDisplay.stopDisplay(); + } + + return result; +} diff --git a/cli/src/execution/parallel.ts b/cli/src/execution/parallel.ts index 5318088e..fdfa0488 100644 --- a/cli/src/execution/parallel.ts +++ b/cli/src/execution/parallel.ts @@ -1,11 +1,18 @@ -import { copyFileSync, cpSync, existsSync, mkdirSync } from "node:fs"; -import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { join, resolve, sep } from "node:path"; import simpleGit from "simple-git"; -import { PROGRESS_FILE, RALPHY_DIR } from "../config/loader.ts"; import { logTaskProgress } from "../config/writer.ts"; -import type { AIEngine, AIResult } from "../engines/types.ts"; +import type { AIEngine } from "../engines/types.ts"; import { getCurrentBranch, returnToBaseBranch } from "../git/branch.ts"; -import { syncPrdToIssue } from "../git/issue-sync.ts"; import { abortMerge, analyzePreMerge, @@ -13,230 +20,311 @@ import { mergeAgentBranch, sortByConflictLikelihood, } from "../git/merge.ts"; -import { - canUseWorktrees, - cleanupAgentWorktree, - createAgentWorktree, - getWorktreeBase, -} from "../git/worktree.ts"; -import type { Task, TaskSource } from "../tasks/types.ts"; -import { formatDuration, logDebug, logError, logInfo, logSuccess, logWarn } from "../ui/logger.ts"; +import { canUseWorktrees, cleanupAgentWorktree, getWorktreeBase } from "../git/worktree.ts"; +import type { Task } from "../tasks/types.ts"; +import { logDebug, logError, logInfo, logSuccess, logWarn } from "../ui/logger.ts"; import { notifyTaskComplete, notifyTaskFailed } from "../ui/notify.ts"; +import { StaticAgentDisplay } from "../ui/static-agent-display.ts"; +import { type AgentRunnerOptions, runAgentInSandbox, runAgentInWorktree } from "./agent-runner.ts"; import { resolveConflictsWithAI } from "./conflict-resolution.ts"; import { clearDeferredTask, recordDeferredTask } from "./deferred.ts"; -import { buildParallelPrompt } from "./prompt.ts"; -import { isRetryableError, withRetry } from "./retry.ts"; +import { + type PlannedTask, + batchByColor, + buildConflictGraph, + colorGraph, +} from "./graph-coloring.ts"; +import { isRetryableError } from "./retry.ts"; import { commitSandboxChanges } from "./sandbox-git.ts"; -import { cleanupSandbox, createSandbox, getModifiedFiles, getSandboxBase } from "./sandbox.ts"; +import { cleanupSandbox, getModifiedFiles, getSandboxBase } from "./sandbox.ts"; import type { ExecutionOptions, ExecutionResult } from "./sequential.ts"; +import { type StateFormat, TaskState, TaskStateManager, detectStateFormat } from "./task-state.ts"; + +const GLOBAL_MERGE_LOCK_TIMEOUT_MS = 300000; // 5 minutes timeout for merge operations +const WORKTREE_TRACKING_FILE = ".ralphy-worktrees/tracked.json"; -interface ParallelAgentResult { - task: Task; - agentNum: number; +interface TrackedWorktree { worktreeDir: string; branchName: string; - result: AIResult | null; - error?: string; - /** Whether this agent used sandbox mode */ - usedSandbox?: boolean; + createdAt: number; + pid: number; } /** - * Run a single agent in a worktree + * Read tracking file atomically using rename trick */ -async function runAgentInWorktree( - engine: AIEngine, - task: Task, - agentNum: number, - baseBranch: string, - worktreeBase: string, - originalDir: string, - prdSource: string, - prdFile: string, - prdIsFolder: boolean, - maxRetries: number, - retryDelay: number, - skipTests: boolean, - skipLint: boolean, - browserEnabled: "auto" | "true" | "false", - modelOverride?: string, - engineArgs?: string[], -): Promise { - let worktreeDir = ""; - let branchName = ""; - +function readTrackingFile(wd: string): TrackedWorktree[] { + const trackingFile = join(wd, WORKTREE_TRACKING_FILE); + if (!existsSync(trackingFile)) return []; try { - // Create worktree - const worktree = await createAgentWorktree( - task.title, - agentNum, - baseBranch, - worktreeBase, - originalDir, - ); - worktreeDir = worktree.worktreeDir; - branchName = worktree.branchName; - - logDebug(`Agent ${agentNum}: Created worktree at ${worktreeDir}`); + return JSON.parse(readFileSync(trackingFile, "utf8")); + } catch (err) { + logWarn(`Tracking file unreadable/corrupted: ${err}`); + throw err; + } +} - // Copy PRD file or folder to worktree - if (prdSource === "markdown" || prdSource === "yaml" || prdSource === "json") { - const srcPath = join(originalDir, prdFile); - const destPath = join(worktreeDir, prdFile); - if (existsSync(srcPath)) { - copyFileSync(srcPath, destPath); - } - } else if (prdSource === "markdown-folder" && prdIsFolder) { - const srcPath = join(originalDir, prdFile); - const destPath = join(worktreeDir, prdFile); - if (existsSync(srcPath)) { - cpSync(srcPath, destPath, { recursive: true }); - } +/** + * Write tracking file atomically using rename (atomic on POSIX, best effort on Windows) + */ +function writeTrackingFile(wd: string, tracked: TrackedWorktree[]): void { + const trackingFile = join(wd, WORKTREE_TRACKING_FILE); + const trackingDir = join(wd, ".ralphy-worktrees"); + const tempFile = join(wd, `${WORKTREE_TRACKING_FILE}.tmp.${Date.now()}.${process.pid}`); + try { + if (!existsSync(trackingDir)) { + mkdirSync(trackingDir, { recursive: true }); + } + writeFileSync(tempFile, JSON.stringify(tracked, null, 2)); + renameSync(tempFile, trackingFile); + } catch (err) { + // Clean up temp file if it exists + try { + if (existsSync(tempFile)) unlinkSync(tempFile); + } catch { + /* ignore cleanup failure */ } + // Fallback: direct write if rename fails + try { + writeFileSync(trackingFile, JSON.stringify(tracked, null, 2)); + } catch (writeErr) { + logDebug(`Failed to write tracking file: ${writeErr}`); + } + } +} - // Ensure .ralphy/ exists in worktree - const ralphyDir = join(worktreeDir, RALPHY_DIR); - if (!existsSync(ralphyDir)) { - mkdirSync(ralphyDir, { recursive: true }); +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + const code = + error && typeof error === "object" && "code" in error + ? (error as { code?: string }).code + : undefined; + // EPERM means process exists but we don't have permission to signal it. + if (code === "EPERM") { + return true; } + return false; + } +} - // Build prompt - const prompt = buildParallelPrompt({ - task: task.title, - progressFile: PROGRESS_FILE, - prdFile, - skipTests, - skipLint, - browserEnabled, - }); +function isTrackedWorktreePathSafe(workDir: string, worktreeDir: string): boolean { + const worktreeBase = getWorktreeBase(workDir); + const baseResolved = resolve(worktreeBase); + const targetResolved = resolve(worktreeDir); + return targetResolved !== baseResolved && targetResolved.startsWith(`${baseResolved}${sep}`); +} - // Execute with retry - const engineOptions = { - ...(modelOverride && { modelOverride }), - ...(engineArgs && engineArgs.length > 0 && { engineArgs }), - }; - const result = await withRetry( - async () => { - const res = await engine.execute(prompt, worktreeDir, engineOptions); - if (!res.success && res.error && isRetryableError(res.error)) { - throw new Error(res.error); - } - return res; - }, - { maxRetries, retryDelay }, - ); +/** + * Track a worktree for cleanup tracking (persistent across process crashes) + */ +function trackWorktree(wd: string, worktreeDir: string, branchName: string): void { + try { + const tracked = readTrackingFile(wd); + tracked.push({ + worktreeDir, + branchName, + createdAt: Date.now(), + pid: process.pid, + }); + writeTrackingFile(wd, tracked); + } catch (err) { + logDebug(`Failed to track worktree: ${err}`); + } +} - return { task, agentNum, worktreeDir, branchName, result }; - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { task, agentNum, worktreeDir, branchName, result: null, error: errorMsg }; +/** + * Untrack a worktree after successful cleanup + */ +function untrackWorktree(wd: string, worktreeDir: string): void { + try { + const tracked = readTrackingFile(wd); + const filtered = tracked.filter((t) => t.worktreeDir !== worktreeDir); + writeTrackingFile(wd, filtered); + } catch (err) { + logDebug(`Failed to untrack worktree: ${err}`); } } /** - * Run a single agent in a lightweight sandbox. - * - * Sandboxes use symlinks for read-only dependencies (node_modules, .git, etc.) - * and copy source files. This is much faster than git worktrees for large repos. + * Cleanup orphaned worktrees from previous runs */ -async function runAgentInSandbox( - engine: AIEngine, - task: Task, - agentNum: number, - sandboxBase: string, - originalDir: string, - prdSource: string, - prdFile: string, - prdIsFolder: boolean, - maxRetries: number, - retryDelay: number, - skipTests: boolean, - skipLint: boolean, - browserEnabled: "auto" | "true" | "false", - modelOverride?: string, - engineArgs?: string[], -): Promise { - const uniqueSuffix = Math.random().toString(36).substring(2, 8); - const sandboxDir = join(sandboxBase, `agent-${agentNum}-${uniqueSuffix}`); - const branchName = ""; +async function cleanupOrphanedWorktrees(wd: string): Promise { + const trackingFile = join(wd, WORKTREE_TRACKING_FILE); + if (!existsSync(trackingFile)) return; try { - // Create sandbox - const sandboxResult = await createSandbox({ - originalDir, - sandboxDir, - agentNum, - }); + const tracked: TrackedWorktree[] = JSON.parse(readFileSync(trackingFile, "utf8")); - logDebug( - `Agent ${agentNum}: Created sandbox (${sandboxResult.symlinksCreated} symlinks, ${sandboxResult.filesCopied} copies)`, - ); + let cleaned = 0; + const remaining: TrackedWorktree[] = []; - // Copy PRD file or folder to sandbox (same as worktree mode) - if (prdSource === "markdown" || prdSource === "yaml" || prdSource === "json") { - const srcPath = join(originalDir, prdFile); - const destPath = join(sandboxDir, prdFile); - if (existsSync(srcPath)) { - copyFileSync(srcPath, destPath); + for (const entry of tracked) { + if (!isTrackedWorktreePathSafe(wd, entry.worktreeDir)) { + logWarn(`Skipping unsafe tracked worktree path: ${entry.worktreeDir}`); + continue; } - } else if (prdSource === "markdown-folder" && prdIsFolder) { - const srcPath = join(originalDir, prdFile); - const destPath = join(sandboxDir, prdFile); - if (existsSync(srcPath)) { - cpSync(srcPath, destPath, { recursive: true }); + + // Check if worktree directory still exists + if (existsSync(entry.worktreeDir)) { + if (isProcessAlive(entry.pid)) { + // Process still running, keep tracking + remaining.push(entry); + continue; + } + + logDebug(`Process ${entry.pid} not running, cleaning worktree`); + // Process dead - clean up stale worktree + try { + const git = simpleGit(wd); + try { + await git.raw(["worktree", "remove", "-f", entry.worktreeDir]); + if (entry.branchName) { + try { + await git.raw(["branch", "-D", entry.branchName]); + } catch (branchErr) { + logDebug(`Branch cleanup skipped for ${entry.branchName}: ${branchErr}`); + } + } + } catch (gitErr) { + logDebug(`Git worktree remove failed: ${gitErr}`); + if (existsSync(entry.worktreeDir)) { + rmSync(entry.worktreeDir, { recursive: true, force: true }); + } + } + cleaned++; + } catch (cleanupErr) { + logDebug(`Failed to cleanup worktree ${entry.worktreeDir}: ${cleanupErr}`); + // Keep in tracking if cleanup failed + remaining.push(entry); + } } } - // Ensure .ralphy/ exists in sandbox - const ralphyDir = join(sandboxDir, RALPHY_DIR); - if (!existsSync(ralphyDir)) { - mkdirSync(ralphyDir, { recursive: true }); + if (cleaned > 0) { + logInfo(`Cleaned up ${cleaned} orphaned worktrees`); } - // Build prompt - const prompt = buildParallelPrompt({ - task: task.title, - progressFile: PROGRESS_FILE, - prdFile, - skipTests, - skipLint, - browserEnabled, - allowCommit: false, + writeTrackingFile(wd, remaining); + } catch (err) { + logDebug(`Failed to cleanup orphaned worktrees: ${err}`); + } +} + +/** + * Acquire a global merge lock for cross-process coordination. + * Uses atomic file operations to prevent race conditions between simultaneous ralphy runs. + */ +function acquireGlobalMergeLock(workDir: string): { release: () => void } | null { + const lockDir = join(workDir, ".ralphy"); + const lockFile = join(lockDir, ".global-merge.lock"); + + if (!existsSync(lockDir)) { + mkdirSync(lockDir, { recursive: true }); + } + + try { + writeFileSync(lockFile, JSON.stringify({ pid: process.pid, timestamp: Date.now() }), { + flag: "wx", }); + logDebug(`Acquired global merge lock: ${lockFile}`); - // Execute with retry - const engineOptions = { - ...(modelOverride && { modelOverride }), - ...(engineArgs && engineArgs.length > 0 && { engineArgs }), - }; - const result = await withRetry( - async () => { - const res = await engine.execute(prompt, sandboxDir, engineOptions); - if (!res.success && res.error && isRetryableError(res.error)) { - throw new Error(res.error); + return { + release: () => { + try { + if (existsSync(lockFile)) { + unlinkSync(lockFile); + logDebug("Released global merge lock"); + } + } catch (err) { + logDebug(`Failed to release global merge lock: ${err}`); } - return res; }, - { maxRetries, retryDelay }, - ); - - return { task, agentNum, worktreeDir: sandboxDir, branchName, result, usedSandbox: true }; - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - return { - task, - agentNum, - worktreeDir: sandboxDir, - branchName, - result: null, - error: errorMsg, - usedSandbox: true, }; + } catch (_error) { + try { + if (existsSync(lockFile)) { + const staleFile = join(lockDir, `.stale-merge-${Date.now()}.lock`); + try { + renameSync(lockFile, staleFile); + + const content = readFileSync(staleFile, "utf8"); + let lockData: { timestamp?: number } = {}; + try { + const parsed = JSON.parse(content) as unknown; + if (parsed && typeof parsed === "object" && "timestamp" in parsed) { + const candidate = parsed as { timestamp?: unknown }; + if (typeof candidate.timestamp === "number") { + lockData = { timestamp: candidate.timestamp }; + } + } + } catch { + logWarn("Global merge lock contained invalid JSON, treating as stale"); + } + + const age = Date.now() - (lockData.timestamp || 0); + if (age > GLOBAL_MERGE_LOCK_TIMEOUT_MS) { + logWarn("Found stale global merge lock, removing and retrying..."); + unlinkSync(staleFile); + return acquireGlobalMergeLock(workDir); + } + + renameSync(staleFile, lockFile); + if (existsSync(staleFile)) { + unlinkSync(staleFile); + } + return null; + } catch { + logDebug("Another process acquired merge lock during stale check"); + return null; + } + } + } catch { + try { + const corruptFile = join(lockDir, `.corrupt-merge-${Date.now()}.lock`); + renameSync(lockFile, corruptFile); + unlinkSync(corruptFile); + return acquireGlobalMergeLock(workDir); + } catch (recoverErr) { + logDebug(`Failed to recover from corrupt lock: ${recoverErr}`); + } + } + + return null; + } +} + +/** + * Convert Task to PlannedTask for graph coloring by extracting file information. + * Uses planning analysis if available, otherwise defaults to empty file list. + */ +function taskToPlannedTask(task: Task, planningAnalysis?: string): PlannedTask { + let files: string[] = []; + + if (planningAnalysis) { + try { + const fileMatch = planningAnalysis.match(/files:?\s*\[([^\]]+)\]/i); + if (fileMatch) { + files = fileMatch[1].split(",").map((f) => f.trim().replace(/['"]/g, "")); + } + } catch (e) { + logDebug(`Failed to extract files from planning analysis: ${e}`); + } } + + return { + task, + files, + }; } /** * Run tasks in parallel using worktrees or sandboxes + * + * @param options - Execution options including maxParallel, taskSource, etc. + * @returns Execution result with completed/failed task counts */ export async function runParallel( options: ExecutionOptions & { @@ -244,6 +332,7 @@ export async function runParallel( prdSource: string; prdFile: string; prdIsFolder?: boolean; + taskStateManager?: TaskStateManager; }, ): Promise { const { @@ -266,14 +355,15 @@ export async function runParallel( skipMerge, useSandbox = false, engineArgs, - syncIssue, + debug, + debugOpenCode, + allowOpenCodeSandboxAccess, + planningModel, + testModel, + noGitParallel, + taskStateManager: externalTaskStateManager, } = options; - const shouldFallbackToSandbox = (error: string | undefined): boolean => { - if (!error) return false; - return error.includes(".git/worktrees") || error.toLowerCase().includes("invalid path"); - }; - const result: ExecutionResult = { tasksCompleted: 0, tasksFailed: 0, @@ -281,15 +371,43 @@ export async function runParallel( totalOutputTokens: 0, }; + // Cleanup orphaned worktrees from previous runs + await cleanupOrphanedWorktrees(workDir); + + // Initialize task state manager if not provided + let taskStateManager: TaskStateManager; + if (externalTaskStateManager) { + taskStateManager = externalTaskStateManager; + } else { + // Detect format from prdFile extension + const format: StateFormat = detectStateFormat(prdFile); + + taskStateManager = new TaskStateManager( + workDir, + taskSource.type, + prdFile || "tasks.yaml", + format, + ); + + // Get all tasks and initialize state manager + const allTasks = await taskSource.getAllTasks(); + await taskStateManager.initialize(allTasks); + } + // Determine isolation mode (worktree vs sandbox) let effectiveUseSandbox = useSandbox; + let worktreeFallbackToSandbox = false; if (!effectiveUseSandbox && !canUseWorktrees(workDir)) { logWarn("Worktrees unavailable in this repo; falling back to sandbox mode."); effectiveUseSandbox = true; + worktreeFallbackToSandbox = true; } + const effectiveNoGitParallel = + effectiveUseSandbox && (noGitParallel || worktreeFallbackToSandbox); const isolationBase = effectiveUseSandbox ? getSandboxBase(workDir) : getWorktreeBase(workDir); - logDebug(`${effectiveUseSandbox ? "Sandbox" : "Worktree"} base: ${isolationBase}`); + const isolationMode = effectiveUseSandbox ? "sandbox" : "worktree"; + logDebug(`${isolationMode} base: ${isolationBase}`); if (effectiveUseSandbox) { logInfo("Using lightweight sandbox mode (faster for large repos)"); @@ -306,89 +424,142 @@ export async function runParallel( // Global agent counter to ensure unique numbering across batches let globalAgentNum = 0; + const getNextAgentNum = () => ++globalAgentNum; // Track processed tasks in dry-run mode (since we don't modify the source file) const dryRunProcessedIds = new Set(); + // Static agent display for rich output + const staticAgentDisplay = StaticAgentDisplay.getInstance() || new StaticAgentDisplay(); + staticAgentDisplay.startDisplay(); + // Process tasks in batches let iteration = 0; - while (true) { - // Check iteration limit - if (maxIterations > 0 && iteration >= maxIterations) { - logInfo(`Reached max iterations (${maxIterations})`); - break; - } + try { + while (true) { + // Check iteration limit + if (maxIterations > 0 && iteration >= maxIterations) { + logInfo(`Reached max iterations (${maxIterations})`); + break; + } - // Get tasks for this batch - let tasks: Task[] = []; + // Get pending tasks from state manager + const pendingTasks = taskStateManager.getTasksByState(TaskState.PENDING); - const taskSourceWithGroups = taskSource as TaskSource & { - getParallelGroup?: (title: string) => Promise; - getTasksInGroup?: (group: number) => Promise; - }; + if (pendingTasks.length === 0) { + logSuccess("All tasks completed!"); + break; + } + + // Get all tasks from source to find full task objects + const allSourceTasks = await taskSource.getAllTasks(); - if (taskSourceWithGroups.getParallelGroup && taskSourceWithGroups.getTasksInGroup) { - let nextTask = await taskSource.getNextTask(); - if (dryRun && nextTask && dryRunProcessedIds.has(nextTask.id)) { - const allTasks = await taskSource.getAllTasks(); - nextTask = allTasks.find((task) => !dryRunProcessedIds.has(task.id)) || null; + // Map pending state entries to full task objects (single pass) + const tasks: Task[] = []; + for (const pt of pendingTasks) { + const task = allSourceTasks.find((t) => t.id === pt.id); + if (task) { + tasks.push(task); + } } - if (!nextTask) break; - const group = await taskSourceWithGroups.getParallelGroup(nextTask.title); - if (group > 0) { - tasks = await taskSourceWithGroups.getTasksInGroup(group); - if (dryRun) { - tasks = tasks.filter((task) => !dryRunProcessedIds.has(task.id)); + // Filter out already processed tasks in dry-run mode and tasks exceeding max attempts + const filteredTasks: Task[] = []; + for (const task of tasks) { + // Skip already processed tasks in dry-run mode + if (dryRun && dryRunProcessedIds.has(task.id)) { + continue; + } + + // Filter out tasks that have exceeded max attempts + if (taskStateManager.hasExceededMaxAttempts(task.id, maxRetries)) { + logWarn(`Task "${task.title}" has exceeded max attempts (${maxRetries}), skipping...`); + await taskStateManager.transitionState(task.id, TaskState.SKIPPED); + await taskSource.markComplete(task.id); + result.tasksFailed++; + notifyTaskFailed(task.title, "Exceeded maximum retry attempts"); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } else { + filteredTasks.push(task); } - } else { - tasks = [nextTask]; } - } else { - tasks = await taskSource.getAllTasks(); - if (dryRun) { - tasks = tasks.filter((task) => !dryRunProcessedIds.has(task.id)); + + if (filteredTasks.length === 0) { + // All tasks in this batch were skipped due to max attempts + continue; } - } - if (tasks.length === 0) { - logSuccess("All tasks completed!"); - break; - } + // Use graph coloring to select the next conflict-aware batch when file information is available. + let batch: Task[] = []; + const plannedTasks = filteredTasks.map((t) => taskToPlannedTask(t, t.body || "")); + const tasksWithFiles = plannedTasks.filter((pt) => pt.files.length > 0); + + if (tasksWithFiles.length === filteredTasks.length && tasksWithFiles.length > 1) { + logDebug("Using graph coloring to pick next conflict-aware batch..."); + const graph = buildConflictGraph(plannedTasks); + const colors = colorGraph(plannedTasks, graph); + const batches = batchByColor(plannedTasks, colors, maxParallel); + + const batchKeys = Array.from(batches.keys()).sort((a, b) => a - b); + if (batchKeys.length > 0) { + const firstBatch = batches.get(batchKeys[0]); + batch = firstBatch?.map((pt) => pt.task) || []; + } else { + batch = filteredTasks.slice(0, maxParallel); + } + } else { + batch = filteredTasks.slice(0, maxParallel); + } + iteration++; - // Limit to maxParallel - const batch = tasks.slice(0, maxParallel); - iteration++; + logInfo(`Batch ${iteration}: ${batch.length} tasks in parallel`); - const batchStartTime = Date.now(); - logInfo(`Batch ${iteration}: ${batch.length} tasks in parallel`); + if (dryRun && !options.debugOpenCode) { + logInfo("(dry run) Skipping batch"); + // Track processed tasks to avoid infinite loop + for (const task of batch) { + dryRunProcessedIds.add(task.id); + } + continue; + } - if (dryRun) { - logInfo("(dry run) Skipping batch"); - // Track processed tasks to avoid infinite loop + // Claim tasks for execution before starting + const claimedTasks: Array<{ task: Task; agentNum: number }> = []; for (const task of batch) { - dryRunProcessedIds.add(task.id); + // Pass maxRetries for atomic check-and-claim + const claimed = await taskStateManager.claimTaskForExecution(task.id); + if (claimed) { + const agentNum = getNextAgentNum(); + const initialPhase = planningModel ? "planning" : "execution"; + const initialModel = planningModel ? "planning" : "main"; + staticAgentDisplay.setAgentStatus( + agentNum, + task.title, + "working", + initialPhase, + initialModel, + ); + claimedTasks.push({ task, agentNum }); + } else { + logDebug( + `Task "${task.title}" is already being executed or exceeded max attempts, skipping...`, + ); + } } - continue; - } - - // Log task names being processed - for (const task of batch) { - logInfo(` -> ${task.title}`); - } - // Run agents in parallel (using sandbox or worktree mode) - const promises = batch.map((task) => { - globalAgentNum++; + if (claimedTasks.length === 0) { + // No tasks could be claimed, continue to next batch + continue; + } - const runInSandbox = () => - runAgentInSandbox( + // Parallel execution + const promises = claimedTasks.map(({ task, agentNum }) => { + const agentOptions: AgentRunnerOptions = { engine, task, - globalAgentNum, - getSandboxBase(workDir), - workDir, + agentNum, + originalDir: workDir, prdSource, prdFile, prdIsFolder, @@ -398,267 +569,327 @@ export async function runParallel( skipLint, browserEnabled, modelOverride, + planningModel, + testModel, engineArgs, - ); - - if (effectiveUseSandbox) { - return runInSandbox(); - } + env: options.env, + debug, + debugOpenCode, + allowOpenCodeSandboxAccess, + onWorktreeCreated: (worktreeDir, branchName) => { + trackWorktree(workDir, worktreeDir, branchName); + }, + logThoughts: options.logThoughts, + onProgress: (step) => { + // Detect OpenCode JSON and parse it properly + if ( + step.includes('"type":"tool_use"') || + step.includes('"type":"step_') || + step.includes('"type":"text"') + ) { + staticAgentDisplay.updateAgentFromOpenCode(agentNum, step); + } else { + staticAgentDisplay.updateAgent(agentNum, step); + } + }, + dryRun, + noGitParallel: effectiveNoGitParallel, + }; - return runAgentInWorktree( - engine, - task, - globalAgentNum, - baseBranch, - isolationBase, - workDir, - prdSource, - prdFile, - prdIsFolder, - maxRetries, - retryDelay, - skipTests, - skipLint, - browserEnabled, - modelOverride, - engineArgs, - ).then((res) => { - if (shouldFallbackToSandbox(res.error)) { - logWarn(`Agent ${globalAgentNum}: Worktree unavailable, retrying in sandbox mode.`); - if (res.worktreeDir) { - cleanupAgentWorktree(res.worktreeDir, res.branchName, workDir).catch(() => { - // Ignore cleanup failures during fallback - }); - } - return runInSandbox(); + if (effectiveUseSandbox) { + return runAgentInSandbox(getSandboxBase(workDir), agentOptions); } - return res; + + return runAgentInWorktree(getWorktreeBase(workDir), originalBaseBranch, agentOptions); }); - }); - const results = await Promise.all(promises); - - // Process results and collect worktrees for parallel cleanup - let sawRetryableFailure = false; - const worktreesToCleanup: Array<{ worktreeDir: string; branchName: string }> = []; - - for (const agentResult of results) { - const { - task, - agentNum, - worktreeDir, - result: aiResult, - error, - usedSandbox: agentUsedSandbox, - } = agentResult; - let branchName = agentResult.branchName; - let failureReason: string | undefined = error; - let retryableFailure = false; - let preserveSandbox = false; - - if (!failureReason && aiResult?.success && agentUsedSandbox && worktreeDir) { - try { - const modifiedFiles = await getModifiedFiles(worktreeDir, workDir); - if (modifiedFiles.length > 0) { - const commitResult = await commitSandboxChanges( - workDir, - modifiedFiles, - worktreeDir, - task.title, - agentNum, - originalBaseBranch, + const results = await Promise.allSettled(promises); + + // Process all results + let sawRetryableFailure = false; + const worktreesToCleanup: Array<{ worktreeDir: string; branchName: string }> = []; + const allErrors: Array<{ task: Task; error: string }> = []; + + // Helper to determine if a rejection is planning-related + const isPlanningRejection = (error: string): boolean => { + const planningKeywords = ["planning phase", "plan task files", "file analysis"]; + return planningKeywords.some((keyword) => error.toLowerCase().includes(keyword)); + }; + + for (let i = 0; i < results.length; i++) { + const res = results[i]; + // BUG FIX: Add bounds check to prevent undefined task access + const claimedTask = claimedTasks[i]; + const task = claimedTask?.task; + if (!task) { + logError(`Task index ${i} out of bounds (claimedTasks.length=${claimedTasks.length})`); + continue; + } + + if (res.status === "rejected") { + const error = res.reason; + const errorMessage = String(error); + allErrors.push({ task, error: String(error) }); + logError(`Task "${task.title}" failed: ${error}`); + + // Check if failure is planning-related + if (isPlanningRejection(error)) { + // Planning phase failed - transition to failed state but don't mark complete + logDebug( + `Planning phase failed for task "${task.title}", transitioning to FAILED state`, ); + await taskStateManager.transitionState(task.id, TaskState.FAILED, String(error)); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + continue; + } - if (commitResult.success) { - branchName = commitResult.branchName; - logDebug( - `Agent ${agentNum}: Committed ${commitResult.filesCommitted} files to ${branchName}`, - ); + const retryable = isRetryableError(errorMessage); + if (retryable) { + const deferrals = recordDeferredTask(taskSource.type, task, workDir, prdFile); + sawRetryableFailure = true; + if (deferrals >= maxRetries) { + logError(`Task "${task.title}" failed after ${deferrals} deferrals: ${errorMessage}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, errorMessage); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, errorMessage); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); } else { - failureReason = commitResult.error || "Failed to commit sandbox changes"; - preserveSandbox = true; // Preserve work for manual recovery + logWarn( + `Task "${task.title}" deferred (${deferrals}/${maxRetries}): ${errorMessage}`, + ); + await taskStateManager.transitionState(task.id, TaskState.DEFERRED, errorMessage); } + continue; } - } catch (commitErr) { - failureReason = commitErr instanceof Error ? commitErr.message : String(commitErr); - preserveSandbox = true; // Preserve work for manual recovery - } - } - if (failureReason) { - retryableFailure = isRetryableError(failureReason); - if (retryableFailure) { - const deferrals = recordDeferredTask(taskSource.type, task, workDir, prdFile); - if (deferrals >= maxRetries) { - logError(`Task "${task.title}" failed after ${deferrals} deferrals: ${failureReason}`); - logTaskProgress(task.title, "failed", workDir); - result.tasksFailed++; - notifyTaskFailed(task.title, failureReason); - await taskSource.markComplete(task.id); - clearDeferredTask(taskSource.type, task, workDir, prdFile); - retryableFailure = false; - } else { - logWarn(`Task "${task.title}" deferred (${deferrals}/${maxRetries}): ${failureReason}`); - result.tasksFailed++; - } - } else { - logError(`Task "${task.title}" failed: ${failureReason}`); + // Execution phase failure - transition to failed state + await taskStateManager.transitionState(task.id, TaskState.FAILED, errorMessage); logTaskProgress(task.title, "failed", workDir); result.tasksFailed++; - notifyTaskFailed(task.title, failureReason); - - // Mark failed task as complete to remove it from the queue - // This prevents infinite retry loops - the task has already been retried maxRetries times + notifyTaskFailed(task.title, errorMessage); await taskSource.markComplete(task.id); clearDeferredTask(taskSource.type, task, workDir, prdFile); + continue; } - } else if (aiResult?.success) { - logSuccess(`Task "${task.title}" completed`); - result.totalInputTokens += aiResult.inputTokens; - result.totalOutputTokens += aiResult.outputTokens; - await taskSource.markComplete(task.id); - logTaskProgress(task.title, "completed", workDir); - result.tasksCompleted++; - - notifyTaskComplete(task.title); - clearDeferredTask(taskSource.type, task, workDir, prdFile); + const agentResult = res.value; + const { + agentNum, + worktreeDir, + branchName, + result: aiResult, + error: failureReason, + usedSandbox: agentUsedSandbox, + } = agentResult; + + staticAgentDisplay.agentComplete(agentNum); + + let finalBranchName = branchName; + let finalFailureReason = failureReason; + let preserveSandbox = false; + + // Handle sandbox commit if successful + if (!finalFailureReason && aiResult?.success && agentUsedSandbox && worktreeDir) { + try { + const modifiedFiles = await getModifiedFiles(worktreeDir, workDir); + if (modifiedFiles.length > 0) { + const commitResult = await commitSandboxChanges( + workDir, + modifiedFiles, + worktreeDir, + task.title, + agentNum, + originalBaseBranch, + ); - // Track successful branch for merge phase - if (branchName) { - completedBranches.push(branchName); + if (commitResult.success) { + finalBranchName = commitResult.branchName; + logDebug( + `Agent ${agentNum}: Committed ${commitResult.filesCommitted} files to ${finalBranchName}`, + ); + } else { + finalFailureReason = + commitResult.error && + typeof commitResult.error === "object" && + "message" in commitResult.error + ? (commitResult.error as { message: string }).message + : String(commitResult.error); + preserveSandbox = true; + logWarn(`Sandbox commit failed: ${finalFailureReason}`); + } + } + } catch (commitErr) { + finalFailureReason = commitErr instanceof Error ? commitErr.message : String(commitErr); + preserveSandbox = true; + logDebug(`Sandbox commit error for task "${task.title}": ${commitErr}`); + } } - } else { - const errMsg = aiResult?.error || "Unknown error"; - retryableFailure = isRetryableError(errMsg); - if (retryableFailure) { - const deferrals = recordDeferredTask(taskSource.type, task, workDir, prdFile); - if (deferrals >= maxRetries) { - logError(`Task "${task.title}" failed after ${deferrals} deferrals: ${errMsg}`); + + if (finalFailureReason) { + const retryable = isRetryableError(finalFailureReason); + if (retryable) { + const deferrals = recordDeferredTask(taskSource.type, task, workDir, prdFile); + sawRetryableFailure = true; + if (deferrals >= maxRetries) { + logError( + `Task "${task.title}" failed after ${deferrals} deferrals: ${finalFailureReason}`, + ); + await taskStateManager.transitionState(task.id, TaskState.FAILED, finalFailureReason); + logTaskProgress(task.title, "failed", workDir); + result.tasksFailed++; + notifyTaskFailed(task.title, finalFailureReason); + await taskSource.markComplete(task.id); + clearDeferredTask(taskSource.type, task, workDir, prdFile); + } else { + logWarn( + `Task "${task.title}" deferred (${deferrals}/${maxRetries}): ${finalFailureReason}`, + ); + await taskStateManager.transitionState( + task.id, + TaskState.DEFERRED, + finalFailureReason, + ); + } + } else { + logError(`Task "${task.title}" failed: ${finalFailureReason}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, finalFailureReason); logTaskProgress(task.title, "failed", workDir); result.tasksFailed++; - notifyTaskFailed(task.title, errMsg); - failureReason = errMsg; + notifyTaskFailed(task.title, finalFailureReason); await taskSource.markComplete(task.id); clearDeferredTask(taskSource.type, task, workDir, prdFile); - retryableFailure = false; - } else { - logWarn(`Task "${task.title}" deferred (${deferrals}/${maxRetries}): ${errMsg}`); - result.tasksFailed++; - failureReason = errMsg; } - } else { - logError(`Task "${task.title}" failed: ${errMsg}`); - logTaskProgress(task.title, "failed", workDir); - result.tasksFailed++; - notifyTaskFailed(task.title, errMsg); - failureReason = errMsg; - - // Mark failed task as complete to remove it from the queue - // This prevents infinite retry loops - the task has already been retried maxRetries times + } else if (aiResult?.success) { + logSuccess(`Task "${task.title}" completed`); + result.totalInputTokens += aiResult.inputTokens; + result.totalOutputTokens += aiResult.outputTokens; + + await taskStateManager.transitionState(task.id, TaskState.COMPLETED, undefined, { + branch: finalBranchName || undefined, + }); await taskSource.markComplete(task.id); + logTaskProgress(task.title, "completed", workDir); + result.tasksCompleted++; + notifyTaskComplete(task.title); clearDeferredTask(taskSource.type, task, workDir, prdFile); + + if (finalBranchName) { + completedBranches.push(finalBranchName); + } } - } - // Cleanup sandbox inline or collect worktree for parallel cleanup - if (worktreeDir) { - if (agentUsedSandbox) { - if (failureReason || preserveSandbox) { - logWarn(`Sandbox preserved for manual review: ${worktreeDir}`); + // Cleanup + if (worktreeDir) { + if (agentUsedSandbox) { + if (finalFailureReason || preserveSandbox) { + logWarn(`Sandbox preserved for manual review: ${worktreeDir}`); + } else { + await cleanupSandbox(worktreeDir); + logDebug(`Cleaned up sandbox: ${worktreeDir}`); + } } else { - // Sandbox cleanup is simpler - just delete the directory - await cleanupSandbox(worktreeDir); - logDebug(`Cleaned up sandbox: ${worktreeDir}`); + worktreesToCleanup.push({ worktreeDir, branchName: finalBranchName || "" }); } - } else { - // Collect worktree for parallel cleanup below - worktreesToCleanup.push({ worktreeDir, branchName }); } } - if (retryableFailure) { - sawRetryableFailure = true; - } - } - - // Cleanup all worktrees in parallel - if (worktreesToCleanup.length > 0) { - const cleanupResults = await Promise.all( - worktreesToCleanup.map(({ worktreeDir, branchName }) => - cleanupAgentWorktree(worktreeDir, branchName, workDir).then((cleanup) => ({ - worktreeDir, - leftInPlace: cleanup.leftInPlace, - })), - ), - ); + // Cleanup all worktrees in parallel with coordination + if (worktreesToCleanup.length > 0) { + const cleanupResults = await Promise.allSettled( + worktreesToCleanup.map(({ worktreeDir, branchName }) => + cleanupAgentWorktree(worktreeDir, branchName, workDir).then((cleanup) => ({ + worktreeDir, + leftInPlace: cleanup.leftInPlace, + })), + ), + ); - // Log any worktrees left in place - for (const { worktreeDir, leftInPlace } of cleanupResults) { - if (leftInPlace) { - logInfo(`Worktree left in place (uncommitted changes): ${worktreeDir}`); + for (let i = 0; i < cleanupResults.length; i++) { + const result = cleanupResults[i]; + const { worktreeDir } = worktreesToCleanup[i]; + if (result.status === "fulfilled") { + if (result.value.leftInPlace) { + logInfo(`Worktree left in place (uncommitted changes): ${worktreeDir}`); + } else { + // Successfully cleaned up - remove from tracking + untrackWorktree(workDir, worktreeDir); + } + } else { + logWarn(`Failed to cleanup worktree ${worktreeDir}: ${result.reason}`); + } } } - } - - // Sync PRD to GitHub issue once per batch (after all tasks processed) - // This prevents multiple concurrent syncs and reduces API calls - if (syncIssue && prdFile && result.tasksCompleted > 0) { - await syncPrdToIssue(prdFile, syncIssue, workDir); - } - - // Log batch completion time - const batchDuration = formatDuration(Date.now() - batchStartTime); - logInfo(`Batch ${iteration} completed in ${batchDuration}`); - // If any retryable failure occurred, stop the run to allow retry later - if (sawRetryableFailure) { - logWarn("Stopping early due to retryable errors. Try again later."); - break; - } - } - // Merge phase: merge completed branches back to base branch - if (!skipMerge && !dryRun && completedBranches.length > 0) { - const git = simpleGit(workDir); - let stashed = false; - try { - const status = await git.status(); - const hasChanges = status.files.length > 0 || status.not_added.length > 0; - if (hasChanges) { - await git.stash(["push", "-u", "-m", "ralphy-merge-stash"]); - stashed = true; - logDebug("Stashed local changes before merge phase"); + if (sawRetryableFailure) { + logWarn("Stopping early due to retryable errors. Try again later."); + break; } - } catch (stashErr) { - logWarn(`Failed to stash local changes: ${stashErr}`); } - try { - await mergeCompletedBranches( - completedBranches, - originalBaseBranch, - engine, - workDir, - modelOverride, - engineArgs, - ); - - // Restore starting branch if we're not already on it - const currentBranch = await getCurrentBranch(workDir); - if (currentBranch !== startingBranch) { - logDebug(`Restoring starting branch: ${startingBranch}`); - await returnToBaseBranch(startingBranch, workDir); + // Merge phase: merge completed branches back to base branch + if (!skipMerge && !dryRun && completedBranches.length > 0) { + // CRITICAL FIX: Use global cross-process lock for merge coordination + const globalLock = acquireGlobalMergeLock(workDir); + if (!globalLock) { + logWarn("Could not acquire global merge lock, another ralphy instance may be merging"); + logInfo("Skipping merge phase - branches are preserved for manual merge"); + logInfo(`Branches to merge: ${completedBranches.join(", ")}`); + // BUG FIX: Stop display before early return to prevent resource leak + staticAgentDisplay.stopDisplay(); + // Don't throw - just skip the merge and preserve branches + return result; } - } finally { - if (stashed) { + + try { + const git = simpleGit(workDir); + let stashed = false; try { - await git.stash(["pop"]); - logDebug("Restored stashed changes after merge phase"); + const status = await git.status(); + const hasChanges = status.files.length > 0 || status.not_added.length > 0; + if (hasChanges) { + await git.stash(["push", "-u", "-m", "ralphy-merge-stash"]); + stashed = true; + logDebug("Stashed local changes before merge phase"); + } } catch (stashErr) { - logWarn(`Failed to restore stashed changes: ${stashErr}`); + logWarn(`Failed to stash local changes: ${stashErr}`); + } + + try { + await mergeCompletedBranches( + completedBranches, + originalBaseBranch, + engine, + workDir, + modelOverride, + engineArgs, + ); + + const currentBranch = await getCurrentBranch(workDir); + if (currentBranch !== startingBranch) { + logDebug(`Restoring starting branch: ${startingBranch}`); + await returnToBaseBranch(startingBranch, workDir); + } + } finally { + if (stashed) { + try { + await git.stash(["pop"]); + logDebug("Restored local changes after merge phase"); + } catch (popErr) { + logWarn(`Failed to restore local changes: ${popErr}`); + } + } } + } finally { + globalLock.release(); } } + } finally { + // Stop static display + staticAgentDisplay.stopDisplay(); } return result; @@ -666,12 +897,6 @@ export async function runParallel( /** * Merge completed branches back to the base branch. - * - * Optimized merge phase: - * 1. Parallel pre-merge analysis (git diff doesn't require locks) - * 2. Sort branches by conflict likelihood (merge clean ones first) - * 3. Sequential merges (git locking requirement) - * 4. Parallel branch deletion */ async function mergeCompletedBranches( branches: string[], @@ -685,26 +910,30 @@ async function mergeCompletedBranches( return; } - const mergeStartTime = Date.now(); logInfo(`\nMerge phase: merging ${branches.length} branch(es) into ${targetBranch}`); - // Stage 1: Parallel pre-merge analysis - // Run git diff for all branches in parallel (doesn't require locks) logDebug("Analyzing branches for potential conflicts..."); - const analyses = await Promise.all( + const analysesResults = await Promise.allSettled( branches.map((branch) => analyzePreMerge(branch, targetBranch, workDir)), ); + const analyses = analysesResults + .map((result, index) => { + if (result.status === "fulfilled") { + return result.value; + } + logWarn(`Failed to analyze branch ${branches[index]}: ${result.reason}`); + return null; + }) + .filter((a): a is NonNullable => a !== null); - // Stage 2: Sort by conflict likelihood (merge clean ones first) - // This reduces the chance of early conflicts blocking later clean merges const sortedAnalyses = sortByConflictLikelihood(analyses); const sortedBranches = sortedAnalyses.map((a) => a.branch); - if (sortedBranches[0] !== branches[0]) { + // BUG FIX: Check array bounds before accessing first element + if (sortedBranches.length > 0 && branches.length > 0 && sortedBranches[0] !== branches[0]) { logDebug("Reordered branches to minimize conflicts"); } - // Stage 3: Sequential merges (git operations require this) const merged: string[] = []; const failed: string[] = []; @@ -719,7 +948,6 @@ async function mergeCompletedBranches( logSuccess(`Merged ${branch}`); merged.push(branch); } else if (mergeResult.hasConflicts && mergeResult.conflictedFiles) { - // Try AI-assisted conflict resolution logWarn(`Merge conflict in ${branch}, attempting AI resolution...`); const resolved = await resolveConflictsWithAI( @@ -745,27 +973,29 @@ async function mergeCompletedBranches( } } - // Stage 4: Parallel branch deletion - // Delete all successfully merged branches in parallel if (merged.length > 0) { - const deleteResults = await Promise.all( + const deleteResults = await Promise.allSettled( merged.map(async (branch) => { const deleted = await deleteLocalBranch(branch, workDir, true); return { branch, deleted }; }), ); - for (const { branch, deleted } of deleteResults) { - if (deleted) { - logDebug(`Deleted merged branch: ${branch}`); + for (let i = 0; i < deleteResults.length; i++) { + const result = deleteResults[i]; + const branch = merged[i]; + if (result.status === "fulfilled") { + if (result.value.deleted) { + logDebug(`Deleted merged branch: ${branch}`); + } + } else { + logWarn(`Failed to delete branch ${branch}: ${result.reason}`); } } } - // Summary - const mergeDuration = formatDuration(Date.now() - mergeStartTime); if (merged.length > 0) { - logSuccess(`Successfully merged ${merged.length} branch(es) in ${mergeDuration}`); + logSuccess(`Successfully merged ${merged.length} branch(es)`); } if (failed.length > 0) { logWarn(`Failed to merge ${failed.length} branch(es): ${failed.join(", ")}`); diff --git a/cli/src/execution/planning.ts b/cli/src/execution/planning.ts new file mode 100644 index 00000000..dc1e4b50 --- /dev/null +++ b/cli/src/execution/planning.ts @@ -0,0 +1,613 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + lstatSync, + readFileSync, + readdirSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { isAbsolute, join, normalize } from "node:path"; +import { gunzipSync, gzipSync } from "node:zlib"; +import { DEFAULT_MAX_REPLANS, PLANNING_CACHE_FILE } from "../config/constants.ts"; +import { RALPHY_DIR } from "../config/loader.ts"; +import type { AIEngine, AIResult } from "../engines/types.ts"; +import type { Task } from "../tasks/types.ts"; +import { logDebug, logWarn } from "../ui/logger.ts"; +import { extractTaskKeywords, getRelevantFilesForTask } from "../utils/file-indexer.ts"; +import type { PlanningProgressCallback, PlanningProgressEvent } from "./progress-types.ts"; +import { buildPlanningPrompt } from "./prompt.ts"; + +// Re-export PlanningProgressEvent from ui module for backward compatibility +export type { PlanningProgressCallback, PlanningProgressEvent } from "./progress-types.ts"; + +export function getPlanningCacheFile(workDir: string): string { + return join(workDir, RALPHY_DIR, PLANNING_CACHE_FILE); +} + +interface RepoFingerprint { + fileStates: Map; + dirHash: string; + timestamp: number; +} + +const fingerprintCache = new Map(); + +export function generateRepoFingerprint(workDir: string): string { + const cached = fingerprintCache.get(workDir); + const now = Date.now(); + + // Check if cache is very recent (1 minute) for high-frequency calls + if (cached && now - cached.timestamp < 60000) { + return cached.dirHash; + } + + const keyFiles = [ + "package.json", + "pyproject.toml", + "Cargo.toml", + "go.mod", + "requirements.txt", + "pnpm-lock.yaml", + "package-lock.json", + "yarn.lock", + ]; + const fileStates = new Map(); + let changed = !cached; + + for (const file of keyFiles) { + const filePath = join(workDir, file); + if (existsSync(filePath)) { + try { + const stat = lstatSync(filePath); + const mtime = stat.mtimeMs; + const size = stat.size; + + const cachedState = cached?.fileStates.get(file); + if (cachedState && cachedState.mtime === mtime && cachedState.size === size) { + fileStates.set(file, cachedState); + } else { + const content = readFileSync(filePath); + const hash = createHash("sha256").update(content).digest("hex"); + fileStates.set(file, { mtime, size, hash }); + changed = true; + } + } catch { + // Ignore errors + } + } + } + + // Also factor in top-level directory structure changes + let dirFingerprint = ""; + try { + const entries = readdirSync(workDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); + dirFingerprint = entries.join(","); + if (cached && cached.fileStates?.get("dirs")?.hash !== dirFingerprint) { + changed = true; + } + fileStates.set("dirs", { mtime: 0, size: 0, hash: dirFingerprint }); + } catch { + // Ignore errors + } + + if (!changed && cached) { + // Update timestamp but keep dirHash + cached.timestamp = now; + return cached.dirHash; + } + + const combinedHashes = Array.from(fileStates.entries()) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([file, state]) => `${file}:${state.hash}`) + .join("|"); + + const dirHash = createHash("sha256").update(combinedHashes).digest("hex"); + + fingerprintCache.set(workDir, { + fileStates, + dirHash, + timestamp: now, + }); + + return dirHash; +} + +export function loadPlanningCache( + workDir: string, +): Map { + const cacheFile = getPlanningCacheFile(workDir); + const compressedCacheFile = `${cacheFile}.gz`; + + if (existsSync(compressedCacheFile)) { + try { + const compressed = readFileSync(compressedCacheFile); + const data = JSON.parse(gunzipSync(compressed).toString("utf-8")); + return new Map(Object.entries(data)); + } catch (error) { + logWarn(`Failed to load compressed planning cache: ${error}`); + // Fall through + } + } + + if (!existsSync(cacheFile)) { + return new Map(); + } + + try { + const data = JSON.parse(readFileSync(cacheFile, "utf-8")); + return new Map(Object.entries(data)); + } catch (error) { + logWarn(`Failed to load planning cache: ${error}`); + return new Map(); + } +} + +export function savePlanningCache( + workDir: string, + cache: Map, +): void { + const cacheFile = getPlanningCacheFile(workDir); + const compressedCacheFile = `${cacheFile}.gz`; + const data = Object.fromEntries(cache); + const jsonStr = JSON.stringify(data); + + try { + const compressed = gzipSync(Buffer.from(jsonStr, "utf-8")); + writeFileSync(compressedCacheFile, compressed); + + if (existsSync(cacheFile)) { + try { + unlinkSync(cacheFile); + } catch {} + } + } catch { + writeFileSync(cacheFile, JSON.stringify(data, null, 2)); + } +} + +export function generateTaskHash(task: Task): string { + const raw = `${task.id}:${task.title}`; + return createHash("sha256").update(raw).digest("hex").slice(0, 16); +} + +export function normalizePlannedPath(filePath: string): string { + let processed = filePath.trim(); + + // Strip leading bullets (*, -, +) + processed = processed.replace(/^[*\-+]\s+/, ""); + + // Strip leading numbering (1., 1), etc.) + processed = processed.replace(/^\d+[.)]\s+/, ""); + + // Strip wrapping backticks if present + processed = processed.replace(/^`+|`+$/g, ""); + + // Remove leading ./ + if (processed.startsWith("./")) { + processed = processed.substring(2); + } + + // Normalize path separators + processed = normalize(processed); + + if (!processed || isAbsolute(processed) || processed.startsWith("..")) { + return ""; + } + + return processed; +} + +export function parsePlannedFiles(response: string): string[] { + const files = new Set(); + + // Robust Regex approach for blocks + const filesMatch = response.match(/([\s\S]*?)<\/FILES>/i); + if (filesMatch) { + const content = filesMatch[1]; + const lines = content.split(/\r?\n/); + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed && !trimmed.startsWith("#") && !trimmed.startsWith("<")) { + const normalizedPath = normalizePlannedPath(trimmed); + if (normalizedPath) { + files.add(normalizedPath); + } + } + } + } else { + // Fallback: look for lines that look like paths if the block tags are missing/broken + const lines = response.split(/\r?\n/); + let inManualBlock = false; + for (const line of lines) { + const trimmed = line.trim(); + if ( + trimmed.toUpperCase().includes("FILES") && + (trimmed.includes("<") || trimmed.includes("[")) + ) { + inManualBlock = true; + continue; + } + if (inManualBlock && trimmed === "") continue; + if ( + inManualBlock && + (trimmed.startsWith("/") || + trimmed.startsWith("./") || + trimmed.startsWith("../") || + /^[a-zA-Z0-9_\-.]+\/[a-zA-Z0-9_\-./]+/.test(trimmed)) + ) { + const normalizedPath = normalizePlannedPath(trimmed); + if (normalizedPath) { + files.add(normalizedPath); + } + } + } + } + + return Array.from(files); +} + +function parseEnhancedPlanning(response: string): { + analysis?: string; + plan?: string[]; + optimization?: string; +} { + // Use robust regex approach for tags + const analysisMatch = response.match(/([\s\S]*?)<\/ANALYSIS>/i); + const planMatch = response.match(/([\s\S]*?)<\/PLAN>/i); + const optimizationMatch = response.match(/([\s\S]*?)<\/OPTIMIZATION>/i); + + const analysis = analysisMatch ? analysisMatch[1].trim() : undefined; + const optimization = optimizationMatch ? optimizationMatch[1].trim() : undefined; + + let plan: string[] | undefined; + if (planMatch) { + const content = planMatch[1]; + const lines = content.split(/\r?\n/); + const planSteps: string[] = []; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed) { + // Parse numbered steps (strip bullet points if present) + let stepText = trimmed; + if (stepText.startsWith("- ")) { + stepText = stepText.substring(2); + } + const stepMatch = stepText.match(/^\d+\.\s*(.*)/); + if (stepMatch) { + planSteps.push(stepMatch[1]); + } else if (!line.startsWith("<") && !line.startsWith(" 0) { + planSteps.push(stepText); + } + } + } + if (planSteps.length > 0) { + plan = planSteps; + } + } + + return { + analysis, + plan, + optimization, + }; +} + +export interface PlanningResult { + files: string[]; + analysis?: string; + plan?: string[]; + optimization?: string; + noFilesNeeded?: boolean; + error?: string; +} + +export async function planTaskFiles( + engine: AIEngine, + task: Task, + workDir: string, + modelOverride?: string, + maxReplans = DEFAULT_MAX_REPLANS, + planningModel?: string, + fullTasksContext?: string, + debug?: boolean, + onProgress?: PlanningProgressCallback, + debugOpenCode?: boolean, + logThoughts?: boolean, + engineArgs?: string[], +): Promise { + const taskId = task.title && task.title !== "No title" ? task.title : task.id || "unknown"; + + // Use semantic chunking to get relevant files for this task + let relevantFiles: string[] = []; + try { + const taskDescription = `${task.title || ""} ${task.description || ""}`; + relevantFiles = await getRelevantFilesForTask(workDir, taskDescription, { + maxFiles: 50, + minRelevance: 0.1, + }); + logDebug(`Semantic chunking found ${relevantFiles.length} relevant files for task "${taskId}"`); + + // Log extracted keywords for debugging + const keywords = extractTaskKeywords(taskDescription); + logDebug(`Task keywords: ${keywords.join(", ")}`); + } catch (error) { + logDebug(`Failed to get relevant files for task: ${error}`); + // Continue without semantic chunking - planning will use full codebase + } + + // Build prompt with relevant files context if available + const prompt = buildPlanningPrompt(task, fullTasksContext, relevantFiles); + + // Emit planning started + if (onProgress) { + try { + onProgress({ + taskId, + status: "started", + timestamp: Date.now(), + message: + relevantFiles.length > 0 + ? `Planning with ${relevantFiles.length} relevant files...` + : "Planning...", + }); + } catch (error) { + // Don't let progress callback errors break planning + logDebug(`Progress callback error: ${error}`); + } + } + + // Use planningModel if provided, otherwise default to modelOverride or engine default + const options = { + modelOverride: planningModel || modelOverride || undefined, + ...(debugOpenCode && { debugOpenCode }), + ...(logThoughts !== undefined && { logThoughts }), + ...(engineArgs && engineArgs.length > 0 && { engineArgs }), + }; + + let result: AIResult; + if (onProgress && engine.executeStreaming) { + // Emit starting status + try { + onProgress({ + taskId, + status: "started", + timestamp: Date.now(), + message: "Starting planning analysis...", + }); + } catch (error) { + logDebug(`Progress callback error: ${error}`); + } + + // Create wrapper for streaming progress + const streamingCallback = (step: string) => { + try { + // Parse step to determine status and extract meaningful action + let status: PlanningProgressEvent["status"] = "thinking"; + let message = step; + + // Detect specific actions for better display + if (step.includes("analyzing") || step.includes("I need to") || step.includes("I should")) { + status = "analyzing"; + } else if ( + step.includes("planning") || + step.includes("I'll create") || + step.includes("Let me create") + ) { + status = "planning"; + } else if ( + step.includes("Reading") || + step.includes("Looking at") || + step.includes("Let me examine") + ) { + status = "analyzing"; + message = "Reading project structure and files"; + } else if ( + step.includes("identifying") || + step.includes("found") || + step.includes("need to modify") + ) { + status = "planning"; + message = "Identifying files that need changes"; + } else if (step.includes("completed") || step.includes("done") || step.includes("ready")) { + status = "completed"; + message = "Planning complete - ready to implement"; + } else if (step.includes("failed") || step.includes("error")) { + status = "failed"; + message = "Planning encountered an issue"; + } + + // Extract reward if present in step (e.g., "reward: 0.85") + const rewardMatch = step.match(/reward:\s*([0-9.]+)/i); + const reward = rewardMatch ? Number.parseFloat(rewardMatch[1]) : undefined; + + onProgress({ + taskId, + status, + reward, + message: message, + timestamp: Date.now(), + }); + } catch (error) { + logDebug(`Streaming progress callback error: ${error}`); + } + }; + result = await engine.executeStreaming(prompt, workDir, streamingCallback, options); + } else { + // Non-streaming: emit thinking status before execution + if (onProgress) { + try { + onProgress({ + taskId, + status: "thinking", + timestamp: Date.now(), + message: "Processing planning request...", + }); + } catch (error) { + logDebug(`Progress callback error: ${error}`); + } + } + result = await engine.execute(prompt, workDir, options); + } + + if (!result.success) { + const rawResponse = result.response || ""; + const error = result.error || "Planning failed"; + + // Detect if AI returned raw tool_use JSON instead of structured planning + const isRawToolUse = rawResponse.trim().startsWith('{"type":"tool_use"'); + + if (isRawToolUse) { + // Try to extract what file/tool the AI was trying to access + let toolInfo = ""; + try { + const parsed = JSON.parse(rawResponse); + if (parsed.part?.tool) { + toolInfo = ` (tool: ${parsed.part.tool})`; + } + if (parsed.part?.state?.input?.filePath) { + toolInfo += ` file: ${parsed.part.state.input.filePath}`; + } + } catch { + // Ignore parse errors + } + + const helpfulError = `Planning failed: AI returned tool output${toolInfo} instead of planning analysis. The AI may have started executing prematurely. This usually indicates the planning prompt was too complex or the AI engine interrupted the planning phase.`; + logDebug( + `Raw tool use detected instead of planning format. Response: ${rawResponse.substring(0, 500)}...`, + ); + + if (onProgress) { + try { + onProgress({ + taskId, + status: "failed", + timestamp: Date.now(), + message: helpfulError, + }); + } catch (err) { + logDebug(`Progress callback error: ${err}`); + } + } + + if (maxReplans > 0) { + logDebug( + `Planning failed with malformed response, retrying... (${maxReplans} attempts left)`, + ); + return planTaskFiles( + engine, + task, + workDir, + modelOverride, + maxReplans - 1, + planningModel, + fullTasksContext, + debug, + onProgress, + debugOpenCode, + logThoughts, + engineArgs, + ); + } + return { files: [], error: helpfulError }; + } + + // Regular failure - emit failed status + if (onProgress) { + try { + onProgress({ + taskId, + status: "failed", + timestamp: Date.now(), + message: error, + }); + } catch (err) { + logDebug(`Progress callback error: ${err}`); + } + } + + if (maxReplans > 0) { + // Check if this is a connection error that warrants a longer retry delay + const isConnectionError = + /connection|network|timeout|unable to connect|internet connection|econnrefused|econnreset/i.test( + error, + ); + const attemptNumber = DEFAULT_MAX_REPLANS - maxReplans + 1; + + if (isConnectionError) { + // Exponential backoff for connection errors: 2s, 4s, 8s + const delayMs = Math.min(2000 * 2 ** (attemptNumber - 1), 10000); + logWarn( + `Connection error detected. Retrying in ${delayMs}ms... (${maxReplans} attempts left)`, + ); + + if (onProgress) { + try { + onProgress({ + taskId, + status: "thinking", + timestamp: Date.now(), + message: `Connection error. Retrying in ${delayMs}ms... (${maxReplans} attempts left)`, + }); + } catch (err) { + logDebug(`Progress callback error: ${err}`); + } + } + + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } else { + logDebug(`Planning failed, retrying... (${maxReplans} attempts left). Error: ${error}`); + } + + return planTaskFiles( + engine, + task, + workDir, + modelOverride, + maxReplans - 1, + planningModel, + fullTasksContext, + debug, + onProgress, + debugOpenCode, + logThoughts, + engineArgs, + ); + } + return { files: [], error }; + } + + const files = parsePlannedFiles(result.response || ""); + const parsed = parseEnhancedPlanning(result.response || ""); + + // Emit completed status + if (onProgress) { + try { + onProgress({ + taskId, + status: "completed", + timestamp: Date.now(), + message: `Planned ${files.length} files with ${parsed.plan?.length || 0} steps`, + metadata: { + fileCount: files.length, + files: files.slice(0, 10), + hasAnalysis: !!parsed.analysis, + hasPlan: !!parsed.plan, + hasOptimization: !!parsed.optimization, + }, + }); + } catch (error) { + logDebug(`Progress callback error: ${error}`); + } + } + + return { + files, + analysis: parsed.analysis, + plan: parsed.plan, + optimization: parsed.optimization, + }; +} diff --git a/cli/src/execution/progress-types.ts b/cli/src/execution/progress-types.ts new file mode 100644 index 00000000..9dba9c07 --- /dev/null +++ b/cli/src/execution/progress-types.ts @@ -0,0 +1,42 @@ +/** + * High-level execution phase - stable throughout the workflow + */ +export type ExecutionPhase = "planning" | "execution" | "testing"; + +/** + * Detailed current activity - for display purposes only, shown below + */ +export type CurrentActivity = "analyzing" | "reading" | "writing" | "thinking" | "running-tests" | "debugging" | "idle"; + +export interface AgentProgress { + agentNum: number; + taskTitle: string; + worktreeDir: string; + status: "planning" | "working" | "completed" | "failed"; + /** High-level phase: PLANNING → EXECUTION → TESTING */ + phase?: ExecutionPhase; + /** Which model is currently running (e.g., "main", "planning", "test") */ + modelName?: string; + /** Detailed current action shown below */ + currentActivity?: string; + progress?: string; + currentStep?: string; + recentSteps?: string[]; + /** Steps the agent plans to do (extracted from agent's output) */ + plannedSteps?: string[]; + /** The model's thought pipeline - what it's thinking, goals, what it needs to do */ + thoughtPipeline?: string[]; + startTime: number; +} + + +export interface PlanningProgressEvent { + taskId: string; + status: "started" | "thinking" | "completed" | "error" | string; + timestamp: number; + message?: string; + metadata?: Record; + reward?: number; +} + +export type PlanningProgressCallback = (event: PlanningProgressEvent) => void; diff --git a/cli/src/execution/prompt.ts b/cli/src/execution/prompt.ts index 19690778..3162e534 100644 --- a/cli/src/execution/prompt.ts +++ b/cli/src/execution/prompt.ts @@ -1,7 +1,60 @@ -import { existsSync } from "node:fs"; +import type { Dirent } from "node:fs"; +import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; +import process from "node:process"; import { loadBoundaries, loadProjectContext, loadRules } from "../config/loader.ts"; +import type { Task } from "../tasks/types.ts"; +import { logDebug } from "../ui/logger.ts"; import { getBrowserInstructions, isBrowserAvailable } from "./browser.ts"; +import { getSkillsAsCsv } from "./skill-compress.ts"; + +// ============================================================================= +// CONSTANTS +// ============================================================================= + +const RALPHY_PROTECTED_PATHS = [ + ".ralphy/progress.txt", + ".ralphy-worktrees", + ".ralphy-sandboxes", +] as const; + +const SKILL_DIRECTORIES = [".opencode/skills", ".claude/skills", ".skills"] as const; + +const PLANNING_SECTIONS = [ + "", + " - Problem: [What is the actual problem being solved?]", + " - Goal: [What is the desired end state?]", + " - Complexity: [low/medium/high]", + " - Risks: [Potential challenges or edge cases]", + "", + "", + "", + "1. [Step 1: What to do first]", + "2. [Step 2: Analysis or research needed]", + "3. [Step 3: Implementation approach]", + "4. [Step 4: Testing/validation]", + "5. [Step 5: Final integration or cleanup]", + "", + "", + "", + "path/to/file1.ext", + "path/to/file2.ext", + "...", + "", + "", + "", + " - Most efficient approach: [How to implement this optimally]", + " - Key considerations: [Technical factors to remember]", + " - Potential shortcuts: [Ways to accomplish this faster/better]", + "", +] as const; + +// Default rules that should always be included +const DEFAULT_RULES = ["Keep changes focused and minimal. Do not refactor unrelated code."]; + +// ============================================================================= +// TYPES +// ============================================================================= interface PromptOptions { task: string; @@ -11,226 +64,522 @@ interface PromptOptions { skipTests?: boolean; skipLint?: boolean; prdFile?: string; + progressFile?: string; } -/** - * Detect skill/playbook directories that can guide the agent. - * We keep this engine-agnostic: OpenCode can load skills via `skill` tool, - * other engines can still read these docs as repo guidance. - */ -function detectAgentSkills(workDir: string): string[] { - const candidates = [ - join(workDir, ".opencode", "skills"), - join(workDir, ".claude", "skills"), - join(workDir, ".github", "skills"), - join(workDir, ".skills"), - ]; +interface ParallelPromptOptions extends PromptOptions { + allowCommit?: boolean; + planningAnalysis?: string; + planningSteps?: string[]; + enableOrchestrator?: boolean; +} - return candidates.filter((p) => existsSync(p)); +interface EnvironmentInfo { + language?: string; + framework?: string; + buildTool?: string; + testFramework?: string; + projectType?: string; + packageManager?: string; } -/** - * Build the full prompt with project context, rules, boundaries, and task - */ -export function buildPrompt(options: PromptOptions): string { - const { - task, - autoCommit = true, - workDir = process.cwd(), - browserEnabled = "auto", - skipTests = false, - skipLint = false, - prdFile, - } = options; +// ============================================================================= +// CACHE +// ============================================================================= - const parts: string[] = []; +const envCache = new Map(); - // Add project context if available - const context = loadProjectContext(workDir); - if (context) { - parts.push(`## Project Context\n${context}`); +// ============================================================================= +// ENVIRONMENT DETECTION +// ============================================================================= + +export function detectEnvironment(workDir: string): EnvironmentInfo { + const cached = envCache.get(workDir); + if (cached) return cached; + + const result: EnvironmentInfo = {}; + + const packageJsonPath = join(workDir, "package.json"); + if (existsSync(packageJsonPath)) { + try { + const pkg = JSON.parse(readFileSync(packageJsonPath, "utf-8")); + Object.assign(result, extractEnvironmentInfo(pkg)); + } catch (error) { + logDebug(`Failed to parse package.json: ${error}`); + } } - // Add rules if available - const rules = loadRules(workDir); - const codeChangeRules = [ - "Keep changes focused and minimal. Do not refactor unrelated code.", - "One logical change per commit. If a task is too large, break it into subtasks.", - "Write concise code. Avoid over-engineering.", - "Don't leave dead code. Delete unused code completely.", - "Quality over speed. Small steps compound into big progress.", - ...rules, - ]; - if (codeChangeRules.length > 0) { - parts.push( - `## Rules (you MUST follow these)\n${codeChangeRules.map((r) => `- ${r}`).join("\n")}`, - ); + if (existsSync(join(workDir, "pyproject.toml"))) { + result.language = "Python"; + result.buildTool = "setuptools/poetry"; + result.packageManager = "pip/poetry"; + } else if (existsSync(join(workDir, "go.mod"))) { + result.language = "Go"; + result.packageManager = "go mod"; + } else if (existsSync(join(workDir, "Cargo.toml"))) { + result.language = "Rust"; + result.packageManager = "cargo"; } - // Add boundaries - combine system boundaries with user-defined boundaries - // System boundaries come first to ensure they are prominently visible - const userBoundaries = loadBoundaries(workDir); - const systemBoundaries = [ - prdFile || "the PRD file", - ".ralphy/progress.txt", - ".ralphy-worktrees", - ".ralphy-sandboxes", - ]; - const allBoundaries = [...systemBoundaries, ...userBoundaries]; - parts.push( - `## Boundaries\nDo NOT modify these files/directories:\n${allBoundaries.map((b) => `- ${b}`).join("\n")}`, - ); + envCache.set(workDir, result); + return result; +} - // Agent skills/playbooks (optional) - const skillRoots = detectAgentSkills(workDir); - if (skillRoots.length > 0) { - parts.push( - [ - "## Agent Skills", - "This repo includes skill/playbook docs that describe preferred patterns, workflows, or tooling:", - ...skillRoots.map((p) => `- ${p}`), - "", - "Before you start coding:", - "- Read and follow any relevant skill docs from the paths above.", - "- If your engine supports a `skill` tool (e.g. OpenCode), use it to load the relevant skills before implementing.", - "- If none apply, continue normally.", - ].join("\n"), +function extractEnvironmentInfo(pkg: { + dependencies?: Record; + devDependencies?: Record; + scripts?: Record; + private?: boolean; + workspaces?: unknown; + bin?: unknown; + bun?: unknown; + packageManager?: string; +}): Partial { + const deps = { ...pkg.dependencies, ...pkg.devDependencies }; + const scripts = pkg.scripts || {}; + + return { + language: detectLanguage(deps), + framework: detectFramework(pkg.dependencies || {}), + buildTool: detectBuildTool(scripts), + testFramework: detectTestFramework(deps, scripts), + projectType: detectProjectType(pkg), + packageManager: detectPackageManager(pkg), + }; +} + +function detectLanguage(deps: Record): string | undefined { + if (deps.typescript || deps["@types/node"] || deps["@types/react"]) { + return "TypeScript/JavaScript"; + } + if (deps.react || deps.vue || deps.angular || deps.express || deps.fastify) { + return "TypeScript/JavaScript"; + } + const pyPackages = ["numpy", "pandas", "django", "flask", "fastapi", "pytest"]; + if (pyPackages.some((p) => deps[p])) return "Python"; + return undefined; +} + +function detectFramework(deps: Record): string | undefined { + if (deps.next) return "Next.js"; + if (deps.nuxt) return "Nuxt.js"; + if (deps["@remix-run/react"]) return "Remix"; + if (deps["@astrojs/astro"]) return "Astro"; + if (deps.react || deps["react-dom"]) return "React"; + if (deps.vue) return "Vue.js"; + if (deps.svelte) return "Svelte"; + if (deps.angular) return "Angular"; + if (deps.express) return "Express.js"; + if (deps.fastify) return "Fastify"; + return undefined; +} + +function detectBuildTool(scripts: Record): string | undefined { + if (scripts.vite || scripts["vite build"]) return "Vite"; + if (scripts.webpack) return "Webpack"; + if (scripts.rollup) return "Rollup"; + if (scripts.esbuild) return "esbuild"; + if (scripts["next build"]) return "Next.js Build"; + if (scripts["nuxt build"]) return "Nuxt.js Build"; + if (scripts.tsc || scripts.build?.includes("tsc")) return "TypeScript Compiler"; + if (scripts.build?.includes("bun")) return "Bun"; + return undefined; +} + +function detectTestFramework( + deps: Record, + scripts: Record, +): string | undefined { + if (deps.vitest || scripts.test?.includes("vitest")) return "Vitest"; + if (deps.jest || scripts.test?.includes("jest")) return "Jest"; + if (deps.cypress) return "Cypress"; + if (deps["@playwright/test"]) return "Playwright"; + if (deps.pytest) return "Pytest"; + return undefined; +} + +function detectProjectType(pkg: { + private?: boolean; + workspaces?: unknown; + bin?: unknown; +}): string | undefined { + if (pkg.private) return "Private Package"; + if (pkg.workspaces) return "Monorepo"; + if (pkg.bin) return "CLI Tool/Library"; + return undefined; +} + +function detectPackageManager(pkg: { bun?: unknown; packageManager?: string }): string { + if (pkg.bun) return "Bun"; + if (pkg.packageManager?.startsWith("pnpm")) return "pnpm"; + if (pkg.packageManager?.startsWith("yarn")) return "Yarn"; + return "npm"; +} + +// ============================================================================= +// UTILITY FUNCTIONS +// ============================================================================= + +function detectSymlinks(workDir: string): string[] { + if (!existsSync(workDir)) return []; + + let dirents: Dirent[]; + try { + dirents = readdirSync(workDir, { withFileTypes: true }) as Dirent[]; + } catch { + return []; + } + + return dirents + .filter((d) => { + try { + return lstatSync(join(workDir, d.name as string)).isSymbolicLink(); + } catch { + return false; + } + }) + .map((d) => d.name as string); +} + +function buildEnvironmentSection(workDir: string): string { + const env = detectEnvironment(workDir); + const lines: string[] = []; + + const envFields = [ + ["Language", env.language], + ["Framework", env.framework], + ["Build Tool", env.buildTool], + ["Test Framework", env.testFramework], + ["Project Type", env.projectType], + ["Package Manager", env.packageManager], + ].filter(([, val]) => val) as [string, string][]; + + if (envFields.length > 0) { + lines.push("## Environment Detection", ""); + for (const [label, value] of envFields) { + lines.push(`**${label}:** ${value}`); + } + lines.push( + "", + "Use this information to:", + "- Choose appropriate build/test commands based on detected framework", + "- Consider framework-specific patterns and best practices", + "- Understand project structure and conventions", + "", ); } - // Add browser instructions if available - if (isBrowserAvailable(browserEnabled)) { - parts.push(getBrowserInstructions()); + const symlinks = detectSymlinks(workDir); + if (symlinks.length > 0) { + lines.push( + "## Symlink Analysis", + "", + `**Detected ${symlinks.length} symlink(s):**`, + ...symlinks.map((s) => `- ${s}`), + "", + "Note: Symlinks can affect file system operations and tool behavior.", + "", + ); } - // Add the task - parts.push(`## Task\n${task}`); + return lines.join("\n"); +} - // Add instructions - const instructions = ["1. Implement the task described above"]; +function buildSkillsSection(workDir: string): string { + const skillsCsv = getSkillsAsCsv(workDir); + if (skillsCsv) { + return `## Agent Skills +This repo includes compressed skill/playbook documentation for token efficiency: +${skillsCsv} + +Before you start coding: +- Read and follow any relevant skill docs from compressed list above. +- If your engine supports a \`skill\` tool (e.g. OpenCode), use it to load relevant skills before implementing. +- If none apply, continue normally.`; + } + + const skillRoots = SKILL_DIRECTORIES.map((dir) => join(workDir, dir)).filter(existsSync); + if (skillRoots.length > 0) { + return `## Agent Skills +This repo includes skill/playbook docs that describe preferred patterns, workflows, or tooling: +${skillRoots.map((p) => `- ${p}`).join("\n")} + +Before you start coding: +- Read and follow any relevant skill docs from paths above. +- If your engine supports a \`skill\` tool (e.g. OpenCode), use it to load relevant skills before implementing. +- If none apply, continue normally.`; + } + + return ""; +} + +function buildInstructions(options: { + skipTests: boolean; + skipLint: boolean; + autoCommit: boolean; + progressFile: string; +}): string[] { + const { skipTests, skipLint, autoCommit, progressFile } = options; + const instructions: string[] = []; + let step = 1; + + instructions.push(`${step++}. Implement the task described above`); - let step = 2; if (!skipTests) { - instructions.push(`${step}. Write tests for the feature`); - step++; - instructions.push(`${step}. Run tests and ensure they pass before proceeding`); - step++; + instructions.push(`${step++}. Write tests for the feature`); + instructions.push(`${step++}. Run tests and ensure they pass before proceeding`); } if (!skipLint) { - instructions.push(`${step}. Run linting and ensure it passes`); - step++; + instructions.push(`${step++}. Run linting and ensure it passes`); } - instructions.push(`${step}. Ensure the code works correctly`); - step++; + instructions.push(`${step++}. Update ${progressFile} with what you did`); if (autoCommit) { - instructions.push(`${step}. Commit your changes with a descriptive message`); + instructions.push(`${step++}. Commit your changes with a descriptive message`); + } else { + instructions.push(`${step++}. Do NOT run git commit; changes will be collected automatically`); } - parts.push(`## Instructions\n${instructions.join("\n")}`); + return instructions; +} - return parts.join("\n\n"); +function buildProtectedPathsWarning(prdFile?: string, boundaries: string[] = []): string { + const systemPaths = [ + `- ${prdFile || "the PRD file"}`, + ...RALPHY_PROTECTED_PATHS.map((p) => `- ${p}`), + ]; + const userPaths = boundaries.map((b) => (b.startsWith("- ") ? b : `- ${b}`)); + return [...systemPaths, ...userPaths].join("\n"); } -interface ParallelPromptOptions { - task: string; - progressFile: string; - prdFile?: string; - workDir?: string; - skipTests?: boolean; - skipLint?: boolean; - browserEnabled?: "auto" | "true" | "false"; - allowCommit?: boolean; +// ============================================================================= +// MAIN PROMPT BUILDERS +// ============================================================================= + +export function buildPrompt(options: PromptOptions): string { + const { + task, + autoCommit = true, + workDir = process.cwd(), + browserEnabled = "auto", + skipTests = false, + skipLint = false, + prdFile, + progressFile = "progress.txt", + } = options; + + const instructions = buildInstructions({ skipTests, skipLint, autoCommit, progressFile }); + const boundaries = loadBoundaries(workDir); + const sections = [ + buildEnvironmentSection(workDir), + buildContextSection(workDir), + buildSkillsSection(workDir), + isBrowserAvailable(browserEnabled) ? getBrowserInstructions() : "", + `## Boundaries\nDo NOT modify these files/directories:\n${buildProtectedPathsWarning(prdFile, boundaries)}`, + `## Task\n${task}`, + `## Instructions\n${instructions.join("\n")}`, + ].filter(Boolean); + + return `You are working on a specific task. Focus ONLY on this task: + +TASK: ${task} + +${sections.join("\n\n")} + +Protected paths are listed in the Boundaries section. +Do NOT Read, Glob, or Search inside .ralphy-sandboxes or .ralphy-worktrees. +Do NOT mark tasks complete - that will be handled separately. +Focus only on implementing: ${task}`; } -/** - * Build a prompt for parallel agent execution - */ -export function buildParallelPrompt(options: ParallelPromptOptions): string { +function buildContextSection(workDir: string): string { + const context = loadProjectContext(workDir); + const rules = loadRules(workDir); + + const sections: string[] = []; + if (context) sections.push(`## Project Context\n${context}`); + + // Always include rules section with default rules + const allRules = [...DEFAULT_RULES, ...rules]; + sections.push(`## Rules (you MUST follow these)\n${allRules.join("\n")}`); + + // Boundaries are included in the protected paths warning section. + + return sections.join("\n\n"); +} + +export function buildExecutionPrompt(options: ParallelPromptOptions): string { const { task, progressFile, prdFile, - workDir = process.cwd(), skipTests = false, skipLint = false, browserEnabled = "auto", allowCommit = true, + planningAnalysis, + planningSteps, + enableOrchestrator, + workDir = process.cwd(), } = options; + const instructions = buildInstructions({ + skipTests, + skipLint, + autoCommit: allowCommit, + progressFile: progressFile || ".progress.json", + }); - // Parallel execution typically runs in a worktree - const skillRoots = detectAgentSkills(workDir); - const skillsSection = - skillRoots.length > 0 - ? `\n\nAgent Skills:\nThis repo includes skill/playbook docs:\n${skillRoots - .map((p) => `- ${p}`) - .join( - "\n", - )}\nBefore coding, read relevant skills. If your engine supports a \`skill\` tool, load them before implementing.` - : ""; - - const browserSection = isBrowserAvailable(browserEnabled) - ? `\n\n${getBrowserInstructions()}` - : ""; - - // Load rules from config + const context = loadProjectContext(workDir); const rules = loadRules(workDir); - const codeChangeRules = [ - "Keep changes focused and minimal. Do not refactor unrelated code.", - "One logical change per commit. If a task is too large, break it into subtasks.", - "Write concise code. Avoid over-engineering.", - "Don't leave dead code. Delete unused code completely.", - "Quality over speed. Small steps compound into big progress.", - ...rules, - ]; - const rulesSection = - codeChangeRules.length > 0 - ? `\n\nRules (you MUST follow these):\n${codeChangeRules.map((r) => `- ${r}`).join("\n")}` - : ""; - - // Build boundaries section - combine system boundaries with user-defined boundaries - // System boundaries come first to ensure they are prominently visible - const userBoundaries = loadBoundaries(workDir); + const boundaries = loadBoundaries(workDir); + + // Build sections in the order tests expect + const sections: string[] = []; + + // Task at the top + sections.push(`TASK: ${task}`); + + // Environment section + const envSection = buildEnvironmentSection(workDir); + if (envSection) sections.push(envSection); + + // Context section + if (context) sections.push(`## Project Context\n${context}`); + + // Rules section with specific format for tests + const allRules = [...DEFAULT_RULES, ...rules]; + sections.push(`Rules (you MUST follow these):\n${allRules.join("\n")}`); + + // Boundaries section with specific format for tests - system first, then user const systemBoundaries = [ - prdFile || "the PRD file", - ".ralphy/progress.txt", - ".ralphy-worktrees", - ".ralphy-sandboxes", + `- ${prdFile || "the PRD file"}`, + "- .ralphy/progress.txt", + "- .ralphy-worktrees", + "- .ralphy-sandboxes", ]; + const userBoundaries = boundaries.map((b) => (b.startsWith("- ") ? b : `- ${b}`)); const allBoundaries = [...systemBoundaries, ...userBoundaries]; - const boundariesSection = `\n\nBoundaries - Do NOT modify:\n${allBoundaries.map((b) => `- ${b}`).join("\n")}\n\nDo NOT mark tasks complete - that will be handled separately.`; - - const instructions = ["1. Implement this specific task completely"]; + sections.push(`Boundaries - Do NOT modify:\n${allBoundaries.join("\n")}`); - let step = 2; - if (!skipTests) { - instructions.push(`${step}. Write tests for the feature`); - step++; - instructions.push(`${step}. Run tests and ensure they pass before proceeding`); - step++; + // Planning section if provided + if (planningAnalysis && planningSteps) { + sections.push(buildPlanningSection(planningAnalysis, planningSteps)); } - if (!skipLint) { - instructions.push(`${step}. Run linting and ensure it passes`); - step++; + // Skills section + const skillsSection = buildSkillsSection(workDir); + if (skillsSection) sections.push(skillsSection); + + // Browser instructions + if (isBrowserAvailable(browserEnabled)) { + sections.push(getBrowserInstructions()); } - instructions.push(`${step}. Update ${progressFile} with what you did`); - step++; - if (allowCommit) { - instructions.push(`${step}. Commit your changes with a descriptive message`); - } else { - instructions.push(`${step}. Do NOT run git commit; changes will be collected automatically`); + // Instructions section with specific format for tests + const instructionLines = instructions.map((line) => + line.replace("Implement the task described above", "Implement this specific task completely"), + ); + sections.push(`Instructions:\n${instructionLines.join("\n")}`); + + // Orchestrator section if enabled + if (enableOrchestrator) { + sections.push(buildOrchestratorSection()); } return `You are working on a specific task. Focus ONLY on this task: -TASK: ${task}${rulesSection}${boundariesSection}${browserSection}${skillsSection} - -Instructions: -${instructions.join("\n")} +${sections.join("\n\n")} +Do NOT mark tasks complete - that will be handled separately. Focus only on implementing: ${task}`; } + +function buildPlanningSection(analysis: string, steps: string[]): string { + return `## Planning Analysis (Completed Earlier) +${analysis} + +## Planned Implementation Steps +${steps.map((s, i) => `${i + 1}. ${s}`).join("\n")} + +Follow these steps. If they don't apply to the current situation, explain why and propose an alternative approach.`; +} + +function buildOrchestratorSection(): string { + return `## Test Delegation (Orchestrator Mode Enabled) + +You have access to a specialized test model. When you need tests run, use these markers: + +### Quick Test Request +Use [RUN_TESTS] or [RUN_TESTS:command] to request tests: +- \`[RUN_TESTS]\` - Run default test command +- \`[RUN_TESTS:npm test]\` - Run specific command + +### Detailed Test Request +For complex testing scenarios, use: +\`\`\` +[TEST_REQUEST] +command: npm test -- --grep "feature name" +files: src/feature.ts, tests/feature.test.ts +context: Brief context about what to test +[/TEST_REQUEST] +\`\`\` + +### Completion +When done, signal completion with: +\`\`\` +[TEST_COMPLETE] +Your final summary here +[/TEST_COMPLETE] +\`\`\` + +The test model will analyze results and return them to you. You can iterate: implement → request tests → review results → fix → request tests again.`; +} + +export function buildPlanningPrompt( + task: Task, + fullTasksContext?: string, + relevantFiles?: string[], +): string { + const relevantFilesSection = relevantFiles?.length + ? `\nRELEVANT FILES (prioritize these in your analysis):\n${relevantFiles + .slice(0, 30) + .map((f) => `- ${f}`) + .join("\n")}\n` + : ""; + + return `You are a senior engineering planner. Your job is to create a comprehensive plan for this task. + +TASK: ${task.title || task.id} +${task.description ? `DESCRIPTION: ${task.description}` : ""} +${task.dependencies?.length ? `DEPENDENCIES: ${task.dependencies.join(", ")}` : ""} +${relevantFilesSection} + +${fullTasksContext ? `FULL PROJECT TASKS CONTEXT:\n${fullTasksContext}\n\n` : ""} + +First, analyze this task thoroughly and provide structured output in this format: + +${PLANNING_SECTIONS.join("\n")} + +IMPORTANT INSTRUCTIONS FOR PLANNING PHASE: +1. You may use read/glob/grep tools to EXPLORE the codebase and understand the task +2. DO NOT write, edit, create, or modify any files during planning +3. DO NOT execute any implementation - this is a planning-only phase +4. After exploring, return the structured plan above in your final response +5. Your entire response must contain the , , , and tags +6. Return ONLY the planning analysis, not partial results from tool exploration + +Think step by step, explaining your reasoning clearly. Use tools to explore the codebase before finalizing your plan.`; +} + +// Backward compatibility +export function buildParallelPrompt(options: ParallelPromptOptions): string { + const { planningAnalysis, planningSteps, ...rest } = options; + + if (planningAnalysis && planningSteps) { + return buildExecutionPrompt({ ...rest, planningAnalysis, planningSteps }); + } + + return buildExecutionPrompt(rest); +} diff --git a/cli/src/execution/retry.ts b/cli/src/execution/retry.ts index 9eb4f293..a1d95bec 100644 --- a/cli/src/execution/retry.ts +++ b/cli/src/execution/retry.ts @@ -1,15 +1,175 @@ -import { logDebug, logWarn } from "../ui/logger.ts"; +import { logDebug, logError, logWarn } from "../ui/logger.ts"; +import { isRetryableError, standardizeError } from "../utils/errors.ts"; interface RetryOptions { maxRetries: number; - retryDelay: number; // base delay in seconds - onRetry?: (attempt: number, error?: string, nextDelayMs?: number) => void; - /** Use exponential backoff (default: true) */ + retryDelay: number; // in seconds + onRetry?: (attempt: number, error: string, delayMs: number) => void; + /** Enable exponential backoff for connection errors */ exponentialBackoff?: boolean; /** Maximum delay in seconds (default: 60) */ maxDelay?: number; /** Add random jitter to delay (default: true) */ jitter?: boolean; + /** Optional task ID for tracking connection state */ + taskId?: string; +} + +/** + * Circuit breaker states + */ +type CircuitState = "CLOSED" | "OPEN" | "HALF_OPEN"; + +interface CircuitBreakerConfig { + /** Number of failures before opening the circuit */ + failureThreshold: number; + /** Time in ms before attempting to close the circuit */ + resetTimeoutMs: number; + /** Half-open max attempts to test if service recovered */ + halfOpenMaxAttempts: number; +} + +/** + * Connection state manager to track global connection health + * Prevents infinite retries when connection is consistently failing + */ +class ConnectionStateManager { + private static instance: ConnectionStateManager; + private circuitState: CircuitState = "CLOSED"; + private consecutiveFailures = 0; + private lastFailureTime: number | null = null; + + private halfOpenAttempts = 0; + + private readonly config: CircuitBreakerConfig = { + failureThreshold: 3, // Open after 3 consecutive failures + resetTimeoutMs: 30000, // Wait 30s before trying again + halfOpenMaxAttempts: 2, // Try 2 times in half-open state + }; + + static getInstance(): ConnectionStateManager { + if (!ConnectionStateManager.instance) { + ConnectionStateManager.instance = new ConnectionStateManager(); + } + return ConnectionStateManager.instance; + } + + /** + * Check if we should attempt a request (circuit allows it) + */ + canAttempt(): { allowed: boolean; reason?: string } { + const now = Date.now(); + + switch (this.circuitState) { + case "CLOSED": + return { allowed: true }; + + case "OPEN": { + // Check if we should transition to half-open + if (this.lastFailureTime && now - this.lastFailureTime > this.config.resetTimeoutMs) { + this.circuitState = "HALF_OPEN"; + this.halfOpenAttempts = 0; + logWarn("Circuit breaker entering HALF_OPEN state - testing connection..."); + return { allowed: true }; + } + const remainingMs = this.config.resetTimeoutMs - (now - (this.lastFailureTime || 0)); + return { + allowed: false, + reason: `Connection circuit OPEN - too many failures. Waiting ${Math.ceil(remainingMs / 1000)}s before retry...`, + }; + } + + case "HALF_OPEN": + if (this.halfOpenAttempts >= this.config.halfOpenMaxAttempts) { + // BUG FIX: Too many attempts in half-open, go back to open and BLOCK the request + this.circuitState = "OPEN"; + this.lastFailureTime = now; + return { + allowed: false, + reason: `Connection circuit OPEN - service still unavailable after ${this.config.halfOpenMaxAttempts} test attempts`, + }; + } + this.halfOpenAttempts++; + return { allowed: true }; + } + } + + /** + * Record a successful request + */ + recordSuccess(): void { + if (this.circuitState === "HALF_OPEN") { + // Success in half-open closes the circuit + this.circuitState = "CLOSED"; + this.consecutiveFailures = 0; + this.halfOpenAttempts = 0; + logWarn("Circuit breaker CLOSED - connection restored"); + } else { + this.consecutiveFailures = 0; + } + } + + /** + * Record a failed request + */ + recordFailure(error: Error): void { + const isConnectionError = this.isConnectionRelatedError(error); + + if (!isConnectionError) { + // Non-connection errors don't affect circuit breaker + return; + } + + this.consecutiveFailures++; + this.lastFailureTime = Date.now(); + + if (this.circuitState === "HALF_OPEN") { + // Failure in half-open goes back to open + this.circuitState = "OPEN"; + logWarn( + `Circuit breaker OPEN - connection failed in half-open state (failure ${this.consecutiveFailures})`, + ); + } else if (this.consecutiveFailures >= this.config.failureThreshold) { + this.circuitState = "OPEN"; + logError( + `Circuit breaker OPEN - ${this.consecutiveFailures} consecutive connection failures. Stopping retries for ${this.config.resetTimeoutMs / 1000}s`, + ); + } + } + + /** + * Check if error is connection-related + */ + private isConnectionRelatedError(error: Error): boolean { + return ( + isRetryableError(error) && + /connection|network|timeout|unable to connect|internet connection|econnrefused|econnreset|socket hang up|dns|ENOTFOUND/i.test( + error.message, + ) + ); + } + + /** + * Get current circuit state for debugging + */ + getState(): { state: CircuitState; consecutiveFailures: number; lastFailureTime: number | null } { + return { + state: this.circuitState, + consecutiveFailures: this.consecutiveFailures, + lastFailureTime: this.lastFailureTime, + }; + } + + /** + * Force reset the circuit (for manual recovery) + */ + reset(): void { + this.circuitState = "CLOSED"; + this.consecutiveFailures = 0; + this.halfOpenAttempts = 0; + this.lastFailureTime = null; + logWarn("Circuit breaker manually reset to CLOSED"); + } } /** @@ -20,42 +180,82 @@ export function sleep(ms: number): Promise { } /** - * Calculate delay with exponential backoff and optional jitter - * - * @param attempt - Current attempt number (1-based) - * @param baseDelayMs - Base delay in milliseconds - * @param maxDelayMs - Maximum delay cap in milliseconds - * @param useJitter - Add random jitter (0-25% of delay) + * Global circuit breaker instance + */ +export const circuitBreaker = ConnectionStateManager.getInstance(); + +/** + * Check if connection is healthy enough to attempt requests */ -export function calculateBackoffDelay( +export function canMakeConnectionAttempt(): { allowed: boolean; reason?: string } { + return circuitBreaker.canAttempt(); +} + +/** + * Reset connection circuit breaker (for manual recovery) + */ +export function resetConnectionCircuit(): void { + circuitBreaker.reset(); +} + +/** + * Get current connection health status + */ +export function getConnectionHealth(): { + state: CircuitState; + consecutiveFailures: number; + lastFailureTime: number | null; +} { + return circuitBreaker.getState(); +} + +/** + * Calculate delay with exponential backoff for connection errors + */ +function calculateDelay( + baseDelaySeconds: number, attempt: number, - baseDelayMs: number, - maxDelayMs: number, + error: Error, + exponentialBackoff: boolean, + maxDelaySeconds: number, useJitter: boolean, ): number { - // Exponential backoff: baseDelay * 2^(attempt-1) - let delay = baseDelayMs * Math.pow(2, attempt - 1); + const maxDelayMs = maxDelaySeconds * 1000; + const baseDelayMs = baseDelaySeconds * 1000; + + if (!exponentialBackoff) { + const delay = Math.min(baseDelayMs, maxDelayMs); + if (!useJitter) return delay; + const jitter = Math.floor(delay * 0.25 * Math.random()); + return Math.min(delay + jitter, maxDelayMs); + } - // Cap at maximum delay - delay = Math.min(delay, maxDelayMs); + // Check if this is a connection/network error + const isConnectionError = + isRetryableError(error) && + /connection|network|timeout|unable to connect|internet connection|econnrefused|econnreset|socket hang up/i.test( + error.message, + ); - // Add jitter (0-25% of delay) to prevent thundering herd - if (useJitter) { - const jitter = delay * 0.25 * Math.random(); - delay += jitter; + if (isConnectionError) { + // Exponential backoff: 2s, 4s, 8s, max 30s + let delayMs = Math.min(2000 * 2 ** (attempt - 1), maxDelayMs); + if (useJitter) { + delayMs = Math.min(delayMs + Math.floor(delayMs * 0.25 * Math.random()), maxDelayMs); + } + logDebug(`Connection error detected, using exponential backoff: ${delayMs}ms`); + return delayMs; } - return Math.floor(delay); + let delay = Math.min(baseDelayMs, maxDelayMs); + if (useJitter) { + delay = Math.min(delay + Math.floor(delay * 0.25 * Math.random()), maxDelayMs); + } + return delay; } /** - * Execute a function with retry logic and exponential backoff - * - * Features: - * - Exponential backoff (2^attempt * baseDelay) - * - Optional jitter to prevent thundering herd - * - Configurable maximum delay cap - * - Progress callbacks with next delay info + * Execute a function with retry logic and circuit breaker */ export async function withRetry(fn: () => Promise, options: RetryOptions): Promise { const { @@ -65,32 +265,66 @@ export async function withRetry(fn: () => Promise, options: RetryOptions): exponentialBackoff = true, maxDelay = 60, jitter = true, + taskId, } = options; - - const baseDelayMs = retryDelay * 1000; - const maxDelayMs = maxDelay * 1000; let lastError: Error | null = null; + // Check circuit breaker before attempting + const circuitCheck = circuitBreaker.canAttempt(); + if (!circuitCheck.allowed) { + logError(`Circuit breaker preventing retry: ${circuitCheck.reason}`); + throw new Error(circuitCheck.reason || "Connection circuit open - too many failures"); + } + for (let attempt = 1; attempt <= maxRetries; attempt++) { try { - return await fn(); + const result = await fn(); + // Success - record it to close circuit if in half-open + circuitBreaker.recordSuccess(); + return result; } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); + lastError = standardizeError(error); + + // Record failure for circuit breaker tracking + if (!lastError) { + continue; + } + circuitBreaker.recordFailure(lastError); if (attempt < maxRetries) { const errorMsg = lastError.message; - // Calculate delay with exponential backoff - const delayMs = exponentialBackoff - ? calculateBackoffDelay(attempt, baseDelayMs, maxDelayMs, jitter) - : baseDelayMs; + // Check if circuit is now open + const currentState = circuitBreaker.canAttempt(); + if (!currentState.allowed) { + logError(`Connection circuit opened after ${attempt} attempts: ${currentState.reason}`); + // Don't throw immediately - finish current retry loop but warn user + if (taskId) { + logWarn(`Task ${taskId} will be paused due to connection issues`); + } + } + + const delayMs = calculateDelay( + retryDelay, + attempt, + lastError, + exponentialBackoff, + maxDelay, + jitter, + ); - const delaySecs = (delayMs / 1000).toFixed(1); - logWarn(`Attempt ${attempt}/${maxRetries} failed: ${errorMsg}`); + logWarn( + `Attempt ${attempt}/${maxRetries} failed: ${errorMsg}. Retrying in ${delayMs}ms...`, + ); onRetry?.(attempt, errorMsg, delayMs); - logDebug(`Waiting ${delaySecs}s before retry (exponential backoff)...`); await sleep(delayMs); + + // Re-check circuit state before next attempt + const recheck = circuitBreaker.canAttempt(); + if (!recheck.allowed) { + throw new Error(recheck.reason || "Connection circuit open - stopping retries"); + } } } } @@ -99,32 +333,94 @@ export async function withRetry(fn: () => Promise, options: RetryOptions): } /** - * Check if an error is retryable (e.g., rate limit, network error) + * Connection fallback options for graceful degradation */ -export function isRetryableError(error: string): boolean { - const retryablePatterns = [ - /rate limit/i, - /rate_limit/i, - /hit your limit/i, - /quota/i, - /too many requests/i, - /429/, - /timeout/i, - /network/i, - /connection/i, - /ECONNRESET/, - /ETIMEDOUT/, - /ENOTFOUND/, - /overloaded/i, - ]; +export interface ConnectionFallbackOptions { + /** Save task state when connection fails */ + saveState?: () => Promise; + /** Skip current task and continue with next */ + skipTask?: () => void; + /** Pause execution and wait for manual intervention */ + pauseExecution?: () => void; +} + +/** + * Handle connection failure with graceful degradation + * This is called when all retries are exhausted due to connection issues + */ +export async function handleConnectionFailure( + taskId: string, + error: Error, + options?: ConnectionFallbackOptions, +): Promise<{ action: "retry" | "skip" | "pause" | "abort"; message: string }> { + const state = circuitBreaker.getState(); - return retryablePatterns.some((pattern) => pattern.test(error)); + logError(`Connection failure for task ${taskId}: ${error.message}`); + logError(`Circuit state: ${state.state}, Failures: ${state.consecutiveFailures}`); + + // If circuit is open, we should not retry immediately + if (state.state === "OPEN") { + const message = `Connection lost. Circuit breaker OPEN. ${state.consecutiveFailures} consecutive failures.\nWaiting ${30000 / 1000}s before next attempt.\nYou can:\n1. Wait for automatic retry\n2. Press Ctrl+C to stop and resume later\n3. Check your internet connection`; + + logWarn(message); + + // Try to save state if provided + if (options?.saveState) { + try { + await options.saveState(); + logWarn("Task state saved for later resumption"); + } catch (saveError) { + logError(`Failed to save task state: ${saveError}`); + } + } + + return { action: "pause", message }; + } + + // For other cases, return the error + return { + action: "abort", + message: `Connection failure after maximum retries: ${error.message}`, + }; } +/** + * Wait for connection to be restored with timeout + */ +export async function waitForConnectionRestore(timeoutMs = 300000): Promise { + const checkInterval = 5000; // Check every 5 seconds + const startTime = Date.now(); + + logWarn("Waiting for connection to be restored..."); + + while (Date.now() - startTime < timeoutMs) { + const state = circuitBreaker.canAttempt(); + + if (state.allowed) { + logWarn("Connection restored - resuming execution"); + return true; + } + + const elapsed = Math.floor((Date.now() - startTime) / 1000); + const remaining = Math.floor((timeoutMs - (Date.now() - startTime)) / 1000); + logWarn( + `Still waiting for connection... (${elapsed}s elapsed, ${remaining}s timeout remaining)`, + ); + + await sleep(checkInterval); + } + + logError("Connection restore timeout reached"); + return false; +} + +/** + * Re-export isRetryableError from utils/errors.ts for backward compatibility + */ +export { isRetryableError } from "../utils/errors.ts"; + /** * Check if an error is fatal and should abort all remaining tasks. - * Fatal errors indicate a configuration or authentication problem that - * will affect all subsequent tasks. */ export function isFatalError(error: string): boolean { const fatalPatterns = [ @@ -138,7 +434,7 @@ export function isFatalError(error: string): boolean { /\b403\b/i, /command not found/i, /not installed/i, - /is not recognized/i, // Windows "command not recognized" + /is not recognized/i, ]; return fatalPatterns.some((pattern) => pattern.test(error)); diff --git a/cli/src/execution/runner-types.ts b/cli/src/execution/runner-types.ts new file mode 100644 index 00000000..45b96ac1 --- /dev/null +++ b/cli/src/execution/runner-types.ts @@ -0,0 +1,56 @@ +import type { AIEngine, AIResult } from "../engines/types.ts"; +import type { Task } from "../tasks/types.ts"; + +export interface AgentRunnerOptions { + engine: AIEngine; + task: Task; + agentNum: number; + originalDir: string; + prdSource: string; + prdFile: string; + prdIsFolder: boolean; + maxRetries: number; + retryDelay: number; + skipTests: boolean; + skipLint: boolean; + browserEnabled: "auto" | "true" | "false"; + modelOverride?: string; + planningModel?: string; + testModel?: string; + planningAnalysis?: string; + planningSteps?: string[]; + engineArgs?: string[]; + env?: Record; + debug?: boolean; + debugOpenCode?: boolean; + /** Allow OpenCode to access sandbox directories without permission prompts (default: true) */ + allowOpenCodeSandboxAccess?: boolean; + logThoughts?: boolean; + onProgress?: (step: string) => void; + dryRun?: boolean; + /** Files to specifically copy into the isolation directory (for planning-based mode) */ + filesToCopy?: string[]; + /** Skip git parallel execution (don't symlink .git in sandboxes) */ + noGitParallel?: boolean; + /** Use semantic chunking to select relevant files (default: true) */ + useSemanticChunking?: boolean; + /** Called right after worktree creation for crash-safe tracking */ + onWorktreeCreated?: (worktreeDir: string, branchName: string) => void; +} + +export interface ParallelAgentResult { + task: Task; + agentNum: number; + worktreeDir: string; + branchName: string; + result: AIResult | null; + error?: string; + /** Whether this agent used sandbox mode */ + usedSandbox?: boolean; + /** Optional performance metrics */ + metrics?: { + inputTokens: number; + outputTokens: number; + durationMs: number; + }; +} diff --git a/cli/src/execution/sandbox-git.ts b/cli/src/execution/sandbox-git.ts index 45ceb1e9..8be2aeac 100644 --- a/cli/src/execution/sandbox-git.ts +++ b/cli/src/execution/sandbox-git.ts @@ -1,28 +1,69 @@ +import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { dirname, join, relative, resolve, sep } from "node:path"; import simpleGit, { type SimpleGit } from "simple-git"; import { slugify } from "../git/branch.ts"; -import { logDebug } from "../ui/logger.ts"; +import { logDebug, logWarn } from "../ui/logger.ts"; +import { standardizeError } from "../utils/errors.ts"; +import { validatePath } from "./sandbox.ts"; /** - * Simple mutex to serialize git operations across sandbox agents. + * Simple mutex to serialize git operations and file writes across sandbox agents. * Prevents race conditions when multiple agents commit through shared .git. */ class GitMutex { - private queue: Promise = Promise.resolve(); + private queue: Array<() => Promise> = []; + private active = false; + private readonly GIT_MUTEX_MAX_QUEUE_SIZE = 1000; async acquire(fn: () => Promise): Promise { - let release: () => void; - const next = new Promise((resolve) => { - release = resolve; + // Check queue size limit + if (this.queue.length >= this.GIT_MUTEX_MAX_QUEUE_SIZE) { + const error = new Error( + `Git mutex queue full (${this.queue.length}/${this.GIT_MUTEX_MAX_QUEUE_SIZE})`, + ); + logWarn(error.message); + throw error; + } + + return new Promise((resolve, reject) => { + const operation = async () => { + try { + const result = await fn(); + resolve(result); + } catch (err) { + reject(err); + } finally { + this.processNext(); + } + }; + + this.queue.push(operation); + this.processNext(); }); - const prev = this.queue; - this.queue = next; - await prev; - try { - return await fn(); - } finally { - release!(); + } + + private processNext(): void { + if (this.active || this.queue.length === 0) { + return; + } + + this.active = true; + const nextOp = this.queue.shift(); + + if (nextOp) { + nextOp() + .catch((err) => { + logDebug(`Git operation failed: ${err}`); + }) + .finally(() => { + this.active = false; + this.processNext(); + }); + } else { + // Queue might have more items now, check and process + this.active = false; + this.processNext(); } } } @@ -33,9 +74,7 @@ const gitMutex = new GitMutex(); * Generate a unique identifier for branch names */ function generateUniqueId(): string { - const timestamp = Date.now(); - const random = Math.random().toString(36).substring(2, 8); - return `${timestamp}-${random}`; + return randomUUID(); } /** @@ -50,12 +89,6 @@ export interface SandboxCommitResult { /** * Commit changes from a sandbox to a new branch in the original repo. - * - * This: - * 1. Creates a new branch from the base branch - * 2. Copies modified files from sandbox to original - * 3. Stages and commits the changes - * 4. Returns to the original branch */ export async function commitSandboxChanges( originalDir: string, @@ -79,6 +112,7 @@ export async function commitSandboxChanges( // Serialize git operations to prevent race conditions return gitMutex.acquire(async () => { const git: SimpleGit = simpleGit(originalDir); + const copiedFiles: string[] = []; try { // Save current branch @@ -87,10 +121,26 @@ export async function commitSandboxChanges( // Create and checkout new branch from base await git.checkout(["-B", branchName, baseBranch]); - // Copy modified files from sandbox to original + // Copy modified files from sandbox to original (protected by mutex) for (const relPath of modifiedFiles) { - const sandboxPath = join(sandboxDir, relPath); - const originalPath = join(originalDir, relPath); + // Validate paths before copying + const sandboxPath = validatePath(sandboxDir, relPath); + const originalPath = validatePath(originalDir, relPath); + + if (!sandboxPath || !originalPath) { + logDebug(`Security: Invalid path rejected: ${relPath}`); + continue; + } + + // Additional validation: ensure file is within sandbox + const resolvedSandboxPath = resolve(sandboxPath); + const resolvedSandboxBase = resolve(sandboxDir); + const resolvedRelative = relative(resolvedSandboxBase, resolvedSandboxPath); + + if (resolvedRelative.startsWith("..") || resolvedRelative.startsWith(`${sep}..`)) { + logDebug(`Security: File outside sandbox: ${relPath}`); + continue; + } if (existsSync(sandboxPath)) { const parentDir = dirname(originalPath); @@ -101,17 +151,30 @@ export async function commitSandboxChanges( // Read from sandbox and write to original const content = readFileSync(sandboxPath); writeFileSync(originalPath, content); + copiedFiles.push(relPath); + logDebug(`Copied back validated file: ${relPath}`); } } + if (copiedFiles.length === 0) { + logWarn(`Agent ${agentNum}: No valid files copied from sandbox for commit`); + await git.checkout(currentBranch); + return { + success: false, + branchName, + filesCommitted: 0, + error: "No valid sandbox files to commit", + }; + } + // Stage all modified files - await git.add(modifiedFiles); + await git.add(copiedFiles); // Commit const commitMessage = `feat: ${taskName}\n\nAutomated commit by Ralphy agent ${agentNum}`; await git.commit(commitMessage); - logDebug(`Agent ${agentNum}: Committed ${modifiedFiles.length} files to ${branchName}`); + logDebug(`Agent ${agentNum}: Committed ${copiedFiles.length} files to ${branchName}`); // Return to original branch await git.checkout(currentBranch); @@ -119,14 +182,13 @@ export async function commitSandboxChanges( return { success: true, branchName, - filesCommitted: modifiedFiles.length, + filesCommitted: copiedFiles.length, }; } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); + const errorMsg = standardizeError(error).message; // Try to return to a safe state try { - const git: SimpleGit = simpleGit(originalDir); const branches = await git.branch(); if (branches.current !== baseBranch) { await git.checkout(baseBranch); @@ -147,32 +209,24 @@ export async function commitSandboxChanges( /** * Check if there are uncommitted changes in a sandbox. - * Since sandboxes don't have proper git, we check if any files - * were modified compared to original. */ export async function hasSandboxChanges( - sandboxDir: string, - originalDir: string, + _sandboxDir: string, + _originalDir: string, modifiedFiles: string[], ): Promise { return modifiedFiles.length > 0; } /** - * Initialize git configuration in sandbox (if needed). - * This is mainly for agents that require git to be present. + * Initialize git configuration in sandbox. */ export async function initSandboxGit(sandboxDir: string, originalDir: string): Promise { - // The .git directory should already be symlinked from createSandbox - // This function is here for any additional git setup needed - const gitDir = join(sandboxDir, ".git"); if (!existsSync(gitDir)) { - // If .git wasn't symlinked, create a minimal git init const git: SimpleGit = simpleGit(sandboxDir); await git.init(); - // Copy user config from original if available const originalGit: SimpleGit = simpleGit(originalDir); try { const userName = await originalGit.getConfig("user.name"); diff --git a/cli/src/execution/sandbox.ts b/cli/src/execution/sandbox.ts index d27e76c9..f7b0d045 100644 --- a/cli/src/execution/sandbox.ts +++ b/cli/src/execution/sandbox.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { copyFileSync, cpSync, @@ -6,53 +7,303 @@ import { mkdirSync, readdirSync, readlinkSync, + realpathSync, rmSync, statSync, symlinkSync, utimesSync, } from "node:fs"; -import { dirname, join, sep } from "node:path"; -import { logDebug, logWarn } from "../ui/logger.ts"; +import { tmpdir } from "node:os"; +import { dirname, join, normalize, relative, resolve, sep } from "node:path"; +import { + DEFAULT_IGNORE_PATTERNS, + SANDBOX_BACKGROUND_CLEANUP_DELAY_MS, + SANDBOX_DIR_PREFIX, + SANDBOX_STALE_THRESHOLD_MS, + SANDBOX_SUFFIX, +} from "../config/constants.ts"; + +export { + DEFAULT_IGNORE_PATTERNS, + SANDBOX_BACKGROUND_CLEANUP_DELAY_MS, + SANDBOX_DIR_PREFIX, + SANDBOX_STALE_THRESHOLD_MS, + SANDBOX_SUFFIX, +}; + +import { logDebug } from "../ui/logger.ts"; +import { copyAndCompressSkillFolders } from "./skill-compress.ts"; + +const MAX_SYNC_DEPTH = 100; /** - * Robustly remove a directory or file, retrying on EBUSY/EPERM errors. - * This is critical on Windows where file locks (e.g. anti-virus, indexing, open handles) - * frequently cause spurious cleanup failures. - * - * It attempts to delete 5 times with exponential backoff. - * If it ultimately fails, it LOGS A WARNING but DOES NOT CRASH. - * This prevents the entire runner from failing just because a temp folder is locked. + * Smartly sync a directory from source to destination. + * Only copies files that have changed (based on size/mtime) or are new. + * Removes files in dest that are not in source. */ -export async function rmRF(path: string): Promise { - if (!existsSync(path)) return; +function syncDirectory( + src: string, + dest: string, + ignorePatterns: (item: string) => boolean, + currentDepth = 0, + rootSrc: string = src, +): { filesCopied: number; filesDeleted: number } { + // Prevent stack overflow from deeply nested directories + if (currentDepth > MAX_SYNC_DEPTH) { + logDebug(`Max sync depth ${MAX_SYNC_DEPTH} exceeded for ${src}, skipping subdirectories`); + return { filesCopied: 0, filesDeleted: 0 }; + } - const retries = 5; - for (let i = 0; i < retries; i++) { - try { - // Using force: true and recursive: true is standard - rmSync(path, { recursive: true, force: true }); - return; - } catch (err: any) { - const isLockError = err.code === "EBUSY" || err.code === "EPERM" || err.code === "ENOTEMPTY"; + let filesCopied = 0; + let filesDeleted = 0; - if (isLockError && i < retries - 1) { - // Wait with exponential backoff: 500, 1000, 2000, 4000... - const delay = 500 * Math.pow(2, i); - await new Promise((resolve) => setTimeout(resolve, delay)); - continue; + if (!existsSync(dest)) { + mkdirSync(dest, { recursive: true }); + } + + const srcItems = new Set(readdirSync(src)); + const destItems = readdirSync(dest); + + // 1. Remove items in dest that are not in src + for (const item of destItems) { + if (!srcItems.has(item) && !ignorePatterns(item)) { + const destPath = join(dest, item); + rmSync(destPath, { recursive: true, force: true }); + filesDeleted++; + } + } + + // 2. Sync items from src to dest + for (const item of srcItems) { + if (ignorePatterns(item)) continue; + + const srcPath = join(src, item); + const destPath = join(dest, item); + + // Skip if source is invalid (e.g. broken symlink) + if (!existsSync(srcPath)) continue; + + const srcStat = lstatSync(srcPath); + + if (srcStat.isDirectory()) { + if (existsSync(destPath) && !lstatSync(destPath).isDirectory()) { + rmSync(destPath, { force: true }); + } + const subResult = syncDirectory(srcPath, destPath, ignorePatterns, currentDepth + 1, rootSrc); + filesCopied += subResult.filesCopied; + filesDeleted += subResult.filesDeleted; + } else if (srcStat.isFile()) { + let shouldCopy = true; + if (existsSync(destPath)) { + const destStat = lstatSync(destPath); + if ( + destStat.isFile() && + destStat.size === srcStat.size && + destStat.mtimeMs === srcStat.mtimeMs + ) { + shouldCopy = false; + } + } + + if (shouldCopy) { + copyFileSync(srcPath, destPath); + try { + utimesSync(destPath, srcStat.atime, srcStat.mtime); + } catch (error) { + logDebug(`Failed to set timestamp: ${error}`); + } + filesCopied++; + } + } else if (srcStat.isSymbolicLink()) { + let shouldRecreate = true; + if (existsSync(destPath) && lstatSync(destPath).isSymbolicLink()) { + if (readlinkSync(srcPath) === readlinkSync(destPath)) { + shouldRecreate = false; + } + } + if (shouldRecreate) { + if (existsSync(destPath)) rmSync(destPath, { force: true }); + const target = readlinkSync(srcPath); + + // Validate symlink target to prevent sandbox escape + const resolvedTarget = resolve(dirname(srcPath), target); + const resolvedSrcBase = resolve(rootSrc); + const relativeTarget = relative(resolvedSrcBase, resolvedTarget); + if ( + relativeTarget.startsWith("..") || + relativeTarget.includes("/..") || + relativeTarget.includes("\\..") + ) { + logDebug(`Security: Symlink target escapes base directory, skipping: ${target}`); + continue; + } + + symlinkSync(target, destPath); + } + } + } + + return { filesCopied, filesDeleted }; +} + +/** + * Validate and canonicalize a path to prevent path traversal attacks. + * Returns null if the path is invalid or escapes the base directory. + */ +export function validatePath(baseDir: string, targetPath: string, maxDepth = 10): string | null { + // Validate baseDir exists and is a string + if (!baseDir || typeof baseDir !== "string") { + logDebug(`Security: Invalid base directory: ${baseDir}`); + return null; + } + + // Validate targetPath is a string + if (typeof targetPath !== "string") { + logDebug(`Security: Invalid target path type: ${typeof targetPath}`); + return null; + } + + // Reject null bytes which can be used to bypass path validation + if (targetPath.includes("\0")) { + logDebug(`Security: Null byte detected in path: ${targetPath}`); + return null; + } + + // Reject paths that try to escape via URL encoding + if (targetPath.includes("%") && /%[0-9a-fA-F]{2}/.test(targetPath)) { + logDebug(`Security: URL encoding detected in path: ${targetPath}`); + return null; + } + + const absoluteBase = realpathSync(resolve(baseDir)); + const candidateTarget = resolve(absoluteBase, targetPath); + + // SECURITY: Resolve existing targets with realpath. For non-existent targets, + // require parent directories to resolve inside baseDir to prevent symlink escapes. + let absoluteTarget: string; + if (existsSync(candidateTarget)) { + absoluteTarget = realpathSync(candidateTarget); + } else { + const parentDir = dirname(candidateTarget); + const resolvedParent = existsSync(parentDir) ? realpathSync(parentDir) : null; + if (!resolvedParent) { + logDebug(`Security: Parent directory does not exist for path: ${targetPath}`); + return null; + } + if (resolvedParent !== absoluteBase && !resolvedParent.startsWith(`${absoluteBase}${sep}`)) { + logDebug(`Security: Parent directory escapes base after symlink resolution: ${targetPath}`); + return null; + } + absoluteTarget = join(resolvedParent, relative(parentDir, candidateTarget)); + } + + // Check if the resolved path is within the base directory + const relativePath = relative(absoluteBase, absoluteTarget); + + // If relative path starts with .., it escapes the base directory + if (relativePath.startsWith("..") || relativePath.startsWith(`${sep}..`)) { + logDebug(`Security: Path traversal attempt detected: ${targetPath}`); + return null; + } + + // Check for absolute path injection (paths starting with / or \ or drive letters) + if (targetPath.startsWith("/") || targetPath.startsWith("\\") || /^[a-zA-Z]:/.test(targetPath)) { + logDebug(`Security: Absolute path injection attempt detected: ${targetPath}`); + return null; + } + + // SECURITY: Double-check with startsWith after realpath resolution + // This catches any remaining traversal attempts after symlink resolution + if (!absoluteTarget.startsWith(absoluteBase + sep) && absoluteTarget !== absoluteBase) { + logDebug(`Security: Path escapes base directory after symlink resolution: ${targetPath}`); + return null; + } + + // Recursive symlink validation with depth limit and circular detection + return validatePathRecursive(absoluteBase, absoluteTarget, 0, maxDepth, new Set()); +} + +function validatePathRecursive( + baseDir: string, + targetPath: string, + currentDepth: number, + maxDepth: number, + visited: Set, +): string | null { + // Prevent infinite loops + if (currentDepth > maxDepth) { + logDebug(`Security: Symlink chain too deep (${currentDepth} levels): ${targetPath}`); + return null; + } + + if (visited.has(targetPath)) { + logDebug(`Security: Circular symlink detected: ${targetPath}`); + return null; + } + visited.add(targetPath); + + // Check if target itself is a symlink + try { + const stat = lstatSync(targetPath); + if (stat.isSymbolicLink()) { + // BUG FIX: Use realpathSync for atomic symlink resolution to prevent TOCTOU + // This resolves the symlink target atomically, preventing race conditions + const resolvedTarget = realpathSync(targetPath); + const resolvedRelative = relative(baseDir, resolvedTarget); + + if (resolvedRelative.startsWith("..") || resolvedRelative.startsWith(`${sep}..`)) { + logDebug(`Security: Symlink path traversal detected: ${targetPath}`); + return null; + } + + // Recursively check the symlink target (which is now fully resolved) + return validatePathRecursive( + baseDir, + resolvedTarget, + currentDepth + 1, + maxDepth, + new Set(visited), + ); + } + + // Check parent directory for symlinks using realpathSync for atomicity + const parentDir = dirname(targetPath); + if (existsSync(parentDir)) { + const parentReal = realpathSync(parentDir); + const parentRelative = relative(baseDir, parentReal); + + if (parentRelative.startsWith("..") || parentRelative.startsWith(`${sep}..`)) { + logDebug(`Security: Parent symlink path traversal: ${parentDir}`); + return null; } - // On final failure for lock errors, log warning and swallow. - // For non-lock errors (any time), throw immediately. - if (isLockError && i === retries - 1) { - logWarn( - `Failed to clean up ${path} after ${retries} attempts: ${err.message}. This may be due to a file lock. Proceeding anyway.`, + // Recursively check parent if it's different from original + if (parentReal !== parentDir) { + return validatePathRecursive( + baseDir, + parentReal, + currentDepth + 1, + maxDepth, + new Set(visited), ); - } else { - throw err; } } + } catch (err) { + // Path might not exist yet, validate parent + logDebug(`Path validation error for ${targetPath}: ${err}`); + const parentDir = dirname(targetPath); + if (existsSync(parentDir)) { + return validatePathRecursive( + baseDir, + parentDir, + currentDepth + 1, + maxDepth, + new Set(visited), + ); + } } + + return targetPath; } /** @@ -110,6 +361,14 @@ export const DEFAULT_COPY_PATTERNS = [ "pyproject.toml", ]; +export function shouldIgnore(item: string): boolean { + if (DEFAULT_IGNORE_PATTERNS.includes(item)) return true; + for (const pattern of DEFAULT_IGNORE_PATTERNS) { + if (pattern.endsWith("*") && item.startsWith(pattern.slice(0, -1))) return true; + } + return false; +} + export interface SandboxOptions { /** Original working directory */ originalDir: string; @@ -152,79 +411,147 @@ export async function createSandbox(options: SandboxOptions): Promise(); - // Step 1: Create symlinks for read-only dependencies + // Step 1: Create/Update symlinks for read-only dependencies for (const item of items) { if (symlinkDirs.includes(item)) { const originalPath = join(originalDir, item); const sandboxPath = join(sandboxDir, item); - if (existsSync(originalPath)) { - try { - // Create symlink (use 'junction' on Windows for directories) - const stat = lstatSync(originalPath); - const type = stat.isDirectory() ? "junction" : "file"; + if (!existsSync(originalPath)) { + // Clean up dead symlink in sandbox if it exists + if (existsSync(sandboxPath)) rmSync(sandboxPath, { force: true }); + continue; + } + + try { + const stat = lstatSync(originalPath); + // Use "junction" on Windows for directories, "dir" on Unix-like platforms + const type = stat.isDirectory() + ? process.platform === "win32" + ? "junction" + : "dir" + : "file"; + + // Check if symlink needs update + let needsUpdate = true; + if (existsSync(sandboxPath)) { + const sandboxStat = lstatSync(sandboxPath); + if (sandboxStat.isSymbolicLink()) { + const currentTarget = readlinkSync(sandboxPath); + // Ideally we'd compare resolved paths, but strict string eq is safer/faster here + if (currentTarget === originalPath) needsUpdate = false; + } else { + rmSync(sandboxPath, { recursive: true, force: true }); + } + } + + if (needsUpdate) { + if (existsSync(sandboxPath)) rmSync(sandboxPath, { force: true }); symlinkSync(originalPath, sandboxPath, type); + + // Verify + if (!existsSync(sandboxPath)) throw new Error(`Symlink creation failed: ${item}`); symlinksCreated++; - handled.add(item); - logDebug(`Agent ${agentNum}: Symlinked ${item}`); - } catch (err) { - // Symlink failed, will copy instead - logDebug(`Agent ${agentNum}: Symlink failed for ${item}, will copy`); + createdSymlinks.push(sandboxPath); } + + handled.add(item); + } catch (err) { + logDebug(`Agent ${agentNum}: Symlink failed for ${item} (${err}), will try copy`); } } } - // Step 2: Copy everything else + // Step 2: Copy/Sync everything else for (const item of items) { if (handled.has(item)) continue; + if (shouldIgnore(item)) continue; const originalPath = join(originalDir, item); const sandboxPath = join(sandboxDir, item); + const resolvedOriginalPath = resolve(originalPath); + const resolvedSandboxDir = resolve(sandboxDir); - // Skip if it's a symlink pointing outside (like node_modules might be) - try { - const stat = lstatSync(originalPath); - - if (stat.isSymbolicLink()) { - // Validate and copy symlink only if target exists - const target = readlinkSync(originalPath); - const resolvedTarget = join(dirname(originalPath), target); - if (existsSync(resolvedTarget)) { - symlinkSync(target, sandboxPath); - symlinksCreated++; - } else { - logDebug(`Agent ${agentNum}: Skipping broken symlink ${item} -> ${target}`); + // Extra check: ensure we don't try to copy the sandbox base itself if it's in the originalDir + if ( + resolvedOriginalPath === resolvedSandboxDir || + resolvedSandboxDir.startsWith(`${resolvedOriginalPath}${sep}`) + ) { + continue; + } + + // Use syncDirectory for recursive optimization + const ignoreFunc = (_name: string) => false; // Already filtered at top level + + if (lstatSync(originalPath).isDirectory()) { + const syncRes = syncDirectory(originalPath, sandboxPath, ignoreFunc); + filesCopied += syncRes.filesCopied; + } else { + // Single file copy logic + const srcStat = lstatSync(originalPath); + let shouldCopy = true; + if (existsSync(sandboxPath)) { + const destStat = lstatSync(sandboxPath); + if (destStat.size === srcStat.size && destStat.mtimeMs === srcStat.mtimeMs) { + shouldCopy = false; } - } else if (stat.isDirectory()) { - // Copy directory recursively, preserving timestamps for change detection - cpSync(originalPath, sandboxPath, { recursive: true, preserveTimestamps: true }); - filesCopied++; - } else if (stat.isFile()) { - // Copy file and preserve timestamps for change detection + } + if (shouldCopy) { copyFileSync(originalPath, sandboxPath); try { - utimesSync(sandboxPath, stat.atime, stat.mtime); - } catch (utimeErr) { - logDebug(`Agent ${agentNum}: Failed to preserve timestamps for ${item}: ${utimeErr}`); + utimesSync(sandboxPath, srcStat.atime, srcStat.mtime); + } catch (err) { + logDebug(`Agent ${agentNum}: Failed to set timestamps on ${sandboxPath}: ${err}`); } filesCopied++; } - } catch (err) { - logDebug(`Agent ${agentNum}: Failed to copy ${item}: ${err}`); } } @@ -235,7 +562,33 @@ export async function createSandbox(options: SandboxOptions): Promise= 0; i--) { + const dirPath = createdDirs[i]; + try { + if (existsSync(dirPath)) { + rmSync(dirPath, { recursive: true, force: true }); + logDebug(`Agent ${agentNum}: Cleaned up directory: ${dirPath}`); + } + } catch (cleanupErr) { + logDebug(`Agent ${agentNum}: Failed to cleanup directory ${dirPath}: ${cleanupErr}`); + } + } + throw err; } } @@ -250,12 +603,30 @@ export function verifySandboxIsolation(sandboxDir: string, symlinkDirs: string[] if (existsSync(sandboxPath)) { try { const stat = lstatSync(sandboxPath); - if (stat.isSymbolicLink()) { - // Good - it's a symlink + if (!stat.isSymbolicLink()) { + logDebug(`Warning: ${dir} is not a symlink as expected`); continue; } - } catch { - // Error checking - assume not isolated + + // Verify symlink target exists + const linkTarget = readlinkSync(sandboxPath); + const resolvedTarget = resolve(dirname(sandboxPath), linkTarget); + + if (!existsSync(resolvedTarget)) { + logDebug(`Warning: Symlink ${dir} has broken target: ${linkTarget}`); + return false; + } + + // Verify target is not a symlink itself (to avoid chains) + const targetStat = lstatSync(resolvedTarget); + if (targetStat.isSymbolicLink()) { + logDebug(`Warning: Symlink ${dir} points to another symlink: ${linkTarget}`); + return false; + } + + logDebug(`Verified symlink: ${dir} -> ${linkTarget}`); + } catch (err) { + logDebug(`Error verifying symlink ${dir}: ${err}`); return false; } } @@ -273,15 +644,31 @@ export async function getModifiedFiles( symlinkDirs: string[] = DEFAULT_SYMLINK_DIRS, ): Promise { const modified: string[] = []; + const HASH_THRESHOLD_SIZE = 1024 * 1024; // 1MB - threshold for potential hash verification + const MAX_SCAN_DEPTH = 100; + const visitedInodes = new Set(); + + function scanDir(relPath: string, currentDepth: number) { + // Prevent stack overflow and infinite loops + if (currentDepth > MAX_SCAN_DEPTH) { + logDebug(`Max scan depth ${MAX_SCAN_DEPTH} exceeded at ${relPath}, stopping`); + return; + } - function scanDir(relPath: string) { const sandboxPath = join(sandboxDir, relPath); - const originalPath = join(originalDir, relPath); if (!existsSync(sandboxPath)) return; const stat = lstatSync(sandboxPath); + // Detect and prevent symlink cycles using inode + const inode = `${stat.dev}-${stat.ino}`; + if (visitedInodes.has(inode)) { + logDebug(`Cycle detected (inode ${inode}), skipping: ${relPath}`); + return; + } + visitedInodes.add(inode); + // Skip symlinks (they're shared, not modified) if (stat.isSymbolicLink()) return; @@ -292,18 +679,41 @@ export async function getModifiedFiles( if (stat.isDirectory()) { const items = readdirSync(sandboxPath); for (const item of items) { - scanDir(join(relPath, item)); + scanDir(join(relPath, item), currentDepth + 1); } } else if (stat.isFile()) { - // Check if file is new or modified + let isModified = false; + const originalPath = join(originalDir, relPath); + if (!existsSync(originalPath)) { - modified.push(relPath); + isModified = true; } else { const originalStat = statSync(originalPath); - if (stat.mtimeMs !== originalStat.mtimeMs || stat.size !== originalStat.size) { - modified.push(relPath); + + // Check mtime and size + const mtimeDifferent = stat.mtimeMs !== originalStat.mtimeMs; + const sizeDifferent = stat.size !== originalStat.size; + + if (mtimeDifferent || sizeDifferent) { + // For close mtime matches on small files, verify with hash + if ( + mtimeDifferent && + Math.abs(stat.mtimeMs - originalStat.mtimeMs) < 1000 && + stat.size < HASH_THRESHOLD_SIZE + ) { + // This is async, but we're in a sync function + // For now, just use mtime/size difference + isModified = true; + logDebug(`Modified file detected by mtime/size: ${relPath}`); + } else { + isModified = true; + } } } + + if (isModified) { + modified.push(relPath); + } } } @@ -316,9 +726,9 @@ export async function getModifiedFiles( if (itemStat.isSymbolicLink()) continue; if (itemStat.isDirectory()) { - scanDir(item); + scanDir(item, 1); } else if (itemStat.isFile()) { - scanDir(item); + scanDir(item, 1); } } @@ -355,20 +765,364 @@ export async function syncSandboxToOriginal( return synced; } +/** + * Copy back only planned files from sandbox to original directory. + * This is used in parallel execution mode where we only want to copy + * files that were identified as needed during the planning phase. + */ +export async function copyBackPlannedFilesParallel( + originalDir: string, + sandboxDir: string, + files: string[], +): Promise { + const pendingChanges: Array<{ originalPath: string; sandboxPath: string; relPath: string }> = []; + + // Phase 1: Validate and prepare all changes + for (const relPath of files) { + const sandboxPath = validatePath(sandboxDir, relPath); + const originalPath = validatePath(originalDir, relPath); + + if (!sandboxPath || !originalPath) { + logDebug(`Security: Invalid path rejected: ${relPath}`); + continue; + } + + if (!existsSync(sandboxPath)) { + logDebug(`File not found in sandbox: ${relPath}`); + continue; + } + + pendingChanges.push({ originalPath, sandboxPath, relPath }); + } + + // Phase 2: Ensure all parent directories exist + const directoriesToCreate = new Set(); + for (const change of pendingChanges) { + directoriesToCreate.add(dirname(change.originalPath)); + } + + for (const dir of directoriesToCreate) { + if (!existsSync(dir)) { + try { + mkdirSync(dir, { recursive: true }); + } catch (err) { + logDebug(`Failed to create directory ${dir}: ${err}`); + // Rollback: remove any directories we created + for (const createdDir of directoriesToCreate) { + if (existsSync(createdDir)) { + try { + rmSync(createdDir, { recursive: true, force: true }); + } catch (rollbackErr) { + logDebug(`Failed to rollback directory ${createdDir}: ${rollbackErr}`); + } + } + } + throw new Error(`Failed to create directory structure: ${err}`); + } + } + } + + // Phase 3: Copy files with TOCTOU protection + // SECURITY: Re-validate paths immediately before copy to prevent symlink attacks + let synced = 0; + for (const change of pendingChanges) { + try { + // Re-validate paths right before use to prevent TOCTOU attacks + const sandboxPath = validatePath(sandboxDir, change.relPath); + const originalPath = validatePath(originalDir, change.relPath); + + if (!sandboxPath || !originalPath) { + logDebug(`Security: Path re-validation failed for ${change.relPath}`); + continue; + } + + // Verify file still exists and hasn't been swapped + if (!existsSync(sandboxPath)) { + logDebug(`Security: File disappeared or was swapped: ${change.relPath}`); + continue; + } + + copyFileSync(sandboxPath, originalPath); + synced++; + logDebug(`Copied back: ${change.relPath}`); + } catch (err) { + logDebug(`Failed to copy back ${change.relPath}: ${err}`); + // Continue with other files + } + } + + return synced; +} + /** * Clean up a sandbox directory. */ export async function cleanupSandbox(sandboxDir: string): Promise { - await rmRF(sandboxDir); + const allowedBase = resolve(join(tmpdir(), "ralphy-sandboxes")); + const resolvedSandbox = resolve(sandboxDir); + if (resolvedSandbox === allowedBase || !resolvedSandbox.startsWith(`${allowedBase}${sep}`)) { + logDebug(`Security: refusing to cleanup path outside sandbox base: ${sandboxDir}`); + return; + } + + if (existsSync(resolvedSandbox)) { + rmSync(resolvedSandbox, { recursive: true, force: true }); + } } /** * Get the base directory for sandboxes. + * Uses system temp directory to ensure complete isolation. */ export function getSandboxBase(workDir: string): string { - const sandboxBase = join(workDir, ".ralphy-sandboxes"); + const projectHash = createHash("sha256").update(resolve(workDir)).digest("hex"); + const sandboxBase = join(tmpdir(), "ralphy-sandboxes", projectHash); + if (!existsSync(sandboxBase)) { mkdirSync(sandboxBase, { recursive: true }); } return sandboxBase; } + +/** + * Symlink shared resources from original directory to sandbox. + * This is used to create symlinks for directories that should be shared + * between sandboxes (e.g., node_modules, .git). + */ +export function symlinkSharedResources( + originalDir: string, + sandboxDir: string, + resources: string[], +): void { + for (const resource of resources) { + const originalPath = join(originalDir, resource); + const sandboxPath = join(sandboxDir, resource); + + if (!existsSync(originalPath)) { + logDebug(`Shared resource not found: ${resource}`); + continue; + } + + try { + // Create symlink with platform-specific handling + const stat = lstatSync(originalPath); + const isDir = stat.isDirectory(); + + if (isDir) { + // For directories, use 'junction' on Windows (more permissive) or 'dir' on Unix + const type = process.platform === "win32" ? "junction" : "dir"; + symlinkSync(originalPath, sandboxPath, type); + } else { + // For files, use 'file' type on all platforms + // On Windows, this may require Developer Mode or admin privileges + // If it fails, the caller should handle the error and fall back to copying + symlinkSync(originalPath, sandboxPath, "file"); + } + logDebug(`Symlinked shared resource: ${resource}`); + } catch (err) { + // On Windows, file symlinks often fail without admin privileges + // Log the error but don't crash - caller can fall back to copying + logDebug(`Failed to symlink shared resource ${resource}: ${err}`); + // Re-throw so caller knows to fall back + throw err; + } + } +} + +/** + * Copy skill/playbook folders from original directory to sandbox. + * This ensures that skill documentation is available in the sandbox. + * Uses compression to reduce token usage when skills are loaded by AI. + */ +export function copySkillFolders(originalDir: string, sandboxDir: string): void { + const saved = copyAndCompressSkillFolders(originalDir, sandboxDir); + if (saved > 0) { + logDebug(`Skill folders compressed, saved ~${saved} characters`); + } +} + +/** + * Copy only the planned files to a sandbox directory. + * This is used in parallel execution mode to create an isolated environment + * with only the files that were identified as needed during planning. + */ +export async function copyPlannedFilesIsolated( + originalDir: string, + sandboxDir: string, + filesToCopy: string[], +): Promise { + const copiedFiles: string[] = []; + const rejectedFiles: string[] = []; + + // CLEAN SYNC: Remove files in sandbox that are NOT in the plan + // This prevents "wandering off" by ensuring the agent only sees what it should + if (existsSync(sandboxDir)) { + const plannedSet = new Set(filesToCopy.map((f) => normalize(f))); + + // Helper to recursively scan and clean + function cleanUnplanned(dir: string, base: string) { + try { + const items = readdirSync(dir); + for (const item of items) { + const fullPath = join(dir, item); + const relPath = relative(base, fullPath); + + // Skip protected directories + if (item === ".git" || item === "node_modules" || item === ".ralphy") continue; + if (DEFAULT_SYMLINK_DIRS.includes(item)) continue; + + const stat = lstatSync(fullPath); + + if (stat.isDirectory()) { + // Check if any planned file is inside this directory + const isParentOfPlan = Array.from(plannedSet).some((p) => p.startsWith(relPath + sep)); + if (isParentOfPlan) { + cleanUnplanned(fullPath, base); + // If directory is empty after cleaning, remove it? keeping it is safer/faster + } else { + // Entire directory is unplanned + rmSync(fullPath, { recursive: true, force: true }); + } + } else { + // If file is not in plan, delete it + if (!plannedSet.has(relPath)) { + rmSync(fullPath, { force: true }); + } + } + } + } catch (e) { + logDebug(`Failed to clean sandbox: ${e}`); + } + } + + cleanUnplanned(sandboxDir, sandboxDir); + } + + for (const relPath of filesToCopy) { + // Validate paths to prevent traversal attacks + let validatedPath = validatePath(originalDir, relPath); + + if (!validatedPath) { + logDebug(`Security: Invalid path rejected: ${relPath}`); + rejectedFiles.push(relPath); + continue; + } + + // SECURITY FIX: Re-validate path immediately before file operations to prevent TOCTOU attacks + // This ensures the path hasn't been swapped with a symlink between validation and use + validatedPath = validatePath(originalDir, relPath); + if (!validatedPath) { + logDebug(`Security: Path re-validation failed for ${relPath}`); + rejectedFiles.push(relPath); + continue; + } + + if (!existsSync(validatedPath)) { + logDebug(`File not found in original directory: ${relPath}`); + continue; + } + + const sandboxPath = join(sandboxDir, relPath); + + try { + // Ensure parent directory exists + const parentDir = dirname(sandboxPath); + if (!existsSync(parentDir)) { + mkdirSync(parentDir, { recursive: true }); + } + + // FINAL SECURITY CHECK: Re-validate immediately before copy to prevent TOCTOU + // This is the last line of defense against path manipulation + const finalPath = validatePath(originalDir, relPath); + if (finalPath !== validatedPath) { + logDebug(`Security: Path changed between validation and copy for ${relPath}`); + rejectedFiles.push(relPath); + continue; + } + validatedPath = finalPath; + + // Copy file preserving timestamps + const stat = lstatSync(validatedPath); + if (stat.isDirectory()) { + cpSync(validatedPath, sandboxPath, { recursive: true, preserveTimestamps: true }); + } else if (stat.isFile()) { + copyFileSync(validatedPath, sandboxPath); + try { + utimesSync(sandboxPath, stat.atime, stat.mtime); + } catch (utimeErr) { + logDebug(`Failed to preserve timestamps for ${relPath}: ${utimeErr}`); + } + } + + copiedFiles.push(relPath); + } catch (err) { + logDebug(`Failed to copy file ${relPath}: ${err}`); + rejectedFiles.push(relPath); + } + } + + logDebug(`Copied ${copiedFiles.length} planned files to sandbox`); + if (rejectedFiles.length > 0) { + logDebug(`Rejected ${rejectedFiles.length} invalid files: ${rejectedFiles.join(", ")}`); + } +} + +/** + * Schedule background cleanup of stale sandboxes. + * This runs after a delay to allow parallel tasks to complete. + */ +// Track scheduled cleanup timers for potential cancellation +const scheduledCleanupTimers = new Set(); + +export function scheduleBackgroundCleanup(sandboxBase: string): NodeJS.Timeout { + // Schedule cleanup after 5 minutes + const timer = setTimeout(() => { + scheduledCleanupTimers.delete(timer); + cleanupStaleSandboxes(sandboxBase); + }, SANDBOX_BACKGROUND_CLEANUP_DELAY_MS); + + // BUG FIX: Track timer for cleanup on exit + scheduledCleanupTimers.add(timer); + return timer; +} + +/** + * Cancel all scheduled background cleanup timers. + * Call this on process exit to prevent timers from keeping the process alive. + */ +export function cancelScheduledCleanups(): void { + for (const timer of scheduledCleanupTimers) { + clearTimeout(timer); + } + scheduledCleanupTimers.clear(); +} + +/** + * Clean up stale sandbox directories. + */ +export function cleanupStaleSandboxes(sandboxBase: string): void { + if (!existsSync(sandboxBase)) { + return; + } + + const now = Date.now(); + + try { + const items = readdirSync(sandboxBase); + + for (const item of items) { + const itemPath = join(sandboxBase, item); + try { + const stat = lstatSync(itemPath); + if (stat.isDirectory() && now - stat.mtimeMs > SANDBOX_STALE_THRESHOLD_MS) { + rmSync(itemPath, { recursive: true, force: true }); + logDebug(`Cleaned up stale sandbox: ${item}`); + } + } catch (err) { + logDebug(`Failed to cleanup sandbox ${item}: ${err}`); + } + } + } catch (err) { + logDebug(`Failed to cleanup stale sandboxes: ${err}`); + } +} diff --git a/cli/src/execution/sequential.ts b/cli/src/execution/sequential.ts index 813bc859..750b92ef 100644 --- a/cli/src/execution/sequential.ts +++ b/cli/src/execution/sequential.ts @@ -7,9 +7,11 @@ import type { Task, TaskSource } from "../tasks/types.ts"; import { logDebug, logError, logInfo, logSuccess, logWarn } from "../ui/logger.ts"; import { notifyTaskComplete, notifyTaskFailed } from "../ui/notify.ts"; import { ProgressSpinner } from "../ui/spinner.ts"; +import { standardizeError } from "../utils/errors.ts"; import { clearDeferredTask, recordDeferredTask } from "./deferred.ts"; import { buildPrompt } from "./prompt.ts"; -import { isFatalError, isRetryableError, sleep, withRetry } from "./retry.ts"; +import { isFatalError, isRetryableError, withRetry } from "./retry.ts"; +import { type StateFormat, TaskState, TaskStateManager, detectStateFormat } from "./task-state.ts"; export interface ExecutionOptions { engine: AIEngine; @@ -34,11 +36,31 @@ export interface ExecutionOptions { modelOverride?: string; /** Skip automatic branch merging after parallel execution */ skipMerge?: boolean; + /** Additional environment variables for the engine CLI */ + env?: Record; /** Use lightweight sandboxes instead of git worktrees for parallel execution */ useSandbox?: boolean; /** Additional arguments to pass to the engine CLI */ engineArgs?: string[]; - /** GitHub issue number to sync PRD with on each iteration */ + /** Separate model for planning phase (cheaper/faster) */ + planningModel?: string; + /** Separate model for test-related tasks (cheaper/faster) */ + testModel?: string; + /** Force non-git parallel execution (sandboxes) even in git repos */ + noGitParallel?: boolean; + /** Log AI thoughts/reasoning to console */ + logThoughts?: boolean; + /** Enable full debug logging (cli errors, full ai responses) */ + debug?: boolean; + /** Enable comprehensive OpenCode debugging */ + debugOpenCode?: boolean; + /** Allow OpenCode to access sandbox directories without permission prompts */ + allowOpenCodeSandboxAccess?: boolean; + /** Progress callback for progress reporting */ + onProgress?: (step: string) => void; + /** Task state manager for centralized state tracking */ + taskStateManager?: TaskStateManager; + /** Optional GitHub issue number to sync progress to */ syncIssue?: number; } @@ -72,7 +94,7 @@ export async function runSequential(options: ExecutionOptions): Promise(); + + for (const task of await taskSource.getAllTasks()) { + taskIndex.set(task.id, task); + } + // BUG FIX: Safety counter to prevent infinite loops + let safetyCounter = 0; + const MAX_SAFETY_ITERATIONS = 10000; while (true) { + // Safety check to prevent infinite loops + if (safetyCounter++ > MAX_SAFETY_ITERATIONS) { + throw new Error("Safety limit exceeded - possible infinite loop in sequential execution"); + } // Check iteration limit if (maxIterations > 0 && iteration >= maxIterations) { logInfo(`Reached max iterations (${maxIterations})`); break; } - // Get next task - const task = await taskSource.getNextTask(); - if (!task) { + // Get next pending task from state manager + const pendingTask = taskStateManager.getNextPendingTask(); + if (!pendingTask) { logSuccess("All tasks completed!"); break; } + // Find the full task in the source + let task = taskIndex.get(pendingTask.id); + if (!task) { + for (const refreshedTask of await taskSource.getAllTasks()) { + taskIndex.set(refreshedTask.id, refreshedTask); + } + task = taskIndex.get(pendingTask.id); + } + if (!task) { + logError(`Task ${pendingTask.id} not found in source`); + await taskStateManager.transitionState(pendingTask.id, TaskState.SKIPPED); + continue; + } + + // BUG FIX: Check max attempts and claim atomically in claimTaskForExecution + // to prevent race condition where state could change between check and claim + const claimed = await taskStateManager.claimTaskForExecution(task.id); + if (!claimed) { + // Task could be: already running, completed, or exceeded max attempts + // Check if it was max attempts + if (taskStateManager.hasExceededMaxAttempts(task.id, maxRetries)) { + logWarn(`Task "${task.title}" has exceeded max attempts (${maxRetries}), skipping...`); + await taskStateManager.transitionState(task.id, TaskState.SKIPPED); + await taskSource.markComplete(task.id); + result.tasksFailed++; + notifyTaskFailed(task.title, "Exceeded maximum retry attempts"); + clearDeferredTask(taskSource.type, task, workDir, options.prdFile); + } else { + logDebug(`Task "${task.title}" is already being executed, skipping...`); + } + continue; + } + iteration++; - const remaining = await taskSource.countRemaining(); + const remaining = taskStateManager.countPending(); logInfo(`Task ${iteration}: ${task.title} (${remaining} remaining)`); // Create branch if needed @@ -129,7 +215,7 @@ export async function runSequential(options: ExecutionOptions): Promise 0 && { engineArgs }), + ...(options.debugOpenCode && { debugOpenCode: options.debugOpenCode }), + ...(options.logThoughts !== undefined && { logThoughts: options.logThoughts }), + ...(dryRun && { dryRun: true }), }; + if (engine.executeStreaming) { return await engine.executeStreaming( prompt, @@ -170,21 +260,24 @@ export async function runSequential(options: ExecutionOptions): Promise= maxRetries) { logError(`Task "${task.title}" failed after ${deferrals} deferrals: ${errMsg}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, errMsg); logTaskProgress(task.title, "failed", workDir); result.tasksFailed++; notifyTaskFailed(task.title, errMsg); @@ -217,34 +315,42 @@ export async function runSequential(options: ExecutionOptions): Promise= maxRetries) { logError(`Task "${task.title}" failed after ${deferrals} deferrals: ${errorMsg}`); + await taskStateManager.transitionState(task.id, TaskState.FAILED, errorMsg); logTaskProgress(task.title, "failed", workDir); result.tasksFailed++; notifyTaskFailed(task.title, errorMsg); @@ -252,23 +358,30 @@ export async function runSequential(options: ExecutionOptions): Promise { + if (segment.startsWith("```")) { + return segment; + } + + return segment + .replace(/\n{3,}/g, "\n\n") + .replace(/[ \t]+$/gm, "") + .replace(/^\s+$/gm, "") + .replace(/Please note that /gi, "Note: ") + .replace(/In order to /gi, "To ") + .replace(/Make sure to /gi, "") + .replace(/You should /gi, "") + .replace(/You must /gi, "Must ") + .replace(/It is important to /gi, "") + .replace(/Keep in mind that /gi, "") + .replace(/\*\*Note\*\*:/g, "Note:") + .replace(/\*\*Important\*\*:/g, "Important:") + .replace(/\bimplementation\b/gi, "impl") + .replace(/\binformation\b/gi, "info") + .replace(/\bdirectory\b/gi, "dir") + .replace(/\bdirectories\b/gi, "dirs") + .replace(/\binitialization\b/gi, "init") + .replace(/\bconfiguration\b/gi, "config") + .replace(/\bparameters\b/gi, "params") + .replace(/\benvironment\b/gi, "env") + .replace(/\bdocumentation\b/gi, "docs"); + }) + .join(""); + + return compressed.trim(); +} + +function csvEscape(value: string): string { + const escaped = value.replace(/"/g, '""'); + if (/[",\n\r]/.test(escaped)) { + return `"${escaped}"`; + } + return escaped; +} + +/** + * Copy and compress skill folders + */ +export function copyAndCompressSkillFolders(originalDir: string, sandboxDir: string): number { + const skillDirs = [".opencode/skills", ".claude/skills", ".skills"]; + let totalSaved = 0; + + for (const dir of skillDirs) { + const srcPath = join(originalDir, dir); + if (!existsSync(srcPath)) continue; + + const destPath = join(sandboxDir, dir); + mkdirSync(destPath, { recursive: true }); + + const saved = compressDirectory(srcPath, destPath); + totalSaved += saved; + } + + if (totalSaved > 0) { + logDebug(`[SKILLS] Compressed skill files, saved ~${totalSaved} chars`); + } + + return totalSaved; +} + +/** + * Recursively compress markdown files in a directory + */ +function compressDirectory(srcDir: string, destDir: string): number { + let saved = 0; + // Handle case where srcDir doesn't exist (though checked above) + if (!existsSync(srcDir)) return 0; + + const entries = readdirSync(srcDir, { withFileTypes: true }); + + for (const entry of entries) { + const srcPath = join(srcDir, entry.name); + const destPath = join(destDir, entry.name); + + if (entry.isDirectory()) { + mkdirSync(destPath, { recursive: true }); + saved += compressDirectory(srcPath, destPath); + } else if (entry.name.endsWith(".md")) { + const original = readFileSync(srcPath, "utf-8"); + const compressed = compressMarkdown(original); + writeFileSync(destPath, compressed, "utf-8"); + saved += original.length - compressed.length; + } else { + // Copy non-markdown files as-is + const content = readFileSync(srcPath); + writeFileSync(destPath, content); + } + } + + return saved; +} + +/** + * Get all skills as a compact CSV string for LLM context + * Format: SkillName,Instructions + */ +export function getSkillsAsCsv(workDir: string): string { + const skillDirs = [".opencode/skills", ".claude/skills", ".skills"]; + const rows: string[] = []; + + for (const dir of skillDirs) { + const srcPath = join(workDir, dir); + if (!existsSync(srcPath)) continue; + + const entries = readdirSync(srcPath, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith(".md")) { + const content = readFileSync(join(srcPath, entry.name), "utf-8"); + const compressed = compressMarkdown(content).replace(/\n/g, " "); + + const name = entry.name.replace(".md", ""); + const nameFinal = csvEscape(name); + const contentFinal = csvEscape(compressed); + + rows.push(`${nameFinal},${contentFinal}`); + } + } + } + + if (rows.length === 0) return ""; + return `Name,Instructions\n${rows.join("\n")}`; +} diff --git a/cli/src/execution/task-state.ts b/cli/src/execution/task-state.ts new file mode 100644 index 00000000..989d5625 --- /dev/null +++ b/cli/src/execution/task-state.ts @@ -0,0 +1,773 @@ +/** + * Task State Manager + * + * Centralized state management for task execution. + * Provides a single source of truth for task states across all execution modes. + * State is persisted in the same format as the input source (YAML, JSON, CSV, MD). + */ + +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import YAML from "yaml"; +import { RALPHY_DIR } from "../config/loader.ts"; +import type { Task, TaskSourceType } from "../tasks/types.ts"; +import { logDebug, logError } from "../ui/logger.ts"; + +export enum TaskState { + PENDING = "pending", + RUNNING = "running", + COMPLETED = "completed", + FAILED = "failed", + DEFERRED = "deferred", + SKIPPED = "skipped", +} + +export interface TaskStateEntry { + id: string; + title: string; + state: TaskState; + attemptCount: number; + lastAttemptTime?: number; + errorHistory: string[]; + executionContext?: { + branch?: string; + worktree?: string; + sandbox?: string; + }; +} + +interface StateFileFormat { + version: number; + lastUpdated: string; + tasks: Record; +} + +export type StateFormat = "yaml" | "json" | "csv" | "md"; + +export function detectStateFormat(filePath: string | undefined): StateFormat { + if (!filePath) return "yaml"; + if (filePath.endsWith(".json")) return "json"; + if (filePath.endsWith(".csv")) return "csv"; + if (filePath.endsWith(".md")) return "md"; + return "yaml"; +} + +export class TaskStateManager { + private stateFilePath: string; + private tasks: Map = new Map(); + private format: StateFormat; + private sourceType: TaskSourceType; + private sourcePath: string; + private static readonly STATE_VERSION = 1; + private static readonly CLAIM_LOCK_TIMEOUT_MS = 5000; + private static readonly CLAIM_LOCK_RETRY_MS = 50; + + constructor( + workDir: string, + sourceType: TaskSourceType, + sourcePath: string, + format: StateFormat = "yaml", + ) { + this.sourceType = sourceType; + this.sourcePath = sourcePath; + this.format = format; + this.stateFilePath = join(workDir, RALPHY_DIR, `task-state.${format}`); + } + + /** + * Initialize the state manager with tasks from the source. + * Loads existing state if available, or creates new state from tasks. + */ + async initialize(tasksFromSource: Task[]): Promise { + // Ensure directory exists + const dir = dirname(this.stateFilePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + + // Try to load existing state + if (existsSync(this.stateFilePath)) { + await this.loadState(); + } + + // Reset any RUNNING tasks to PENDING (they were interrupted) + // and any DEFERRED tasks that have exceeded max deferrals + let resetCount = 0; + for (const [_key, task] of this.tasks) { + if (task.state === TaskState.RUNNING) { + logDebug(`Resetting interrupted task ${task.id} from RUNNING to PENDING`); + task.state = TaskState.PENDING; + // Reset attemptCount on fresh program start to prevent accumulation + // This ensures retries don't persist across program restarts + task.attemptCount = 0; + resetCount++; + } + } + if (resetCount > 0) { + logDebug(`Reset ${resetCount} interrupted tasks to PENDING (attempt counts cleared)`); + } + + // Merge with new tasks from source + for (const task of tasksFromSource) { + const key = this.buildTaskKey(task.id); + const existing = this.tasks.get(key); + + if (!existing) { + // New task - add with pending state + this.tasks.set(key, { + id: task.id, + title: task.title, + state: TaskState.PENDING, + attemptCount: 0, + errorHistory: [], + }); + } else { + // Existing task - update title if changed + existing.title = task.title; + } + } + + // Remove tasks that no longer exist in source + const validKeys = new Set(tasksFromSource.map((t) => this.buildTaskKey(t.id))); + for (const key of this.tasks.keys()) { + if (!validKeys.has(key)) { + this.tasks.delete(key); + } + } + + await this.persistState(); + logDebug(`TaskStateManager initialized with ${this.tasks.size} tasks`); + } + + /** + * Atomically claim a task for execution. + * Returns true if the task was claimed (was in PENDING state), false otherwise. + */ + async claimTaskForExecution(taskId: string): Promise { + const release = await this.acquireClaimLock(); + + try { + await this.loadState(); + + const key = this.buildTaskKey(taskId); + const task = this.tasks.get(key); + + if (!task) { + logError(`Task ${taskId} not found in state manager`); + return false; + } + + // Only allow claiming if task is pending + if (task.state !== TaskState.PENDING) { + logDebug(`Task ${taskId} cannot be claimed - state is ${task.state}`); + return false; + } + + // Atomically transition to running + task.state = TaskState.RUNNING; + task.attemptCount++; + task.lastAttemptTime = Date.now(); + await this.persistState(); + + logDebug(`Task ${taskId} claimed for execution (attempt ${task.attemptCount})`); + return true; + } finally { + release(); + } + } + + /** + * Transition a task to a new state. + */ + async transitionState( + taskId: string, + newState: TaskState, + error?: string, + executionContext?: TaskStateEntry["executionContext"], + ): Promise { + const key = this.buildTaskKey(taskId); + const task = this.tasks.get(key); + + if (!task) { + logError(`Task ${taskId} not found in state manager`); + return; + } + + const oldState = task.state; + task.state = newState; + + if (error) { + task.errorHistory.push(error); + } + + if (executionContext) { + task.executionContext = { ...task.executionContext, ...executionContext }; + } + + await this.persistState(); + logDebug(`Task ${taskId} transitioned from ${oldState} to ${newState}`); + } + + /** + * Get the next pending task that can be executed. + */ + getNextPendingTask(): TaskStateEntry | null { + for (const task of this.tasks.values()) { + if (task.state === TaskState.PENDING) { + return task; + } + } + return null; + } + + /** + * Get all tasks in a specific state. + */ + getTasksByState(state: TaskState): TaskStateEntry[] { + return Array.from(this.tasks.values()).filter((t) => t.state === state); + } + + /** + * Get the current state of a task. + */ + getTaskState(taskId: string): TaskState | null { + const key = this.buildTaskKey(taskId); + return this.tasks.get(key)?.state ?? null; + } + + /** + * Check if a task has exceeded the maximum number of attempts. + */ + hasExceededMaxAttempts(taskId: string, maxRetries: number): boolean { + const key = this.buildTaskKey(taskId); + const task = this.tasks.get(key); + if (!task) return false; + return task.attemptCount >= maxRetries; + } + + private async acquireClaimLock(): Promise<() => void> { + const lockPath = `${this.stateFilePath}.claim.lock`; + const deadline = Date.now() + TaskStateManager.CLAIM_LOCK_TIMEOUT_MS; + + while (true) { + try { + const fd = openSync(lockPath, "wx"); + writeFileSync(fd, String(process.pid)); + + return () => { + try { + closeSync(fd); + } catch { + // ignore close errors + } + try { + if (existsSync(lockPath)) { + unlinkSync(lockPath); + } + } catch { + // ignore unlock errors + } + }; + } catch { + if (Date.now() >= deadline) { + throw new Error("Timeout acquiring task-state claim lock"); + } + await new Promise((resolve) => setTimeout(resolve, TaskStateManager.CLAIM_LOCK_RETRY_MS)); + } + } + } + + /** + * Get the number of remaining pending tasks. + */ + countPending(): number { + return this.getTasksByState(TaskState.PENDING).length; + } + + /** + * Get summary statistics. + */ + getStats(): { + total: number; + pending: number; + running: number; + completed: number; + failed: number; + deferred: number; + skipped: number; + } { + return { + total: this.tasks.size, + pending: this.getTasksByState(TaskState.PENDING).length, + running: this.getTasksByState(TaskState.RUNNING).length, + completed: this.getTasksByState(TaskState.COMPLETED).length, + failed: this.getTasksByState(TaskState.FAILED).length, + deferred: this.getTasksByState(TaskState.DEFERRED).length, + skipped: this.getTasksByState(TaskState.SKIPPED).length, + }; + } + + /** + * Reset a task to pending state (for retrying failed/skipped tasks). + * Also resets the attempt count so retries don't accumulate across program restarts. + */ + async resetTask(taskId: string): Promise { + const key = this.buildTaskKey(taskId); + const task = this.tasks.get(key); + + if (!task) { + logError(`Task ${taskId} not found in state manager`); + return; + } + + task.state = TaskState.PENDING; + task.attemptCount = 0; + task.errorHistory = []; + await this.persistState(); + logDebug(`Task ${taskId} reset to pending state`); + } + + /** + * Reset all failed/skipped tasks to pending. + * Also resets the attempt count so retries don't accumulate across program restarts. + */ + async resetAllFailed(): Promise { + let count = 0; + for (const [_key, task] of this.tasks) { + if (task.state === TaskState.FAILED || task.state === TaskState.SKIPPED) { + task.state = TaskState.PENDING; + task.attemptCount = 0; + task.errorHistory = []; + count++; + } + } + if (count > 0) { + await this.persistState(); + } + logDebug(`Reset ${count} failed/skipped tasks to pending`); + return count; + } + + /** + * Reset attempt counts for all tasks when starting a fresh run. + * This ensures retries don't persist across program restarts. + */ + async resetAllAttemptCounts(): Promise { + for (const task of this.tasks.values()) { + task.attemptCount = 0; + } + await this.persistState(); + logDebug("Reset all task attempt counts"); + } + + /** + * Build a unique key for a task. + */ + private buildTaskKey(taskId: string): string { + return `${this.sourceType}:${this.sourcePath}:${taskId}`; + } + + /** + * Check for prototype pollution keys in data + */ + private hasPrototypePollution(data: unknown): boolean { + if (data === null || typeof data !== "object") { + return false; + } + + const pollutionKeys = ["__proto__", "constructor", "prototype"]; + + for (const key of Object.keys(data)) { + if (pollutionKeys.includes(key)) { + return true; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const value = (data as Record)[key]; + if (typeof value === "object" && value !== null) { + if (this.hasPrototypePollution(value)) { + return true; + } + } + } + + return false; + } + + /** + * Persist state to disk in the appropriate format. + */ + private async persistState(): Promise { + // Validate format before proceeding + if (!this.format || !["yaml", "json", "csv", "md"].includes(this.format)) { + throw new Error(`Invalid state format: ${this.format}`); + } + + // Check for prototype pollution before persisting + const rawTasks = Object.fromEntries(this.tasks); + if (this.hasPrototypePollution(rawTasks)) { + throw new Error("State contains potentially malicious prototype pollution keys"); + } + + const data: StateFileFormat = { + version: TaskStateManager.STATE_VERSION, + lastUpdated: new Date().toISOString(), + tasks: rawTasks, + }; + + const tempPath = `${this.stateFilePath}.tmp`; + + try { + let content: string; + + switch (this.format) { + case "yaml": + content = YAML.stringify(data); + break; + case "json": + content = JSON.stringify(data, null, 2); + break; + case "csv": + content = this.toCSV(data); + break; + case "md": + content = this.toMarkdown(data); + break; + default: + content = YAML.stringify(data); + } + + // Write to temp file first, then rename for atomicity (TOCTOU-safe) + writeFileSync(tempPath, content, "utf-8"); + renameSync(tempPath, this.stateFilePath); + } catch (error) { + // Clean up temp file on error to prevent stale file accumulation + try { + if (existsSync(tempPath)) { + unlinkSync(tempPath); + } + } catch { + // Ignore cleanup errors + } + logError(`Failed to persist task state: ${error}`); + throw error; + } + } + + /** + * Load state from disk. + */ + private async loadState(): Promise { + // Validate format before proceeding + if (!this.format || !["yaml", "json", "csv", "md"].includes(this.format)) { + logError(`Invalid state format: ${this.format}`); + this.tasks = new Map(); + return; + } + + try { + const content = readFileSync(this.stateFilePath, "utf-8"); + let data: StateFileFormat; + + switch (this.format) { + case "yaml": + data = YAML.parse(content) as StateFileFormat; + break; + case "json": + // SECURITY: Parse JSON safely and check for prototype pollution + try { + data = JSON.parse(content) as StateFileFormat; + } catch (parseError) { + throw new Error(`Invalid JSON in state file: ${parseError}`); + } + break; + case "csv": + data = this.fromCSV(content); + break; + case "md": + data = this.fromMarkdown(content); + break; + default: + data = YAML.parse(content) as StateFileFormat; + } + + // Validate data structure before using + if (!data || typeof data !== "object") { + throw new Error("State file contains invalid data structure"); + } + + // Validate no prototype pollution keys using deep check + if (this.hasPrototypePollution(data)) { + throw new Error("State file contains potentially malicious prototype pollution keys"); + } + + if (data.version !== TaskStateManager.STATE_VERSION) { + logDebug( + `Migrating state file from version ${data.version} to ${TaskStateManager.STATE_VERSION}`, + ); + } + + // Validate tasks is an object before creating Map + if (!data.tasks || typeof data.tasks !== "object") { + logDebug("State file has no tasks or invalid tasks structure"); + this.tasks = new Map(); + return; + } + + this.tasks = new Map(Object.entries(data.tasks)); + logDebug(`Loaded ${this.tasks.size} tasks from state file`); + } catch (error) { + logError(`Failed to load task state: ${error}`); + this.tasks = new Map(); + } + } + + /** + * Convert state to CSV format. + */ + private toCSV(data: StateFileFormat): string { + const headers = [ + "key", + "id", + "title", + "state", + "attemptCount", + "lastAttemptTime", + "errorHistory", + ]; + const rows = Object.entries(data.tasks).map(([key, task]) => [ + key, + task.id, + task.title, + task.state, + task.attemptCount, + task.lastAttemptTime ?? "", + task.errorHistory.join("|"), + ]); + + return [ + headers.map((h) => this.csvField(h)).join(","), + ...rows.map((r) => r.map((v) => this.csvField(v)).join(",")), + ].join("\n"); + } + + /** + * Parse state from CSV format. + */ + private fromCSV(content: string): StateFileFormat { + const records = this.parseCsvRecords(content); + if (records.length < 2) { + return { + version: TaskStateManager.STATE_VERSION, + lastUpdated: new Date().toISOString(), + tasks: {}, + }; + } + + const tasks: Record = {}; + for (let i = 1; i < records.length; i++) { + const line = records[i]; + if (!line || line.trim().length === 0) { + continue; + } + + const parts = this.parseCSVLine(line); + if (parts.length >= 4) { + const key = parts[0]; + const id = parts[1]; + const title = parts[2]; + const state = parts[3] as TaskState; + const attemptCount = parts[4]; + const lastAttemptTime = parts[5]; + const errorHistory = parts[6]; + + // Skip entries with invalid key + if (!key) continue; + + // Validate state is a valid TaskState + const validStates = Object.values(TaskState); + if (!validStates.includes(state)) { + logDebug(`Skipping CSV row with invalid state: ${state}`); + continue; + } + + tasks[key] = { + id: id || key, + title: title || "Unknown", + state: state, + attemptCount: attemptCount ? Number.parseInt(attemptCount, 10) || 0 : 0, + lastAttemptTime: lastAttemptTime + ? Number.parseInt(lastAttemptTime, 10) || undefined + : undefined, + errorHistory: errorHistory ? errorHistory.split("|").filter(Boolean) : [], + }; + } + } + + return { + version: TaskStateManager.STATE_VERSION, + lastUpdated: new Date().toISOString(), + tasks, + }; + } + + private csvField(value: string | number): string { + const str = String(value); + if (str.includes(",") || str.includes('"') || str.includes("\n")) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; + } + + private parseCsvRecords(content: string): string[] { + const normalized = content.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + const records: string[] = []; + let current = ""; + let inQuotes = false; + + for (let i = 0; i < normalized.length; i++) { + const char = normalized[i]; + + if (char === '"') { + if (inQuotes && normalized[i + 1] === '"') { + current += '""'; + i++; + } else { + inQuotes = !inQuotes; + current += char; + } + continue; + } + + if (char === "\n" && !inQuotes) { + records.push(current); + current = ""; + continue; + } + + current += char; + } + + if (current.length > 0) { + records.push(current); + } + + return records.filter((record) => record.trim().length > 0); + } + + private parseCSVLine(line: string): string[] { + const parts: string[] = []; + let current = ""; + let inQuotes = false; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + + if (char === '"') { + if (inQuotes && line[i + 1] === '"') { + current += '"'; + i++; + } else { + inQuotes = !inQuotes; + } + continue; + } + + if (char === "," && !inQuotes) { + parts.push(current.trim()); + current = ""; + continue; + } + + current += char; + } + + parts.push(current.trim()); + return parts; + } + + /** + * Convert state to Markdown format. + */ + private toMarkdown(data: StateFileFormat): string { + const lines = ["# Task State", "", `Last Updated: ${data.lastUpdated}`, ""]; + + for (const [key, task] of Object.entries(data.tasks)) { + lines.push(`## ${task.title} (${key})`); + lines.push(""); + lines.push(`- **State**: ${task.state}`); + lines.push(`- **Attempt Count**: ${task.attemptCount}`); + if (task.lastAttemptTime) { + lines.push(`- **Last Attempt**: ${new Date(task.lastAttemptTime).toISOString()}`); + } + if (task.errorHistory.length > 0) { + lines.push(`- **Errors**: ${task.errorHistory.join(", ")}`); + } + lines.push(""); + } + + return lines.join("\n"); + } + + /** + * Parse state from Markdown format. + */ + private fromMarkdown(content: string): StateFileFormat { + const tasks: Record = {}; + const sections = content.split(/\n## /); + + for (const section of sections.slice(1)) { + const lines = section.split("\n"); + const titleMatch = lines[0].match(/(.+) \((.+)\)/); + if (!titleMatch) continue; + + const [, title, key] = titleMatch; + const task: TaskStateEntry = { + id: "", + title, + state: TaskState.PENDING, + attemptCount: 0, + errorHistory: [], + }; + + for (const line of lines) { + if (line.startsWith("- **State**: ")) { + task.state = line.replace("- **State**: ", "").trim() as TaskState; + } else if (line.startsWith("- **Attempt Count**: ")) { + task.attemptCount = Number.parseInt(line.replace("- **Attempt Count**: ", ""), 10) || 0; + } else if (line.startsWith("- **Last Attempt**: ")) { + const dateStr = line.replace("- **Last Attempt**: ", "").trim(); + task.lastAttemptTime = new Date(dateStr).getTime(); + } else if (line.startsWith("- **Errors**: ")) { + task.errorHistory = line + .replace("- **Errors**: ", "") + .split(", ") + .map((s) => s.trim()) + .filter(Boolean); + } + } + + // Extract ID from key + const idMatch = key.match(/[^:]+:[^:]+:(.+)/); + if (idMatch) { + task.id = idMatch[1]; + } + + tasks[key] = task; + } + + return { + version: TaskStateManager.STATE_VERSION, + lastUpdated: new Date().toISOString(), + tasks, + }; + } +} diff --git a/cli/src/tasks/cached-task-source.ts b/cli/src/tasks/cached-task-source.ts index 13ed5ab4..6d215b46 100644 --- a/cli/src/tasks/cached-task-source.ts +++ b/cli/src/tasks/cached-task-source.ts @@ -1,5 +1,4 @@ import { logError } from "../ui/logger.ts"; -import { JsonTaskSource } from "./json.ts"; import type { Task, TaskSource, TaskSourceType } from "./types.ts"; import { YamlTaskSource } from "./yaml.ts"; @@ -56,20 +55,6 @@ export class CachedTaskSource implements TaskSource { return this.inner instanceof YamlTaskSource; } - /** - * Check if the inner source is a JsonTaskSource - */ - isJsonSource(): boolean { - return this.inner instanceof JsonTaskSource; - } - - /** - * Check if the inner source supports parallel groups (YAML or JSON) - */ - supportsParallelGroups(): boolean { - return this.inner instanceof YamlTaskSource || this.inner instanceof JsonTaskSource; - } - async getAllTasks(): Promise { if (!this.cachedTasks) { this.cachedTasks = await this.inner.getAllTasks(); @@ -111,16 +96,13 @@ export class CachedTaskSource implements TaskSource { } /** - * Get the parallel group of a task (YAML or JSON sources) + * Get the parallel group of a task (YamlTaskSource only) */ async getParallelGroup(title: string): Promise { - if (this.inner instanceof YamlTaskSource) { - return this.inner.getParallelGroup(title); - } - if (this.inner instanceof JsonTaskSource) { - return this.inner.getParallelGroup(title); + if (!(this.inner instanceof YamlTaskSource)) { + return 0; } - return 0; + return this.inner.getParallelGroup(title); } /** @@ -212,9 +194,7 @@ export class CachedTaskSource implements TaskSource { ); this.scheduleFlush(); // Retry on failure } else { - logError( - `CachedTaskSource: Failed to flush after ${CachedTaskSource.MAX_FLUSH_RETRIES} retries: ${err}`, - ); + logError(`CachedTaskSource: Failed to flush after ${CachedTaskSource.MAX_FLUSH_RETRIES} retries: ${err}`); this.flushRetryCount = 0; } }); diff --git a/cli/src/tasks/types.ts b/cli/src/tasks/types.ts index a4ba6379..0c5c6c4f 100644 --- a/cli/src/tasks/types.ts +++ b/cli/src/tasks/types.ts @@ -10,6 +10,10 @@ export interface Task { body?: string; /** Parallel group number (0 = sequential, >0 = can run in parallel with same group) */ parallelGroup?: number; + /** Optional description from PRD */ + description?: string; + /** Optional dependencies (task IDs) */ + dependencies?: string[]; /** Whether the task is completed */ completed: boolean; } @@ -17,7 +21,7 @@ export interface Task { /** * Task source type */ -export type TaskSourceType = "markdown" | "markdown-folder" | "yaml" | "json" | "github"; +export type TaskSourceType = "markdown" | "markdown-folder" | "yaml" | "csv" | "github" | "json"; /** * Task source interface - one per format @@ -37,4 +41,6 @@ export interface TaskSource { countCompleted(): Promise; /** Get tasks in a specific parallel group */ getTasksInGroup?(group: number): Promise; + /** Get compact format of all tasks (for planning context) */ + toCompactFormat?(): Promise; } diff --git a/cli/src/telemetry/collector.ts b/cli/src/telemetry/collector.ts index 37b38fe9..b2273dd2 100644 --- a/cli/src/telemetry/collector.ts +++ b/cli/src/telemetry/collector.ts @@ -6,6 +6,7 @@ */ import { randomUUID } from "node:crypto"; +import { sanitizeSecrets } from "../utils/sanitization.ts"; import type { Session, SessionFull, @@ -29,6 +30,26 @@ function getCliVersion(): string { return cachedVersion; } +function sanitizeTelemetryValue(value: unknown): unknown { + if (typeof value === "string") { + return sanitizeSecrets(value); + } + + if (Array.isArray(value)) { + return value.map((item) => sanitizeTelemetryValue(item)); + } + + if (value && typeof value === "object") { + const sanitized: Record = {}; + for (const [key, nested] of Object.entries(value)) { + sanitized[key] = sanitizeTelemetryValue(nested); + } + return sanitized; + } + + return value; +} + /** * Internal tool call tracking (includes timing data) */ @@ -116,8 +137,8 @@ export class TelemetryCollector { // Store prompts/responses for full mode if (this.level === "full") { - if (prompt) this.prompts.push(prompt); - if (response) this.responses.push(response); + if (prompt) this.prompts.push(sanitizeSecrets(prompt)); + if (response) this.responses.push(sanitizeSecrets(response)); } } @@ -131,7 +152,10 @@ export class TelemetryCollector { startTime: Date.now(), toolName, parameterKeys: parameters ? Object.keys(parameters) : undefined, - parameters: this.level === "full" ? parameters : undefined, + parameters: + this.level === "full" + ? (sanitizeTelemetryValue(parameters) as Record | undefined) + : undefined, }; // Track file paths in full mode @@ -164,7 +188,7 @@ export class TelemetryCollector { // Add full mode data if (this.level === "full") { toolCall.parameters = this.activeToolCall.parameters; - if (result) toolCall.result = result; + if (result) toolCall.result = sanitizeSecrets(result); } this.toolCalls.push(toolCall); @@ -199,8 +223,10 @@ export class TelemetryCollector { }; if (this.level === "full") { - toolCall.parameters = options?.parameters; - toolCall.result = options?.result; + toolCall.parameters = sanitizeTelemetryValue(options?.parameters) as + | Record + | undefined; + toolCall.result = options?.result ? sanitizeSecrets(options.result) : undefined; // Track file paths if (options?.parameters) { @@ -303,7 +329,7 @@ export class TelemetryCollector { fullSession.response = this.responses.join("\n\n---\n\n"); } if (this.filePaths.size > 0) { - fullSession.filePaths = Array.from(this.filePaths); + fullSession.filePaths = Array.from(this.filePaths).map((path) => sanitizeSecrets(path)); } return { session: fullSession, toolCalls: this.toolCalls }; } diff --git a/cli/src/telemetry/exporter.ts b/cli/src/telemetry/exporter.ts index c4b4a2be..bef8331c 100644 --- a/cli/src/telemetry/exporter.ts +++ b/cli/src/telemetry/exporter.ts @@ -161,7 +161,7 @@ export class TelemetryExporter { await this.ensureExportsDir(); const filePath = outputPath || join(this.exportsDir, "openai-evals.jsonl"); - await writeFile(filePath, entries.join("\n") + "\n", "utf-8"); + await writeFile(filePath, `${entries.join("\n")}\n`, "utf-8"); return filePath; } @@ -194,7 +194,7 @@ export class TelemetryExporter { await this.ensureExportsDir(); const filePath = outputPath || join(this.exportsDir, "raw-telemetry.jsonl"); - const lines = entries.map((e) => JSON.stringify(e)).join("\n") + "\n"; + const lines = `${entries.map((e) => JSON.stringify(e)).join("\n")}\n`; await writeFile(filePath, lines, "utf-8"); return filePath; diff --git a/cli/src/telemetry/types.ts b/cli/src/telemetry/types.ts index 41650f3c..11424fa8 100644 --- a/cli/src/telemetry/types.ts +++ b/cli/src/telemetry/types.ts @@ -78,6 +78,51 @@ export interface ToolCall { */ export type TelemetryLevel = "anonymous" | "full"; +/** + * Full session data for webhook + */ +export interface WebhookSessionData { + sessionId: string; + engine: string; + mode: string; + cliVersion: string; + platform: string; + totalTokensIn: number; + totalTokensOut: number; + totalDurationMs: number; + taskCount: number; + successCount: number; + failedCount: number; + toolCalls: { + toolName: string; + callCount: number; + successCount: number; + failedCount: number; + avgDurationMs: number; + }[]; + tags?: string[]; +} + +/** + * Full session details for webhook (full privacy mode) + */ +export interface WebhookSessionDetails { + prompt?: string; + response?: string; + filePaths?: string[]; +} + +/** + * Telemetry webhook payload + */ +export interface TelemetryWebhookPayload { + event: string; + version: string; + timestamp: string; + session: WebhookSessionData; + details?: WebhookSessionDetails; +} + /** * Telemetry configuration */ diff --git a/cli/src/telemetry/webhook.ts b/cli/src/telemetry/webhook.ts index c305e0e0..b68299e6 100644 --- a/cli/src/telemetry/webhook.ts +++ b/cli/src/telemetry/webhook.ts @@ -76,11 +76,10 @@ export async function sendTelemetryWebhook( } const payload = buildPayload(session, level); + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout - const response = await fetch(webhookUrl, { method: "POST", headers: { @@ -90,14 +89,19 @@ export async function sendTelemetryWebhook( signal: controller.signal, }); - clearTimeout(timeoutId); - if (!response.ok) { const text = await response.text().catch(() => ""); throw new Error(`HTTP ${response.status}${text ? `: ${text}` : ""}`); } - logDebug(`Telemetry webhook sent successfully to ${webhookUrl}`); + const safeTarget = (() => { + try { + return new URL(webhookUrl).host; + } catch { + return "configured endpoint"; + } + })(); + logDebug(`Telemetry webhook sent successfully to ${safeTarget}`); } catch (error) { if (error instanceof Error && error.name === "AbortError") { logError("Telemetry webhook timed out after 10 seconds"); @@ -107,5 +111,7 @@ export async function sendTelemetryWebhook( ); } // Don't throw - webhook failures shouldn't break the session + } finally { + clearTimeout(timeoutId); } } diff --git a/cli/src/telemetry/writer.ts b/cli/src/telemetry/writer.ts index ab64c48f..8b0d9569 100644 --- a/cli/src/telemetry/writer.ts +++ b/cli/src/telemetry/writer.ts @@ -7,6 +7,7 @@ import { existsSync } from "node:fs"; import { appendFile, mkdir, readFile, readdir } from "node:fs/promises"; import { dirname, join } from "node:path"; +import { logDebug } from "../ui/logger.ts"; import type { Session, SessionFull, ToolCall } from "./types.js"; const DEFAULT_OUTPUT_DIR = ".ralphy/telemetry"; @@ -56,7 +57,7 @@ export class TelemetryWriter { async writeSession(session: Session | SessionFull): Promise { await this.ensureDir(); const path = join(this.outputDir, SESSIONS_FILE); - const line = JSON.stringify(session) + "\n"; + const line = `${JSON.stringify(session)}\n`; await appendFile(path, line, "utf-8"); } @@ -68,7 +69,7 @@ export class TelemetryWriter { await this.ensureDir(); const path = join(this.outputDir, TOOL_CALLS_FILE); - const lines = toolCalls.map((call) => JSON.stringify(call)).join("\n") + "\n"; + const lines = `${toolCalls.map((call) => JSON.stringify(call)).join("\n")}\n`; await appendFile(path, lines, "utf-8"); } @@ -91,8 +92,16 @@ export class TelemetryWriter { const content = await readFile(path, "utf-8"); const lines = content.trim().split("\n").filter(Boolean); + const sessions: Array = []; + for (const line of lines) { + try { + sessions.push(JSON.parse(line) as Session | SessionFull); + } catch (error) { + logDebug(`Skipping invalid telemetry session line: ${error}`); + } + } - return lines.map((line) => JSON.parse(line) as Session | SessionFull); + return sessions; } /** @@ -107,8 +116,16 @@ export class TelemetryWriter { const content = await readFile(path, "utf-8"); const lines = content.trim().split("\n").filter(Boolean); + const toolCalls: ToolCall[] = []; + for (const line of lines) { + try { + toolCalls.push(JSON.parse(line) as ToolCall); + } catch (error) { + logDebug(`Skipping invalid telemetry tool-call line: ${error}`); + } + } - return lines.map((line) => JSON.parse(line) as ToolCall); + return toolCalls; } /** diff --git a/cli/src/ui/logger.ts b/cli/src/ui/logger.ts index 324e719d..1655dced 100644 --- a/cli/src/ui/logger.ts +++ b/cli/src/ui/logger.ts @@ -1,49 +1,407 @@ +import { appendFileSync } from "node:fs"; +import path from "node:path"; import pc from "picocolors"; +import { sanitizeSecrets } from "../utils/sanitization.ts"; -let verboseMode = false; +// Use a module-level object for state to avoid direct mutable exports +const loggerState = { + verboseMode: false, + debugMode: false, +}; + +// Allowed log directory - logs can only be written here +const ALLOWED_LOG_DIR = "logs"; + +/** + * Validate log file path to prevent path traversal attacks + */ +function validateLogPath(filePath: string): string { + const resolved = path.resolve(filePath); + const allowedDir = path.resolve(process.cwd(), ALLOWED_LOG_DIR); + const relative = path.relative(allowedDir, resolved); + + if (relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error(`Invalid log file path: ${filePath} must be within ${ALLOWED_LOG_DIR}`); + } + + return resolved; +} + +/** + * Get current verbose mode state + */ +export function isVerbose(): boolean { + return loggerState.verboseMode; +} + +/** + * Get current debug mode state + */ +export function isDebug(): boolean { + return loggerState.debugMode; +} /** * Set verbose mode */ export function setVerbose(verbose: boolean): void { - verboseMode = verbose; + loggerState.verboseMode = verbose; + verboseMode = loggerState.verboseMode; +} + +/** + * Set debug mode (implies verbose) + */ +export function setDebug(debug: boolean): void { + loggerState.debugMode = debug; + if (debug) { + loggerState.verboseMode = true; + } + verboseMode = loggerState.verboseMode; +} + +// BUG FIX: Export a getter function instead of a stale primitive value +// This ensures consumers always get the current state, not the initial value +export function getVerboseMode(): boolean { + return loggerState.verboseMode; +} + +// Keep backward compatibility export but mark as deprecated +/** @deprecated Use getVerboseMode() instead for live value */ +export let verboseMode = loggerState.verboseMode; + +/** + * Log levels for structured logging + */ +export type LogLevel = "debug" | "info" | "success" | "warn" | "error"; + +/** + * Structured log entry interface + */ +export interface LogEntry { + timestamp: string; + level: LogLevel; + component: string; + message: string; + context?: Record; +} + +/** + * Log sink interface for extensible logging + */ +export interface LogSink { + write(entry: LogEntry): void; +} + +/** + * Default console log sink with colors + */ +class ConsoleLogSink implements LogSink { + write(entry: LogEntry): void { + // Defensive: validate entry has required fields + if (!entry || typeof entry !== "object") { + console.error("[Logger] Invalid log entry"); + return; + } + const timestamp = entry.timestamp ?? new Date().toISOString(); + const level = entry.level ?? "info"; + const component = entry.component ?? "ralphy"; + const message = entry.message ?? ""; + const prefix = `[${timestamp}] [${level.toUpperCase()}]`; + + switch (level) { + case "error": + console.error(pc.red(`${prefix} ${component ? `[${component}] ` : ""}${message}`)); + break; + case "warn": + console.warn(pc.yellow(`${prefix} ${component ? `[${component}] ` : ""}${message}`)); + break; + case "success": + console.log(pc.green(`[OK] ${message}`)); + break; + case "info": + console.log(pc.blue(`${prefix} ${component ? `[${component}] ` : ""}${message}`)); + break; + case "debug": + console.log(pc.gray(`${prefix} ${component ? `[${component}] ` : ""}${message}`)); + break; + default: + console.log(`${prefix} ${component ? `[${component}] ` : ""}${message}`); + } + } +} + +// Global log sink instance +let logSink: LogSink = new ConsoleLogSink(); + +/** + * Set a custom log sink for extensible logging + */ +export function setLogSink(sink: LogSink): void { + logSink = sink; +} + +/** + * Get current log sink + */ +export function getLogSink(): LogSink { + return logSink; +} + +/** + * Internal function to create log entry + * Sanitizes secrets from logged data + */ +function createLogEntry(level: LogLevel, component: string | undefined, args: unknown[]): LogEntry { + const rawMessage = args + .map((arg) => (typeof arg === "string" ? arg : JSON.stringify(arg))) + .join(" "); + // Sanitize secrets from the message + const message = sanitizeSecrets(rawMessage); + + return { + timestamp: new Date().toISOString(), + level, + component: component || "ralphy", + message, + }; +} + +/** + * Core logging function + */ +function log(level: LogLevel, component: string | undefined, ...args: unknown[]): void { + // Debug messages only show in verbose or debug mode + if (level === "debug" && !loggerState.verboseMode) { + return; + } + + const entry = createLogEntry(level, component, args); + logSink.write(entry); } /** * Log info message */ export function logInfo(...args: unknown[]): void { - console.log(pc.blue("[INFO]"), ...args); + log("info", undefined, ...args); +} + +/** + * Log info message with component context + */ +export function logInfoContext(component: string, ...args: unknown[]): void { + log("info", component, ...args); } /** * Log success message */ export function logSuccess(...args: unknown[]): void { - console.log(pc.green("[OK]"), ...args); + log("success", undefined, ...args); +} + +/** + * Log success message with component context + */ +export function logSuccessContext(component: string, ...args: unknown[]): void { + log("success", component, ...args); } /** * Log warning message */ export function logWarn(...args: unknown[]): void { - console.log(pc.yellow("[WARN]"), ...args); + log("warn", undefined, ...args); +} + +/** + * Log warning message with component context + */ +export function logWarnContext(component: string, ...args: unknown[]): void { + log("warn", component, ...args); } /** * Log error message */ export function logError(...args: unknown[]): void { - console.error(pc.red("[ERROR]"), ...args); + log("error", undefined, ...args); +} + +/** + * Log error message with component context + */ +export function logErrorContext(component: string, ...args: unknown[]): void { + log("error", component, ...args); } /** * Log debug message (only in verbose mode) */ export function logDebug(...args: unknown[]): void { - if (verboseMode) { - console.log(pc.dim("[DEBUG]"), ...args); + log("debug", undefined, ...args); +} + +/** + * Log debug message with component context + */ +export function logDebugContext(component: string, ...args: unknown[]): void { + log("debug", component, ...args); +} + +/** + * JSON file log sink for structured logging to file + */ +export class JsonFileLogSink implements LogSink { + private filePath: string; + private buffer: LogEntry[] = []; + private flushInterval: number; + private maxBufferSize: number; + // BUG FIX: Use proper nullable type instead of type cast hack + private flushTimer: NodeJS.Timeout | null = null; + + constructor(filePath: string, options?: { flushIntervalMs?: number; maxBufferSize?: number }) { + // Validate path to prevent path traversal attacks + this.filePath = validateLogPath(filePath); + this.flushInterval = options?.flushIntervalMs ?? 1000; + this.maxBufferSize = options?.maxBufferSize ?? 100; + + // Auto-flush buffer periodically + this.flushTimer = setInterval(() => { + try { + this.flush(); + } catch (err) { + console.error(`Failed to flush log buffer: ${err}`); + } + }, this.flushInterval); } + + private isFlushing = false; + + write(entry: LogEntry): void { + this.buffer.push(entry); + + if (this.buffer.length >= this.maxBufferSize) { + this.flush(); + } + } + + private flush(): void { + // Prevent concurrent flushes (race condition fix) + if (this.isFlushing || this.buffer.length === 0) return; + + this.isFlushing = true; + let currentBuffer: LogEntry[] = []; + + try { + // ATOMIC: Swap buffers instead of copy-then-clear + // This prevents race conditions where write() is called between copy and clear + currentBuffer = this.buffer; + this.buffer = []; // New empty buffer assigned atomically + const lines = `${currentBuffer.map((entry) => JSON.stringify(entry)).join("\n")}\n`; + appendFileSync(this.filePath, lines, "utf-8"); + } catch (error) { + console.error(`Failed to write to log file: ${error}`); + // Limit buffer size to prevent memory exhaustion on persistent write failures + const MAX_BUFFER_SIZE = 10000; + const combined = [...currentBuffer, ...this.buffer]; + // Keep only the most recent entries, discard oldest to prevent memory leak + if (combined.length > MAX_BUFFER_SIZE) { + this.buffer = combined.slice(-MAX_BUFFER_SIZE); + console.warn(`Log buffer truncated to ${MAX_BUFFER_SIZE} entries due to write failure`); + } else { + this.buffer = combined; + } + } finally { + this.isFlushing = false; + } + } + + /** + * Dispose of the file log sink, stopping the flush timer. + * Call this when done logging to prevent memory leaks. + */ + dispose(): void { + // BUG FIX: Proper nullable type check without type cast hack + if (this.flushTimer !== null) { + clearInterval(this.flushTimer); + this.flushTimer = null; + } + // Final flush to ensure all logs are written + this.flush(); + } +} + +/** + * Multi-sink that writes to multiple log sinks + */ +export class MultiLogSink implements LogSink { + private sinks: LogSink[]; + + constructor(sinks: LogSink[]) { + this.sinks = sinks; + } + + write(entry: LogEntry): void { + for (const sink of this.sinks) { + try { + sink.write(entry); + } catch (error) { + console.error(`Log sink failed: ${error}`); + } + } + } + + addSink(sink: LogSink): void { + this.sinks.push(sink); + } +} + +/** + * Filtered log sink that only passes certain log levels + */ +export class FilteredLogSink implements LogSink { + private sink: LogSink; + private minLevel: LogLevel; + private levelPriority: Record = { + debug: 0, + info: 1, + success: 2, + warn: 3, + error: 4, + }; + + constructor(sink: LogSink, minLevel: LogLevel) { + this.sink = sink; + this.minLevel = minLevel; + } + + write(entry: LogEntry): void { + if (this.levelPriority[entry.level] >= this.levelPriority[this.minLevel]) { + this.sink.write(entry); + } + } +} + +/** + * Initialize structured logging with file output + * @param logFilePath - Path to JSON log file (optional) + * @param minLevel - Minimum log level to record (default: "info") + */ +export function initializeStructuredLogging( + logFilePath?: string, + minLevel: LogLevel = "info", +): void { + const sinks: LogSink[] = [new ConsoleLogSink()]; + + if (logFilePath) { + const fileSink = new JsonFileLogSink(logFilePath); + const filteredFileSink = new FilteredLogSink(fileSink, minLevel); + sinks.push(filteredFileSink); + } + + setLogSink(new MultiLogSink(sinks)); } /** diff --git a/cli/src/ui/static-agent-display.ts b/cli/src/ui/static-agent-display.ts new file mode 100644 index 00000000..7f69214f --- /dev/null +++ b/cli/src/ui/static-agent-display.ts @@ -0,0 +1,285 @@ +import type { AgentProgress, ExecutionPhase } from "../execution/progress-types.ts"; +import { formatDuration } from "./logger.ts"; + +const c = { + rst: "\x1b[0m", + bld: "\x1b[1m", + dim: "\x1b[2m", + red: "\x1b[31m", + grn: "\x1b[32m", + yel: "\x1b[33m", + blu: "\x1b[34m", + mag: "\x1b[35m", + cyn: "\x1b[36m", + wht: "\x1b[37m", + gry: "\x1b[90m", +}; + +export class StaticAgentDisplay { + private static instance: StaticAgentDisplay | null = null; + private agentProgressMap = new Map(); + private displayInterval: NodeJS.Timeout | null = null; + + constructor() { + StaticAgentDisplay.instance?.stopDisplay(); + StaticAgentDisplay.instance = this; + } + static getInstance(): StaticAgentDisplay | null { + return StaticAgentDisplay.instance; + } + + log(_message: string): void { + // Logs interrupt display - will be redrawn + } + + updateAgent(agentNum: number, step: string): void { + const current = this.agentProgressMap.get(agentNum); + if (!current) return; + if (!current.recentSteps) current.recentSteps = []; + + const cleanStep = step + .trim() + .replace(/^\[RAW OPENCODE OUTPUT\]\s*/i, "") + .replace(/^Thinking:\s*/i, ""); + + // Skip garbled/encoded content + if (cleanStep.match(/^[A-Za-z0-9+/]{30,}$/)) return; + if (!cleanStep || cleanStep.length < 3) return; + + if (current.recentSteps[current.recentSteps.length - 1] === cleanStep) return; + + current.recentSteps.push(cleanStep); + if (current.recentSteps.length > 5) current.recentSteps.shift(); + } + + updateAgentFromOpenCode(agentNum: number, jsonLine: string): void { + try { + // Defensive: ensure jsonLine is a string + if (typeof jsonLine !== "string") { + return; + } + // Defensive: check for empty or whitespace-only strings + if (!jsonLine || jsonLine.trim().length === 0) { + return; + } + const parsed = JSON.parse(jsonLine); + // Defensive: validate parsed is an object + if (!parsed || typeof parsed !== "object") { + return; + } + if (parsed.type === "text" && parsed.part?.text) { + const text = parsed.part.text.trim(); + if (text && text.length > 3 && !text.startsWith("{")) { + this.updateAgent(agentNum, text); + } + } else if (parsed.type === "tool_use" && parsed.part?.tool) { + const tool = parsed.part.tool; + const input = parsed.part.state?.input || {}; + const file = input.filePath || input.path || ""; + this.updateAgent(agentNum, file ? `${tool}: ${file}` : tool); + } else if (parsed.type === "step_finish" && parsed.part?.tokens) { + const t = parsed.part.tokens; + // Defensive: validate token values are numbers + const inputTokens = typeof t.input === "number" ? t.input : 0; + const outputTokens = typeof t.output === "number" ? t.output : 0; + this.updateAgent(agentNum, `${inputTokens}→${outputTokens} tokens`); + } + } catch { + // Not JSON - ignore silently + } + } + + startDisplay(): void { + if (this.displayInterval) return; + this.render(); + this.displayInterval = setInterval(() => this.render(), 1000); + } + + stopDisplay(): void { + if (this.displayInterval) { + clearInterval(this.displayInterval); + this.displayInterval = null; + } + this.render(); + this.agentProgressMap.clear(); + } + + private render(): void { + const agents = Array.from(this.agentProgressMap.values()); + if (agents.length === 0) return; + + // Get current phase from first agent + const currentPhase = agents[0]?.phase || "execution"; + + const width = process.stdout.columns || 80; + + // Clear screen and move to top + process.stdout.write("\x1b[2J\x1b[0;0H"); + + // Workflow bar + console.log(); + console.log(this.renderWorkflowLine(currentPhase, width)); + console.log(); + + // Header + const title = " AGENTS "; + const side = Math.floor((width - title.length) / 2); + console.log( + `${c.cyn}${"─".repeat(side)}${c.bld}${title}${c.rst}${c.cyn}${"─".repeat(width - side - title.length)}${c.rst}`, + ); + console.log(); + + // Each agent with 5 numbered steps + for (const agent of agents) { + console.log(this.renderAgentLine(agent)); + + const steps = agent.recentSteps || []; + // Pad to always show 5 lines + for (let i = 0; i < 5; i++) { + const num = i + 1; + if (i < steps.length) { + const formatted = this.formatStepWithColors(steps[steps.length - 1 - i]); + console.log(` ${c.gry}${num}.${c.rst} ${formatted}`); + } else { + console.log(` ${c.gry}${num}.${c.rst}`); + } + } + console.log(); + } + + // Instructions at bottom + console.log(`${c.gry}Press Ctrl+C to stop${c.rst}`); + } + + private renderWorkflowLine(phase: ExecutionPhase, width: number): string { + const phases: ExecutionPhase[] = ["planning", "execution", "testing"]; + const phaseIndex = phases.indexOf(phase); + + const parts: string[] = []; + for (let i = 0; i < phases.length; i++) { + const p = phases[i]; + const isActive = i === phaseIndex; + const isPast = i < phaseIndex; + + if (isActive) { + const color = p === "planning" ? c.cyn : p === "execution" ? c.mag : c.yel; + parts.push(`${c.bld}${color}▓▓▓ ${p.toUpperCase()} ▓▓▓${c.rst}`); + } else if (isPast) { + parts.push(`${c.gry}░ ${p.toUpperCase()} ░${c.rst}`); + } else { + parts.push(`${c.gry}${c.dim} ${p.toUpperCase()} ${c.rst}`); + } + + if (i < phases.length - 1) { + parts.push(isPast ? `${c.cyn} → ${c.rst}` : `${c.gry} → ${c.rst}`); + } + } + + const content = parts.join(""); + const pad = Math.max(0, Math.floor((width - this.stripAnsi(content).length) / 2)); + return " ".repeat(pad) + content; + } + + private stripAnsi(str: string): string { + // biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escape sequences are intentional + return str.replace(/\x1b\[[0-9;]*m/g, ""); + } + + private formatStepWithColors(step: string): string { + // Match patterns like "Tool: bash: command" or "Glob: pattern" or "Read: filepath" + const toolMatch = step.match( + /^(Tool|Read|Write|Edit|Create|Delete|Glob|Grep|Search|Analyze|Run|Test|Execute|Build|Fix|Debug)\s*:\s*(.+)/i, + ); + if (toolMatch) { + const action = toolMatch[1]; + const rest = toolMatch[2]; + // Split rest by first colon if present (e.g., "bash: ls -la") + const subMatch = rest.match(/^([^:]+):\s*(.+)/); + if (subMatch) { + const tool = subMatch[1]; + const args = subMatch[2]; + // Color the action type + const actionColor = this.getActionColor(action); + return `${actionColor}${action}${c.rst}: ${c.cyn}${tool}${c.rst}: ${c.gry}${args.slice(0, 50)}${c.rst}`; + } + // No sub-colon, just action: rest + const actionColor = this.getActionColor(action); + return `${actionColor}${action}${c.rst}: ${c.gry}${rest.slice(0, 55)}${c.rst}`; + } + // For plain text steps, return as-is (will be white) + return `${c.wht}${step.slice(0, 60)}${c.rst}`; + } + + private getActionColor(action: string): string { + const lower = action.toLowerCase(); + if (lower === "tool" || lower === "run" || lower === "execute") return c.yel; + if ( + lower === "read" || + lower === "glob" || + lower === "grep" || + lower === "search" || + lower === "analyze" + ) + return c.blu; + if (lower === "write" || lower === "edit" || lower === "create" || lower === "delete") + return c.mag; + if (lower === "test" || lower === "build") return c.grn; + if (lower === "fix" || lower === "debug") return c.red; + return c.wht; + } + + private renderAgentLine(agent: AgentProgress): string { + const phase = agent.phase || "execution"; + const model = agent.modelName || "main"; + const elapsed = formatDuration(Date.now() - agent.startTime); + const status = + agent.status === "completed" + ? `${c.grn}✓${c.rst}` + : agent.status === "failed" + ? `${c.red}✗${c.rst}` + : `${c.cyn}●${c.rst}`; + + const phaseColor = phase === "planning" ? c.cyn : phase === "execution" ? c.mag : c.yel; + const phaseTag = `${phaseColor}[${phase.toUpperCase()}]${c.rst}`; + const modelTag = `${c.gry}[${c.blu}${model}${c.gry}]${c.rst}`; + const title = + agent.taskTitle.length > 30 ? `${agent.taskTitle.slice(0, 27)}...` : agent.taskTitle; + + return `${status} ${c.bld}Agent ${agent.agentNum}${c.rst} ${phaseTag} ${c.wht}${title}${c.rst} ${modelTag} ${c.gry}${elapsed}${c.rst}`; + } + + setAgentStatus( + agentNum: number, + taskTitle: string, + status: "planning" | "working" | "completed" | "failed", + phase?: ExecutionPhase, + modelName?: string, + ): void { + const current = this.agentProgressMap.get(agentNum); + if (!current) { + this.agentProgressMap.set(agentNum, { + agentNum, + taskTitle, + status, + phase: phase || "execution", + modelName: modelName || "main", + worktreeDir: "", + startTime: Date.now(), + recentSteps: [], + }); + } else { + current.status = status; + if (phase) current.phase = phase; + if (modelName) current.modelName = modelName; + } + } + + clearAgentSteps(agentNum: number): void { + const current = this.agentProgressMap.get(agentNum); + if (current) current.recentSteps = []; + } + + agentComplete(agentNum: number): void { + this.agentProgressMap.delete(agentNum); + } +} diff --git a/cli/src/utils/cleanup.ts b/cli/src/utils/cleanup.ts new file mode 100644 index 00000000..211458cb --- /dev/null +++ b/cli/src/utils/cleanup.ts @@ -0,0 +1,157 @@ +import type { ChildProcess } from "node:child_process"; +import { spawnSync } from "node:child_process"; +import { logDebug, logWarn } from "../ui/logger.ts"; + +type CleanupFn = () => Promise | void; + +const cleanupRegistry: Set = new Set(); +const trackedProcesses: Set = new Set(); +let isCleaningUp = false; + +function isProcessRunning(proc: ChildProcess): boolean { + return proc.exitCode === null && proc.signalCode === null; +} + +/** + * Register a function to be called on process exit or manual cleanup + */ +export function registerCleanup(fn: CleanupFn): () => void { + cleanupRegistry.add(fn); + return () => cleanupRegistry.delete(fn); +} + +/** + * Register a child process to be tracked and killed on exit + */ +export function registerProcess(proc: ChildProcess): () => void { + trackedProcesses.add(proc); + + const remove = () => trackedProcesses.delete(proc); + + proc.on("exit", remove); + proc.on("error", remove); + + return remove; +} + +/** + * Run all registered cleanup functions and kill tracked processes + */ +export async function runCleanup(): Promise { + if (isCleaningUp) return; + isCleaningUp = true; + + // 1. Kill all tracked child processes with verification + for (const proc of trackedProcesses) { + try { + if (proc.pid && isProcessRunning(proc)) { + const pid = proc.pid; + + if (process.platform === "win32") { + // Windows needs taskkill for robust child tree termination + const result = spawnSync("taskkill", ["/pid", String(pid), "/f", "/t"], { + stdio: "pipe", + }); + + // Verify the process was actually killed + // Status 128 = process already exited, which is fine + if (result.status !== 0 && result.status !== 128) { + logWarn(`taskkill may have failed for PID ${pid} (exit code: ${result.status})`); + if (result.stderr) { + logDebug(`taskkill stderr: ${result.stderr.toString()}`); + } + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + if (isProcessRunning(proc)) { + logWarn(`Process ${pid} may still be running after taskkill`); + } + } else { + // Try graceful termination first + proc.kill("SIGTERM"); + + // Wait a bit and verify it's dead + await new Promise((resolve) => setTimeout(resolve, 1000)); + + // Check if process is still running + if (isProcessRunning(proc)) { + proc.kill("SIGKILL"); + + // Final verification + await new Promise((resolve) => setTimeout(resolve, 500)); + if (isProcessRunning(proc)) { + logWarn(`Failed to terminate process ${pid} after SIGKILL`); + } + } + } + } + } catch (err) { + // Process termination failed, continue cleanup + logDebug(`Failed to terminate process ${proc.pid}: ${err}`); + } + } + trackedProcesses.clear(); + + // 2. Run registered cleanup functions + const promises: Promise[] = []; + for (const fn of cleanupRegistry) { + try { + const result = fn(); + if (result instanceof Promise) { + promises.push(result); + } + } catch (err) { + // Log sync errors but continue with other cleanup functions + promises.push(Promise.reject(err)); + } + } + + const results = await Promise.allSettled(promises); + for (const result of results) { + if (result.status === "rejected") { + logWarn(`Cleanup task failed: ${result.reason}`); + } + } + cleanupRegistry.clear(); + isCleaningUp = false; +} + +let isShuttingDown = false; +let handlersRegistered = false; + +/** + * Setup process signal handlers for cleanup + */ +export function setupSignalHandlers(): void { + if (handlersRegistered) { + return; + } + handlersRegistered = true; + + const signals: NodeJS.Signals[] = ["SIGINT", "SIGTERM"]; + + for (const signal of signals) { + process.on(signal, async () => { + // Prevent duplicate cleanup runs + if (isShuttingDown) { + process.stdout.write(`\nReceived ${signal}, cleanup already in progress...\n`); + return; + } + isShuttingDown = true; + + // Use writeSync to avoid event loop issues during exit + process.stdout.write(`\nReceived ${signal}, cleaning up processes and files...\n`); + + try { + await runCleanup(); + process.exit(0); + } catch (error) { + process.stderr.write(`\nCleanup failed: ${error}\n`); + process.exit(1); + } + }); + } + + // Note: uncaughtException is handled in cli/src/index.ts for the main process + // This avoids duplicate handlers and ensures consistent error handling +} diff --git a/cli/src/utils/errors.ts b/cli/src/utils/errors.ts new file mode 100644 index 00000000..e03a8aea --- /dev/null +++ b/cli/src/utils/errors.ts @@ -0,0 +1,145 @@ +/** + * Standardized error handling utilities for consistent error types across the codebase + */ + +export class RalphyError extends Error { + public readonly code: string; + public readonly context?: Record; + + constructor(message: string, code = "RALPHY_ERROR", context?: Record) { + super(message); + this.name = "RalphyError"; + this.code = code; + this.context = context; + + // Maintains proper stack trace for where our error was thrown (only available on V8) + if (Error.captureStackTrace) { + Error.captureStackTrace(this, RalphyError); + } + } +} + +export class ValidationError extends RalphyError { + constructor(message: string, context?: Record) { + super(message, "VALIDATION_ERROR", context); + this.name = "ValidationError"; + } +} + +export class TimeoutError extends RalphyError { + constructor(message: string, context?: Record) { + super(message, "TIMEOUT_ERROR", context); + this.name = "TimeoutError"; + } +} + +export class LockError extends RalphyError { + constructor(message: string, context?: Record) { + super(message, "LOCK_ERROR", context); + this.name = "LockError"; + } +} + +export class ProcessError extends RalphyError { + constructor(message: string, context?: Record) { + super(message, "PROCESS_ERROR", context); + this.name = "ProcessError"; + } +} + +export class SandboxError extends RalphyError { + constructor(message: string, context?: Record) { + super(message, "SANDBOX_ERROR", context); + this.name = "SandboxError"; + } +} + +/** + * Convert any error to a standardized format + */ +export function standardizeError(error: unknown): RalphyError { + if (error instanceof RalphyError) { + return error; + } + + if (error instanceof Error) { + return new RalphyError(error.message, "UNKNOWN_ERROR", { + originalName: error.name, + originalStack: error.stack, + }); + } + + if (typeof error === "string") { + return new RalphyError(error, "STRING_ERROR"); + } + + return new RalphyError(String(error), "UNKNOWN_ERROR", { originalType: typeof error }); +} + +/** + * Check if an error is retryable + */ +export function isRetryableError(error: unknown): boolean { + const standardized = standardizeError(error); + + const retryableCodes = [ + "TIMEOUT_ERROR", + "LOCK_ERROR", + "PROCESS_ERROR", + "NETWORK_ERROR", + "RATE_LIMIT_ERROR", + ]; + + const retryableMessages = [ + "timeout", + "connection refused", + "network", + "rate limit", + "too many requests", + "temporary failure", + "try again", + "locked", + "conflict", + "connection error", + "unable to connect", + "internet connection", + "econnrefused", + "econnreset", + "socket hang up", + "fetch failed", + "hit your limit", + "quota", + "429", + "enotfound", + "overloaded", + ]; + + const message = standardized.message.toLowerCase(); + + // Check error code + if (retryableCodes.includes(standardized.code)) { + return true; + } + + // Check error message + return retryableMessages.some((pattern) => message.includes(pattern)); +} + +/** + * Create error with context for logging + */ +export function createErrorWithContext( + error: unknown, + context: Record, +): RalphyError { + const standardized = standardizeError(error); + + if (standardized.context) { + return new RalphyError(standardized.message, standardized.code, { + ...standardized.context, + ...context, + }); + } + + return new RalphyError(standardized.message, standardized.code, context); +} diff --git a/cli/src/utils/file-indexer.ts b/cli/src/utils/file-indexer.ts new file mode 100644 index 00000000..024dd3ce --- /dev/null +++ b/cli/src/utils/file-indexer.ts @@ -0,0 +1,1025 @@ +/** + * File Indexer Module + * + * Provides semantic chunking for large codebases and file hash caching for unchanged files. + * This module indexes the codebase with file metadata (path, hash, size, mtime, keywords) + * and provides semantic search to find relevant files based on task keywords. + */ + +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join, relative } from "node:path"; +import { DEFAULT_IGNORE_PATTERNS, MAX_FILE_SIZE_FOR_HASH } from "../config/constants.ts"; +import { RALPHY_DIR } from "../config/loader.ts"; +import { logDebug } from "../ui/logger.ts"; + +// Constants +const FILE_INDEX_CACHE = "file-index.json"; +const MAX_KEYWORDS_PER_FILE = 20; +const MAX_CONTENT_PREVIEW_LENGTH = 500; +const RELEVANCE_THRESHOLD = 0.1; + +/** + * Maximum glob pattern length to prevent ReDoS attacks + */ +const MAX_GLOB_PATTERN_LENGTH = 1000; + +/** + * File metadata entry in the index + */ +export interface FileIndexEntry { + /** Relative path from workspace root */ + path: string; + /** File content hash (sha256, first 16 chars) */ + hash: string; + /** File size in bytes */ + size: number; + /** Last modification time (ms since epoch) */ + mtime: number; + /** Extracted keywords from path and content */ + keywords: string[]; + /** Content preview for semantic analysis */ + preview?: string; + /** File extension */ + extension: string; + /** Directory depth */ + depth: number; +} + +/** + * The complete file index for a workspace + */ +export interface FileIndex { + /** Version for cache invalidation */ + version: number; + /** Timestamp of index creation */ + timestamp: number; + /** Workspace root path */ + workDir: string; + /** Map of relative paths to file entries */ + files: Map; + /** Total files indexed */ + totalFiles: number; + /** Total size of all indexed files */ + totalSize: number; +} + +/** + * Serialized version of FileIndex for JSON storage + */ +interface SerializedFileIndex { + version: number; + timestamp: number; + workDir: string; + files: Record; + totalFiles: number; + totalSize: number; +} + +// In-memory cache of file indexes +const indexCache = new Map(); + +// Track promises for workspaces being indexed to allow waiting +const indexingPromises = new Map>(); + +/** + * Deep clone a FileIndex to return an immutable copy + * Prevents callers from modifying the shared cache + */ +function cloneFileIndex(index: FileIndex): FileIndex { + return { + version: index.version, + timestamp: index.timestamp, + workDir: index.workDir, + files: new Map(index.files), + totalFiles: index.totalFiles, + totalSize: index.totalSize, + }; +} + +/** + * Get the path to the file index cache + */ +function getIndexCachePath(workDir: string): string { + return join(workDir, RALPHY_DIR, FILE_INDEX_CACHE); +} + +/** + * Check if a file should be ignored based on patterns + */ +function shouldIgnoreFile(filePath: string, ignorePatterns: string[]): boolean { + const normalizedPath = filePath.replace(/\\/g, "/"); + + for (const pattern of ignorePatterns) { + if (matchesGlob(normalizedPath, pattern)) { + return true; + } + } + + return false; +} + +/** + * Convert glob pattern to regex + */ +function matchesGlob(filePath: string, pattern: string): boolean { + // Handle ** patterns properly + const regexPattern = globToRegex(pattern); + return regexPattern.test(filePath); +} + +/** + * Convert glob pattern to regex + * + * SECURITY NOTE: This function includes protections against ReDoS attacks: + * - Input length is limited to MAX_GLOB_PATTERN_LENGTH + * - Uses non-backtracking patterns where possible + */ +function globToRegex(pattern: string): RegExp { + const safePattern = + pattern.length > MAX_GLOB_PATTERN_LENGTH + ? pattern.slice(0, MAX_GLOB_PATTERN_LENGTH) + : pattern; + + // Limit pattern length to prevent ReDoS attacks + if (safePattern.length < pattern.length) { + logDebug(`Glob pattern too long (${pattern.length} > ${MAX_GLOB_PATTERN_LENGTH}), truncating`); + } + + // Escape special regex characters except * and ? + // Use a bounded approach to prevent catastrophic backtracking + let regex = safePattern + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*\*/g, "\0DOUBLESTAR\0") // Temporarily mark ** + .replace(/\*/g, "[^/]*") // Single * matches anything except / + .replace(/\?/g, "[^/]"); // ? matches single char except / + + // Handle ** (match any number of directories) using non-capturing group + // The (?:.*/)? pattern is bounded - it won't cause catastrophic backtracking + regex = regex.replace(/\0DOUBLESTAR\0/g, "(?:.*/)?"); + + // Handle directory separators + regex = regex.replace(/\//g, "[/\\\\]"); + + // Anchor to start + regex = `^${regex}`; + + // Match at end if pattern doesn't end with /** + if (!safePattern.endsWith("/**")) { + regex += "$"; + } + + return new RegExp(regex, "i"); +} + +/** + * Extract keywords from a file path + */ +function extractPathKeywords(filePath: string): string[] { + const keywords = new Set(); + + // Split path into components + const parts = filePath.split(/[/\\]/); + + for (const part of parts) { + // Skip empty parts and common non-descriptive names + if (!part || part === "." || part === "..") continue; + + // Extract words from camelCase, PascalCase, snake_case, kebab-case + const words = part + .replace(/\.[^.]+$/, "") // Remove extension + .split(/[_-]/) // Split by underscore and hyphen + .flatMap((word) => { + // Split camelCase/PascalCase + return word + .replace(/([a-z])([A-Z])/g, "$1 $2") + .split(/\s+/) + .filter((w) => w.length > 2); + }); + + for (const word of words) { + const lower = word.toLowerCase(); + if (isSignificantKeyword(lower)) { + keywords.add(lower); + } + } + + // Add the full filename (without extension) as a keyword + const nameWithoutExt = part.replace(/\.[^.]+$/, "").toLowerCase(); + if (nameWithoutExt.length > 2 && !isCommonWord(nameWithoutExt)) { + keywords.add(nameWithoutExt); + } + } + + // Add extension as keyword + const ext = filePath.split(".").pop()?.toLowerCase(); + if (ext && ext !== filePath) { + keywords.add(ext); + } + + return Array.from(keywords); +} + +/** + * Extract keywords from file content + */ +function extractContentKeywords(content: string, maxKeywords = 10): string[] { + const keywords = new Set(); + + // Extract function/class/variable names from code + const patterns = [ + // Function declarations + /(?:function|def|fn|func)\s+(\w+)/g, + // Class declarations + /(?:class|interface|type|struct)\s+(\w+)/g, + // Variable declarations (const, let, var) + /(?:const|let|var)\s+(\w+)\s*[=:]/g, + // Export declarations + /export\s+(?:default\s+)?(?:class|function|const|let|var)?\s*(\w+)/g, + // Import statements - extract imported names + /import\s+{([^}]+)}/g, + // Python imports + /from\s+\S+\s+import\s+([^\n]+)/g, + // Go/Rust function signatures + /fn\s+(\w+)\s*\(/g, + // React components (PascalCase functions) + /const\s+([A-Z][a-zA-Z0-9]*)\s*[:=]/g, + ]; + + for (const pattern of patterns) { + let match: RegExpExecArray | null = null; + // biome-ignore lint/suspicious/noAssignInExpressions: Standard regex loop pattern + while ((match = pattern.exec(content)) !== null) { + const names = match[1] + .split(/[,\s]+/) + .map((n) => n.trim()) + .filter((n) => n.length > 2 && isSignificantKeyword(n.toLowerCase())); + + for (const name of names) { + keywords.add(name.toLowerCase()); + } + } + } + + // Extract common words that appear frequently + const words = content.toLowerCase().match(/\b[a-z]{3,}\b/g) || []; + + const wordFreq = new Map(); + for (const word of words) { + if (!isCommonWord(word) && isSignificantKeyword(word)) { + wordFreq.set(word, (wordFreq.get(word) || 0) + 1); + } + } + + // Add most frequent words + const sortedWords = Array.from(wordFreq.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, maxKeywords); + + for (const [word] of sortedWords) { + keywords.add(word); + } + + return Array.from(keywords).slice(0, maxKeywords); +} + +/** + * Check if a word is a common/insignificant word + */ +function isCommonWord(word: string): boolean { + const commonWords = new Set([ + "the", + "and", + "for", + "are", + "but", + "not", + "you", + "all", + "can", + "had", + "her", + "was", + "one", + "our", + "out", + "day", + "get", + "has", + "him", + "his", + "how", + "its", + "may", + "new", + "now", + "old", + "see", + "two", + "who", + "boy", + "did", + "she", + "use", + "way", + "many", + "oil", + "sit", + "set", + "run", + "eat", + "far", + "sea", + "eye", + "ago", + "off", + "too", + "any", + "say", + "man", + "try", + "ask", + "end", + "why", + "let", + "put", + "own", + "tell", + "very", + "when", + "come", + "here", + "just", + "like", + "long", + "make", + "over", + "such", + "take", + "than", + "them", + "well", + "were", + "will", + "with", + "have", + "from", + "they", + "know", + "want", + "been", + "good", + "much", + "some", + "time", + "this", + "that", + "would", + "there", + "their", + "what", + "said", + "each", + "which", + "about", + "could", + "other", + "after", + "first", + "never", + "these", + "think", + "where", + "being", + "every", + "great", + "might", + "shall", + "still", + "those", + "while", + "true", + "false", + "null", + "undefined", + "return", + "import", + "export", + "default", + "async", + "await", + "yield", + "throw", + "catch", + "finally", + "break", + "continue", + "switch", + "case", + "try", + "new", + ]); + return commonWords.has(word.toLowerCase()); +} + +/** + * Check if a keyword is significant (not too short, not numeric) + */ +function isSignificantKeyword(word: string): boolean { + if (word.length < 3) return false; + if (/^\d+$/.test(word)) return false; + if (/^[0-9a-f]{8,}$/i.test(word)) return false; // Likely a hash + return true; +} + +/** + * Extract keywords from a task description + */ +export function extractTaskKeywords(taskDescription: string): string[] { + const keywords = new Set(); + + // Extract file paths mentioned in the task + const pathMatches = taskDescription.match(/[\w\-./\\]+\.[\w]+/g) || []; + for (const path of pathMatches) { + const pathKeywords = extractPathKeywords(path); + for (const kw of pathKeywords) { + keywords.add(kw); + } + } + + // Extract camelCase/PascalCase words (likely identifiers) + const identifierMatches = taskDescription.match(/\b[a-z]+[A-Z][a-zA-Z0-9]*\b/g) || []; + for (const id of identifierMatches) { + const words = id + .replace(/([a-z])([A-Z])/g, "$1 $2") + .split(/\s+/) + .filter((w) => w.length > 2); + for (const word of words) { + keywords.add(word.toLowerCase()); + } + } + + // Extract technical terms and concepts + const techTerms = taskDescription.match(/\b[A-Z][a-z]+[A-Z][a-zA-Z]+\b/g) || []; + for (const term of techTerms) { + keywords.add(term.toLowerCase()); + } + + // Extract words that look like file names or components + const componentMatches = + taskDescription.match( + /\b[A-Z][a-zA-Z0-9]*(?:Component|Module|Service|Handler|Controller|Model|View|Util|Helper|Manager|Store|Context|Provider|Hook)\b/g, + ) || []; + for (const comp of componentMatches) { + keywords.add(comp.toLowerCase()); + } + + // Extract all significant words + const allWords = taskDescription.toLowerCase().match(/\b[a-z]{3,}\b/g) || []; + + for (const word of allWords) { + if (!isCommonWord(word) && isSignificantKeyword(word)) { + keywords.add(word); + } + } + + return Array.from(keywords); +} + +/** + * Calculate relevance score between task keywords and file entry + */ +function calculateRelevanceScore(taskKeywords: string[], fileEntry: FileIndexEntry): number { + let score = 0; + const fileKeywords = new Set(fileEntry.keywords); + + for (const taskKw of taskKeywords) { + // Exact match in file keywords + if (fileKeywords.has(taskKw)) { + score += 1.0; + continue; + } + + // Partial match (task keyword is substring of file keyword or vice versa) + for (const fileKw of fileKeywords) { + if (fileKw.includes(taskKw) || taskKw.includes(fileKw)) { + score += 0.5; + break; + } + } + + // Check if keyword appears in path + if (fileEntry.path.toLowerCase().includes(taskKw)) { + score += 0.3; + } + } + + // Normalize by number of task keywords + return taskKeywords.length > 0 ? score / taskKeywords.length : 0; +} + +/** + * Create a file index entry for a single file + */ +function createFileIndexEntry( + filePath: string, + relPath: string, + maxSizeForContent = MAX_FILE_SIZE_FOR_HASH, +): FileIndexEntry | null { + try { + const stat = statSync(filePath); + + if (!stat.isFile()) return null; + + // Calculate hash + let hash = ""; + let preview = ""; + let contentKeywords: string[] = []; + + if (stat.size <= maxSizeForContent) { + try { + const content = readFileSync(filePath, "utf-8"); + hash = createHash("sha256").update(content).digest("hex").slice(0, 16); + preview = content.slice(0, MAX_CONTENT_PREVIEW_LENGTH); + contentKeywords = extractContentKeywords(content, 10); + } catch { + // Binary or unreadable file - use mtime+size as pseudo-hash + hash = createHash("sha256").update(`${stat.mtimeMs}-${stat.size}`).digest("hex").slice(0, 16); + } + } else { + // Large file - use mtime+size as pseudo-hash + hash = createHash("sha256").update(`${stat.mtimeMs}-${stat.size}`).digest("hex").slice(0, 16); + } + + // Extract path keywords + const pathKeywords = extractPathKeywords(relPath); + + // Combine keywords + const allKeywords = [...new Set([...pathKeywords, ...contentKeywords])].slice(0, MAX_KEYWORDS_PER_FILE); + + // Get extension + const ext = relPath.split(".").pop()?.toLowerCase() || ""; + + // Calculate depth + const depth = relPath.split(/[/\\]/).length - 1; + + return { + path: relPath, + hash, + size: stat.size, + mtime: stat.mtimeMs, + keywords: allKeywords, + preview, + extension: ext, + depth, + }; + } catch (error) { + logDebug(`Failed to index file ${filePath}: ${error}`); + return null; + } +} + +/** + * Index all files in a directory recursively + * + * Thread-safe: Returns a cloned copy to prevent cache corruption. + * Concurrent calls for the same workspace will wait for a single indexing operation. + */ +export async function indexWorkspace( + workDir: string, + options: { + ignorePatterns?: string[]; + forceRebuild?: boolean; + maxDepth?: number; + } = {}, +): Promise { + const { ignorePatterns = DEFAULT_IGNORE_PATTERNS, forceRebuild = false, maxDepth = 50 } = options; + + // Check memory cache first - return a clone to prevent mutation + const cached = indexCache.get(workDir); + if (!forceRebuild && cached) { + return cloneFileIndex(cached); + } + + // Check if another operation is already indexing this workspace + const existingPromise = indexingPromises.get(workDir); + if (existingPromise) { + logDebug(`Waiting for concurrent indexing of ${workDir}...`); + const result = await existingPromise; + // Return a clone even from the concurrent operation's result + return cloneFileIndex(result); + } + + // Create the indexing promise to lock this workspace + const indexingPromise = performIndexing(workDir, ignorePatterns, forceRebuild, maxDepth); + indexingPromises.set(workDir, indexingPromise); + + try { + const result = await indexingPromise; + // Return a cloned copy to prevent cache corruption + return cloneFileIndex(result); + } finally { + // Always clean up the promise lock + indexingPromises.delete(workDir); + } +} + +/** + * Perform the actual indexing operation + */ +async function performIndexing( + workDir: string, + ignorePatterns: string[], + forceRebuild: boolean, + maxDepth: number, +): Promise { + // Double-check cache after acquiring lock (another thread may have completed) + const cached = indexCache.get(workDir); + if (!forceRebuild && cached) { + return cached; + } + + // Try to load from disk cache + if (!forceRebuild) { + const diskCache = loadIndexFromDisk(workDir); + if (diskCache) { + // Perform incremental update + const updated = await incrementalUpdateIndex(workDir, diskCache, ignorePatterns, maxDepth); + indexCache.set(workDir, updated); + saveIndexToDisk(workDir, updated); + return updated; + } + } + + // Build fresh index + const index: FileIndex = { + version: 1, + timestamp: Date.now(), + workDir, + files: new Map(), + totalFiles: 0, + totalSize: 0, + }; + + // Ensure .ralphy directory exists + const ralphyDir = join(workDir, RALPHY_DIR); + if (!existsSync(ralphyDir)) { + mkdirSync(ralphyDir, { recursive: true }); + } + + // Collect all files + const filesToIndex: string[] = []; + + function collectFiles(dir: string, currentDepth: number) { + if (currentDepth > maxDepth) return; + + try { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + const relPath = relative(workDir, fullPath); + + if (shouldIgnoreFile(relPath, ignorePatterns)) { + continue; + } + + if (entry.isDirectory()) { + collectFiles(fullPath, currentDepth + 1); + } else if (entry.isFile()) { + filesToIndex.push(fullPath); + } + } + } catch (error) { + logDebug(`Failed to read directory ${dir}: ${error}`); + } + } + + collectFiles(workDir, 0); + + // Index all collected files + for (const filePath of filesToIndex) { + const relPath = relative(workDir, filePath); + const entry = createFileIndexEntry(filePath, relPath); + if (entry) { + index.files.set(relPath, entry); + index.totalFiles++; + index.totalSize += entry.size; + } + } + + // Cache and save + indexCache.set(workDir, index); + saveIndexToDisk(workDir, index); + + logDebug(`Indexed ${index.totalFiles} files (${(index.totalSize / 1024 / 1024).toFixed(2)} MB)`); + + return index; +} + +/** + * Perform incremental update of file index + */ +async function incrementalUpdateIndex( + workDir: string, + existingIndex: FileIndex, + ignorePatterns: string[], + maxDepth: number, +): Promise { + const updatedIndex: FileIndex = { + version: existingIndex.version, + timestamp: Date.now(), + workDir, + files: new Map(existingIndex.files), + totalFiles: 0, + totalSize: 0, + }; + + const currentFiles = new Set(); + let reindexedCount = 0; + let unchangedCount = 0; + let removedCount = 0; + + function scanDirectory(dir: string, currentDepth: number) { + if (currentDepth > maxDepth) return; + + try { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + const relPath = relative(workDir, fullPath); + + if (shouldIgnoreFile(relPath, ignorePatterns)) { + continue; + } + + if (entry.isDirectory()) { + scanDirectory(fullPath, currentDepth + 1); + } else if (entry.isFile()) { + currentFiles.add(relPath); + + const existingEntry = updatedIndex.files.get(relPath); + const stat = statSync(fullPath); + + if (existingEntry && existingEntry.mtime === stat.mtimeMs && existingEntry.size === stat.size) { + // File unchanged - keep existing entry + unchangedCount++; + } else { + // File changed or new - reindex + const newEntry = createFileIndexEntry(fullPath, relPath); + if (newEntry) { + updatedIndex.files.set(relPath, newEntry); + reindexedCount++; + } + } + } + } + } catch (error) { + logDebug(`Failed to scan directory ${dir}: ${error}`); + } + } + + scanDirectory(workDir, 0); + + // Remove deleted files from index + for (const [relPath] of updatedIndex.files) { + if (!currentFiles.has(relPath)) { + updatedIndex.files.delete(relPath); + removedCount++; + } + } + + // Recalculate totals + for (const entry of updatedIndex.files.values()) { + updatedIndex.totalFiles++; + updatedIndex.totalSize += entry.size; + } + + logDebug( + `Incremental index update: ${unchangedCount} unchanged, ${reindexedCount} reindexed, ${removedCount} removed`, + ); + + return updatedIndex; +} + +/** + * Load index from disk cache + */ +function loadIndexFromDisk(workDir: string): FileIndex | null { + const cachePath = getIndexCachePath(workDir); + + if (!existsSync(cachePath)) { + return null; + } + + try { + const content = readFileSync(cachePath, "utf-8"); + const serialized: SerializedFileIndex = JSON.parse(content); + + return { + version: serialized.version, + timestamp: serialized.timestamp, + workDir: serialized.workDir, + files: new Map(Object.entries(serialized.files)), + totalFiles: serialized.totalFiles, + totalSize: serialized.totalSize, + }; + } catch (error) { + logDebug(`Failed to load file index from disk: ${error}`); + return null; + } +} + +/** + * Save index to disk cache + */ +function saveIndexToDisk(workDir: string, index: FileIndex): void { + const cachePath = getIndexCachePath(workDir); + + try { + const serialized: SerializedFileIndex = { + version: index.version, + timestamp: index.timestamp, + workDir: index.workDir, + files: Object.fromEntries(index.files), + totalFiles: index.totalFiles, + totalSize: index.totalSize, + }; + + writeFileSync(cachePath, JSON.stringify(serialized, null, 2)); + } catch (error) { + logDebug(`Failed to save file index to disk: ${error}`); + } +} + +/** + * Get relevant files for a task based on semantic matching + */ +export async function getRelevantFilesForTask( + workDir: string, + taskDescription: string, + options: { + maxFiles?: number; + minRelevance?: number; + includeExtensions?: string[]; + excludeExtensions?: string[]; + } = {}, +): Promise { + const { + maxFiles = 50, + minRelevance = RELEVANCE_THRESHOLD, + includeExtensions, + excludeExtensions = ["log", "lock", "map", "min.js", "min.css"], + } = options; + + // Get or build file index + const index = await indexWorkspace(workDir); + + // Extract keywords from task + const taskKeywords = extractTaskKeywords(taskDescription); + logDebug(`Task keywords: ${taskKeywords.join(", ")}`); + + if (taskKeywords.length === 0) { + // No keywords extracted - return most recently modified files as fallback + return Array.from(index.files.values()) + .sort((a, b) => b.mtime - a.mtime) + .slice(0, maxFiles) + .map((e) => e.path); + } + + // Score all files + const scoredFiles: Array<{ path: string; score: number; entry: FileIndexEntry }> = []; + + for (const [path, entry] of index.files) { + // Filter by extension + if (includeExtensions && !includeExtensions.includes(entry.extension)) { + continue; + } + if (excludeExtensions.includes(entry.extension)) { + continue; + } + + const score = calculateRelevanceScore(taskKeywords, entry); + if (score >= minRelevance) { + scoredFiles.push({ path, score, entry }); + } + } + + // Sort by score (descending), then by mtime (most recent first for ties) + scoredFiles.sort((a, b) => { + if (b.score !== a.score) { + return b.score - a.score; + } + return b.entry.mtime - a.entry.mtime; + }); + + // Take top N files + const relevantFiles = scoredFiles.slice(0, maxFiles).map((s) => s.path); + + logDebug(`Found ${relevantFiles.length} relevant files for task (scored ${scoredFiles.length} total)`); + + return relevantFiles; +} + +/** + * Get file hash from index (useful for caching unchanged files) + */ +export async function getFileHashFromIndex(workDir: string, relPath: string): Promise { + const index = await indexWorkspace(workDir); + const entry = index.files.get(relPath); + return entry?.hash ?? null; +} + +/** + * Check if a file has changed based on index + */ +export async function hasFileChanged(workDir: string, relPath: string, expectedHash: string): Promise { + const currentHash = await getFileHashFromIndex(workDir, relPath); + if (currentHash === null) { + return true; // File not in index, assume changed + } + return currentHash !== expectedHash; +} + +/** + * Get file metadata from index + */ +export async function getFileMetadata(workDir: string, relPath: string): Promise { + const index = await indexWorkspace(workDir); + return index.files.get(relPath) ?? null; +} + +/** + * Clear the file index cache (both memory and disk) + */ +export function clearFileIndexCache(workDir: string): void { + indexCache.delete(workDir); + const cachePath = getIndexCachePath(workDir); + try { + if (existsSync(cachePath)) { + rmSync(cachePath); + } + } catch (error) { + logDebug(`Failed to clear file index cache: ${error}`); + } +} + +/** + * Get index statistics + */ +export async function getIndexStats(workDir: string): Promise<{ + totalFiles: number; + totalSize: number; + avgFileSize: number; + lastUpdated: number; +}> { + const index = await indexWorkspace(workDir); + return { + totalFiles: index.totalFiles, + totalSize: index.totalSize, + avgFileSize: index.totalFiles > 0 ? index.totalSize / index.totalFiles : 0, + lastUpdated: index.timestamp, + }; +} + +/** + * Force rebuild the file index + */ +export async function rebuildFileIndex(workDir: string): Promise { + clearFileIndexCache(workDir); + return indexWorkspace(workDir, { forceRebuild: true }); +} + +/** + * Find files by keyword (simple search) + */ +export async function findFilesByKeyword( + workDir: string, + keyword: string, + options: { maxResults?: number } = {}, +): Promise { + const { maxResults = 20 } = options; + const index = await indexWorkspace(workDir); + const results: FileIndexEntry[] = []; + const lowerKeyword = keyword.toLowerCase(); + + for (const entry of index.files.values()) { + // Check if keyword is in path + if (entry.path.toLowerCase().includes(lowerKeyword)) { + results.push(entry); + continue; + } + + // Check if keyword is in keywords + if (entry.keywords.some((k) => k.includes(lowerKeyword) || lowerKeyword.includes(k))) { + results.push(entry); + continue; + } + + // Check preview for code files + if (entry.preview?.toLowerCase().includes(lowerKeyword)) { + results.push(entry); + } + } + + return results.slice(0, maxResults); +} diff --git a/cli/src/utils/sanitization.ts b/cli/src/utils/sanitization.ts new file mode 100644 index 00000000..5f2fa5e6 --- /dev/null +++ b/cli/src/utils/sanitization.ts @@ -0,0 +1,50 @@ +/** + * Sanitization utilities for removing sensitive data + * + * SECURITY: All patterns use bounded quantifiers to prevent ReDoS attacks + */ + +/** + * Maximum input length for secret sanitization to prevent ReDoS + */ +const MAX_SANITIZE_INPUT_LENGTH = 1000000; // 1MB + +/** + * Sanitize sensitive data (API keys, passwords, etc.) from string input + * + * SECURITY NOTE: This function includes protections against ReDoS attacks: + * - Input length is limited to MAX_SANITIZE_INPUT_LENGTH + * - All regex patterns use bounded quantifiers (e.g., {48}, {36}) + * - Patterns are applied sequentially with early exit if input becomes too large + * + * @param input - The string to sanitize + * @returns Sanitized string with secrets redacted + */ +export function sanitizeSecrets(input: string): string { + // Limit input length to prevent ReDoS attacks + if (input.length > MAX_SANITIZE_INPUT_LENGTH) { + // For very large inputs, truncate and add warning + const truncated = input.slice(0, MAX_SANITIZE_INPUT_LENGTH); + return `${truncated}\n\n[WARNING: Content truncated due to size limits during secret sanitization]`; + } + + // All patterns use bounded quantifiers to prevent ReDoS + // Patterns are designed to match specific token formats with fixed lengths + const patterns = [ + { regex: /sk-[a-zA-Z0-9]{48}/g, replacement: "[API_KEY_REDACTED]" }, + { regex: /sk-ant-[a-zA-Z0-9_-]{32,128}/g, replacement: "[ANTHROPIC_TOKEN_REDACTED]" }, + { regex: /ghp_[a-zA-Z0-9]{36}/g, replacement: "[GITHUB_TOKEN_REDACTED]" }, + { regex: /gho_[a-zA-Z0-9]{52}/g, replacement: "[GITHUB_OAUTH_REDACTED]" }, + { regex: /AKIA[0-9A-Z]{16}/g, replacement: "[AWS_KEY_REDACTED]" }, + // For hex secrets, use a bounded length and require word boundaries to prevent + // matching large hex strings that could cause performance issues + { regex: /\b[0-9a-f]{64}\b/g, replacement: "[HEX_SECRET_REDACTED]" }, + ]; + + let result = input; + for (const { regex, replacement } of patterns) { + result = result.replace(regex, replacement); + } + + return result; +}