Trace Recorder — TraceStore interface, MySQL sink, DP fields

package.json

@@ -94,7 +94,7 @@
     "copy:web": "node -e \"const fs=require('fs');const p=require('path');const s=p.join('src','gateway','web','dist');const d=p.join('dist','gateway','web','dist');if(fs.existsSync(s)){fs.cpSync(s,d,{recursive:true});console.log('Copied web UI to '+d)}\"",
     "prepublishOnly": "npm run build:web && npm run build",
     "dev": "tsx src/cli-main.ts",
-    "dev:gateway": "npm run build:web && tsx src/gateway-main.ts",
+    "dev:gateway": "[ -f .siclaw/trace-env.sh ] && . .siclaw/trace-env.sh; npm run build:web && tsx src/gateway-main.ts",
     "dev:agentbox": "tsx src/agentbox-main.ts",
     "start": "node siclaw-tui.mjs",
     "start:gateway": "node siclaw-gateway.mjs",

siclaw-trace-db-full.yaml

@@ -0,0 +1,114 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: siclaw-trace-db-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: local-hostpath
+  volumeName: siclaw-trace-db-pv
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: siclaw-trace-db
+  labels:
+    app: siclaw-trace-db
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: siclaw-trace-db
+  template:
+    metadata:
+      labels:
+        app: siclaw-trace-db
+    spec:
+      nodeName: cpu-10-208-55-85
+      initContainers:
+      - name: init-mkdir
+        image: busybox
+        command: ["sh", "-c", "mkdir -p /data/siclaw-trace-db && chmod 777 /data/siclaw-trace-db"]
+        volumeMounts:
+        - name: host-data
+          mountPath: /data
+        securityContext:
+          privileged: true
+      containers:
+      - name: siclaw-trace-db
+        image: mysql:8.0
+        ports:
+        - containerPort: 3306
+        env:
+        - name: MYSQL_ROOT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: siclaw-trace-db-secret
+              key: mysql-root-password
+        - name: MYSQL_DATABASE
+          valueFrom:
+            secretKeyRef:
+              name: siclaw-trace-db-secret
+              key: mysql-database
+        - name: MYSQL_USER
+          valueFrom:
+            secretKeyRef:
+              name: siclaw-trace-db-secret
+              key: mysql-user
+        - name: MYSQL_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: siclaw-trace-db-secret
+              key: mysql-password
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "250m"
+          limits:
+            memory: "1Gi"
+            cpu: "500m"
+        volumeMounts:
+        - name: mysql-storage
+          mountPath: /var/lib/mysql
+        livenessProbe:
+          exec:
+            command:
+            - mysqladmin
+            - ping
+            - -h
+            - localhost
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - mysqladmin
+            - ping
+            - -h
+            - localhost
+          initialDelaySeconds: 15
+          periodSeconds: 5
+      volumes:
+      - name: mysql-storage
+        persistentVolumeClaim:
+          claimName: siclaw-trace-db-pvc
+      - name: host-data
+        hostPath:
+          path: /data
+          type: DirectoryOrCreate
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: siclaw-trace-db
+spec:
+  selector:
+    app: siclaw-trace-db
+  ports:
+  - protocol: TCP
+    port: 3306
+    targetPort: 3306
+  type: ClusterIP

siclaw-trace-db-pv.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: siclaw-trace-db-pv
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: local-hostpath
+  local:
+    path: /data/siclaw-trace-db
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - cpu-10-208-55-85

src/agentbox/http-server.ts

@@ -241,7 +241,7 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
    * The message is sent to the Agent, and responses are returned via SSE stream.
    */
   addRoute("POST", "/api/prompt", async (req, res) => {
-    const body = (await parseJsonBody(req)) as { sessionId?: string; text?: string; mode?: SessionMode; modelProvider?: string; modelId?: string; brainType?: BrainType; systemPromptTemplate?: string; modelConfig?: Record<string, unknown>; credentials?: { manifest: Array<Record<string, unknown>>; files: Array<{ name: string; content: string; mode?: number }> } };
+    const body = (await parseJsonBody(req)) as { sessionId?: string; text?: string; mode?: SessionMode; modelProvider?: string; modelId?: string; brainType?: BrainType; systemPromptTemplate?: string; modelConfig?: Record<string, unknown>; credentials?: { manifest: Array<Record<string, unknown>>; files: Array<{ name: string; content: string; mode?: number }> }; username?: string };
 
     if (!body.text) {
       sendJson(res, 400, { error: "Missing 'text' field" });
@@ -321,7 +321,11 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
     // Subscribe to buffer events so SSE can replay them even if it connects late
     const brainUnsub = managed.brain.subscribe((event) => {
       if (!managed._promptDone) {
-        managed._eventBuffer.push(event);
+        // Stamp with server time when emitted so replayed events have accurate timestamps
+        const tsEvent = typeof event === "object" && event !== null
+          ? { ...(event as object), ts: Date.now() }
+          : event;
+        managed._eventBuffer.push(tsEvent);
       }
       // Null dpState.checklist when deep_search completes — this is the exit signal
       // for the SDK brain's auto-continue loop in claude-sdk-brain.ts.
@@ -343,6 +347,7 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
           type: "tool_progress",
           toolName: "deep_search",
           progress: event,
+          ts: Date.now(),
         });
       }
       // Sync phase events to SDK brain's dpState so the auto-continue loop
@@ -436,6 +441,19 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
       } catch { /* best-effort, don't block prompt */ }
     }
 
+    // Mark the explicit trace boundary: ONE user prompt = ONE trace file, even
+    // if pi-agent internally fires multiple agent_start/end cycles (retry or
+    // auto-compaction). beginPrompt is the start, endPrompt is called below in
+    // actuallyFinish() after the whole prompt (including any retries) settles.
+    // Also forward the displayable username so filenames use "admin" instead of
+    // the internal hex userId.
+    if (managed._traceRecorder) {
+      try {
+        if (typeof body.username === "string" && body.username) managed._traceRecorder.setUsername(body.username);
+        if (typeof body.text === "string") await managed._traceRecorder.beginPrompt(body.text);
+      } catch (err) { console.warn("[agentbox-http] trace-recorder beginPrompt failed:", err); }
+    }
+
     // Execute prompt asynchronously; notify SSE to close on completion
     console.log(`[agentbox-http] Starting prompt for session ${managed.id} [lang=${detectedLang}]`);
 
@@ -461,6 +479,18 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
         userId: sessionManager.userId,
       });
 
+      // Flush the explicit trace — fires ONCE per user prompt, even if pi-agent
+      // internally executed multiple agent_start/end cycles (retry, compaction).
+      // actuallyFinish() is the definitive "prompt is truly done" point (it waits
+      // for auto_compaction_end / auto_retry_end before firing).
+      if (managed._traceRecorder) {
+        // Fire-and-forget: actuallyFinish() is a sync callback, and we don't
+        // want to block the SSE stream close on DB flush. Failures are warned.
+        managed._traceRecorder.endPrompt(promptOutcome).catch((err) => {
+          console.warn("[agentbox-http] trace-recorder endPrompt failed:", err);
+        });
+      }
+
       // Stop buffering
       if (managed._bufferUnsub) {
         managed._bufferUnsub();
@@ -551,7 +581,11 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
       if (closed || res.writableEnded) return;
       try {
         sseEventCount++;
-        const data = JSON.stringify(event);
+        // Add server timestamp if not already present (buffered events carry their original ts)
+        const out = typeof event === "object" && event !== null && !("ts" in (event as object))
+          ? { ...(event as object), ts: Date.now() }
+          : event;
+        const data = JSON.stringify(out);
         res.write(`data: ${data}\n\n`);
       } catch (err) {
         console.warn(`[agentbox-http] SSE write error for session ${sessionId}:`, err);
@@ -674,6 +708,14 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
     }
 
     console.log(`[agentbox-http] Steering session ${sessionId}: ${body.text.slice(0, 80)}`);
+    // Record a standalone trace row for this steer BEFORE the brain consumes
+    // it. Steer messages bypass the /api/prompt path that normally triggers
+    // beginPrompt(), so without this the DP button clicks ([DP_CONFIRM],
+    // [DP_ADJUST], [DP_SKIP], [DP_REINVESTIGATE]) leave zero audit trail.
+    if (managed._traceRecorder) {
+      try { await managed._traceRecorder.recordSteerEvent(body.text); }
+      catch (err) { console.warn("[agentbox-http] recordSteerEvent failed:", err); }
+    }
     try {
       await managed.brain.steer(body.text);
       sendJson(res, 200, { ok: true });
@@ -993,6 +1035,11 @@ export function createHttpServer(sessionManager: AgentBoxSessionManager): http.S
     }
   });
 
+  // NOTE: /api/traces routes are intentionally NOT registered here.
+  // They live on the Gateway (src/gateway/server.ts) because querying the
+  // trace DB is a pure read that must not depend on the lazy-spawned
+  // AgentBox — you shouldn't have to send a prompt before you can query.
+
   // ==================== Server ====================
 
   /** Main request handler shared by HTTP and HTTPS servers */

src/agentbox/session.ts

@@ -24,6 +24,7 @@ import { saveSessionKnowledge } from "../memory/session-summarizer.js";
 import type { DpState } from "../tools/workflow/dp-tools.js";
 import { loadConfig, getEmbeddingConfig } from "../core/config.js";
 import { emitDiagnostic } from "../shared/diagnostic-events.js";
+import { maybeCreateTraceRecorder, type TraceRecorder } from "../core/trace-recorder.js";
 // topic-consolidator import removed — consolidation disabled
 
 export interface ManagedSession {
@@ -70,6 +71,10 @@ export interface ManagedSession {
   _lastSavedMessageCount: number;
   /** Pending release timer (cleared when a new prompt arrives before TTL expires) */
   _releaseTimer: ReturnType<typeof setTimeout> | null;
+  /** Trace recorder — writes per-prompt JSON to .siclaw/traces. null when disabled. */
+  _traceRecorder?: TraceRecorder | null;
+  /** Unsubscribe fn for trace recorder's brain subscription. */
+  _traceUnsub?: (() => void) | null;
 }
 
 export interface PersistedDpStateSnapshot {
@@ -262,6 +267,27 @@ export class AgentBoxSessionManager {
     this.sessions.set(id, managed);
     emitDiagnostic({ type: "session_created", sessionId: id });
 
+    // Trace recorder — writes per-prompt JSON traces to .siclaw/traces for
+    // offline retrospective. Filesystem only, not exposed via HTTP/SSE/WS.
+    // Disable with SICLAW_TRACE_DISABLE=1; override path with SICLAW_TRACE_DIR.
+    try {
+      const recorder = await maybeCreateTraceRecorder({
+        sessionId: id,
+        userId: this.userId,
+        mode: effectiveMode,
+        brainType: effectiveBrainType,
+        getSessionStats: () => managed!.brain.getSessionStats(),
+        getModel: () => managed!.brain.getModel(),
+        dpStateRef: result.dpStateRef,
+      });
+      if (recorder) {
+        managed._traceRecorder = recorder;
+        managed._traceUnsub = recorder.attach(managed.brain);
+      }
+    } catch (err) {
+      console.warn(`[agentbox-session] Trace recorder setup failed for ${id}:`, err);
+    }
+
     // Tool execution timing (for tool_call diagnostic events).
     // NOTE: tool_execution_start/end events depend on the brain implementation.
     // claude-sdk brain emits them reliably; pi-agent brain depends on the SDK's
@@ -491,6 +517,18 @@ export class AgentBoxSessionManager {
       console.warn(`[agentbox-session] Memory auto-save failed for ${sessionId}:`, err);
     }
 
+    // 1b. Close trace recorder — flushes any in-flight trace to disk.
+    if (managed._traceUnsub) {
+      try { managed._traceUnsub(); } catch { /* ignore */ }
+      managed._traceUnsub = null;
+    }
+    if (managed._traceRecorder) {
+      try { await managed._traceRecorder.close(); } catch (err) {
+        console.warn(`[agentbox-session] Trace recorder close failed for ${sessionId}:`, err);
+      }
+      managed._traceRecorder = null;
+    }
+
     // 2. Shutdown per-session MCP connections
     if (managed.mcpManager) {
       try {

src/cli-main.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import {
   InteractiveMode,
@@ -13,6 +14,7 @@ import { saveSessionKnowledge } from "./memory/session-summarizer.js";
 // topic-consolidator import removed — consolidation disabled
 import type { BrainType } from "./core/brain-session.js";
 import { debugPodGC, debugPodCache } from "./tools/infra/debug-pod.js";
+import { maybeCreateTraceRecorder } from "./core/trace-recorder.js";
 
 
 // Parse arguments
@@ -116,6 +118,41 @@ if (memoryIndexer) {
     .catch(err => console.warn("[siclaw] Startup maintenance failed:", err));
 }
 
+// Trace recorder — writes per-prompt JSON traces to .siclaw/traces for offline
+// retrospective. Not exposed via HTTP/SSE. Disable with SICLAW_TRACE_DISABLE=1.
+const osUsername = (() => { try { return os.userInfo().username; } catch { return process.env.USER ?? "unknown"; } })();
+const traceRecorder = await maybeCreateTraceRecorder({
+  sessionId: sessionManager.getSessionId?.() ?? `cli-${Date.now()}`,
+  userId: osUsername,
+  username: osUsername,
+  mode: "cli",
+  brainType: brain.brainType,
+  getSessionStats: () => brain.getSessionStats(),
+  getModel: () => brain.getModel(),
+});
+if (traceRecorder) {
+  traceRecorder.attach(brain);
+  const traceDir = process.env.SICLAW_TRACE_DIR ?? path.join(process.cwd(), ".siclaw", "traces");
+  console.log(`[siclaw] Trace recording → ${path.relative(process.cwd(), traceDir) || traceDir}`);
+
+  // Wrap session.prompt (what InteractiveMode calls) so each user-initiated
+  // prompt yields exactly ONE trace file, even if pi-agent internally runs
+  // multiple agent_start/end cycles (empty-response retry, auto-compaction).
+  const origSessionPrompt = session.prompt.bind(session);
+  (session as unknown as { prompt: (text: string) => Promise<void> }).prompt = async (text: string) => {
+    await traceRecorder.beginPrompt(text);
+    let outcome: "completed" | "error" = "completed";
+    try {
+      await origSessionPrompt(text);
+    } catch (err) {
+      outcome = "error";
+      throw err;
+    } finally {
+      await traceRecorder.endPrompt(outcome);
+    }
+  };
+}
+
 // Debug: subscribe to all session events and write to log file
 if (debugMode) {
   const logFile = path.join(process.cwd(), "siclaw-debug.log");
@@ -228,6 +265,10 @@ if (session.sessionFile) {
   }
 }
 
+// Close trace recorder — flushes any in-flight trace.
+if (traceRecorder) {
+  try { await traceRecorder.close(); } catch { /* ignore */ }
+}
 // Clean up cached debug pods
 try { await debugPodCache.evictAll(); } catch { /* ignore */ }
 // Shutdown MCP connections