tinyfish-io · giaphutran12 · May 22, 2026
diff --git a/backend/BigSet_Data_Collection_Agent/src/models/schemas.ts b/backend/BigSet_Data_Collection_Agent/src/models/schemas.ts
@@ -102,6 +102,22 @@ export const agentGoalSchema = z.object({
 
 export type AgentGoal = z.infer<typeof agentGoalSchema>;
 
+export const browserActionReportSchema = z.object({
+  action: z.string().optional(),
+  url: z.string().optional(),
+  selector: z.string().optional(),
+  target_text: z.string().optional(),
+  targetText: z.string().optional(),
+  value_description: z.string().optional(),
+  valueDescription: z.string().optional(),
+  status: z.string().optional(),
+  error: z.string().optional(),
+  phase: z.string().optional(),
+  label: z.string().optional(),
+});
+
+export type BrowserActionReport = z.infer<typeof browserActionReportSchema>;
+
 export const agentRunRecordSchema = z.object({
   url: z.string(),
   status: sourceStatusSchema,
@@ -110,6 +126,7 @@ export const agentRunRecordSchema = z.object({
   goal: z.string(),
   records_extracted: z.number(),
   error: z.string().optional(),
+  browser_actions: z.array(browserActionReportSchema).optional(),
 });
 
 export type AgentRunRecord = z.infer<typeof agentRunRecordSchema>;
@@ -152,6 +169,7 @@ export const repairLoopReportSchema = z.object({
   loop_index: z.number().int().positive(),
   diagnosis_summary: z.string().optional(),
   repair_queries: z.array(z.string()),
+  agent_browser_actions: z.array(browserActionReportSchema).optional(),
   rationale: z.string().optional(),
   missing_fields: z.array(z.string()),
   records_before: z.number(),
@@ -198,6 +216,7 @@ export const runReportSchema = z.object({
     search_queries: z.array(z.string()),
     fetched_urls: z.array(z.string()),
     failed_urls: z.array(z.string()),
+    agent_browser_actions: z.array(browserActionReportSchema).optional(),
   }),
   repair: repairReportSchema,
   search_queries: z.array(z.string()),

diff --git a/backend/BigSet_Data_Collection_Agent/src/orchestrator/browser-actions.ts b/backend/BigSet_Data_Collection_Agent/src/orchestrator/browser-actions.ts
@@ -0,0 +1,99 @@
+import {
+  browserActionReportSchema,
+  type AgentRunRecord,
+  type BrowserActionReport,
+} from "../models/schemas.js";
+
+const EXPLICIT_BROWSER_ACTION_ARRAY_KEYS = [
+  "browser_actions",
+  "agent_browser_actions",
+] as const;
+
+export function explicitBrowserActionsFromAgentResult(
+  input: {
+    agentResult: Record<string, unknown> | null;
+    pageUrl: string;
+  }
+): BrowserActionReport[] {
+  if (!input.agentResult) {
+    return [];
+  }
+
+  const actions: BrowserActionReport[] = [];
+  for (const key of EXPLICIT_BROWSER_ACTION_ARRAY_KEYS) {
+    actions.push(...browserActionsFromValue(input.agentResult[key], input.pageUrl));
+  }
+  return dedupeBrowserActions(actions);
+}
+
+export function explicitBrowserActionsFromAgentRuns(
+  agentRuns: AgentRunRecord[]
+): BrowserActionReport[] {
+  return dedupeBrowserActions(
+    agentRuns.flatMap((run) => run.browser_actions ?? [])
+  );
+}
+
+function browserActionsFromValue(
+  value: unknown,
+  pageUrl: string
+): BrowserActionReport[] {
+  if (Array.isArray(value)) {
+    return value
+      .map((item) => browserActionFromValue(item, pageUrl))
+      .filter((action): action is BrowserActionReport => Boolean(action));
+  }
+  const action = browserActionFromValue(value, pageUrl);
+  return action ? [action] : [];
+}
+
+function browserActionFromValue(
+  value: unknown,
+  pageUrl: string
+): BrowserActionReport | undefined {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return undefined;
+  }
+  const parsed = browserActionReportSchema.safeParse(value);
+  if (!parsed.success || !hasReplayAnchor(parsed.data)) {
+    return undefined;
+  }
+  return {
+    ...parsed.data,
+    url: parsed.data.url ?? pageUrl,
+  };
+}
+
+function hasReplayAnchor(action: BrowserActionReport): boolean {
+  return Boolean(
+    action.url ||
+    action.selector ||
+    action.target_text ||
+    action.targetText
+  );
+}
+
+function dedupeBrowserActions(
+  actions: BrowserActionReport[]
+): BrowserActionReport[] {
+  const seen = new Set<string>();
+  const deduped: BrowserActionReport[] = [];
+  for (const action of actions) {
+    const key = JSON.stringify([
+      action.action ?? "",
+      action.url ?? "",
+      action.selector ?? "",
+      action.target_text ?? action.targetText ?? "",
+      action.status ?? "",
+      action.error ?? "",
+      action.phase ?? "",
+      action.label ?? "",
+    ]);
+    if (seen.has(key)) {
+      continue;
+    }
+    seen.add(key);
+    deduped.push(action);
+  }
+  return deduped;
+}
diff --git a/backend/BigSet_Data_Collection_Agent/src/orchestrator/pipeline.ts b/backend/BigSet_Data_Collection_Agent/src/orchestrator/pipeline.ts
@@ -48,6 +48,7 @@ import {
   type RunPaths,
 } from "../storage/run-store.js";
 import { normalizeUrl } from "../utils/url.js";
+import { explicitBrowserActionsFromAgentRuns } from "./browser-actions.js";
 
 export interface PipelineOptions {
   prompt: string;
@@ -545,6 +546,9 @@ async function executeRunPipeline(
   const visualizationCount = benchmarkVisualizationRecords.length;
 
   const llmUsage = getCurrentLlmUsage();
+  const initialAgentBrowserActions = explicitBrowserActionsFromAgentRuns(
+    initialAcquisition.agentRuns,
+  );
 
   const report: RunReport = {
     run_id: runId,
@@ -586,6 +590,7 @@ async function executeRunPipeline(
       search_queries: initialQueries,
       fetched_urls: initialAcquisition.fetchedUrls,
       failed_urls: initialAcquisition.failedUrls,
+      agent_browser_actions: initialAgentBrowserActions,
     },
     repair: repairReport,
     search_queries: allSearchQueries,

diff --git a/backend/BigSet_Data_Collection_Agent/src/orchestrator/process-pages.ts b/backend/BigSet_Data_Collection_Agent/src/orchestrator/process-pages.ts
@@ -26,6 +26,7 @@ import {
 } from "../queue/pools.js";
 import { saveJson, type RunPaths } from "../storage/run-store.js";
 import { getDomain } from "../utils/url.js";
+import { explicitBrowserActionsFromAgentResult } from "./browser-actions.js";
 import { join } from "node:path";
 
 export interface AgentDeferredEntry {
@@ -408,6 +409,11 @@ export async function processFetchedPages(options: {
           return;
         }
 
+        const browserActions = explicitBrowserActionsFromAgentResult({
+          agentResult: run.result,
+          pageUrl,
+        });
+
         try {
           const agentRecords = await extractFromAgentResult({
             spec: options.spec,
@@ -432,6 +438,9 @@ export async function processFetchedPages(options: {
             agent_status: run.status,
             goal: job.goal,
             records_extracted: agentRecords.length,
+            browser_actions: browserActions.length > 0
+              ? browserActions
+              : undefined,
           });
 
           options.log(
@@ -450,6 +459,9 @@ export async function processFetchedPages(options: {
             goal: job.goal,
             records_extracted: 0,
             error: msg,
+            browser_actions: browserActions.length > 0
+              ? browserActions
+              : undefined,
           });
         }
       },

diff --git a/backend/BigSet_Data_Collection_Agent/src/orchestrator/repair-loop.ts b/backend/BigSet_Data_Collection_Agent/src/orchestrator/repair-loop.ts
@@ -29,6 +29,7 @@ import {
   runAcquisitionPhase,
   type AcquisitionResult,
 } from "./acquisition.js";
+import { explicitBrowserActionsFromAgentRuns } from "./browser-actions.js";
 
 export interface RepairLoopContext {
   userPrompt: string;
@@ -235,6 +236,9 @@ export async function runRepairLoops(options: {
       loop_index: loopIndex,
       diagnosis_summary: diagnosis.summary,
       repair_queries: repairPlan.repair_queries,
+      agent_browser_actions: explicitBrowserActionsFromAgentRuns(
+        acquisition.agentRuns
+      ),
       rationale: repairPlan.rationale,
       missing_fields: coverage.field_gaps.map((gap) => gap.column),
       records_before: recordsBeforeLoop.length,

diff --git a/backend/test/collection-browser-actions.test.ts b/backend/test/collection-browser-actions.test.ts
@@ -0,0 +1,158 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+  explicitBrowserActionsFromAgentResult,
+  explicitBrowserActionsFromAgentRuns,
+} from "../BigSet_Data_Collection_Agent/src/orchestrator/browser-actions.js";
+import {
+  agentRunRecordSchema,
+  runReportSchema,
+} from "../BigSet_Data_Collection_Agent/src/models/schemas.js";
+
+test("explicit browser actions are copied from Agent results without generic inference", () => {
+  const actions = explicitBrowserActionsFromAgentResult({
+    pageUrl: "https://example.com/start",
+    agentResult: {
+      browser_actions: [
+        {
+          action: "navigate",
+          url: "https://example.com/start",
+          status: "succeeded",
+          phase: "initial",
+        },
+        "not an action",
+      ],
+      agent_browser_actions: [{
+        action: "click",
+        selector: "button[type=submit]",
+        target_text: "Submit",
+        value_description: "redacted",
+        status: "succeeded",
+      }],
+      actions: [{
+        action: "click",
+        selector: "#generic-actions-are-ignored",
+      }],
+    },
+  });
+
+  assert.equal(actions.length, 2);
+  assert.deepEqual(actions[0], {
+    action: "navigate",
+    url: "https://example.com/start",
+    status: "succeeded",
+    phase: "initial",
+  });
+  assert.deepEqual(actions[1], {
+    action: "click",
+    url: "https://example.com/start",
+    selector: "button[type=submit]",
+    target_text: "Submit",
+    value_description: "redacted",
+    status: "succeeded",
+  });
+});
+
+test("Agent run records and run reports persist browser action arrays", () => {
+  const browserActions = [{
+    action: "click",
+    url: "https://example.com/start",
+    selector: "button[type=submit]",
+    target_text: "Submit",
+    value_description: "redacted",
+    status: "succeeded",
+    phase: "initial",
+  }];
+  const agentRun = agentRunRecordSchema.parse({
+    url: "https://example.com/start",
+    status: "requires_form_submission",
+    run_id: "run-1",
+    agent_status: "COMPLETED",
+    goal: "Submit the form and extract the result.",
+    records_extracted: 1,
+    browser_actions: browserActions,
+  });
+
+  assert.deepEqual(
+    explicitBrowserActionsFromAgentRuns([agentRun]),
+    browserActions
+  );
+
+  const report = runReportSchema.parse({
+    run_id: "run-1",
+    prompt: "Find form-backed data.",
+    target_rows: 1,
+    started_at: "2026-05-23T00:00:00.000Z",
+    finished_at: "2026-05-23T00:00:01.000Z",
+    duration_ms: 1_000,
+    dataset_spec: datasetSpec(),
+    stats: {
+      ...phaseStats(),
+      records_after_merge: 1,
+      visualization_records: 1,
+    },
+    initial: {
+      ...phaseStats(),
+      search_queries: ["example form"],
+      fetched_urls: ["https://example.com/start"],
+      failed_urls: [],
+      agent_browser_actions: browserActions,
+    },
+    repair: {
+      attempted: true,
+      total_loops: 1,
+      loops: [{
+        loop_index: 1,
+        repair_queries: ["example form details"],
+        agent_browser_actions: browserActions,
+        missing_fields: [],
+        records_before: 0,
+        records_after: 1,
+        fields_filled: {},
+        stats: phaseStats(),
+      }],
+      missing_fields: [],
+      repair_queries: ["example form details"],
+      records_before: 0,
+      records_after: 1,
+      fields_filled: {},
+      stats: phaseStats(),
+    },
+    search_queries: ["example form", "example form details"],
+    fetched_urls: ["https://example.com/start"],
+    failed_urls: [],
+    errors: [],
+  });
+
+  assert.deepEqual(report.initial.agent_browser_actions, browserActions);
+  assert.deepEqual(report.repair.loops[0]?.agent_browser_actions, browserActions);
+});
+
+function datasetSpec() {
+  return {
+    intent_summary: "Find form-backed data.",
+    target_row_count: 1,
+    row_grain: "company",
+    columns: [{
+      name: "entity_name",
+      type: "string",
+      description: "Entity name",
+      required: true,
+    }],
+    dedupe_keys: ["entity_name"],
+    search_queries: ["example form"],
+    extraction_hints: "Use source-backed rows.",
+  };
+}
+
+function phaseStats() {
+  return {
+    search_queries_executed: 1,
+    search_results_collected: 1,
+    unique_urls_selected: 1,
+    pages_fetched: 1,
+    pages_failed: 0,
+    raw_records_extracted: 1,
+  };
+}