Skip to content

Commit b239464

Browse files
authored
Merge pull request #688 from ThinkEx-OSS/codex/interactive-browser-handoff
feat: improve collaboration startup and add interactive browser handoff
2 parents 707a49c + b0c39cd commit b239464

33 files changed

Lines changed: 1842 additions & 230 deletions

docs/concepts/ai-chat.mdx

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ The AI tool surface is intentionally named and bounded:
1818
Search, read, create, and update workspace content through workspace access checks and `WorkspaceKernel` commands.
1919
</Accordion>
2020
<Accordion title="Web tools">
21-
Search the public web, read rendered pages as Markdown, and extract links through policy-checked public HTTP(S) URLs.
21+
Search the public web, read rendered pages as Markdown, and extract links through policy-checked public HTTP(S) URLs. For interactive sites, the AI can open a task-scoped Browser Run session that appears in the chat.
22+
</Accordion>
23+
<Accordion title="Interactive browser">
24+
Watch the active browser from the chat, expand its Live View, or take control when the AI asks for login, MFA, CAPTCHA, private input, or confirmation of a consequential action. Select **Done** or **Failed** to hand the same session back to the AI. The unrecorded session remains available briefly after the response so you can inspect it, then expires after inactivity; select **Stop** to discard it immediately.
2225
</Accordion>
2326
<Accordion title="Research tools">
2427
Discover and deepen research sources when the workspace task needs external literature or source discovery.

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@
138138
"tw-animate-css": "^1.3.6",
139139
"unified": "^11.0.5",
140140
"unist-util-visit": "^5.1.0",
141+
"y-indexeddb": "9.0.12",
141142
"y-partyserver": "^2.2.0",
142143
"y-protocols": "^1.0.7",
143144
"yjs": "^13.6.31",

pnpm-lock.yaml

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { generateTypes } from "@cloudflare/codemode/ai";
2+
import type { ToolSet } from "ai";
3+
4+
import { requireAIThreadToolRuntime } from "#/features/workspaces/ai/ai-thread-tool";
5+
6+
/**
7+
* The TypeScript block the model writes Code Mode against.
8+
*
9+
* The tool a provider sees is deliberately lossy: `getModelToolDefinition`
10+
* drops `outputSchema` so it never has to survive a provider's JSON Schema
11+
* dialect, and `modelInputSchema` flattens unions that some providers reject.
12+
* Code Mode is not a provider — it hands the model TypeScript, which
13+
* represents both faithfully — so generate types from the runtime's real
14+
* schemas. Without this, every method reads as `Promise<unknown>` and every
15+
* flattened union loses the fields it makes conditional.
16+
*
17+
* This mirrors what `AIThreadToolSetConnector` publishes as the connector's
18+
* JSON Schema, so the types and the contract agree.
19+
*/
20+
export function generateAIThreadCodemodeTypes(toolSet: ToolSet, namespace: string) {
21+
const typedToolSet = Object.fromEntries(
22+
Object.entries(toolSet).map(([toolName, aiTool]) => {
23+
const runtime = requireAIThreadToolRuntime(toolName, aiTool);
24+
return [
25+
toolName,
26+
{ ...aiTool, inputSchema: runtime.inputSchema, outputSchema: runtime.outputSchema },
27+
];
28+
}),
29+
);
30+
31+
return generateTypes(typedToolSet, namespace);
32+
}

src/features/workspaces/ai/ai-codemode-types.worker.test.ts

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1-
import { generateTypes } from "@cloudflare/codemode/ai";
1+
import type { ToolSet } from "ai";
22
import { describe, expect, it } from "vitest";
33

44
import {
55
AI_TOOL_REGISTRY,
66
requireAiToolDefinition,
77
} from "#/features/workspaces/ai/ai-tool-registry";
8+
import { generateAIThreadCodemodeTypes } from "#/features/workspaces/ai/ai-codemode-types";
9+
import { createAIThreadBrowserHandoffTool } from "#/features/workspaces/ai/ai-thread-browser";
810
import { createAIThreadCodeRunTools } from "#/features/workspaces/ai/code-run-tools";
911
import { createAIThreadResearchTools } from "#/features/workspaces/ai/research-tools";
1012
import { createAIThreadTimeTools } from "#/features/workspaces/ai/time-tools";
@@ -13,13 +15,17 @@ import { createAIThreadWebTools } from "#/features/workspaces/ai/web-tools";
1315
describe("AI Code Mode type generation", () => {
1416
it("publishes concrete output types for every non-workspace nested tool", () => {
1517
const env = {} as Cloudflare.Env;
16-
const tools = {
18+
const tools: ToolSet = {
19+
browser_handoff: createAIThreadBrowserHandoffTool(),
1720
...createAIThreadCodeRunTools({ env, sandboxId: "test-thread" }),
1821
...createAIThreadResearchTools(env),
1922
...createAIThreadTimeTools(),
2023
...createAIThreadWebTools(env),
2124
};
22-
const declarations = generateTypes(tools, "tools");
25+
// Exercise the generator the connector actually calls: the raw ToolSet has
26+
// no output schemas, so asserting against `generateTypes` directly would
27+
// pass on a type block the model never sees.
28+
const declarations = generateAIThreadCodemodeTypes(tools, "tools");
2329

2430
expect(declarations).not.toMatch(/type \w+Output = unknown/);
2531
for (const toolName of Object.keys(tools)) {
@@ -35,15 +41,20 @@ describe("AI Code Mode type generation", () => {
3541

3642
it("keeps every runtime tool factory synchronized with the registry", () => {
3743
const env = {} as Cloudflare.Env;
38-
const tools = {
44+
const tools: ToolSet = {
45+
browser_handoff: createAIThreadBrowserHandoffTool(),
3946
...createAIThreadCodeRunTools({ env, sandboxId: "registry-test" }),
4047
...createAIThreadResearchTools(env),
4148
...createAIThreadTimeTools(),
4249
...createAIThreadWebTools(env),
4350
};
4451
const runtimeNames = ["sandbox_bash", ...Object.keys(tools)].sort();
45-
const registeredRuntimeNames = Object.keys(AI_TOOL_REGISTRY)
46-
.filter((name) => name !== "orchestrate" && !name.startsWith("workspace_"))
52+
const registeredRuntimeNames = Object.entries(AI_TOOL_REGISTRY)
53+
.filter(
54+
([name, definition]) =>
55+
name !== "orchestrate" && !name.startsWith("workspace_") && !definition.presentationOnly,
56+
)
57+
.map(([name]) => name)
4758
.sort();
4859

4960
for (const name of runtimeNames) {
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
export type AIThreadBrowserActivityStatus = "completed" | "failed" | "running";
2+
3+
interface AIThreadBrowserActivityCall {
4+
args: unknown;
5+
method: string;
6+
result?: unknown;
7+
}
8+
9+
interface AIThreadBrowserDestination {
10+
site: string;
11+
title?: string;
12+
url: string;
13+
}
14+
15+
/**
16+
* Turns low-level CDP traffic into one user-facing destination summary.
17+
*
18+
* Only explicit navigations count. Protocol setup such as `attachToTarget`
19+
* carries no useful destination, and successful traffic without a destination
20+
* is omitted instead of rendering a generic "Browsed web" row.
21+
*/
22+
export function summarizeAIThreadBrowserActivity(
23+
calls: readonly AIThreadBrowserActivityCall[],
24+
status: AIThreadBrowserActivityStatus,
25+
): string | undefined {
26+
const destinations = getBrowserDestinations(calls);
27+
const sites = unique(destinations.map((destination) => destination.site));
28+
29+
if (status === "failed") {
30+
if (sites.length === 1) {
31+
return `Browser failed on ${sites[0]}`;
32+
}
33+
return sites.length > 1 ? `Browser failed across ${sites.length} sites` : "Browser failed";
34+
}
35+
36+
if (sites.length === 0) {
37+
return undefined;
38+
}
39+
40+
const verb = status === "running" ? "Browsing" : "Browsed";
41+
if (sites.length === 1) {
42+
const pageCount = unique(destinations.map((destination) => destination.url)).length;
43+
const pageTitles = unique(
44+
destinations
45+
.map((destination) => destination.title)
46+
.filter((title): title is string => Boolean(title)),
47+
);
48+
if (pageCount === 1 && pageTitles.length === 1 && pageTitles[0] !== sites[0]) {
49+
return `${verb} ${sites[0]}${pageTitles[0]}`;
50+
}
51+
return pageCount > 1 ? `${verb} ${pageCount} pages on ${sites[0]}` : `${verb} ${sites[0]}`;
52+
}
53+
if (sites.length === 2) {
54+
return `${verb} ${sites[0]} and ${sites[1]}`;
55+
}
56+
return `${verb} ${sites.length} sites`;
57+
}
58+
59+
function getBrowserDestinations(calls: readonly AIThreadBrowserActivityCall[]) {
60+
const destinations = calls.flatMap((call) => {
61+
if (call.method !== "send") {
62+
return [];
63+
}
64+
65+
const args = asRecord(call.args);
66+
const method = getString(args.method);
67+
if (method !== "Target.createTarget" && method !== "Page.navigate") {
68+
return [];
69+
}
70+
71+
const url = getString(asRecord(args.params).url);
72+
const destination = url ? getBrowserDestination(url) : undefined;
73+
return destination ? [destination] : [];
74+
});
75+
76+
if (unique(destinations.map((destination) => destination.url)).length === 1) {
77+
const title = getLastEvaluatedPageTitle(calls);
78+
if (title) {
79+
for (const destination of destinations) {
80+
destination.title ??= title;
81+
}
82+
}
83+
}
84+
85+
return destinations;
86+
}
87+
88+
function getLastEvaluatedPageTitle(calls: readonly AIThreadBrowserActivityCall[]) {
89+
for (let index = calls.length - 1; index >= 0; index -= 1) {
90+
const call = calls[index];
91+
const args = asRecord(call?.args);
92+
if (call?.method !== "send" || getString(args.method) !== "Runtime.evaluate") {
93+
continue;
94+
}
95+
96+
const params = asRecord(args.params);
97+
const expression = getString(params.expression);
98+
if (!expression?.includes("document.title")) {
99+
continue;
100+
}
101+
102+
const value = asRecord(asRecord(call.result).result).value;
103+
const title =
104+
typeof value === "string"
105+
? formatPageTitle(value)
106+
: formatPageTitle(getString(asRecord(value).title));
107+
if (title) {
108+
return title;
109+
}
110+
}
111+
return undefined;
112+
}
113+
114+
function getBrowserDestination(value: string): AIThreadBrowserDestination | undefined {
115+
try {
116+
const url = new URL(value);
117+
if (url.protocol !== "http:" && url.protocol !== "https:") {
118+
return undefined;
119+
}
120+
121+
url.hash = "";
122+
const hostname = url.hostname.toLowerCase().replace(/^www\./, "");
123+
return {
124+
site: formatBrowserSite(hostname),
125+
url: url.toString(),
126+
};
127+
} catch {
128+
return undefined;
129+
}
130+
}
131+
132+
function formatPageTitle(value: string | undefined) {
133+
const title = value?.replace(/\s+/g, " ").trim();
134+
if (!title) {
135+
return undefined;
136+
}
137+
return title.length > 80 ? `${title.slice(0, 79).trimEnd()}…` : title;
138+
}
139+
140+
function formatBrowserSite(hostname: string) {
141+
if (hostname === "instructure.com" || hostname.endsWith(".instructure.com")) {
142+
const tenant = hostname.slice(0, -".instructure.com".length).split(".").filter(Boolean).at(-1);
143+
return tenant && tenant !== "canvas" ? `${formatSiteWord(tenant)} Canvas` : "Canvas";
144+
}
145+
return hostname;
146+
}
147+
148+
function formatSiteWord(value: string) {
149+
return value.length <= 5
150+
? value.toUpperCase()
151+
: `${value.charAt(0).toUpperCase()}${value.slice(1)}`;
152+
}
153+
154+
function unique<T>(values: readonly T[]) {
155+
return [...new Set(values)];
156+
}
157+
158+
function asRecord(value: unknown): Record<string, unknown> {
159+
return value && typeof value === "object" && !Array.isArray(value)
160+
? (value as Record<string, unknown>)
161+
: {};
162+
}
163+
164+
function getString(value: unknown) {
165+
return typeof value === "string" && value.trim() ? value.trim() : undefined;
166+
}

0 commit comments

Comments
 (0)