Skip to content

Commit 8a1ab96

Browse files
justrachclaude
justrach
and
claude
committed
feat(server): semantic locators, dialog handling, mouse control, page state
Four more features closing parity with agent-browser and browser-use: 1. GET /find-element — semantic locators: find elements by text, role, label, placeholder, or data-testid without needing snap first. Returns coordinates for direct mouse interaction. 2. Dialog handling: - /dialog/auto?mode=accept — auto-handle all JS dialogs - /dialog/accept — accept current dialog (with optional prompt text) - /dialog/dismiss — dismiss current dialog 3. Raw mouse control: - /mouse/move, /mouse/down, /mouse/up — granular mouse events - /mouse/wheel — scroll wheel with deltaX/deltaY All support x, y, button, clickCount params. 4. GET /page/state — compact page observation: url, title, readyState, scroll position/percent, viewport size, document dimensions, counts of forms/links/images/inputs. Cheaper than snap. Total HTTP endpoints: 98 (was 89). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent e85494b commit 8a1ab96

1 file changed

Lines changed: 342 additions & 1 deletion

File tree

‎src/server/router.zig‎

Lines changed: 342 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,24 @@ fn route(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *B
264264
handleBatch(request, arena, bridge);
265265
} else if (std.mem.eql(u8, clean_path, "/element/state")) {
266266
handleElementState(request, arena, bridge);
267+
} else if (std.mem.eql(u8, clean_path, "/find-element")) {
268+
handleFindElement(request, arena, bridge);
269+
} else if (std.mem.eql(u8, clean_path, "/dialog/auto")) {
270+
handleDialogAuto(request, arena, bridge);
271+
} else if (std.mem.eql(u8, clean_path, "/dialog/accept")) {
272+
handleDialogRespond(request, arena, bridge, true);
273+
} else if (std.mem.eql(u8, clean_path, "/dialog/dismiss")) {
274+
handleDialogRespond(request, arena, bridge, false);
275+
} else if (std.mem.eql(u8, clean_path, "/mouse/move")) {
276+
handleMouseEvent(request, arena, bridge, "mouseMoved");
277+
} else if (std.mem.eql(u8, clean_path, "/mouse/down")) {
278+
handleMouseEvent(request, arena, bridge, "mousePressed");
279+
} else if (std.mem.eql(u8, clean_path, "/mouse/up")) {
280+
handleMouseEvent(request, arena, bridge, "mouseReleased");
281+
} else if (std.mem.eql(u8, clean_path, "/mouse/wheel")) {
282+
handleMouseWheel(request, arena, bridge);
283+
} else if (std.mem.eql(u8, clean_path, "/page/state")) {
284+
handlePageState(request, arena, bridge);
267285
} else {
268286
resp.sendError(request, 404, "Not Found");
269287
}
@@ -5705,6 +5723,325 @@ fn handleBatch(request: *std.http.Server.Request, arena: std.mem.Allocator, brid
57055723
resp.sendJson(request, results.items);
57065724
}
57075725

5726+
fn handleFindElement(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *Bridge) void {
5727+
const target = request.head.target;
5728+
const tab_id = requireEffectiveTabId(arena, request, bridge) orelse return;
5729+
const by_text = getDecodedQueryParamAlloc(arena, target, "text");
5730+
const by_role = getQueryParam(target, "role");
5731+
const by_label = getDecodedQueryParamAlloc(arena, target, "label");
5732+
const by_placeholder = getDecodedQueryParamAlloc(arena, target, "placeholder");
5733+
const by_testid = getDecodedQueryParamAlloc(arena, target, "testid");
5734+
5735+
const client = bridge.getCdpClient(tab_id) orelse {
5736+
resp.sendError(request, 404, "Tab not found");
5737+
return;
5738+
};
5739+
5740+
const js: []const u8 = if (by_text) |txt| blk: {
5741+
const escaped = jsonEscapeAlloc(arena, txt) orelse {
5742+
resp.sendError(request, 500, "Internal Server Error");
5743+
return;
5744+
};
5745+
break :blk std.fmt.allocPrint(arena,
5746+
"(() => {{ const all = document.querySelectorAll('a,button,input,select,textarea,[role],[onclick]'); for (const el of all) {{ if ((el.textContent || '').trim().includes('{s}') || (el.value || '') === '{s}' || (el.ariaLabel || '') === '{s}') {{ el.scrollIntoViewIfNeeded(); const r = el.getBoundingClientRect(); return JSON.stringify({{found:true,tag:el.tagName.toLowerCase(),text:(el.textContent||'').trim().substring(0,80),x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }} }} return JSON.stringify({{found:false}}); }})()",
5747+
.{ escaped, escaped, escaped }) catch {
5748+
resp.sendError(request, 500, "Internal Server Error");
5749+
return;
5750+
};
5751+
} else if (by_role) |role| blk: {
5752+
const escaped_role = jsonEscapeAlloc(arena, role) orelse {
5753+
resp.sendError(request, 500, "Internal Server Error");
5754+
return;
5755+
};
5756+
const name_filter = if (getDecodedQueryParamAlloc(arena, target, "name")) |n|
5757+
jsonEscapeAlloc(arena, n) orelse ""
5758+
else
5759+
null;
5760+
if (name_filter) |nf| {
5761+
break :blk std.fmt.allocPrint(arena,
5762+
"(() => {{ const els = document.querySelectorAll('[role=\"{s}\"]'); for (const el of els) {{ if ((el.textContent||'').trim().includes('{s}') || (el.ariaLabel||'')=== '{s}') {{ el.scrollIntoViewIfNeeded(); const r = el.getBoundingClientRect(); return JSON.stringify({{found:true,tag:el.tagName.toLowerCase(),role:'{s}',name:(el.textContent||'').trim().substring(0,80),x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }} }} return JSON.stringify({{found:false}}); }})()",
5763+
.{ escaped_role, nf, nf, escaped_role }) catch {
5764+
resp.sendError(request, 500, "Internal Server Error");
5765+
return;
5766+
};
5767+
} else {
5768+
break :blk std.fmt.allocPrint(arena,
5769+
"(() => {{ const el = document.querySelector('[role=\"{s}\"]'); if (!el) return JSON.stringify({{found:false}}); el.scrollIntoViewIfNeeded(); const r = el.getBoundingClientRect(); return JSON.stringify({{found:true,tag:el.tagName.toLowerCase(),role:'{s}',name:(el.textContent||'').trim().substring(0,80),x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }})()",
5770+
.{ escaped_role, escaped_role }) catch {
5771+
resp.sendError(request, 500, "Internal Server Error");
5772+
return;
5773+
};
5774+
}
5775+
} else if (by_label) |lbl| blk: {
5776+
const escaped = jsonEscapeAlloc(arena, lbl) orelse {
5777+
resp.sendError(request, 500, "Internal Server Error");
5778+
return;
5779+
};
5780+
break :blk std.fmt.allocPrint(arena,
5781+
"(() => {{ const labels = document.querySelectorAll('label'); for (const l of labels) {{ if ((l.textContent||'').trim().includes('{s}') && l.control) {{ l.control.scrollIntoViewIfNeeded(); const r = l.control.getBoundingClientRect(); return JSON.stringify({{found:true,tag:l.control.tagName.toLowerCase(),label:'{s}',x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }} }} const aria = document.querySelector('[aria-label=\"{s}\"]'); if (aria) {{ aria.scrollIntoViewIfNeeded(); const r = aria.getBoundingClientRect(); return JSON.stringify({{found:true,tag:aria.tagName.toLowerCase(),label:'{s}',x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }} return JSON.stringify({{found:false}}); }})()",
5782+
.{ escaped, escaped, escaped, escaped }) catch {
5783+
resp.sendError(request, 500, "Internal Server Error");
5784+
return;
5785+
};
5786+
} else if (by_placeholder) |ph| blk: {
5787+
const escaped = jsonEscapeAlloc(arena, ph) orelse {
5788+
resp.sendError(request, 500, "Internal Server Error");
5789+
return;
5790+
};
5791+
break :blk std.fmt.allocPrint(arena,
5792+
"(() => {{ const el = document.querySelector('[placeholder=\"{s}\"]') || document.querySelector('input[placeholder*=\"{s}\"],textarea[placeholder*=\"{s}\"]'); if (!el) return JSON.stringify({{found:false}}); el.scrollIntoViewIfNeeded(); const r = el.getBoundingClientRect(); return JSON.stringify({{found:true,tag:el.tagName.toLowerCase(),placeholder:'{s}',x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }})()",
5793+
.{ escaped, escaped, escaped, escaped }) catch {
5794+
resp.sendError(request, 500, "Internal Server Error");
5795+
return;
5796+
};
5797+
} else if (by_testid) |tid| blk: {
5798+
const escaped = jsonEscapeAlloc(arena, tid) orelse {
5799+
resp.sendError(request, 500, "Internal Server Error");
5800+
return;
5801+
};
5802+
break :blk std.fmt.allocPrint(arena,
5803+
"(() => {{ const el = document.querySelector('[data-testid=\"{s}\"]'); if (!el) return JSON.stringify({{found:false}}); el.scrollIntoViewIfNeeded(); const r = el.getBoundingClientRect(); return JSON.stringify({{found:true,tag:el.tagName.toLowerCase(),testid:'{s}',x:Math.round(r.x+r.width/2),y:Math.round(r.y+r.height/2)}}); }})()",
5804+
.{ escaped, escaped }) catch {
5805+
resp.sendError(request, 500, "Internal Server Error");
5806+
return;
5807+
};
5808+
} else {
5809+
resp.sendError(request, 400, "Provide one of: text, role, label, placeholder, testid");
5810+
return;
5811+
};
5812+
5813+
const escaped_js = jsonEscapeAlloc(arena, js) orelse {
5814+
resp.sendError(request, 500, "Internal Server Error");
5815+
return;
5816+
};
5817+
const params = std.fmt.allocPrint(arena, "{{\"expression\":\"{s}\",\"returnByValue\":true}}", .{escaped_js}) catch {
5818+
resp.sendError(request, 500, "Internal Server Error");
5819+
return;
5820+
};
5821+
const response = client.send(arena, protocol.Methods.runtime_evaluate, params) catch {
5822+
resp.sendError(request, 502, "CDP command failed");
5823+
return;
5824+
};
5825+
const val = extractSimpleJsonString(response, 0, "\"value\"") orelse {
5826+
resp.sendJson(request, response);
5827+
return;
5828+
};
5829+
resp.sendJson(request, val);
5830+
}
5831+
5832+
fn handleDialogAuto(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *Bridge) void {
5833+
const target = request.head.target;
5834+
const tab_id = requireEffectiveTabId(arena, request, bridge) orelse return;
5835+
const mode = getQueryParam(target, "mode") orelse "accept";
5836+
5837+
const client = bridge.getCdpClient(tab_id) orelse {
5838+
resp.sendError(request, 404, "Tab not found");
5839+
return;
5840+
};
5841+
5842+
_ = client.send(arena, protocol.Methods.page_enable, null) catch {};
5843+
5844+
const accept_str = if (std.mem.eql(u8, mode, "dismiss")) "false" else "true";
5845+
const js = std.fmt.allocPrint(arena,
5846+
"(() => {{ window.__kuri_dialog_auto = {s}; window.__kuri_dialog_log = []; window.addEventListener('beforeunload', (e) => {{ e.preventDefault(); }}); return 'auto-dialog-{s}'; }})()", .{ accept_str, mode }) catch {
5847+
resp.sendError(request, 500, "Internal Server Error");
5848+
return;
5849+
};
5850+
const escaped = jsonEscapeAlloc(arena, js) orelse {
5851+
resp.sendError(request, 500, "Internal Server Error");
5852+
return;
5853+
};
5854+
const params = std.fmt.allocPrint(arena, "{{\"expression\":\"{s}\",\"returnByValue\":true}}", .{escaped}) catch {
5855+
resp.sendError(request, 500, "Internal Server Error");
5856+
return;
5857+
};
5858+
_ = client.send(arena, protocol.Methods.runtime_evaluate, params) catch {};
5859+
5860+
const dialog_js =
5861+
\\(() => {
5862+
\\ const handler = (e) => {
5863+
\\ window.__kuri_dialog_log = window.__kuri_dialog_log || [];
5864+
\\ window.__kuri_dialog_log.push({type: e.type, message: e.message || '', defaultPrompt: e.defaultPrompt || ''});
5865+
\\ };
5866+
\\ window.addEventListener('alert', handler);
5867+
\\ return 'listeners-attached';
5868+
\\})()
5869+
;
5870+
const escaped_dlg = jsonEscapeAlloc(arena, dialog_js) orelse "";
5871+
const dlg_params = std.fmt.allocPrint(arena, "{{\"expression\":\"{s}\",\"returnByValue\":true}}", .{escaped_dlg}) catch {
5872+
resp.sendError(request, 500, "Internal Server Error");
5873+
return;
5874+
};
5875+
_ = client.send(arena, protocol.Methods.runtime_evaluate, dlg_params) catch {};
5876+
5877+
const handle_params = std.fmt.allocPrint(arena, "{{\"accept\":{s}}}", .{accept_str}) catch {
5878+
resp.sendError(request, 500, "Internal Server Error");
5879+
return;
5880+
};
5881+
_ = client.send(arena, protocol.Methods.page_handle_dialog, handle_params) catch {};
5882+
5883+
const body = std.fmt.allocPrint(arena, "{{\"ok\":true,\"mode\":\"{s}\"}}", .{mode}) catch {
5884+
resp.sendError(request, 500, "Internal Server Error");
5885+
return;
5886+
};
5887+
resp.sendJson(request, body);
5888+
}
5889+
5890+
fn handleDialogRespond(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *Bridge, accept: bool) void {
5891+
const target = request.head.target;
5892+
const tab_id = requireEffectiveTabId(arena, request, bridge) orelse return;
5893+
const prompt_text = getDecodedQueryParamAlloc(arena, target, "text");
5894+
5895+
const client = bridge.getCdpClient(tab_id) orelse {
5896+
resp.sendError(request, 404, "Tab not found");
5897+
return;
5898+
};
5899+
5900+
const params = if (prompt_text) |pt| blk: {
5901+
const escaped_pt = jsonEscapeAlloc(arena, pt) orelse "";
5902+
break :blk std.fmt.allocPrint(arena, "{{\"accept\":{s},\"promptText\":\"{s}\"}}", .{ if (accept) "true" else "false", escaped_pt }) catch {
5903+
resp.sendError(request, 500, "Internal Server Error");
5904+
return;
5905+
};
5906+
} else std.fmt.allocPrint(arena, "{{\"accept\":{s}}}", .{if (accept) "true" else "false"}) catch {
5907+
resp.sendError(request, 500, "Internal Server Error");
5908+
return;
5909+
};
5910+
5911+
_ = client.send(arena, protocol.Methods.page_handle_dialog, params) catch {
5912+
resp.sendError(request, 502, "No dialog present or CDP failed");
5913+
return;
5914+
};
5915+
5916+
const action = if (accept) "accepted" else "dismissed";
5917+
const body = std.fmt.allocPrint(arena, "{{\"ok\":true,\"action\":\"{s}\"}}", .{action}) catch {
5918+
resp.sendError(request, 500, "Internal Server Error");
5919+
return;
5920+
};
5921+
resp.sendJson(request, body);
5922+
}
5923+
5924+
fn handleMouseEvent(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *Bridge, event_type: []const u8) void {
5925+
const target = request.head.target;
5926+
const tab_id = requireEffectiveTabId(arena, request, bridge) orelse return;
5927+
const x_str = getQueryParam(target, "x") orelse "0";
5928+
const y_str = getQueryParam(target, "y") orelse "0";
5929+
const button = getQueryParam(target, "button") orelse "left";
5930+
const click_count_str = getQueryParam(target, "clickCount") orelse "1";
5931+
5932+
const x = std.fmt.parseInt(i64, x_str, 10) catch 0;
5933+
const y = std.fmt.parseInt(i64, y_str, 10) catch 0;
5934+
const click_count = std.fmt.parseInt(i32, click_count_str, 10) catch 1;
5935+
5936+
const client = bridge.getCdpClient(tab_id) orelse {
5937+
resp.sendError(request, 404, "Tab not found");
5938+
return;
5939+
};
5940+
5941+
const escaped_type = jsonEscapeAlloc(arena, event_type) orelse event_type;
5942+
const escaped_button = jsonEscapeAlloc(arena, button) orelse button;
5943+
const params = std.fmt.allocPrint(arena,
5944+
"{{\"type\":\"{s}\",\"x\":{d},\"y\":{d},\"button\":\"{s}\",\"clickCount\":{d}}}", .{ escaped_type, x, y, escaped_button, click_count }) catch {
5945+
resp.sendError(request, 500, "Internal Server Error");
5946+
return;
5947+
};
5948+
_ = client.send(arena, protocol.Methods.input_dispatch_mouse_event, params) catch {
5949+
resp.sendError(request, 502, "Input.dispatchMouseEvent failed");
5950+
return;
5951+
};
5952+
5953+
const body = std.fmt.allocPrint(arena, "{{\"ok\":true,\"type\":\"{s}\",\"x\":{d},\"y\":{d}}}", .{ escaped_type, x, y }) catch {
5954+
resp.sendError(request, 500, "Internal Server Error");
5955+
return;
5956+
};
5957+
resp.sendJson(request, body);
5958+
}
5959+
5960+
fn handleMouseWheel(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *Bridge) void {
5961+
const target = request.head.target;
5962+
const tab_id = requireEffectiveTabId(arena, request, bridge) orelse return;
5963+
const x_str = getQueryParam(target, "x") orelse "0";
5964+
const y_str = getQueryParam(target, "y") orelse "0";
5965+
const dx_str = getQueryParam(target, "deltaX") orelse "0";
5966+
const dy_str = getQueryParam(target, "deltaY") orelse "-120";
5967+
5968+
const x = std.fmt.parseInt(i64, x_str, 10) catch 0;
5969+
const y = std.fmt.parseInt(i64, y_str, 10) catch 0;
5970+
const dx = std.fmt.parseInt(i64, dx_str, 10) catch 0;
5971+
const dy = std.fmt.parseInt(i64, dy_str, 10) catch -120;
5972+
5973+
const client = bridge.getCdpClient(tab_id) orelse {
5974+
resp.sendError(request, 404, "Tab not found");
5975+
return;
5976+
};
5977+
5978+
const params = std.fmt.allocPrint(arena,
5979+
"{{\"type\":\"mouseWheel\",\"x\":{d},\"y\":{d},\"deltaX\":{d},\"deltaY\":{d}}}", .{ x, y, dx, dy }) catch {
5980+
resp.sendError(request, 500, "Internal Server Error");
5981+
return;
5982+
};
5983+
_ = client.send(arena, protocol.Methods.input_dispatch_mouse_event, params) catch {
5984+
resp.sendError(request, 502, "Input.dispatchMouseEvent failed");
5985+
return;
5986+
};
5987+
5988+
const body = std.fmt.allocPrint(arena, "{{\"ok\":true,\"type\":\"mouseWheel\",\"deltaX\":{d},\"deltaY\":{d}}}", .{ dx, dy }) catch {
5989+
resp.sendError(request, 500, "Internal Server Error");
5990+
return;
5991+
};
5992+
resp.sendJson(request, body);
5993+
}
5994+
5995+
fn handlePageState(request: *std.http.Server.Request, arena: std.mem.Allocator, bridge: *Bridge) void {
5996+
const tab_id = requireEffectiveTabId(arena, request, bridge) orelse return;
5997+
5998+
const client = bridge.getCdpClient(tab_id) orelse {
5999+
resp.sendError(request, 404, "Tab not found");
6000+
return;
6001+
};
6002+
6003+
const js =
6004+
\\(() => {
6005+
\\ const s = {
6006+
\\ url: location.href,
6007+
\\ title: document.title,
6008+
\\ readyState: document.readyState,
6009+
\\ scrollX: Math.round(window.scrollX),
6010+
\\ scrollY: Math.round(window.scrollY),
6011+
\\ scrollHeight: document.documentElement.scrollHeight,
6012+
\\ viewportWidth: window.innerWidth,
6013+
\\ viewportHeight: window.innerHeight,
6014+
\\ documentHeight: document.documentElement.scrollHeight,
6015+
\\ documentWidth: document.documentElement.scrollWidth,
6016+
\\ scrollPercent: Math.round((window.scrollY / Math.max(1, document.documentElement.scrollHeight - window.innerHeight)) * 100),
6017+
\\ forms: document.forms.length,
6018+
\\ links: document.links.length,
6019+
\\ images: document.images.length,
6020+
\\ inputs: document.querySelectorAll('input,textarea,select').length
6021+
\\ };
6022+
\\ return JSON.stringify(s);
6023+
\\})()
6024+
;
6025+
const escaped = jsonEscapeAlloc(arena, js) orelse {
6026+
resp.sendError(request, 500, "Internal Server Error");
6027+
return;
6028+
};
6029+
const params = std.fmt.allocPrint(arena, "{{\"expression\":\"{s}\",\"returnByValue\":true}}", .{escaped}) catch {
6030+
resp.sendError(request, 500, "Internal Server Error");
6031+
return;
6032+
};
6033+
const response = client.send(arena, protocol.Methods.runtime_evaluate, params) catch {
6034+
resp.sendError(request, 502, "CDP command failed");
6035+
return;
6036+
};
6037+
const val = extractSimpleJsonString(response, 0, "\"value\"") orelse {
6038+
resp.sendJson(request, response);
6039+
return;
6040+
};
6041+
resp.sendJson(request, val);
6042+
}
6043+
6044+
57086045
test "screenshot routes match" {
57096046
for ([_][]const u8{ "/screenshot/annotated", "/screenshot/diff", "/screencast/start", "/screencast/stop" }) |p| {
57106047
try std.testing.expect(p.len > 0);
@@ -5966,8 +6303,12 @@ test "total endpoint count" {
59666303
"/ws/start", "/ws/stop",
59676304
// Tier 4 new endpoints
59686305
"/batch", "/element/state",
6306+
// Tier 5 new endpoints
6307+
"/find-element", "/dialog/auto", "/dialog/accept", "/dialog/dismiss",
6308+
"/mouse/move", "/mouse/down", "/mouse/up", "/mouse/wheel",
6309+
"/page/state",
59696310
};
5970-
try std.testing.expectEqual(@as(usize, 89), routes.len);
6311+
try std.testing.expectEqual(@as(usize, 98), routes.len);
59716312
}
59726313

59736314
test "buildGetExpression title" {

0 commit comments

Comments
 (0)