diff --git a/.changeset/frame-snapshot-screenshot-once.md b/.changeset/frame-snapshot-screenshot-once.md new file mode 100644 index 00000000..13483da9 --- /dev/null +++ b/.changeset/frame-snapshot-screenshot-once.md @@ -0,0 +1,8 @@ +--- +'@wdio/devtools-backend': patch +'@wdio/devtools-service': patch +'@wdio/selenium-devtools': patch +'@wdio/nightwatch-devtools': patch +--- + +A trace zip stores each action screenshot once. The `frame-snapshot` event for an action used to carry its screenshot again as base64 inside the page it describes, beside the same screenshot under `resources/`; it now references that file through `resourceOverrides`, which a trace viewer resolves to the stored image. On a WebdriverIO cucumber example trace this takes the zip from 1.63 MB to 1.10 MB and `trace.trace` from 538 KB to 5 KB. The DevTools trace player reads such a zip as before. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fb14d508..8b143269 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -64,7 +64,7 @@ Contains: - `writeTraceZip` / `buildTraceZip` — the zip writer and its resource model (`TraceZipResource`). - `buildActionEvents`: commands → the ordered before/after event stream the player replays, including the `(startTime, sequence, index)` ordering key, with a row's children written inside its span under its callId. - `buildGroupPath` — the suite/test/step hierarchy a row is nested under. -- `buildImageFrameSnapshots` / `upsertRichestSnapshot` / `buildDenseScreencast` / `thinScreencastFrames` — filmstrip and per-action frame snapshots. +- `buildImageFrameSnapshots` + `buildScreenshotResourceSnapshots` / `upsertRichestSnapshot` / `buildDenseScreencast` / `thinScreencastFrames` — filmstrip and per-action frame snapshots (each a page whose image is the stored screenshot, reached through a `resourceOverrides` entry and the one recorded response it patches). - `buildMutationsNdjson` / `reattributeDomAnchors` — the DOM mutation stream and its anchor repositioning. - `buildConsoleEvents`, `networkRequestToHar`, `buildSourceResources` / `sourceResourceName` / `callSourceToStack`, `generateTranscript`, `sha1Hex`. - `parseTraceZip` (`trace-reader.ts` and its `trace-reader-*` siblings) — the reverse direction: a `trace.zip`, ours or a foreign one, back into the `TracePlayerData` the player renders. Imported by subpath (`@wdio/devtools-trace/trace-reader`) and kept out of the package index, so adapters that reach `trace` through `core` never inline it. It reaches no Node built-in, because the hosted player runs it in the browser (`tests/trace-reader-browser-safe.test.ts` walks its imports); the disk read is `readTraceZip` in `read-trace-zip.ts`, and source paths are hashed with the pure `portableSha1Hex` so the reader needs no `node:crypto`; resource bytes keep native `sha1Hex`, ~30x faster. diff --git a/README.md b/README.md index dfe72658..c6978fe6 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ In trace mode no DevTools UI window opens. At session end the adapter writes tra | **Python** | `pytest --devtools-trace`, or `devtools.enable(trace=True)` in a plain script | The trace artifact contains: -- `trace.trace` — NDJSON `context-options` + `before`/`after` action events. When test hooks are available (Mocha's `it()` / Cucumber's `Scenario()`), each test becomes a `Tracing.tracingGroup` span — an open/close `before`/`after` pair with `method: "tracingGroup"` and `params.name` set to the test title. Child actions inside the group carry `parentId` pointing back to the group's `callId`, so timeline viewers render tests as labelled spans wrapping their commands. +- `trace.trace` — NDJSON `context-options` + `before`/`after` action events. When test hooks are available (Mocha's `it()` / Cucumber's `Scenario()`), each test becomes a `Tracing.tracingGroup` span — an open/close `before`/`after` pair with `method: "tracingGroup"` and `params.name` set to the test title. Child actions inside the group carry `parentId` pointing back to the group's `callId`, so timeline viewers render tests as labelled spans wrapping their commands. For viewers that render a DOM snapshot per action, it also carries one `frame-snapshot` event per action: a page holding just that action's screenshot. The image is not stored a second time: its `src` is a placeholder `https://screenshot.wdio-devtools.invalid/` address, which the event's `resourceOverrides` maps to the screenshot's file under `resources/`. - `trace.network` — HAR-style network entries derived from the existing capture - `resources/page@-.jpeg` — screenshot per user-facing action - `resources/page@--elements.json` — flat interactable element list extracted by the page-injected scripts in `@wdio/devtools-core/element-scripts` diff --git a/packages/shared/src/trace-file.ts b/packages/shared/src/trace-file.ts index 0f1fb0c1..bbd382e2 100644 --- a/packages/shared/src/trace-file.ts +++ b/packages/shared/src/trace-file.ts @@ -30,6 +30,8 @@ export const TRACE_EVENT_TYPES = { before: 'before', after: 'after', frameSnapshot: 'frame-snapshot', + /** Here only the response a frame-snapshot's screenshot override patches; captured traffic is in `.network`. */ + resourceSnapshot: 'resource-snapshot', screencastFrame: 'screencast-frame', contextOptions: 'context-options', console: 'console', diff --git a/packages/trace/src/trace-exporter.ts b/packages/trace/src/trace-exporter.ts index 6f6a7b65..714e911f 100644 --- a/packages/trace/src/trace-exporter.ts +++ b/packages/trace/src/trace-exporter.ts @@ -35,8 +35,10 @@ import { import { buildDenseScreencast } from './screencast-trace.js' import { buildImageFrameSnapshots, + buildScreenshotResourceSnapshots, FrameSnapshotIndex, - type FrameSnapshotEvent + type FrameSnapshotEvent, + type ScreenshotResourceEvent } from './trace-frame-snapshots.js' import { buildActionEvents, type ActionEvent } from './trace-action-events.js' import { buildSourceResources } from './trace-sources.js' @@ -138,6 +140,7 @@ type TraceEvent = | ConsoleEvent | StdioEvent | FrameSnapshotEvent + | ScreenshotResourceEvent function shortId(sessionId?: string): string { return (sessionId ?? Math.random().toString(36).slice(2, 10)).slice(0, 8) @@ -255,6 +258,8 @@ function eventTime(e: TraceEvent): number { switch (e.type) { case 'context-options': return -Infinity + case 'resource-snapshot': + return e.snapshot._monotonicTime case 'before': return e.startTime case 'after': @@ -279,6 +284,7 @@ function eventTime(e: TraceEvent): number { function eventOrder(e: TraceEvent): number { switch (e.type) { case 'context-options': + case 'resource-snapshot': return 0 case 'after': return 1 @@ -340,6 +346,7 @@ function buildEventStream( wallTime, viewport ), + ...buildScreenshotResourceSnapshots(snapshotIndex.refs(), pageId), ...buildConsoleEvents(trace.consoleLogs, pageId, wallTime) ] events.sort(compareEvents) diff --git a/packages/trace/src/trace-frame-snapshots.ts b/packages/trace/src/trace-frame-snapshots.ts index af0b9745..c0871795 100644 --- a/packages/trace/src/trace-frame-snapshots.ts +++ b/packages/trace/src/trace-frame-snapshots.ts @@ -1,10 +1,15 @@ -// Image-backed frame-snapshot synthesis: each action screenshot becomes a -// minimal DOM document so standard trace viewers render the action pane. -// Compatibility shim until real DOM snapshots are captured. +// Image-backed frame-snapshots, a shim until real DOM snapshots are captured: each action is a page +// whose resolves, through `resourceOverrides`, to the screenshot already stored under `resources/`. import { imageMime, type ActionSnapshot } from '@wdio/devtools-shared' +import { networkRequestToHar, type ResourceSnapshotEntry } from './trace-har.js' +import { screenshotResourceName } from './trace-snapshots.js' const SNAPSHOT_DOCTYPE = 'html' +// `.invalid` never resolves (RFC 6761), so a viewer that ignores the override fetches nothing. +const SCREENSHOT_URL_ORIGIN = 'https://screenshot.wdio-devtools.invalid' +// Before every snapshot, including one clamped to 0: the viewer only serves a resource loaded before the snapshot. +const SCREENSHOT_RESOURCE_TIME = -1 const FALLBACK_FRAME_URL = 'about:blank' const BODY_STYLE = 'margin:0' const IMAGE_STYLE = 'display:block;width:100vw;height:100vh;object-fit:contain' @@ -40,6 +45,15 @@ export interface FrameSnapshotEvent { snapshot: FrameSnapshot } +/** The response a frame-snapshot's screenshot override patches; the viewer applies an override only to a recorded resource. */ +export interface ScreenshotResourceEvent extends ResourceSnapshotEntry { + snapshot: ResourceSnapshotEntry['snapshot'] & { + _monotonicTime: number + pageref: string + _frameref: string + } +} + export interface FrameSnapshotRef { callId: string snapshotName: string @@ -133,8 +147,13 @@ function frameIdForPage(pageId: string): string { return `frame@${suffix}` } +/** One URL per image type, so the shared response's content type is right for every override of it. */ +export function screenshotUrl(screenshot: string): string { + const extension = imageMime(screenshot).slice('image/'.length) + return `${SCREENSHOT_URL_ORIGIN}/action.${extension}` +} + function imageDocument(snap: ActionSnapshot): FrameSnapshotNode { - const screenshot = snap.screenshot ?? '' return [ 'HTML', {}, @@ -145,7 +164,7 @@ function imageDocument(snap: ActionSnapshot): FrameSnapshotNode { [ 'IMG', { - src: `data:${imageMime(screenshot)};base64,${screenshot}`, + src: screenshotUrl(snap.screenshot ?? ''), style: IMAGE_STYLE } ] @@ -175,8 +194,55 @@ export function buildImageFrameSnapshots( timestamp: Math.max(0, ref.snapshot.timestamp - wallTime), wallTime: ref.snapshot.timestamp, collectionTime: 0, - resourceOverrides: [], + resourceOverrides: [ + { + url: screenshotUrl(ref.snapshot.screenshot ?? ''), + sha1: screenshotResourceName(pageId, ref.snapshot.timestamp) + } + ], isMainFrame: true } })) } + +/** One resource-snapshot per screenshot URL the frame-snapshots reference. */ +export function buildScreenshotResourceSnapshots( + refs: FrameSnapshotRef[], + pageId: string +): ScreenshotResourceEvent[] { + const byUrl = new Map() + for (const { snapshot } of refs) { + const screenshot = snapshot.screenshot ?? '' + const url = screenshotUrl(screenshot) + if (byUrl.has(url)) { + continue + } + const mimeType = imageMime(screenshot) + const har = networkRequestToHar({ + id: url, + url, + method: 'GET', + status: 200, + statusText: 'OK', + timestamp: snapshot.timestamp, + startTime: snapshot.timestamp, + time: 0, + type: 'image', + response: { fromCache: false, headers: {}, mimeType, status: 200 } + }) + har.snapshot.response.content._sha1 = screenshotResourceName( + pageId, + snapshot.timestamp + ) + byUrl.set(url, { + type: 'resource-snapshot', + snapshot: { + ...har.snapshot, + _monotonicTime: SCREENSHOT_RESOURCE_TIME, + pageref: pageId, + _frameref: frameIdForPage(pageId) + } + }) + } + return [...byUrl.values()] +} diff --git a/packages/trace/src/trace-snapshots.ts b/packages/trace/src/trace-snapshots.ts index 9633ea80..99ffc1c0 100644 --- a/packages/trace/src/trace-snapshots.ts +++ b/packages/trace/src/trace-snapshots.ts @@ -70,6 +70,14 @@ export function upsertRichestSnapshot( mergeSnapshotInto(snapshots[idx]!, snap) } +/** Name under `resources/` of the screenshot captured at `timestamp`. */ +export function screenshotResourceName( + pageId: string, + timestamp: number +): string { + return `${pageId}-${timestamp}.jpeg` +} + export function buildSnapshotResources( rawSnapshots: ActionSnapshot[], pageId: string @@ -80,7 +88,7 @@ export function buildSnapshotResources( const base = `${pageId}-${snap.timestamp}` if (snap.screenshot) { out.push({ - resourceName: `${base}.jpeg`, + resourceName: screenshotResourceName(pageId, snap.timestamp), data: Buffer.from(snap.screenshot, 'base64') }) } @@ -110,7 +118,7 @@ function frameForSnapshot( const frame: ScreencastFrameEvent = { type: 'screencast-frame', pageId, - sha1: `${base}.jpeg`, + sha1: screenshotResourceName(pageId, snap.timestamp), width: viewport.width, height: viewport.height, timestamp diff --git a/packages/trace/tests/trace-frame-snapshots.test.ts b/packages/trace/tests/trace-frame-snapshots.test.ts index 26a87eac..e44d91fa 100644 --- a/packages/trace/tests/trace-frame-snapshots.test.ts +++ b/packages/trace/tests/trace-frame-snapshots.test.ts @@ -1,15 +1,31 @@ import { describe, it, expect } from 'vitest' +import { strFromU8, unzipSync } from 'fflate' import { buildActionEvents, buildImageFrameSnapshots, + buildScreenshotResourceSnapshots, + exportTraceZip, FrameSnapshotIndex, + screenshotUrl, type AfterEvent, - type FrameSnapshotRef + type FrameSnapshotEvent, + type FrameSnapshotRef, + type ScreenshotResourceEvent } from '@wdio/devtools-trace' -import type { ActionSnapshot, CommandLog } from '@wdio/devtools-shared' +import { parseTraceZip } from '../src/trace-reader.js' +import { + TraceType, + type ActionSnapshot, + type CommandLog +} from '@wdio/devtools-shared' function snap(overrides: Partial = {}): ActionSnapshot { - return { timestamp: 2000, command: 'click', screenshot: 'AAAA', ...overrides } + return { + timestamp: 2000, + command: 'click', + screenshot: 'AAAA', + ...overrides + } } describe('FrameSnapshotIndex', () => { @@ -211,7 +227,7 @@ describe('buildImageFrameSnapshots', () => { ) }) - it('encodes the screenshot as an image document in node-array format', () => { + it('references the stored screenshot from an image document in node-array format', () => { const [event] = buildImageFrameSnapshots( [ref({ url: 'https://example.test/login' })], pageId, @@ -228,7 +244,7 @@ describe('buildImageFrameSnapshots', () => { [ 'IMG', { - src: 'data:image/jpeg;base64,AAAA', + src: 'https://screenshot.wdio-devtools.invalid/action.jpeg', style: 'display:block;width:100vw;height:100vh;object-fit:contain' } ] @@ -253,11 +269,16 @@ describe('buildImageFrameSnapshots', () => { expect(s.timestamp).toBe(1000) expect(s.wallTime).toBe(2000) expect(s.collectionTime).toBe(0) - expect(s.resourceOverrides).toEqual([]) + expect(s.resourceOverrides).toEqual([ + { + url: 'https://screenshot.wdio-devtools.invalid/action.jpeg', + sha1: 'page@abc123-2000.jpeg' + } + ]) expect(s.isMainFrame).toBe(true) }) - it('sniffs a png data uri from the base64 magic', () => { + it('gives a png screenshot its own url, sniffed from the base64 magic', () => { const [event] = buildImageFrameSnapshots( [ref({ screenshot: 'iVBORw0KGgo' })], pageId, @@ -265,7 +286,24 @@ describe('buildImageFrameSnapshots', () => { viewport ) const html = JSON.stringify(event!.snapshot.html) - expect(html).toContain('data:image/png;base64,iVBORw0KGgo') + expect(html).toContain( + 'https://screenshot.wdio-devtools.invalid/action.png' + ) + expect(event!.snapshot.resourceOverrides[0]!.url).toBe( + 'https://screenshot.wdio-devtools.invalid/action.png' + ) + }) + + it('never inlines the screenshot bytes', () => { + const [event] = buildImageFrameSnapshots( + [ref({ screenshot: 'iVBORw0KGgoAAAANSUhEUg' })], + pageId, + wallTime, + viewport + ) + const json = JSON.stringify(event) + expect(json).not.toContain('data:') + expect(json).not.toContain('iVBORw0KGgoAAAANSUhEUg') }) it('falls back to about:blank when the snapshot has no url', () => { @@ -289,3 +327,157 @@ describe('buildImageFrameSnapshots', () => { expect(event!.snapshot.wallTime).toBe(500) }) }) + +describe('buildScreenshotResourceSnapshots', () => { + const pageId = 'page@abc123' + + function ref(callId: string, snapshot: ActionSnapshot): FrameSnapshotRef { + return { callId, snapshotName: `after@${callId}`, snapshot } + } + + it('records one response per screenshot url, served from the first stored screenshot', () => { + const events = buildScreenshotResourceSnapshots( + [ + ref('call@1', snap({ timestamp: 2000, screenshot: 'iVBORw0KGgo1' })), + ref('call@2', snap({ timestamp: 3000, screenshot: 'iVBORw0KGgo2' })), + ref('call@3', snap({ timestamp: 4000, screenshot: '/9j/4AAQ' })) + ], + pageId + ) + expect( + events.map((e) => [ + e.snapshot.request.url, + e.snapshot.response.content.mimeType, + e.snapshot.response.content._sha1 + ]) + ).toEqual([ + [ + 'https://screenshot.wdio-devtools.invalid/action.png', + 'image/png', + 'page@abc123-2000.jpeg' + ], + [ + 'https://screenshot.wdio-devtools.invalid/action.jpeg', + 'image/jpeg', + 'page@abc123-4000.jpeg' + ] + ]) + }) + + it('is a GET recorded before any snapshot, in the snapshot frame', () => { + // The viewer serves only a GET whose time precedes the snapshot, and the + // first snapshot of a trace is commonly at time 0. + const [event] = buildScreenshotResourceSnapshots( + [ref('call@1', snap())], + pageId + ) + expect(event!.type).toBe('resource-snapshot') + expect(event!.snapshot.request.method).toBe('GET') + expect(event!.snapshot.response.status).toBe(200) + expect(event!.snapshot._monotonicTime).toBeLessThan(0) + expect(event!.snapshot._frameref).toBe('frame@abc123') + expect(event!.snapshot.pageref).toBe(pageId) + }) + + it('records nothing when no screenshot is referenced', () => { + expect(buildScreenshotResourceSnapshots([], pageId)).toEqual([]) + }) +}) + +describe('a written trace stores each screenshot once', () => { + const PNG = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==' + const JPEG = + '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgICAgMCAgIDAwMDBAYEBAQEBAgGBgUGBwkI' + + async function written() { + const zip = await exportTraceZip( + { + mutations: [], + logs: [], + consoleLogs: [], + networkRequests: [], + metadata: { type: TraceType.Standalone }, + commands: [ + { command: 'url', args: ['https://example.test'], timestamp: 1000 }, + { command: 'click', args: ['#go'], timestamp: 2000 }, + { command: 'getText', args: ['#out'], timestamp: 3000 } + ], + sources: {}, + actionSnapshots: [ + { timestamp: 1000, command: 'url', screenshot: PNG }, + { + timestamp: 2000, + command: 'click', + screenshot: JPEG, + snapshotText: '- button "Go"' + } + ] + }, + { sessionId: 'abc12345' } + ) + const files = unzipSync(zip) + const events = strFromU8(files['trace.trace']!) + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { type: string }) + return { zip, files, events } + } + + it('writes no image bytes into the frame-snapshot events', async () => { + const { events } = await written() + const frameSnapshots = events.filter((e) => e.type === 'frame-snapshot') + expect(frameSnapshots).toHaveLength(3) + const json = JSON.stringify(frameSnapshots) + expect(json).not.toContain('data:image') + expect(json).not.toContain(PNG) + expect(json).not.toContain(JPEG) + }) + + it('points every override at a stored resource the src and a recorded response both name', async () => { + const { files, events } = await written() + const responses = events.filter( + (e) => e.type === 'resource-snapshot' + ) as ScreenshotResourceEvent[] + const responseUrls = responses.map((r) => r.snapshot.request.url) + for (const response of responses) { + expect( + files[`resources/${response.snapshot.response.content._sha1}`] + ).toBeDefined() + } + const frameSnapshots = events.filter( + (e) => e.type === 'frame-snapshot' + ) as FrameSnapshotEvent[] + for (const { snapshot } of frameSnapshots) { + const [override] = snapshot.resourceOverrides + expect(snapshot.resourceOverrides).toHaveLength(1) + expect(files[`resources/${override!.sha1}`]).toBeDefined() + expect(JSON.stringify(snapshot.html)).toContain( + `"src":"${override!.url}"` + ) + expect(responseUrls).toContain(override!.url) + } + // The getText row inherits the click's capture, and so its stored file. + expect( + frameSnapshots.map((e) => e.snapshot.resourceOverrides[0]!.sha1) + ).toEqual([ + 'page@abc12345-1000.jpeg', + 'page@abc12345-2000.jpeg', + 'page@abc12345-2000.jpeg' + ]) + expect(responseUrls).toEqual([screenshotUrl(PNG), screenshotUrl(JPEG)]) + }) + + it('reads back unchanged, with the screenshot responses kept out of the network rows', async () => { + const { zip } = await written() + const data = parseTraceZip(zip) + expect(data.trace.networkRequests).toEqual([]) + expect(data.trace.commands.map((c) => c.command)).toEqual([ + 'url', + 'click', + 'getText' + ]) + expect(data.frames.map((f) => f.screenshot)).toEqual([PNG, JPEG]) + expect(data.trace.commands[1]!.snapshotText).toBe('- button "Go"') + }) +})