Merge pull request #513 from webdriverio/fix/505-screenshot-extension

Name trace screenshots by their actual image format

Author: vishnuv688

.changeset/screenshot-resource-extension.md

@@ -0,0 +1,8 @@
+---
+'@wdio/devtools-backend': patch
+'@wdio/devtools-service': patch
+'@wdio/selenium-devtools': patch
+'@wdio/nightwatch-devtools': patch
+---
+
+A trace's screenshots are named by their actual image format. A PNG capture used to be stored as `resources/<page>-<timestamp>.jpeg`, so a viewer that takes the content type from the file name served it as `image/jpeg`; it is now `.png`, and so is a PNG frame of the dense filmstrip. Traces written before this change still open.

README.md

@@ -225,7 +225,7 @@ In trace mode no DevTools UI window opens. At session end the adapter writes tra
 The trace artifact contains:
 - `trace.trace` — NDJSON `context-options` + `before`/`after` action events. When test hooks are available (Mocha's `it()` / Cucumber's `Scenario()`), each test becomes a `Tracing.tracingGroup` span — an open/close `before`/`after` pair with `method: "tracingGroup"` and `params.name` set to the test title. Child actions inside the group carry `parentId` pointing back to the group's `callId`, so timeline viewers render tests as labelled spans wrapping their commands. For viewers that render a DOM snapshot per action, it also carries one `frame-snapshot` event per action: a page holding just that action's screenshot. The image is not stored a second time: its `src` is a placeholder `https://screenshot.wdio-devtools.invalid/` address, which the event's `resourceOverrides` maps to the screenshot's file under `resources/`.
 - `trace.network` — HAR-style network entries derived from the existing capture
-- `resources/page@<id>-<ts>.jpeg` — screenshot per user-facing action
+- `resources/page@<id>-<ts>.png` or `.jpeg`: screenshot per user-facing action, named by its actual image format
 - `resources/page@<id>-<ts>-elements.json` — flat interactable element list extracted by the page-injected scripts in `@wdio/devtools-core/element-scripts`
 - `resources/page@<id>-<ts>-snapshot.txt` — depth-indented accessibility-tree snapshot (AI-friendly)
 - `transcript.md` — human/LLM-readable Markdown transcript of the captured actions, with timing, selectors, and value annotations

packages/shared/src/image.ts

@@ -21,12 +21,17 @@ export interface ImageSize {
   height: number
 }
 
-/** Detect image mime from a base64 string's magic bytes — trace screenshots may
- *  be PNG (polling capture) or JPEG (CDP), and the zip names both `.jpeg`. */
+/** Detect image mime from a base64 string's magic bytes: trace screenshots may
+ *  be PNG (polling capture) or JPEG (CDP). */
 export function imageMime(base64: string): string {
   return base64.startsWith(PNG_BASE64_MAGIC) ? 'image/png' : 'image/jpeg'
 }
 
+/** File extension, without the dot, naming a base64 image's own format. */
+export function imageExtension(base64: string): string {
+  return imageMime(base64).slice('image/'.length)
+}
+
 /** Intrinsic pixel size read out of a base64 PNG or JPEG header, or null when
  *  the bytes name none. A capture is fitted by this rather than by the metadata
  *  viewport: the two disagree on both mobile platforms — Android reports the

packages/shared/src/trace-player.ts

@@ -10,11 +10,11 @@ export const TRACE_API = {
 } as const
 
 /** A single screenshot frame reconstructed from a trace.zip's
- *  `screencast-frame` events + `resources/*.jpeg`. */
+ *  `screencast-frame` events + their `resources/` images. */
 export interface TracePlayerFrame {
   /** Absolute wall-clock ms when the frame was captured. */
   timestamp: number
-  /** Base64-encoded JPEG (no `data:` prefix), matching `CommandLog.screenshot`. */
+  /** Base64-encoded PNG or JPEG (no `data:` prefix), matching `CommandLog.screenshot`. */
   screenshot: string
 }
 

packages/shared/tests/image.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest'
 
-import { imageDimensions, imageMime } from '../src/image.js'
+import { imageDimensions, imageExtension, imageMime } from '../src/image.js'
 
 /** 12x26 and 32x20 RGB PNGs — asymmetric so a swapped axis fails the test. */
 const PORTRAIT_PNG =
@@ -46,6 +46,13 @@ describe('imageMime', () => {
   })
 })
 
+describe('imageExtension', () => {
+  it('names the extension after the sniffed format', () => {
+    expect(imageExtension(PORTRAIT_PNG)).toBe('png')
+    expect(imageExtension(PORTRAIT_JPEG)).toBe('jpeg')
+  })
+})
+
 describe('imageDimensions', () => {
   it("reads a PNG's IHDR", () => {
     expect(imageDimensions(PORTRAIT_PNG)).toEqual({ width: 12, height: 26 })

packages/trace/src/screencast-trace.ts

@@ -13,7 +13,7 @@
  */
 
 import { createHash } from 'node:crypto'
-import type { ScreencastFrame } from '@wdio/devtools-shared'
+import { imageExtension, type ScreencastFrame } from '@wdio/devtools-shared'
 import type { TraceZipResource } from './trace-zip-writer.js'
 import type { ScreencastFrameEvent } from './trace-snapshots.js'
 
@@ -112,7 +112,7 @@ export function buildDenseScreencast(
     }
     const data = Buffer.from(frame.data, 'base64')
     const sha1 = createHash('sha1').update(data).digest('hex')
-    const resourceName = `${sha1}.jpeg`
+    const resourceName = `${sha1}.${imageExtension(frame.data)}`
     if (!seen.has(sha1)) {
       seen.add(sha1)
       resources.push({ resourceName, data })

packages/trace/src/trace-frame-snapshots.ts

@@ -1,7 +1,11 @@
 // Image-backed frame-snapshots, a shim until real DOM snapshots are captured: each action is a page
 // whose <img> resolves, through `resourceOverrides`, to the screenshot already stored under `resources/`.
 
-import { imageMime, type ActionSnapshot } from '@wdio/devtools-shared'
+import {
+  imageExtension,
+  imageMime,
+  type ActionSnapshot
+} from '@wdio/devtools-shared'
 import { networkRequestToHar, type ResourceSnapshotEntry } from './trace-har.js'
 import { screenshotResourceName } from './trace-snapshots.js'
 
@@ -149,8 +153,7 @@ function frameIdForPage(pageId: string): string {
 
 /** One URL per image type, so the shared response's content type is right for every override of it. */
 export function screenshotUrl(screenshot: string): string {
-  const extension = imageMime(screenshot).slice('image/'.length)
-  return `${SCREENSHOT_URL_ORIGIN}/action.${extension}`
+  return `${SCREENSHOT_URL_ORIGIN}/action.${imageExtension(screenshot)}`
 }
 
 function imageDocument(snap: ActionSnapshot): FrameSnapshotNode {
@@ -197,7 +200,11 @@ export function buildImageFrameSnapshots(
       resourceOverrides: [
         {
           url: screenshotUrl(ref.snapshot.screenshot ?? ''),
-          sha1: screenshotResourceName(pageId, ref.snapshot.timestamp)
+          sha1: screenshotResourceName(
+            pageId,
+            ref.snapshot.timestamp,
+            ref.snapshot.screenshot ?? ''
+          )
         }
       ],
       isMainFrame: true
@@ -232,7 +239,8 @@ export function buildScreenshotResourceSnapshots(
     })
     har.snapshot.response.content._sha1 = screenshotResourceName(
       pageId,
-      snapshot.timestamp
+      snapshot.timestamp,
+      screenshot
     )
     byUrl.set(url, {
       type: 'resource-snapshot',

packages/trace/src/trace-snapshots.ts

@@ -2,7 +2,7 @@
 // from them. Split from trace-exporter.ts; the exporter composes these into
 // the trace event stream.
 
-import type { ActionSnapshot } from '@wdio/devtools-shared'
+import { imageExtension, type ActionSnapshot } from '@wdio/devtools-shared'
 import type { TraceZipResource } from './trace-zip-writer.js'
 
 export interface ScreencastFrameEvent {
@@ -70,12 +70,13 @@ export function upsertRichestSnapshot(
   mergeSnapshotInto(snapshots[idx]!, snap)
 }
 
-/** Name under `resources/` of the screenshot captured at `timestamp`. */
+/** Name under `resources/` of a snapshot's screenshot, extension sniffed from its bytes. */
 export function screenshotResourceName(
   pageId: string,
-  timestamp: number
+  timestamp: number,
+  screenshot: string
 ): string {
-  return `${pageId}-${timestamp}.jpeg`
+  return `${pageId}-${timestamp}.${imageExtension(screenshot)}`
 }
 
 export function buildSnapshotResources(
@@ -88,7 +89,11 @@ export function buildSnapshotResources(
     const base = `${pageId}-${snap.timestamp}`
     if (snap.screenshot) {
       out.push({
-        resourceName: screenshotResourceName(pageId, snap.timestamp),
+        resourceName: screenshotResourceName(
+          pageId,
+          snap.timestamp,
+          snap.screenshot
+        ),
         data: Buffer.from(snap.screenshot, 'base64')
       })
     }
@@ -118,7 +123,7 @@ function frameForSnapshot(
   const frame: ScreencastFrameEvent = {
     type: 'screencast-frame',
     pageId,
-    sha1: screenshotResourceName(pageId, snap.timestamp),
+    sha1: screenshotResourceName(pageId, snap.timestamp, snap.screenshot ?? ''),
     width: viewport.width,
     height: viewport.height,
     timestamp

packages/trace/tests/screencast-trace.test.ts

@@ -126,6 +126,28 @@ describe('buildDenseScreencast', () => {
     })
   })
 
+  it('names each frame resource by its own image format', () => {
+    const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'
+    const jpeg = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgICAgMC'
+    const { events, resources } = buildDenseScreencast(
+      [
+        { data: png, timestamp: 0 },
+        { data: jpeg, timestamp: 200 }
+      ],
+      'page@x',
+      0,
+      viewport,
+      { minFrameIntervalMs: 100 }
+    )
+    expect(resources.map((r) => r.resourceName.split('.')[1])).toEqual([
+      'png',
+      'jpeg'
+    ])
+    expect(events.map((e) => e.sha1)).toEqual(
+      resources.map((r) => r.resourceName)
+    )
+  })
+
   it('resource bytes decode back to the source frame data', () => {
     const { resources } = buildDenseScreencast(
       [frame('hello', 0)],

packages/trace/tests/trace-frame-snapshots.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest'
-import { strFromU8, unzipSync } from 'fflate'
+import { strFromU8, strToU8, unzipSync, zipSync } from 'fflate'
 import {
   buildActionEvents,
   buildImageFrameSnapshots,
@@ -10,6 +10,7 @@ import {
   type AfterEvent,
   type FrameSnapshotEvent,
   type FrameSnapshotRef,
+  type ScreencastFrameEvent,
   type ScreenshotResourceEvent
 } from '@wdio/devtools-trace'
 import { parseTraceZip } from '../src/trace-reader.js'
@@ -354,7 +355,7 @@ describe('buildScreenshotResourceSnapshots', () => {
       [
         'https://screenshot.wdio-devtools.invalid/action.png',
         'image/png',
-        'page@abc123-2000.jpeg'
+        'page@abc123-2000.png'
       ],
       [
         'https://screenshot.wdio-devtools.invalid/action.jpeg',
@@ -461,13 +462,49 @@ describe('a written trace stores each screenshot once', () => {
     expect(
       frameSnapshots.map((e) => e.snapshot.resourceOverrides[0]!.sha1)
     ).toEqual([
-      'page@abc12345-1000.jpeg',
+      'page@abc12345-1000.png',
       'page@abc12345-2000.jpeg',
       'page@abc12345-2000.jpeg'
     ])
     expect(responseUrls).toEqual([screenshotUrl(PNG), screenshotUrl(JPEG)])
   })
 
+  it('names each screenshot by its own format in every place that names it', async () => {
+    const { files, events } = await written()
+    const stored = Object.keys(files)
+      .filter((name) => /\.(png|jpeg)$/.test(name))
+      .sort()
+    expect(stored).toEqual([
+      'resources/page@abc12345-1000.png',
+      'resources/page@abc12345-2000.jpeg'
+    ])
+    const filmstrip = (
+      events.filter(
+        (e) => e.type === 'screencast-frame'
+      ) as ScreencastFrameEvent[]
+    ).map((e) => e.sha1)
+    const overrides = (
+      events.filter((e) => e.type === 'frame-snapshot') as FrameSnapshotEvent[]
+    ).map((e) => e.snapshot.resourceOverrides[0]!.sha1)
+    const responses = (
+      events.filter(
+        (e) => e.type === 'resource-snapshot'
+      ) as ScreenshotResourceEvent[]
+    ).map((e) => e.snapshot.response.content)
+    for (const name of [
+      ...filmstrip,
+      ...overrides,
+      ...responses.map((c) => c._sha1)
+    ]) {
+      expect(stored).toContain(`resources/${name}`)
+    }
+    expect(new Set(filmstrip)).toEqual(new Set(overrides))
+    expect(responses.map((c) => [c.mimeType, c._sha1])).toEqual([
+      ['image/png', 'page@abc12345-1000.png'],
+      ['image/jpeg', 'page@abc12345-2000.jpeg']
+    ])
+  })
+
   it('reads back unchanged, with the screenshot responses kept out of the network rows', async () => {
     const { zip } = await written()
     const data = parseTraceZip(zip)
@@ -480,4 +517,30 @@ describe('a written trace stores each screenshot once', () => {
     expect(data.frames.map((f) => f.screenshot)).toEqual([PNG, JPEG])
     expect(data.trace.commands[1]!.snapshotText).toBe('- button "Go"')
   })
+
+  it('still reads a zip written when every screenshot was named .jpeg', async () => {
+    const { files } = await written()
+    const legacy: Record<string, Uint8Array> = {}
+    for (const [name, data] of Object.entries(files)) {
+      legacy[name.replace(/\.png$/, '.jpeg')] = data
+    }
+    legacy['trace.trace'] = strToU8(
+      strFromU8(files['trace.trace']!).replaceAll(
+        'page@abc12345-1000.png',
+        'page@abc12345-1000.jpeg'
+      )
+    )
+    expect(legacy['resources/page@abc12345-1000.jpeg']).toBeDefined()
+    expect(Object.keys(legacy).some((name) => name.endsWith('.png'))).toBe(
+      false
+    )
+
+    const data = parseTraceZip(zipSync(legacy))
+    expect(data.frames.map((f) => f.screenshot)).toEqual([PNG, JPEG])
+    expect(data.trace.commands.map((c) => c.screenshot)).toEqual([
+      PNG,
+      JPEG,
+      JPEG
+    ])
+  })
 })