Check the hosted trace player end to end on every PR

.github/workflows/ci.yml

@@ -96,6 +96,45 @@ jobs:
       - name: 🧩 Component Tests
         run: pnpm test:ui --maxInstances 2
 
+  # Builds the hosted trace player, serves it under the headers vercel.json
+  # sends, and opens the committed older-format zips, a zip from this
+  # checkout's writer and a non-zip through it in headless Chrome, failing on
+  # any console error or CSP violation. Its own job: it needs only the app's
+  # hosted build, not the bundle the other jobs build, so it runs beside
+  # them. Chrome comes preinstalled on the runner image; WebdriverIO fetches
+  # the matching driver itself. scheduled-hosted-player.yml runs the same
+  # check against the deployed page.
+  hosted-player:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Use Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          # the runner cache can hold a 24.x below the engines floor
+          check-latest: true
+      - name: Cache pnpm modules
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        env:
+          cache-name: cache-pnpm-modules
+        with:
+          path: ~/.pnpm-store
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+      - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+        with:
+          version: 10.26.1
+          run_install: false
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Chrome version
+        run: google-chrome --version
+      - name: 🎞️ Hosted trace player end-to-end
+        run: pnpm player:e2e
+
   # Runs one desktop example per adapter in trace mode against a local fixture
   # site and checks each trace zip against
   # packages/conformance/smoke/expectations/. The conformance suite covers the

.github/workflows/scheduled-hosted-player.yml

@@ -0,0 +1,97 @@
+name: Hosted player check
+
+# Opens trace zips through the deployed trace player (vars.HOSTED_PLAYER_URL)
+# with the same check ci.yml's hosted-player job runs against a local build,
+# so a deploy that builds but fails in the browser is caught before a user
+# reports it. Skipped while the variable is unset. A failure opens one issue
+# and comments on it while it stays open; a pass closes it. The page itself
+# gains nothing: no analytics, no tracking.
+
+on:
+  schedule:
+    - cron: '23 */6 * * *'
+  workflow_dispatch:
+  # Vercel's Git integration reports each deploy as a GitHub deployment.
+  deployment_status:
+
+permissions:
+  contents: read
+
+# One run at a time, so two failures cannot each open an issue.
+concurrency:
+  group: scheduled-hosted-player
+  cancel-in-progress: false
+
+env:
+  ISSUE_TITLE: Hosted trace player check is failing
+
+jobs:
+  check:
+    if: >-
+      vars.HOSTED_PLAYER_URL != '' &&
+      (github.event_name != 'deployment_status' ||
+      (github.event.deployment_status.state == 'success' &&
+      github.event.deployment.environment == 'Production'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Use Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          # the runner cache can hold a 24.x below the engines floor
+          check-latest: true
+      - name: Cache pnpm modules
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        env:
+          cache-name: cache-pnpm-modules
+        with:
+          path: ~/.pnpm-store
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+      - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+        with:
+          version: 10.26.1
+          run_install: false
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Chrome version
+        run: google-chrome --version
+      - name: 🎞️ Open traces in ${{ vars.HOSTED_PLAYER_URL }}
+        env:
+          HOSTED_PLAYER_URL: ${{ vars.HOSTED_PLAYER_URL }}
+        run: pnpm exec tsx packages/conformance/hosted-player/run.ts --url "$HOSTED_PLAYER_URL"
+
+  report:
+    needs: check
+    if: ${{ always() && (needs.check.result == 'failure' || needs.check.result == 'success') }}
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+      OUTCOME: ${{ needs.check.result }}
+      HOSTED_PLAYER_URL: ${{ vars.HOSTED_PLAYER_URL }}
+      RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      TRIGGER: ${{ github.event_name }}
+    steps:
+      # Listed rather than searched: the search index lags, and a just-opened issue would be missed.
+      - name: Open, update or close the tracking issue
+        run: |
+          set -o pipefail
+          OPEN=$(gh issue list --state open --limit 500 --json number,title |
+            jq -r --arg title "$ISSUE_TITLE" 'map(select(.title == $title)) | .[0].number // empty')
+          if [ "$OUTCOME" = failure ]; then
+            if [ -n "$OPEN" ]; then
+              gh issue comment "$OPEN" --body "Still failing ($TRIGGER): $RUN_URL"
+            else
+              gh issue create --title "$ISSUE_TITLE" --body "The hosted player check against $HOSTED_PLAYER_URL failed ($TRIGGER): $RUN_URL
+
+          It opens the committed trace fixtures through the deployed page in headless Chrome. The run log names the step and the error; \`pnpm exec tsx packages/conformance/hosted-player/run.ts --url $HOSTED_PLAYER_URL\` reproduces it locally. This issue is commented on while the check keeps failing and closed when it passes."
+            fi
+          elif [ -n "$OPEN" ]; then
+            gh issue close "$OPEN" --comment "Passing again: $RUN_URL"
+          fi

CLAUDE.md

@@ -32,6 +32,7 @@ Run from repo root unless noted.
 | `pnpm smoke:fixtures [--port N]` | Serve the local fixture site the desktop examples drive (a copy of the-internet's login flow: `/`, `/login`, `/secure`, `/logout`; default `http://127.0.0.1:4567`). Every desktop login example reads `DEVTOOLS_EXAMPLE_BASE_URL` (trailing slash ignored); unset, they drive the public site. |
 | `pnpm smoke:assert --expect <name> [--exit-code N] [--log file] <zip-or-dir>...` | Check trace zips against `packages/conformance/smoke/expectations/<name>.json`: the listed rows under each test's own tracing group (or the whole run for an entry with `scope: "run"` and a `reason`), an error row on each deliberately failing test, a retried test's attempt count with its other checks read from its final attempt alone (`attempts`), title fragments such as a failed wait's condition, a call site on every row of a per-test entry (at least one for a run-scoped entry) and every call site inside the example's own `sourceFiles`, the run's exit code, no wait timeout, in any adapter's wording (`TIMEOUT_PATTERNS` in `smoke/constants.ts`), in the trace's stdout or the run log other than the lines of a test marked `allowTimeoutError`, and the zips' combined size against the file's required `traceBudget.maxBytes`. |
 | `pnpm player:hosted` | Build the hosted trace player (the app with `--mode hosted`, no backend) and serve it at `http://localhost:4173` under the headers `vercel.json` sends from trace.webdriver.io. |
+| `pnpm player:e2e [--headed] [--port N]` | Build the hosted trace player, serve it under the `vercel.json` headers and open trace zips through it in headless Chrome with WebdriverIO (`packages/conformance/hosted-player/`). `pnpm exec tsx packages/conformance/hosted-player/run.ts --url <player>` runs the same check against a deployed page without building. |
 | `pnpm docs:capabilities` | Render the per-adapter capability tables from `packages/conformance/src/features.ts` into `README.md`; `--website <webdriverio checkout>` also renders the docs-site tables, `--check` fails instead of writing, and `--parity` lists every feature some adapter lacks plus the evidence counts. |
 | `python3 packages/selenium-devtools-py/scripts/changes.py next-version` | The version a Python-adapter release would publish, from the fragments pending in `changes/`. `check --base <ref>` is the CI gate; `apply` is what the release runs. |
 
@@ -216,6 +217,10 @@ The Python adapter cannot import `core`, so some pure logic exists twice. `packa
 
 The hosted player runs the newest reader against zips any adapter version wrote. `packages/trace/tests/fixtures/trace-formats/` holds real zips from older writers, one per adapter, reduced by `reduce-fixture.ts` beside them (images become 1x1 images of the same format; every other entry keeps its bytes), each with a manifest counted from its raw entries; `trace-format-fixtures.test.ts` checks the current reader still yields what each one held. A release that changes what the writer puts in a zip adds a fixture for it, as the README there describes, and older ones stay.
 
+### Hosted player check
+
+`ci.yml`'s `hosted-player` job runs `pnpm player:e2e`: it serves the hosted build under the `vercel.json` headers and, in headless Chrome, opens each committed older-format fixture and a zip written by this checkout's writer (`hosted-player/current-trace.ts`) through the landing screen's file input, expecting every manifest title as an action row, a filmstrip frame and no error, then a non-zip, expecting `Could not open`. It fails on any console error or uncaught exception the page logs over WebDriver BiDi, and on any CSP violation, which a preload script reports to the console because BiDi carries no CSP message of the browser's own; it also requires the parse worker's script to have been requested. `scheduled-hosted-player.yml` runs the same check against `vars.HOSTED_PLAYER_URL` every six hours, after each Production deployment and on demand, skips while the variable is unset, and keeps one open issue for a failing check, closing it once it passes. A change to the landing screen, the action rows or the filmstrip that moves what `page-scripts.ts` reads updates it in the same PR.
+
 ### Skipping tests that depend on workspace-internal build artifacts
 
 A handful of tests need `@wdio/devtools-script` to be built first (the browser-injected bundle). CI test jobs sometimes run before that build step; those tests gate on `it.skipIf` after probing `createRequire(import.meta.url).resolve('@wdio/devtools-script')`. Locally they run normally.

package.json

@@ -7,6 +7,7 @@
     "build": "pnpm -r build",
     "show-trace": "node packages/backend/dist/show-trace.js",
     "player:hosted": "pnpm --filter @wdio/devtools-app build:hosted && pnpm --filter @wdio/devtools-app preview:hosted",
+    "player:e2e": "pnpm --filter @wdio/devtools-app build:hosted && tsx packages/conformance/hosted-player/run.ts",
     "backend": "node packages/backend/dist/server.js",
     "demo:wdio": "wdio run ./examples/wdio/cucumber/wdio.conf.ts",
     "demo:wdio:mobile": "wdio run ./examples/wdio/mobile/wdio.conf.ts",

packages/conformance/hosted-player/args.ts

@@ -0,0 +1,50 @@
+import { parseArgs } from 'node:util'
+
+import { HOSTED_PREVIEW_PORT } from './constants.js'
+
+export interface CheckArgs {
+  /** A deployed player to check; absent, the local hosted build is served and checked. */
+  url?: string
+  port: number
+  headed: boolean
+}
+
+/** Reads `[--url <https://…>] [--port N] [--headed]`; throws on anything else. */
+export function parseCheckArgs(argv: readonly string[]): CheckArgs {
+  const { values } = parseArgs({
+    args: [...argv],
+    options: {
+      url: { type: 'string' },
+      port: { type: 'string', default: String(HOSTED_PREVIEW_PORT) },
+      headed: { type: 'boolean', default: false }
+    },
+    strict: true,
+    allowPositionals: false
+  })
+  const port = Number(values.port)
+  if (!Number.isInteger(port) || port < 1 || port > 65535) {
+    throw new Error(`--port must be a TCP port, got "${values.port}"`)
+  }
+  return {
+    ...(values.url === undefined ? {} : { url: baseUrl(values.url) }),
+    port,
+    headed: values.headed
+  }
+}
+
+/** The page to open, without a trailing slash, so paths join the same way for a local and a deployed player. */
+function baseUrl(raw: string): string {
+  let url: URL
+  try {
+    url = new URL(raw)
+  } catch {
+    throw new Error(`--url must be an absolute URL, got "${raw}"`)
+  }
+  if (url.protocol !== 'https:' && url.protocol !== 'http:') {
+    throw new Error(`--url must be http or https, got "${raw}"`)
+  }
+  if (url.search || url.hash) {
+    throw new Error(`--url must name the page alone, got "${raw}"`)
+  }
+  return url.href.replace(/\/+$/, '')
+}