Skip to content

Commit fd46bee

Browse files
vishnuv688vishnuv688
authored
Merge pull request #518 from webdriverio/ci/511-hosted-player-e2e
Check the hosted trace player end to end on every PR
2 parents 27392d2 + 3fb27a2 commit fd46bee

19 files changed

Lines changed: 1306 additions & 2 deletions

‎.github/workflows/ci.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,45 @@ jobs:
9696
- name: 🧩 Component Tests
9797
run: pnpm test:ui --maxInstances 2
9898

99+
# Builds the hosted trace player, serves it under the headers vercel.json
100+
# sends, and opens the committed older-format zips, a zip from this
101+
# checkout's writer and a non-zip through it in headless Chrome, failing on
102+
# any console error or CSP violation. Its own job: it needs only the app's
103+
# hosted build, not the bundle the other jobs build, so it runs beside
104+
# them. Chrome comes preinstalled on the runner image; WebdriverIO fetches
105+
# the matching driver itself. scheduled-hosted-player.yml runs the same
106+
# check against the deployed page.
107+
hosted-player:
108+
runs-on: ubuntu-latest
109+
timeout-minutes: 15
110+
steps:
111+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
112+
- name: Use Node.js
113+
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
114+
with:
115+
node-version-file: .nvmrc
116+
# the runner cache can hold a 24.x below the engines floor
117+
check-latest: true
118+
- name: Cache pnpm modules
119+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
120+
env:
121+
cache-name: cache-pnpm-modules
122+
with:
123+
path: ~/.pnpm-store
124+
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
125+
restore-keys: |
126+
${{ runner.os }}-build-${{ env.cache-name }}-
127+
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
128+
with:
129+
version: 10.26.1
130+
run_install: false
131+
- name: Install dependencies
132+
run: pnpm install --frozen-lockfile
133+
- name: Chrome version
134+
run: google-chrome --version
135+
- name: 🎞️ Hosted trace player end-to-end
136+
run: pnpm player:e2e
137+
99138
# Runs one desktop example per adapter in trace mode against a local fixture
100139
# site and checks each trace zip against
101140
# packages/conformance/smoke/expectations/. The conformance suite covers the

‎.github/workflows/scheduled-hosted-player.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
name: Hosted player check
2+
3+
# Opens trace zips through the deployed trace player (vars.HOSTED_PLAYER_URL)
4+
# with the same check ci.yml's hosted-player job runs against a local build,
5+
# so a deploy that builds but fails in the browser is caught before a user
6+
# reports it. Skipped while the variable is unset. A failure opens one issue
7+
# and comments on it while it stays open; a pass closes it. The page itself
8+
# gains nothing: no analytics, no tracking.
9+
10+
on:
11+
schedule:
12+
- cron: '23 */6 * * *'
13+
workflow_dispatch:
14+
# Vercel's Git integration reports each deploy as a GitHub deployment.
15+
deployment_status:
16+
17+
permissions:
18+
contents: read
19+
20+
# One run at a time, so two failures cannot each open an issue.
21+
concurrency:
22+
group: scheduled-hosted-player
23+
cancel-in-progress: false
24+
25+
env:
26+
ISSUE_TITLE: Hosted trace player check is failing
27+
28+
jobs:
29+
check:
30+
if: >-
31+
vars.HOSTED_PLAYER_URL != '' &&
32+
(github.event_name != 'deployment_status' ||
33+
(github.event.deployment_status.state == 'success' &&
34+
github.event.deployment.environment == 'Production'))
35+
runs-on: ubuntu-latest
36+
timeout-minutes: 15
37+
steps:
38+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
39+
- name: Use Node.js
40+
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
41+
with:
42+
node-version-file: .nvmrc
43+
# the runner cache can hold a 24.x below the engines floor
44+
check-latest: true
45+
- name: Cache pnpm modules
46+
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
47+
env:
48+
cache-name: cache-pnpm-modules
49+
with:
50+
path: ~/.pnpm-store
51+
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
52+
restore-keys: |
53+
${{ runner.os }}-build-${{ env.cache-name }}-
54+
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
55+
with:
56+
version: 10.26.1
57+
run_install: false
58+
- name: Install dependencies
59+
run: pnpm install --frozen-lockfile
60+
- name: Chrome version
61+
run: google-chrome --version
62+
- name: 🎞️ Open traces in ${{ vars.HOSTED_PLAYER_URL }}
63+
env:
64+
HOSTED_PLAYER_URL: ${{ vars.HOSTED_PLAYER_URL }}
65+
run: pnpm exec tsx packages/conformance/hosted-player/run.ts --url "$HOSTED_PLAYER_URL"
66+
67+
report:
68+
needs: check
69+
if: ${{ always() && (needs.check.result == 'failure' || needs.check.result == 'success') }}
70+
runs-on: ubuntu-latest
71+
permissions:
72+
issues: write
73+
env:
74+
GH_TOKEN: ${{ github.token }}
75+
GH_REPO: ${{ github.repository }}
76+
OUTCOME: ${{ needs.check.result }}
77+
HOSTED_PLAYER_URL: ${{ vars.HOSTED_PLAYER_URL }}
78+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
79+
TRIGGER: ${{ github.event_name }}
80+
steps:
81+
# Listed rather than searched: the search index lags, and a just-opened issue would be missed.
82+
- name: Open, update or close the tracking issue
83+
run: |
84+
set -o pipefail
85+
OPEN=$(gh issue list --state open --limit 500 --json number,title |
86+
jq -r --arg title "$ISSUE_TITLE" 'map(select(.title == $title)) | .[0].number // empty')
87+
if [ "$OUTCOME" = failure ]; then
88+
if [ -n "$OPEN" ]; then
89+
gh issue comment "$OPEN" --body "Still failing ($TRIGGER): $RUN_URL"
90+
else
91+
gh issue create --title "$ISSUE_TITLE" --body "The hosted player check against $HOSTED_PLAYER_URL failed ($TRIGGER): $RUN_URL
92+
93+
It opens the committed trace fixtures through the deployed page in headless Chrome. The run log names the step and the error; \`pnpm exec tsx packages/conformance/hosted-player/run.ts --url $HOSTED_PLAYER_URL\` reproduces it locally. This issue is commented on while the check keeps failing and closed when it passes."
94+
fi
95+
elif [ -n "$OPEN" ]; then
96+
gh issue close "$OPEN" --comment "Passing again: $RUN_URL"
97+
fi

‎CLAUDE.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ Run from repo root unless noted.
3232
| `pnpm smoke:fixtures [--port N]` | Serve the local fixture site the desktop examples drive (a copy of the-internet's login flow: `/`, `/login`, `/secure`, `/logout`; default `http://127.0.0.1:4567`). Every desktop login example reads `DEVTOOLS_EXAMPLE_BASE_URL` (trailing slash ignored); unset, they drive the public site. |
3333
| `pnpm smoke:assert --expect <name> [--exit-code N] [--log file] <zip-or-dir>...` | Check trace zips against `packages/conformance/smoke/expectations/<name>.json`: the listed rows under each test's own tracing group (or the whole run for an entry with `scope: "run"` and a `reason`), an error row on each deliberately failing test, a retried test's attempt count with its other checks read from its final attempt alone (`attempts`), title fragments such as a failed wait's condition, a call site on every row of a per-test entry (at least one for a run-scoped entry) and every call site inside the example's own `sourceFiles`, the run's exit code, no wait timeout, in any adapter's wording (`TIMEOUT_PATTERNS` in `smoke/constants.ts`), in the trace's stdout or the run log other than the lines of a test marked `allowTimeoutError`, and the zips' combined size against the file's required `traceBudget.maxBytes`. |
3434
| `pnpm player:hosted` | Build the hosted trace player (the app with `--mode hosted`, no backend) and serve it at `http://localhost:4173` under the headers `vercel.json` sends from trace.webdriver.io. |
35+
| `pnpm player:e2e [--headed] [--port N]` | Build the hosted trace player, serve it under the `vercel.json` headers and open trace zips through it in headless Chrome with WebdriverIO (`packages/conformance/hosted-player/`). `pnpm exec tsx packages/conformance/hosted-player/run.ts --url <player>` runs the same check against a deployed page without building. |
3536
| `pnpm docs:capabilities` | Render the per-adapter capability tables from `packages/conformance/src/features.ts` into `README.md`; `--website <webdriverio checkout>` also renders the docs-site tables, `--check` fails instead of writing, and `--parity` lists every feature some adapter lacks plus the evidence counts. |
3637
| `python3 packages/selenium-devtools-py/scripts/changes.py next-version` | The version a Python-adapter release would publish, from the fragments pending in `changes/`. `check --base <ref>` is the CI gate; `apply` is what the release runs. |
3738

@@ -216,6 +217,10 @@ The Python adapter cannot import `core`, so some pure logic exists twice. `packa
216217

217218
The hosted player runs the newest reader against zips any adapter version wrote. `packages/trace/tests/fixtures/trace-formats/` holds real zips from older writers, one per adapter, reduced by `reduce-fixture.ts` beside them (images become 1x1 images of the same format; every other entry keeps its bytes), each with a manifest counted from its raw entries; `trace-format-fixtures.test.ts` checks the current reader still yields what each one held. A release that changes what the writer puts in a zip adds a fixture for it, as the README there describes, and older ones stay.
218219

220+
### Hosted player check
221+
222+
`ci.yml`'s `hosted-player` job runs `pnpm player:e2e`: it serves the hosted build under the `vercel.json` headers and, in headless Chrome, opens each committed older-format fixture and a zip written by this checkout's writer (`hosted-player/current-trace.ts`) through the landing screen's file input, expecting every manifest title as an action row, a filmstrip frame and no error, then a non-zip, expecting `Could not open`. It fails on any console error or uncaught exception the page logs over WebDriver BiDi, and on any CSP violation, which a preload script reports to the console because BiDi carries no CSP message of the browser's own; it also requires the parse worker's script to have been requested. `scheduled-hosted-player.yml` runs the same check against `vars.HOSTED_PLAYER_URL` every six hours, after each Production deployment and on demand, skips while the variable is unset, and keeps one open issue for a failing check, closing it once it passes. A change to the landing screen, the action rows or the filmstrip that moves what `page-scripts.ts` reads updates it in the same PR.
223+
219224
### Skipping tests that depend on workspace-internal build artifacts
220225

221226
A handful of tests need `@wdio/devtools-script` to be built first (the browser-injected bundle). CI test jobs sometimes run before that build step; those tests gate on `it.skipIf` after probing `createRequire(import.meta.url).resolve('@wdio/devtools-script')`. Locally they run normally.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"build": "pnpm -r build",
88
"show-trace": "node packages/backend/dist/show-trace.js",
99
"player:hosted": "pnpm --filter @wdio/devtools-app build:hosted && pnpm --filter @wdio/devtools-app preview:hosted",
10+
"player:e2e": "pnpm --filter @wdio/devtools-app build:hosted && tsx packages/conformance/hosted-player/run.ts",
1011
"backend": "node packages/backend/dist/server.js",
1112
"demo:wdio": "wdio run ./examples/wdio/cucumber/wdio.conf.ts",
1213
"demo:wdio:mobile": "wdio run ./examples/wdio/mobile/wdio.conf.ts",

‎packages/conformance/hosted-player/args.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { parseArgs } from 'node:util'
2+
3+
import { HOSTED_PREVIEW_PORT } from './constants.js'
4+
5+
export interface CheckArgs {
6+
/** A deployed player to check; absent, the local hosted build is served and checked. */
7+
url?: string
8+
port: number
9+
headed: boolean
10+
}
11+
12+
/** Reads `[--url <https://…>] [--port N] [--headed]`; throws on anything else. */
13+
export function parseCheckArgs(argv: readonly string[]): CheckArgs {
14+
const { values } = parseArgs({
15+
args: [...argv],
16+
options: {
17+
url: { type: 'string' },
18+
port: { type: 'string', default: String(HOSTED_PREVIEW_PORT) },
19+
headed: { type: 'boolean', default: false }
20+
},
21+
strict: true,
22+
allowPositionals: false
23+
})
24+
const port = Number(values.port)
25+
if (!Number.isInteger(port) || port < 1 || port > 65535) {
26+
throw new Error(`--port must be a TCP port, got "${values.port}"`)
27+
}
28+
return {
29+
...(values.url === undefined ? {} : { url: baseUrl(values.url) }),
30+
port,
31+
headed: values.headed
32+
}
33+
}
34+
35+
/** The page to open, without a trailing slash, so paths join the same way for a local and a deployed player. */
36+
function baseUrl(raw: string): string {
37+
let url: URL
38+
try {
39+
url = new URL(raw)
40+
} catch {
41+
throw new Error(`--url must be an absolute URL, got "${raw}"`)
42+
}
43+
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
44+
throw new Error(`--url must be http or https, got "${raw}"`)
45+
}
46+
if (url.search || url.hash) {
47+
throw new Error(`--url must name the page alone, got "${raw}"`)
48+
}
49+
return url.href.replace(/\/+$/, '')
50+
}

0 commit comments

Comments
 (0)