From 0eac4fc57f4ee571573b3e6940da0cce824a1809 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Fri, 30 Oct 2026 01:51:22 +0530 Subject: [PATCH 1/2] Fail the examples smoke job when an example's traces outgrow a budget Nothing stopped a change from making traces heavier, and the hosted player work plans features that grow them by design (#494). Each smoke expectation now carries a required traceBudget, checked against the combined size of the zips its run wrote, since how many zips a run writes depends on its granularity. The budgets are the sizes measured on CI plus about 25%. Green runs upload no artifact, so the checker also prints each run's total: that log line is how the next budget gets measured. Closes #496 Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 4 +- packages/conformance/smoke/assert-trace.ts | 59 ++++++++++--- .../smoke/expectations/nightwatch-bdd.json | 5 ++ .../smoke/expectations/python-pytest.json | 5 ++ .../smoke/expectations/selenium-mocha.json | 5 ++ .../smoke/expectations/wdio-mocha.json | 5 ++ .../conformance/smoke/trace-expectations.ts | 88 ++++++++++++++++--- packages/conformance/smoke/types.ts | 17 ++++ .../tests/smoke-trace-expectations.test.ts | 85 +++++++++++++++++- 9 files changed, 243 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 83015171..0c706e35 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ Run from repo root unless noted. | `pnpm demo:wdio:mobile` / `:selenium:mobile` / `:nightwatch:mobile` / `:python:mobile` | The same, against Appium. All four build the same capability bag and drive the Clock app that ships with every Android system image (keying a duration on the timer keypad, never starting a timer), so a native example needs no `.apk`. `DEVTOOLS_MOBILE_PLATFORM=ios` runs the iOS spec instead — all four adapters — which drives Settings because the simulator ships no Clock, from a separate spec per platform rather than a branch. The simulator is chosen by udid and defaults to whichever is already booted, and an unmatched `IOS_DEVICE_NAME` is refused: naming one that does not exist makes the XCUITest driver create and boot it, every run, rather than fail. That refusal and the booted-simulator preflight are **local** policy — `xcrun simctl` enumerates local simulators and nothing else — so `IOS_UDID` and a non-local `APPIUM_HOST` both bypass them, or a real device and a cloud grid would be refused a run they were correctly configured for. `DEVTOOLS_MOBILE=web` drives the device's own browser on both platforms — Chrome on Android, Safari on iOS, which the XCUITest driver serves without a chromedriver. `examples/MOBILE.md` holds the prerequisites and the `DEVTOOLS_MOBILE` / `APPIUM_APP` switches; `DEVTOOLS_MODE=trace` flips any demo to trace mode. | | `pnpm dev` | Run all packages in parallel dev mode. | | `pnpm smoke:fixtures [--port N]` | Serve the local fixture site the desktop examples drive (a copy of the-internet's login flow: `/`, `/login`, `/secure`, `/logout`; default `http://127.0.0.1:4567`). Every desktop login example reads `DEVTOOLS_EXAMPLE_BASE_URL` (trailing slash ignored); unset, they drive the public site. | -| `pnpm smoke:assert --expect [--exit-code N] [--log file] ...` | Check trace zips against `packages/conformance/smoke/expectations/.json`: the listed rows under each test's own tracing group (or the whole run for an entry with `scope: "run"` and a `reason`), an error row on each deliberately failing test, a retried test's attempt count with its other checks read from its final attempt alone (`attempts`), title fragments such as a failed wait's condition, a call site on every row of a per-test entry (at least one for a run-scoped entry) and every call site inside the example's own `sourceFiles`, the run's exit code, and no wait timeout, in any adapter's wording (`TIMEOUT_PATTERNS` in `smoke/constants.ts`), in the trace's stdout or the run log other than the lines of a test marked `allowTimeoutError`. | +| `pnpm smoke:assert --expect [--exit-code N] [--log file] ...` | Check trace zips against `packages/conformance/smoke/expectations/.json`: the listed rows under each test's own tracing group (or the whole run for an entry with `scope: "run"` and a `reason`), an error row on each deliberately failing test, a retried test's attempt count with its other checks read from its final attempt alone (`attempts`), title fragments such as a failed wait's condition, a call site on every row of a per-test entry (at least one for a run-scoped entry) and every call site inside the example's own `sourceFiles`, the run's exit code, no wait timeout, in any adapter's wording (`TIMEOUT_PATTERNS` in `smoke/constants.ts`), in the trace's stdout or the run log other than the lines of a test marked `allowTimeoutError`, and the zips' combined size against the file's required `traceBudget.maxBytes`. | | `pnpm player:hosted` | Build the hosted trace player (the app with `--mode hosted`, no backend) and serve it at `http://localhost:4173` under the headers `vercel.json` sends from trace.webdriver.io. | | `pnpm docs:capabilities` | Render the per-adapter capability tables from `packages/conformance/src/features.ts` into `README.md`; `--website ` also renders the docs-site tables, `--check` fails instead of writing, and `--parity` lists every feature some adapter lacks plus the evidence counts. | | `python3 packages/selenium-devtools-py/scripts/changes.py next-version` | The version a Python-adapter release would publish, from the fragments pending in `changes/`. `check --base ` is the CI gate; `apply` is what the release runs. | @@ -204,7 +204,7 @@ When CI can't run an example (no real browser), the PR description says so expli ### Examples smoke job -`ci.yml`'s `examples-smoke` job runs one desktop example per adapter (WDIO mocha, Selenium mocha, Nightwatch BDD, Python pytest) in trace mode in the runner's headless Chrome, against the fixture site (`pnpm smoke:fixtures`) via `DEVTOOLS_EXAMPLE_BASE_URL`, so no third-party page change can break it. `.github/scripts/examples-smoke/run-example.sh` collects the zips each run wrote and hands them to `pnpm smoke:assert`; each red step names its example and each `FAIL` line names the expectation, and the zips and run logs are uploaded as the `examples-smoke` artifact. It checks that capture still produces the rows the examples imply, not how they render, so dashboard visuals, mobile and Safari still need a person. A change that alters an example's rows updates its expectation file in the same PR. +`ci.yml`'s `examples-smoke` job runs one desktop example per adapter (WDIO mocha, Selenium mocha, Nightwatch BDD, Python pytest) in trace mode in the runner's headless Chrome, against the fixture site (`pnpm smoke:fixtures`) via `DEVTOOLS_EXAMPLE_BASE_URL`, so no third-party page change can break it. `.github/scripts/examples-smoke/run-example.sh` collects the zips each run wrote and hands them to `pnpm smoke:assert`; each red step names its example and each `FAIL` line names the expectation, and the zips and run logs are uploaded as the `examples-smoke` artifact. It checks that capture still produces the rows the examples imply, and that an example's zips stay inside its byte budget, not how they render, so dashboard visuals, mobile and Safari still need a person. A change that alters an example's rows updates its expectation file in the same PR. A budget is the measured total plus about 25% and is lowered, with a new `measuredBytes`, when a change shrinks traces, so the win cannot be undone silently; the artifact is uploaded only on failure, so a passing job's measurement is its log's `trace zips total` line. The venv's editable install of `selenium_devtools` points at the primary checkout, so in a worktree every Python example, and the backend and page collector it resolves through `_find_monorepo_backend`, come from `main` unless `PYTHONPATH=packages/selenium-devtools-py/src` is exported first. A Python run that reports the old behaviour from a worktree is usually this, not the change. diff --git a/packages/conformance/smoke/assert-trace.ts b/packages/conformance/smoke/assert-trace.ts index ccb14c70..312d6b78 100644 --- a/packages/conformance/smoke/assert-trace.ts +++ b/packages/conformance/smoke/assert-trace.ts @@ -5,8 +5,12 @@ import path from 'node:path' import { fileURLToPath } from 'node:url' import { parseArgs } from 'node:util' -import { checkExample, readSmokeTrace } from './trace-expectations.js' -import type { ExampleExpectations, SmokeTrace } from './types.js' +import { + checkExample, + readSmokeTrace, + totalBytes +} from './trace-expectations.js' +import type { ExampleExpectations, SmokeTrace, TraceZip } from './types.js' const EXPECTATIONS_DIR = path.join( path.dirname(fileURLToPath(import.meta.url)), @@ -39,6 +43,40 @@ function escapeAnnotation(text: string): string { return text.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A') } +function readZips(paths: string[]): { + zips: TraceZip[] + traces: SmokeTrace[] +} { + const zips: TraceZip[] = [] + const traces = paths.flatMap((zip) => { + const bytes = fs.readFileSync(zip) + zips.push({ path: zip, bytes: bytes.length }) + return readSmokeTrace(zip, bytes) + }) + return { zips, traces } +} + +function printSummary( + label: string, + expectations: ExampleExpectations, + zips: TraceZip[], + traces: SmokeTrace[] +): void { + const listed = zips.map((zip) => `${zip.path} (${zip.bytes} bytes)`) + console.log( + `${label} ${zips.length} trace zip(s): ${listed.join(', ') || '(none)'}` + ) + for (const trace of traces) { + console.log( + `${label} ${trace.source}: ${trace.rows.length} rows, ${trace.groups.size} groups` + ) + } + // Printed on a pass too: the job uploads its zips only when it fails, so this line is the size record. + console.log( + `${label} trace zips total ${totalBytes(zips)} of a ${expectations.traceBudget.maxBytes}-byte budget` + ) +} + function main(): number { const { values, positionals } = parseArgs({ allowPositionals: true, @@ -55,26 +93,19 @@ function main(): number { return 2 } const expectations = loadExpectations(values.expect) - const zips = positionals.filter((p) => fs.existsSync(p)).flatMap(findZips) - const traces: SmokeTrace[] = zips.flatMap((zip) => - readSmokeTrace(zip, fs.readFileSync(zip)) + const { zips, traces } = readZips( + positionals.filter((p) => fs.existsSync(p)).flatMap(findZips) ) const failures = checkExample(expectations, traces, { exitCode: values['exit-code'] === undefined ? undefined : Number(values['exit-code']), - log: values.log ? fs.readFileSync(values.log, 'utf8') : undefined + log: values.log ? fs.readFileSync(values.log, 'utf8') : undefined, + zips }) const label = `[${expectations.example}]` - console.log( - `${label} ${zips.length} trace zip(s): ${zips.join(', ') || '(none)'}` - ) - for (const trace of traces) { - console.log( - `${label} ${trace.source}: ${trace.rows.length} rows, ${trace.groups.size} groups` - ) - } + printSummary(label, expectations, zips, traces) if (failures.length === 0) { console.log(`${label} PASS (${expectations.tests.length} expectation(s))`) return 0 diff --git a/packages/conformance/smoke/expectations/nightwatch-bdd.json b/packages/conformance/smoke/expectations/nightwatch-bdd.json index c903b376..92dfce0e 100644 --- a/packages/conformance/smoke/expectations/nightwatch-bdd.json +++ b/packages/conformance/smoke/expectations/nightwatch-bdd.json @@ -2,6 +2,11 @@ "example": "nightwatch-bdd", "runFails": true, "sourceFiles": ["examples/nightwatch/bdd/smoke-test.js"], + "traceBudget": { + "maxBytes": 480000, + "measuredBytes": 376379, + "measuredIn": "CI run 40774299701, examples-smoke artifact" + }, "tests": [ { "test": "logs into the secure area with valid credentials", diff --git a/packages/conformance/smoke/expectations/python-pytest.json b/packages/conformance/smoke/expectations/python-pytest.json index a0525fa9..e5ca59cf 100644 --- a/packages/conformance/smoke/expectations/python-pytest.json +++ b/packages/conformance/smoke/expectations/python-pytest.json @@ -2,6 +2,11 @@ "example": "python-pytest", "runFails": false, "sourceFiles": ["examples/selenium-py/pytest/test_login_pytest.py"], + "traceBudget": { + "maxBytes": 1010000, + "measuredBytes": 804136, + "measuredIn": "CI run 40774299701, examples-smoke artifact" + }, "tests": [ { "test": "test_logs_in_with_valid_credentials", diff --git a/packages/conformance/smoke/expectations/selenium-mocha.json b/packages/conformance/smoke/expectations/selenium-mocha.json index 8a3df919..6c8af0fe 100644 --- a/packages/conformance/smoke/expectations/selenium-mocha.json +++ b/packages/conformance/smoke/expectations/selenium-mocha.json @@ -2,6 +2,11 @@ "example": "selenium-mocha", "runFails": true, "sourceFiles": ["examples/selenium-js/mocha/example.js"], + "traceBudget": { + "maxBytes": 200000, + "measuredBytes": 159706, + "measuredIn": "CI run 40774299701, examples-smoke artifact" + }, "tests": [ { "test": "loads the login page and reads the heading", diff --git a/packages/conformance/smoke/expectations/wdio-mocha.json b/packages/conformance/smoke/expectations/wdio-mocha.json index 5fc1cfb6..62e230e5 100644 --- a/packages/conformance/smoke/expectations/wdio-mocha.json +++ b/packages/conformance/smoke/expectations/wdio-mocha.json @@ -8,6 +8,11 @@ "examples/wdio/pageobjects/login.page.ts", "examples/wdio/pageobjects/secure.page.ts" ], + "traceBudget": { + "maxBytes": 1330000, + "measuredBytes": 1059358, + "measuredIn": "CI run 40774299701, examples-smoke artifact" + }, "tests": [ { "test": "logs into the secure area with valid credentials", diff --git a/packages/conformance/smoke/trace-expectations.ts b/packages/conformance/smoke/trace-expectations.ts index 647b116a..d94deb81 100644 --- a/packages/conformance/smoke/trace-expectations.ts +++ b/packages/conformance/smoke/trace-expectations.ts @@ -13,7 +13,9 @@ import type { LocatedRow, SmokeTrace, TestExpectation, - TraceRow + TraceBudget, + TraceRow, + TraceZip } from './types.js' const TRACING_GROUP_CLASS = 'Tracing' @@ -347,17 +349,76 @@ const TEST_SCOPED_ONLY = ['allowTimeoutError', 'attempts'] as const export function validateExpectations( expectations: ExampleExpectations ): string[] { - return expectations.tests.flatMap((test) => { - if (test.scope !== 'run') { - return [] - } - const fields = test as Partial< - Record<(typeof TEST_SCOPED_ONLY)[number], unknown> - > - return TEST_SCOPED_ONLY.filter((key) => fields[key] !== undefined).map( - (key) => `the run: ${key} is only allowed on a test-scoped entry` + return [ + ...validateBudget(expectations.traceBudget as TraceBudget | undefined), + ...expectations.tests.flatMap((test) => { + if (test.scope !== 'run') { + return [] + } + const fields = test as Partial< + Record<(typeof TEST_SCOPED_ONLY)[number], unknown> + > + return TEST_SCOPED_ONLY.filter((key) => fields[key] !== undefined).map( + (key) => `the run: ${key} is only allowed on a test-scoped entry` + ) + }) + ] +} + +const isByteCount = (value: unknown): value is number => + typeof value === 'number' && Number.isInteger(value) && value > 0 + +/** Required, so a new example cannot skip the size check by leaving it out. */ +function validateBudget(budget: TraceBudget | undefined): string[] { + if (!budget) { + return ['traceBudget is required: { maxBytes, measuredBytes, measuredIn }'] + } + const problems: string[] = [] + if (!isByteCount(budget.maxBytes)) { + problems.push( + 'traceBudget.maxBytes must be a positive whole number of bytes' ) - }) + } + if (!isByteCount(budget.measuredBytes)) { + problems.push( + 'traceBudget.measuredBytes must be a positive whole number of bytes' + ) + } else if ( + isByteCount(budget.maxBytes) && + budget.measuredBytes > budget.maxBytes + ) { + problems.push('traceBudget.measuredBytes is already over maxBytes') + } + if (typeof budget.measuredIn !== 'string' || !budget.measuredIn) { + problems.push( + 'traceBudget.measuredIn must say where the baseline was measured' + ) + } + return problems +} + +const formatBytes = (bytes: number): string => bytes.toLocaleString('en-US') + +export function totalBytes(zips: TraceZip[]): number { + return zips.reduce((sum, zip) => sum + zip.bytes, 0) +} + +/** Totalled across the run's zips, since how many a run writes depends on its granularity. */ +export function checkTraceBudget( + budget: TraceBudget, + zips: TraceZip[] +): string[] { + const total = totalBytes(zips) + if (total <= budget.maxBytes) { + return [] + } + const sizes = [...zips] + .sort((a, b) => b.bytes - a.bytes) + .map((zip) => `${zip.path} ${formatBytes(zip.bytes)}`) + .join(', ') + return [ + `trace zips total ${formatBytes(total)} bytes, over the ${formatBytes(budget.maxBytes)}-byte budget (baseline ${formatBytes(budget.measuredBytes)} from ${budget.measuredIn}): ${sizes}` + ] } /** Every unmet expectation as one line; empty means the example passed. */ @@ -385,6 +446,9 @@ export function checkExample( return [ ...failures, ...checkCallSites(expectations, traces), - ...checkTimeouts(expectations, traces, outcome) + ...checkTimeouts(expectations, traces, outcome), + ...(outcome.zips + ? checkTraceBudget(expectations.traceBudget, outcome.zips) + : []) ] } diff --git a/packages/conformance/smoke/types.ts b/packages/conformance/smoke/types.ts index c19d9b63..c08b297b 100644 --- a/packages/conformance/smoke/types.ts +++ b/packages/conformance/smoke/types.ts @@ -39,9 +39,19 @@ export interface ExampleExpectations { allowTimeoutError?: boolean /** Files a row's call site may point into (paths ending in one of these); unset skips the check. */ sourceFiles?: string[] + traceBudget: TraceBudget tests: TestExpectation[] } +/** A ceiling on the bytes of every zip the run wrote, so a change that bloats traces fails its PR. */ +export interface TraceBudget { + maxBytes: number + /** The total the job wrote when the budget was set; `maxBytes` minus this is the headroom left. */ + measuredBytes: number + /** Where `measuredBytes` was read from, e.g. the CI run id. */ + measuredIn: string +} + export interface TraceRow { callId: string parentId?: string @@ -67,7 +77,14 @@ export interface SmokeTrace { stdio: string[] } +export interface TraceZip { + path: string + bytes: number +} + export interface RunOutcome { exitCode?: number log?: string + /** The zips the run wrote; omitted skips the size budget. */ + zips?: TraceZip[] } diff --git a/packages/conformance/tests/smoke-trace-expectations.test.ts b/packages/conformance/tests/smoke-trace-expectations.test.ts index 753ce5d5..d3744dc2 100644 --- a/packages/conformance/tests/smoke-trace-expectations.test.ts +++ b/packages/conformance/tests/smoke-trace-expectations.test.ts @@ -15,6 +15,7 @@ import { exportTraceZip } from '@wdio/devtools-trace' import { checkExample, + checkTraceBudget, findTimeout, readSmokeTrace, validateExpectations @@ -22,7 +23,8 @@ import { import type { ExampleExpectations, RunScopedExpectation, - SmokeTrace + SmokeTrace, + TraceBudget } from '../smoke/types.js' const EXPECTATIONS_DIR = path.join( @@ -33,6 +35,12 @@ const REPO_ROOT = path.join(EXPECTATIONS_DIR, '../../../..') let clock = 1_000 +const traceBudget: TraceBudget = { + maxBytes: 1_000, + measuredBytes: 800, + measuredIn: 'unit' +} + function cmd( command: string, testUid: string, @@ -124,6 +132,7 @@ const failingRun = () => [ const expectations: ExampleExpectations = { example: 'unit', + traceBudget, runFails: true, tests: [ { @@ -324,6 +333,7 @@ describe('checkExample', () => { ]) const byName: ExampleExpectations = { example: 'unit', + traceBudget, runFails: false, tests: [ { test: 'test_rejects_invalid_credentials', rows: ['Page.navigate'] } @@ -351,6 +361,7 @@ describe('checkExample', () => { const trace = await parsed([...passingRun(), ...failingRun()]) const whole: ExampleExpectations = { example: 'unit', + traceBudget, runFails: true, tests: [ { @@ -465,6 +476,7 @@ describe('a test that times out on purpose', () => { allowTimeoutError: boolean | undefined ): ExampleExpectations => ({ example: 'unit', + traceBudget, runFails: true, tests: [ { @@ -533,6 +545,7 @@ describe('a retried test', () => { const retryExpectations = (attempts?: number): ExampleExpectations => ({ example: 'unit', + traceBudget, runFails: true, tests: [ { @@ -604,6 +617,7 @@ describe('call sites', () => { ] const withSources: ExampleExpectations = { example: 'unit', + traceBudget, runFails: false, sourceFiles: ['examples/wdio/mocha/specs/login.e2e.ts'], tests: [{ test: 'logs in with valid credentials' }] @@ -682,6 +696,7 @@ describe('validateExpectations', () => { const typed: RunScopedExpectation = entry const expectations: ExampleExpectations = { example: 'unit', + traceBudget, runFails: true, tests: [typed] } @@ -698,7 +713,12 @@ describe('validateExpectations', () => { // @ts-expect-error attempts is typed `never` on a run-scoped entry const typed: RunScopedExpectation = entry expect( - validateExpectations({ example: 'unit', runFails: true, tests: [typed] }) + validateExpectations({ + example: 'unit', + traceBudget, + runFails: true, + tests: [typed] + }) ).toEqual(['the run: attempts is only allowed on a test-scoped entry']) }) @@ -706,6 +726,7 @@ describe('validateExpectations', () => { expect( validateExpectations({ example: 'unit', + traceBudget, runFails: true, tests: [{ test: 'times out on purpose', allowTimeoutError: true }] }) @@ -713,6 +734,66 @@ describe('validateExpectations', () => { }) }) +describe('trace size budget', () => { + const zips = [ + { path: 'out/1/trace-a.zip', bytes: 400 }, + { path: 'out/2/trace-b.zip', bytes: 600 } + ] + + it('passes zips whose total is at or under the budget', () => { + expect(checkTraceBudget(traceBudget, zips)).toEqual([]) + }) + + it('names every zip, the total, the budget and the baseline when over it', () => { + expect( + checkTraceBudget(traceBudget, [ + ...zips, + { path: 'out/3/trace-c.zip', bytes: 1_200 } + ]) + ).toEqual([ + 'trace zips total 2,200 bytes, over the 1,000-byte budget (baseline 800 from unit): out/3/trace-c.zip 1,200, out/2/trace-b.zip 600, out/1/trace-a.zip 400' + ]) + }) + + it('runs as part of checkExample when the run reports its zips', async () => { + const traces = [await parsed([...passingRun(), ...failingRun()])] + expect(checkExample(expectations, traces, { exitCode: 1, zips })).toEqual( + [] + ) + const over = [{ path: 'trace-smoke.zip', bytes: 1_001 }] + expect( + checkExample(expectations, traces, { exitCode: 1, zips: over }) + ).toEqual([expect.stringMatching(/^trace zips total 1,001 bytes, over/)]) + }) + + it('is required, so an expectation file cannot leave it out', () => { + const { traceBudget: _omitted, ...rest } = expectations + expect(validateExpectations(rest as ExampleExpectations)).toEqual([ + 'traceBudget is required: { maxBytes, measuredBytes, measuredIn }' + ]) + }) + + it('rejects a budget that is not a byte count or sits under its baseline', () => { + const invalid = (budget: Partial>) => + validateExpectations({ + ...expectations, + traceBudget: { ...traceBudget, ...budget } as TraceBudget + }) + expect(invalid({ maxBytes: '1MB' })).toEqual([ + 'traceBudget.maxBytes must be a positive whole number of bytes' + ]) + expect(invalid({ measuredBytes: 0 })).toEqual([ + 'traceBudget.measuredBytes must be a positive whole number of bytes' + ]) + expect(invalid({ measuredBytes: 1_001 })).toEqual([ + 'traceBudget.measuredBytes is already over maxBytes' + ]) + expect(invalid({ measuredIn: '' })).toEqual([ + 'traceBudget.measuredIn must say where the baseline was measured' + ]) + }) +}) + describe('shipped expectation files', () => { const files = fs.readdirSync(EXPECTATIONS_DIR) From 081e976c8dd39d86251341282757d910a69050f0 Mon Sep 17 00:00:00 2001 From: Vishnu Vardhan Date: Fri, 30 Oct 2026 12:20:24 +0530 Subject: [PATCH 2/2] Lower the smoke trace budgets to the sizes after the screenshot dedupe This branch's first CI run measured every example with #508's dedupe in place: WDIO 966,841 bytes, Selenium 158,349, Nightwatch 260,322 and Python 688,296. Each budget is that total plus 25%, so the saving cannot be undone without failing the job. Co-Authored-By: Claude Opus 5.5 --- packages/conformance/smoke/expectations/nightwatch-bdd.json | 6 +++--- packages/conformance/smoke/expectations/python-pytest.json | 6 +++--- packages/conformance/smoke/expectations/selenium-mocha.json | 4 ++-- packages/conformance/smoke/expectations/wdio-mocha.json | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/conformance/smoke/expectations/nightwatch-bdd.json b/packages/conformance/smoke/expectations/nightwatch-bdd.json index 92dfce0e..6b522369 100644 --- a/packages/conformance/smoke/expectations/nightwatch-bdd.json +++ b/packages/conformance/smoke/expectations/nightwatch-bdd.json @@ -3,9 +3,9 @@ "runFails": true, "sourceFiles": ["examples/nightwatch/bdd/smoke-test.js"], "traceBudget": { - "maxBytes": 480000, - "measuredBytes": 376379, - "measuredIn": "CI run 40774299701, examples-smoke artifact" + "maxBytes": 330000, + "measuredBytes": 260322, + "measuredIn": "CI run 40948877371, examples-smoke log (#509, with #508's dedupe)" }, "tests": [ { diff --git a/packages/conformance/smoke/expectations/python-pytest.json b/packages/conformance/smoke/expectations/python-pytest.json index e5ca59cf..ad6e5a70 100644 --- a/packages/conformance/smoke/expectations/python-pytest.json +++ b/packages/conformance/smoke/expectations/python-pytest.json @@ -3,9 +3,9 @@ "runFails": false, "sourceFiles": ["examples/selenium-py/pytest/test_login_pytest.py"], "traceBudget": { - "maxBytes": 1010000, - "measuredBytes": 804136, - "measuredIn": "CI run 40774299701, examples-smoke artifact" + "maxBytes": 870000, + "measuredBytes": 688296, + "measuredIn": "CI run 40948877371, examples-smoke log (#509, with #508's dedupe)" }, "tests": [ { diff --git a/packages/conformance/smoke/expectations/selenium-mocha.json b/packages/conformance/smoke/expectations/selenium-mocha.json index 6c8af0fe..9deabaf7 100644 --- a/packages/conformance/smoke/expectations/selenium-mocha.json +++ b/packages/conformance/smoke/expectations/selenium-mocha.json @@ -4,8 +4,8 @@ "sourceFiles": ["examples/selenium-js/mocha/example.js"], "traceBudget": { "maxBytes": 200000, - "measuredBytes": 159706, - "measuredIn": "CI run 40774299701, examples-smoke artifact" + "measuredBytes": 158349, + "measuredIn": "CI run 40948877371, examples-smoke log (#509, with #508's dedupe)" }, "tests": [ { diff --git a/packages/conformance/smoke/expectations/wdio-mocha.json b/packages/conformance/smoke/expectations/wdio-mocha.json index 62e230e5..159e7134 100644 --- a/packages/conformance/smoke/expectations/wdio-mocha.json +++ b/packages/conformance/smoke/expectations/wdio-mocha.json @@ -9,9 +9,9 @@ "examples/wdio/pageobjects/secure.page.ts" ], "traceBudget": { - "maxBytes": 1330000, - "measuredBytes": 1059358, - "measuredIn": "CI run 40774299701, examples-smoke artifact" + "maxBytes": 1210000, + "measuredBytes": 966841, + "measuredIn": "CI run 40948877371, examples-smoke log (#509, with #508's dedupe)" }, "tests": [ {