diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index cd6cf1e643c1d..0d1300b4add6f 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -132,6 +132,11 @@ Reporter is allowed to override the status and hence affect the exit code of the - `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> Test run status. - `startTime` <[Date]> Test run start wall time. - `duration` <[int]> Test run duration in milliseconds. + - `shards` <[Array]<[Object]>> Only present on merged reports + - `shardIndex` ?<[int]> The index of the shard, one-based. + - `tag` ?<[Array]<[string]>> Global [`property: TestConfig.tag`] that differentiates CI environments + - `startTime` <[Date]> Start wall time of shard. + - `duration` <[int]> Shard run duration in milliseconds. Result of the full test run, `status` can be one of: * `'passed'` - Everything went as expected. diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 278a2dbaf6d5d..eb5f4d726230a 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -103,6 +103,6 @@ The index of the worker between `0` and `workers - 1`. It is guaranteed that wor ## property: TestResult.shardIndex * since: v1.58 -- type: <[int]> +- type: ?<[int]> -The index of the shard between `0` and `shards - 1`. +The index of the shard between `1` and [`shards`](../test-sharding.md). diff --git a/packages/html-reporter/src/barchart.css b/packages/html-reporter/src/barchart.css new file mode 100644 index 0000000000000..08478189a1787 --- /dev/null +++ b/packages/html-reporter/src/barchart.css @@ -0,0 +1,28 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +.barchart-bar { + transition: opacity 0.2s; + cursor: pointer; + outline: none; +} + +.barchart-bar:hover, +.barchart-bar:focus { + opacity: 0.8; + stroke: var(--color-fg-default); + stroke-width: 2; +} diff --git a/packages/html-reporter/src/barchart.tsx b/packages/html-reporter/src/barchart.tsx new file mode 100644 index 0000000000000..a4b52d501c16d --- /dev/null +++ b/packages/html-reporter/src/barchart.tsx @@ -0,0 +1,234 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import './barchart.css'; + +const formatDuration = (ms: number): string => { + const totalSeconds = Math.round(ms / 1000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + if (minutes === 0) + return `${seconds}s`; + return `${minutes}m ${seconds}s`; +}; + +export const GroupedBarChart = ({ + data, + groups, + series, +}: { + data: number[][]; + groups: string[]; + series: string[]; +}) => { + const width = 800; + + // Calculate left margin based on longest group name + // Rough estimate: 7 pixels per character at fontSize 12 + const maxGroupNameLength = Math.max(...groups.map(g => g.length)); + const estimatedTextWidth = maxGroupNameLength * 7; + const leftMargin = Math.min(width * 0.5, Math.max(50, estimatedTextWidth)); + + const margin = { top: 20, right: 20, bottom: 40, left: leftMargin }; + const chartWidth = width - margin.left - margin.right; + + const maxValue = Math.max(...data.flat()); + + let tickInterval: number; + let formatTickLabel: (i: number) => string; + + if (maxValue < 60 * 1000) { + tickInterval = 10 * 1000; + formatTickLabel = i => `${i * 10}s`; + } else if (maxValue < 5 * 60 * 1000) { + tickInterval = 30 * 1000; + formatTickLabel = i => { + const seconds = i * 30; + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return secs === 0 ? `${mins}m` : `${mins}m${secs}s`; + }; + } else if (maxValue < 30 * 60 * 1000) { + tickInterval = 5 * 60 * 1000; + formatTickLabel = i => `${i * 5}m`; + } else { + tickInterval = 10 * 60 * 1000; + formatTickLabel = i => `${i * 10}m`; + } + + const maxRounded = Math.ceil(maxValue / tickInterval) * tickInterval; + const xScale = chartWidth / maxRounded; + + // Calculate the number of actual bars per group (non-zero values) + const barsPerGroup = data.map(group => group.length); + + // Allocate space proportionally based on number of bars + const barHeight = 20; // Fixed bar height + const barSpacing = 4; + const groupPadding = 12; + + // Calculate Y positions for each group based on their bar count + const groupYPositions: number[] = []; + let currentY = 0; + for (let i = 0; i < groups.length; i++) { + groupYPositions.push(currentY); + const groupHeight = barsPerGroup[i] * barHeight + (barsPerGroup[i] - 1) * barSpacing + groupPadding; + currentY += groupHeight; + } + + const contentHeight = currentY; + + const xTicks = []; + const numberOfTicks = Math.ceil(maxRounded / tickInterval); + for (let i = 0; i <= numberOfTicks; i++) { + xTicks.push({ + x: i * tickInterval * xScale, + label: formatTickLabel(i) + }); + } + + const height = contentHeight + margin.top + margin.bottom; + + return ( + + + {xTicks.map(({ x, label }, i) => ( + + ))} + + {groups.map((group, groupIndex) => { + const groupY = groupYPositions[groupIndex]; + let barIndex = 0; + + return ( + + {series.map((seriesName, seriesIndex) => { + const value = data[groupIndex][seriesIndex]; + if (value === undefined || Number.isNaN(value)) + return null; + + const barWidth = value * xScale; + const x = 0; + const y = groupY + barIndex * (barHeight + barSpacing); + barIndex++; + + const colors = ['var(--color-scale-yellow-3)', 'var(--color-scale-orange-4)', 'var(--color-scale-blue-3)', 'var(--color-scale-green-3)']; + const color = colors[seriesIndex % colors.length]; + + return ( + + + {`${seriesName}: ${formatDuration(value)}`} + + + + ); + })} + + ); + })} + + {groups.map((group, groupIndex) => { + const groupY = groupYPositions[groupIndex]; + const actualBars = barsPerGroup[groupIndex]; + const groupHeight = actualBars * barHeight + (actualBars - 1) * barSpacing; + const labelY = groupY + groupHeight / 2; + + return ( + + ); + })} + + + + + + + ); +}; diff --git a/packages/html-reporter/src/speedboard.tsx b/packages/html-reporter/src/speedboard.tsx index b0beb0267a863..b461a2272e449 100644 --- a/packages/html-reporter/src/speedboard.tsx +++ b/packages/html-reporter/src/speedboard.tsx @@ -19,9 +19,12 @@ import { LoadedReport } from './loadedReport'; import { TestFileView } from './testFileView'; import * as icons from './icons'; import { TestCaseSummary } from './types'; +import { AutoChip } from './chip'; +import { GroupedBarChart } from './barchart'; export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) { return <> + ; } @@ -46,3 +49,37 @@ export function SlowestTests({ report, tests }: { report: LoadedReport, tests: T } />; } + +export function Shards({ report }: { report: LoadedReport }) { + const shards = report.json().shards; + if (shards.length === 0) + return null; + + let clash = false; + const bots: Record = {}; + for (const shard of shards) { + const botName = shard.tag.join(' '); + bots[botName] ??= []; + const shardIndex = Math.max((shard.shardIndex ?? 1) - 1, 0); + if (bots[botName][shardIndex] !== undefined) + clash = true; + bots[botName][shardIndex] = shard.duration; + } + + const maxSeries = Math.max(...Object.values(bots).map(shardDurations => shardDurations.length)); + + return + `Shard ${i + 1}`)} + /> + {clash &&
+ + Some shards could not be differentiated because of missing global tags. + Please refer to + the docs + on how to fix this. +
} + ; +} diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts index 270a8cb9838c5..ce913a55a9a5d 100644 --- a/packages/html-reporter/src/types.d.ts +++ b/packages/html-reporter/src/types.d.ts @@ -49,6 +49,12 @@ export type HTMLReport = { projectNames: string[]; startTime: number; duration: number; + shards: { + shardIndex?: number; + tag: string[]; + startTime: number; + duration: number; + }[]; errors: string[]; // Top-level errors that are not attributed to any test. options: HTMLReportOptions; }; diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 657664199ef62..97f5634726e79 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -87,7 +87,7 @@ export type JsonTestResultStart = { retry: number; workerIndex: number; parallelIndex: number; - shardIndex: number; + shardIndex?: number; startTime: number; }; @@ -130,6 +130,12 @@ export type JsonFullResult = { status: reporterTypes.FullResult['status']; startTime: number; duration: number; + shards?: { + shardIndex?: number; + tag: string[]; + startTime: number; + duration: number; + }[]; }; export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent @@ -452,6 +458,12 @@ export class TeleReporterReceiver { status: result.status, startTime: new Date(result.startTime), duration: result.duration, + shards: result.shards?.map(s => ({ + shardIndex: s.shardIndex, + tag: s.tag, + startTime: new Date(s.startTime), + duration: s.duration, + })) ?? [], }); } @@ -725,7 +737,7 @@ export class TeleTestResult implements reporterTypes.TestResult { retry: reporterTypes.TestResult['retry']; parallelIndex: reporterTypes.TestResult['parallelIndex'] = -1; workerIndex: reporterTypes.TestResult['workerIndex'] = -1; - shardIndex: reporterTypes.TestResult['shardIndex'] = -1; + shardIndex: reporterTypes.TestResult['shardIndex']; duration: reporterTypes.TestResult['duration'] = -1; stdout: reporterTypes.TestResult['stdout'] = []; stderr: reporterTypes.TestResult['stderr'] = []; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index b97e571dac9ce..ad1942a88a419 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -298,6 +298,12 @@ class HtmlBuilder { stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) }, errors: topLevelErrors.map(error => formatError(internalScreen, error).message), options: this._options, + shards: result.shards?.map(s => ({ + shardIndex: s.shardIndex, + tag: s.tag, + startTime: s.startTime.getTime(), + duration: s.duration, + })) ?? [] }; htmlReport.files.sort((f1, f2) => { const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky; diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 147a837e52539..d8ae1f3e478bb 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -88,6 +88,7 @@ export class InternalReporter implements ReporterV2 { ...result, startTime: this._startTime!, duration: monotonicTime() - this._monotonicStartTime!, + shards: [], }); } diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index 81d46321c3e55..55624b0714945 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -188,7 +188,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool: const configureEvents: JsonOnConfigureEvent[] = []; const projectEvents: JsonOnProjectEvent[] = []; - const endEvents: JsonOnEndEvent[] = []; + const endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[] = []; const blobs = await extractAndParseReports(dir, shardReportFiles, internalizer, printStatus); // Sort by (report name; shard; file name), so that salt generation below is deterministic when: @@ -235,7 +235,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool: } else if (event.method === 'onProject') { projectEvents.push(event); } else if (event.method === 'onEnd') { - endEvents.push(event); + endEvents.push({ event, metadata, tags }); } } @@ -323,12 +323,13 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig { }; } -function mergeEndEvents(endEvents: JsonOnEndEvent[]): JsonEvent { +function mergeEndEvents(endEvents: { event: JsonOnEndEvent, metadata: BlobReportMetadata, tags: string[] }[]): JsonEvent { let startTime = endEvents.length ? 10000000000000 : Date.now(); let status: JsonFullResult['status'] = 'passed'; let endTime: number = 0; + const shards: JsonFullResult['shards'] = []; - for (const event of endEvents) { + for (const { event, metadata, tags } of endEvents) { const shardResult = event.params.result; if (shardResult.status === 'failed') status = 'failed'; @@ -338,11 +339,19 @@ function mergeEndEvents(endEvents: JsonOnEndEvent[]): JsonEvent { status = 'interrupted'; startTime = Math.min(startTime, shardResult.startTime); endTime = Math.max(endTime, shardResult.startTime + shardResult.duration); + + shards.push({ + shardIndex: metadata.shard?.current, + tag: tags, + startTime: shardResult.startTime, + duration: shardResult.duration, + }); } const result: JsonFullResult = { status, startTime, duration: endTime - startTime, + shards, }; return { method: 'onEnd', diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 4efd401309aa8..7a97d5640e29f 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -167,6 +167,12 @@ export class TeleReporterEmitter implements ReporterV2 { status: result.status, startTime: result.startTime.getTime(), duration: result.duration, + shards: result.shards?.map(s => ({ + shardIndex: s.shardIndex, + tag: s.tag, + startTime: +s.startTime, + duration: s.duration, + })), }; this._messageSink({ method: 'onEnd', diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index e00ba5b459191..0a165ef1d9e09 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -320,7 +320,7 @@ class JobDispatcher { this._dataByTestId.set(test.id, { test, result, steps: new Map() }); result.parallelIndex = this._parallelIndex; result.workerIndex = this._workerIndex; - result.shardIndex = this._config.config.shard?.current ?? 0; + result.shardIndex = this._config.config.shard?.current; result.startTime = new Date(params.startWallTime); this._reporter.onTestBegin?.(test, result); this._currentlyRunning = { test, result }; diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 8ae9b6935dd89..29292ea420be6 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -40,6 +40,31 @@ export interface FullResult { * Test duration in milliseconds. */ duration: number; + + /** + * Only present on merged reports. + */ + shards?: { + /** + * The index of the shard, one-based. + */ + shardIndex?: number; + + /** + * Global tag that differentiates CI environments. + */ + tag: string[]; + + /** + * Start wall time of shard. + */ + startTime: Date; + + /** + * Shard run duration in milliseconds. + */ + duration: number; + }[]; } /** @@ -297,7 +322,7 @@ export interface JSONReportError { export interface JSONReportTestResult { workerIndex: number; parallelIndex: number; - shardIndex: number; + shardIndex?: number; status: TestStatus | undefined; duration: number; error: TestError | undefined; @@ -682,9 +707,9 @@ export interface TestResult { retry: number; /** - * The index of the shard between `0` and `shards - 1`. + * The index of the shard between `1` and [`shards`](https://playwright.dev/docs/test-sharding). */ - shardIndex: number; + shardIndex?: number; /** * Start time of this particular test run. diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 262f927410fe5..e0d502b797ada 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -3306,6 +3306,56 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(page.getByRole('link', { name: 'previous' })).not.toBeVisible(); }); }); + + + test('shard chart', async ({ runInlineTest, writeFiles, showReport, page }) => { + test.skip(!useIntermediateMergeReport); + + await writeFiles({ + 'playwright.config.ts': ` + module.exports = { + fullyParallel: true, + tag: process.env.BOT_TAG, + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + import timers from 'timers/promises'; + test('one', async () => { + await timers.setTimeout(100); + }); + test('two', async () => { + await timers.setTimeout(200); + }); + test('three', async () => { + await timers.setTimeout(300); + }); + `, + }); + + await runInlineTest({}, { reporter: 'dot,html', shard: '1/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' }); + await runInlineTest({}, { reporter: 'dot,html', shard: '2/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' }); + await runInlineTest({}, { reporter: 'dot,html', shard: '3/3', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@linux' }); + + await runInlineTest({}, { reporter: 'dot,html', shard: '1/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' }); + await runInlineTest({}, { reporter: 'dot,html', shard: '2/2', }, { PLAYWRIGHT_HTML_OPEN: 'never', PWTEST_BLOB_DO_NOT_REMOVE: '1', BOT_TAG: '@mac' }); + + await showReport(); + await page.getByRole('link', { name: 'Speedboard' }).click(); + + await expect(page.getByRole('main')).toMatchAriaSnapshot(` + - button "Shard Duration" + - region: + - img: + - list "@linux": + - listitem /Shard 1/ + - listitem /Shard 2/ + - listitem /Shard 3/ + - list "@mac": + - listitem /Shard 1/ + - listitem /Shard 2/ + `); + }); }); } diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index c92e37b006a44..52fd354b0460b 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -39,6 +39,31 @@ export interface FullResult { * Test duration in milliseconds. */ duration: number; + + /** + * Only present on merged reports. + */ + shards?: { + /** + * The index of the shard, one-based. + */ + shardIndex?: number; + + /** + * Global tag that differentiates CI environments. + */ + tag: string[]; + + /** + * Start wall time of shard. + */ + startTime: Date; + + /** + * Shard run duration in milliseconds. + */ + duration: number; + }[]; } export interface Reporter { @@ -110,7 +135,7 @@ export interface JSONReportError { export interface JSONReportTestResult { workerIndex: number; parallelIndex: number; - shardIndex: number; + shardIndex?: number; status: TestStatus | undefined; duration: number; error: TestError | undefined;