chore(reporter): report per-shard durations

docs/src/test-reporter-api/class-reporter.md

@@ -132,6 +132,11 @@ Reporter is allowed to override the status and hence affect the exit code of the
   - `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> Test run status.
   - `startTime` <[Date]> Test run start wall time.
   - `duration` <[int]> Test run duration in milliseconds.
+  - `shards` <[Array]<[Object]>> Only present on merged reports
+    - `shardIndex` <[int]> The index of the shard, one-based.
+    - `tag` ?<[string]> Bot tag that differentiates CI environments
+    - `startTime` <[Date]> start wall time of shard.
+    - `duration` <[int]> Shard run duration in milliseconds.
 
 Result of the full test run, `status` can be one of:
 * `'passed'` - Everything went as expected.

docs/src/test-reporter-api/class-testresult.md

@@ -105,4 +105,4 @@ The index of the worker between `0` and `workers - 1`. It is guaranteed that wor
 * since: v1.58
 - type: <[int]>
 
-The index of the shard between `0` and `shards - 1`.
+The index of the shard between `1` and `shards`.

packages/html-reporter/src/barchart.tsx

@@ -0,0 +1,211 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const formatDuration = (ms: number): string => {
+  const totalSeconds = Math.round(ms / 1000);
+  const minutes = Math.floor(totalSeconds / 60);
+  const seconds = totalSeconds % 60;
+  if (minutes === 0)
+    return `${seconds}s`;
+  return `${minutes}m ${seconds}s`;
+};
+
+export const GroupedBarChart = ({
+  data,
+  groups,
+  series,
+}: {
+  data: number[][];
+  groups: string[];
+  series: string[];
+}) => {
+  const width = 800;
+
+  // Calculate left margin based on longest group name
+  // Rough estimate: 7 pixels per character at fontSize 12
+  const maxGroupNameLength = Math.max(...groups.map(g => g.length));
+  const estimatedTextWidth = maxGroupNameLength * 7;
+  const leftMargin = Math.min(300, Math.max(100, estimatedTextWidth));
+
+  const margin = { top: 20, right: 20, bottom: 40, left: leftMargin };
+  const chartWidth = width - margin.left - margin.right;
+
+  const maxValue = Math.max(...data.flat());
+
+  let tickInterval: number;
+  let formatTickLabel: (i: number) => string;
+
+  if (maxValue < 60 * 1000) {
+    tickInterval = 10 * 1000;
+    formatTickLabel = i => `${i * 10}s`;
+  } else if (maxValue < 5 * 60 * 1000) {
+    tickInterval = 30 * 1000;
+    formatTickLabel = i => {
+      const seconds = i * 30;
+      const mins = Math.floor(seconds / 60);
+      const secs = seconds % 60;
+      return secs === 0 ? `${mins}m` : `${mins}m${secs}s`;
+    };
+  } else if (maxValue < 30 * 60 * 1000) {
+    tickInterval = 5 * 60 * 1000;
+    formatTickLabel = i => `${i * 5}m`;
+  } else {
+    tickInterval = 10 * 60 * 1000;
+    formatTickLabel = i => `${i * 10}m`;
+  }
+
+  const maxRounded = Math.ceil(maxValue / tickInterval) * tickInterval;
+  const xScale = chartWidth / maxRounded;
+
+  // Calculate the number of actual bars per group (non-zero values)
+  const barsPerGroup = data.map(group => group.length);
+
+  // Allocate space proportionally based on number of bars
+  const barHeight = 20; // Fixed bar height
+  const barSpacing = 4;
+  const groupPadding = 12;
+
+  // Calculate Y positions for each group based on their bar count
+  const groupYPositions: number[] = [];
+  let currentY = 0;
+  for (let i = 0; i < groups.length; i++) {
+    groupYPositions.push(currentY);
+    const groupHeight = barsPerGroup[i] * barHeight + (barsPerGroup[i] - 1) * barSpacing + groupPadding;
+    currentY += groupHeight;
+  }
+
+  const contentHeight = currentY;
+
+  const xTicks = [];
+  const numberOfTicks = Math.ceil(maxRounded / tickInterval);
+  for (let i = 0; i <= numberOfTicks; i++) {
+    xTicks.push({
+      x: i * tickInterval * xScale,
+      label: formatTickLabel(i)
+    });
+  }
+
+  const height = contentHeight + margin.top + margin.bottom;
+
+  return (
+    <svg
+      viewBox={`0 0 ${width} ${height}`}
+      preserveAspectRatio='xMidYMid meet'
+      style={{ width: '100%', height: 'auto' }}
+    >
+      <g transform={`translate(${margin.left}, ${margin.top})`}>
+        {xTicks.map(({ x, label }, i) => (
+          <g key={i}>
+            <line
+              x1={x}
+              y1={0}
+              x2={x}
+              y2={contentHeight}
+              stroke='var(--color-border-muted)'
+              strokeWidth='1'
+            />
+            <text
+              x={x}
+              y={contentHeight + 20}
+              textAnchor='middle'
+              dominantBaseline='middle'
+              fontSize='12'
+              fill='var(--color-fg-muted)'
+            >
+              {label}
+            </text>
+          </g>
+        ))}
+
+        {groups.map((group, groupIndex) => {
+          const groupY = groupYPositions[groupIndex];
+          let barIndex = 0;
+
+          return series.map((seriesName, seriesIndex) => {
+            const value = data[groupIndex][seriesIndex];
+
+            const barWidth = value * xScale;
+            const x = 0;
+            const y = groupY + barIndex * (barHeight + barSpacing);
+            barIndex++;
+
+            const colors = ['var(--color-scale-yellow-3)', 'var(--color-scale-orange-4)'];
+            const color = colors[seriesIndex % colors.length];
+
+            return (
+              <rect
+                key={`${groupIndex}-${seriesIndex}`}
+                x={x}
+                y={y}
+                width={barWidth}
+                height={barHeight}
+                fill={color}
+                rx='2'
+                style={{
+                  transition: 'opacity 0.2s',
+                  cursor: 'pointer'
+                }}
+                onMouseEnter={e => e.currentTarget.style.opacity = '0.8'}
+                onMouseLeave={e => e.currentTarget.style.opacity = '1'}
+              >
+                <title>{`${seriesName}: ${formatDuration(value)}`}</title>
+              </rect>
+            );
+          });
+        })}
+
+        {groups.map((group, groupIndex) => {
+          const groupY = groupYPositions[groupIndex];
+          const actualBars = barsPerGroup[groupIndex];
+          const groupHeight = actualBars * barHeight + (actualBars - 1) * barSpacing;
+          const labelY = groupY + groupHeight / 2;
+
+          return (
+            <text
+              key={groupIndex}
+              x={-10}
+              y={labelY}
+              textAnchor='end'
+              dominantBaseline='middle'
+              fontSize='12'
+              fill='var(--color-fg-muted)'
+            >
+              {group}
+            </text>
+          );
+        })}
+
+        <line
+          x1={0}
+          y1={0}
+          x2={0}
+          y2={contentHeight}
+          stroke='var(--color-fg-muted)'
+          strokeWidth='1'
+        />
+
+        <line
+          x1={0}
+          y1={contentHeight}
+          x2={chartWidth}
+          y2={contentHeight}
+          stroke='var(--color-fg-muted)'
+          strokeWidth='1'
+        />
+      </g>
+    </svg>
+  );
+};

packages/html-reporter/src/speedboard.tsx

@@ -19,9 +19,12 @@ import { LoadedReport } from './loadedReport';
 import { TestFileView } from './testFileView';
 import * as icons from './icons';
 import { TestCaseSummary } from './types';
+import { AutoChip } from './chip';
+import { GroupedBarChart } from './barchart';
 
 export function Speedboard({ report, tests }: { report: LoadedReport, tests: TestCaseSummary[] }) {
   return <>
+    <Shards report={report} />
     <SlowestTests report={report} tests={tests} />
   </>;
 }
@@ -46,3 +49,39 @@ export function SlowestTests({ report, tests }: { report: LoadedReport, tests: T
     }
   />;
 }
+
+export function Shards({ report }: { report: LoadedReport }) {
+  const shards = report.json().shards;
+  if (shards.length === 0)
+    return null;
+
+  let clash = false;
+  const bots: Record<string, number[]> = {};
+  for (const shard of shards) {
+    let botName = shard.botName || '';
+    if (botName.startsWith('@'))
+      botName = botName.slice(1);
+    bots[botName] ??= [];
+    const shardIndex = Math.max(shard.shardIndex - 1, 0);
+    if (bots[botName][shardIndex] !== undefined)
+      clash = true;
+    bots[botName][shardIndex] = shard.duration;
+  }
+
+  const maxSeries = Math.max(...Object.values(bots).map(shardDurations => shardDurations.length));
+
+  return <AutoChip header='Shard Duration'>
+    <GroupedBarChart
+      data={Object.values(bots)}
+      groups={Object.keys(bots)}
+      series={Array.from({ length: maxSeries }).map((_, i) => `Shard ${i + 1}`)}
+    />
+    {clash && <div style={{ marginTop: 8 }}>
+      <icons.warning />
+      Some shards could not be differentiated because of missing bot names.
+      Please refer to <a href='https://playwright.dev/docs/test-sharding#merging-reports-from-multiple-environments' target='_blank' rel='noopener noreferrer'>
+        the docs
+      </a> on how to fix this.
+    </div>}
+  </AutoChip>;
+}

packages/html-reporter/src/types.d.ts

@@ -49,6 +49,12 @@ export type HTMLReport = {
   projectNames: string[];
   startTime: number;
   duration: number;
+  shards: {
+    shardIndex: number;
+    botName?: string;
+    startTime: number;
+    duration: number;
+  }[];
   errors: string[];  // Top-level errors that are not attributed to any test.
   options: HTMLReportOptions;
 };

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -130,6 +130,12 @@ export type JsonFullResult = {
   status: reporterTypes.FullResult['status'];
   startTime: number;
   duration: number;
+  shards?: {
+    shardIndex: number;
+    botName?: string;
+    startTime: number;
+    duration: number;
+  }[];
 };
 
 export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent
@@ -452,6 +458,12 @@ export class TeleReporterReceiver {
       status: result.status,
       startTime: new Date(result.startTime),
       duration: result.duration,
+      shards: result.shards?.map(s => ({
+        shardIndex: s.shardIndex,
+        botName: s.botName,
+        startTime: new Date(s.startTime),
+        duration: s.duration,
+      })) ?? [],
     });
   }
 

packages/playwright/src/reporters/html.ts

@@ -298,6 +298,12 @@ class HtmlBuilder {
       stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },
       errors: topLevelErrors.map(error => formatError(internalScreen, error).message),
       options: this._options,
+      shards: result.shards.map(s => ({
+        shardIndex: s.shardIndex,
+        botName: s.botName,
+        startTime: s.startTime.getTime(),
+        duration: s.duration,
+      }))
     };
     htmlReport.files.sort((f1, f2) => {
       const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;

packages/playwright/src/reporters/internalReporter.ts

@@ -88,6 +88,7 @@ export class InternalReporter implements ReporterV2 {
       ...result,
       startTime: this._startTime!,
       duration: monotonicTime() - this._monotonicStartTime!,
+      shards: [],
     });
   }
 

packages/playwright/src/reporters/merge.ts

@@ -188,7 +188,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
 
   const configureEvents: JsonOnConfigureEvent[] = [];
   const projectEvents: JsonOnProjectEvent[] = [];
-  const endEvents: JsonOnEndEvent[] = [];
+  const endEvents: [JsonOnEndEvent, BlobReportMetadata, tags: string[]][] = [];
 
   const blobs = await extractAndParseReports(dir, shardReportFiles, internalizer, printStatus);
   // Sort by (report name; shard; file name), so that salt generation below is deterministic when:
@@ -235,7 +235,7 @@ async function mergeEvents(dir: string, shardReportFiles: string[], stringPool:
       } else if (event.method === 'onProject') {
         projectEvents.push(event);
       } else if (event.method === 'onEnd') {
-        endEvents.push(event);
+        endEvents.push([event, metadata, tags]);
       }
     }
 
@@ -323,12 +323,13 @@ function mergeConfigs(to: JsonConfig, from: JsonConfig): JsonConfig {
   };
 }
 
-function mergeEndEvents(endEvents: JsonOnEndEvent[]): JsonEvent {
+function mergeEndEvents(endEvents: [JsonOnEndEvent, BlobReportMetadata, tags: string[]][]): JsonEvent {
   let startTime = endEvents.length ? 10000000000000 : Date.now();
   let status: JsonFullResult['status'] = 'passed';
   let endTime: number = 0;
+  const shards: JsonFullResult['shards'] = [];
 
-  for (const event of endEvents) {
+  for (const [event, metadata, tags] of endEvents) {
     const shardResult = event.params.result;
     if (shardResult.status === 'failed')
       status = 'failed';
@@ -338,11 +339,19 @@ function mergeEndEvents(endEvents: JsonOnEndEvent[]): JsonEvent {
       status = 'interrupted';
     startTime = Math.min(startTime, shardResult.startTime);
     endTime = Math.max(endTime, shardResult.startTime + shardResult.duration);
+
+    shards.push({
+      shardIndex: metadata.shard?.current ?? -1,
+      botName: metadata.name ?? tags.join('-'),
+      startTime: shardResult.startTime,
+      duration: shardResult.duration,
+    });
   }
   const result: JsonFullResult = {
     status,
     startTime,
     duration: endTime - startTime,
+    shards,
   };
   return {
     method: 'onEnd',