feat: custom sharding weights

docs/src/test-api/class-fullconfig.md

@@ -115,6 +115,7 @@ Run agents to generate the code for [`method: Page.perform`] and similar.
 - type: <[null]|[Object]>
   - `total` <[int]> The total number of shards.
   - `current` <[int]> The index of the shard to execute, one-based.
+  - `weights` ?<[Array]<[int]>> The shard weights.
 
 See [`property: TestConfig.shard`].
 

docs/src/test-api/class-testconfig.md

@@ -525,6 +525,7 @@ Run agents to generate the code for [`method: Page.perform`] and similar.
 - type: ?<[null]|[Object]>
   - `current` <[int]> The index of the shard to execute, one-based.
   - `total` <[int]> The total number of shards.
+  - `weights` ?<[Array]<[int]>> The shard weights.
 
 Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
 

docs/src/test-sharding-js.md

@@ -45,6 +45,12 @@ Without the fullyParallel setting, Playwright Test defaults to file-level granul
 - **Without** `fullyParallel`: Tests are split at the file level, so to balance the shards, it's important to keep your test files small and evenly sized.
 - To ensure the most effective use of sharding, especially in CI environments, it is recommended to use `fullyParallel: true` when aiming for balanced distribution across shards. Otherwise, you may need to manually organize your test files to avoid imbalances.
 
+## Rebalancing Shards
+
+```
+npx playwright test --shard=1/4,26/24/25/25
+```
+
 ## Merging reports from multiple shards
 
 In the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.

packages/playwright/src/common/ipc.ts

@@ -36,7 +36,7 @@ export type ConfigCLIOverrides = {
   retries?: number;
   reporter?: ReporterDescription[];
   additionalReporters?: ReporterDescription[];
-  shard?: { current: number, total: number };
+  shard?: { current: number, total: number, weights?: number[] };
   timeout?: number;
   tsconfig?: string;
   ignoreSnapshots?: boolean;

packages/playwright/src/program.ts

@@ -348,16 +348,22 @@ function resolveShardOption(shard?: string): ConfigCLIOverrides['shard'] {
   if (!shard)
     return undefined;
 
-  const shardPair = shard.split('/');
+  const shardPair = shard.split(',');
+  if (shardPair.length > 2) {
+    throw new Error(
+        `--shard "${shard}", expected format is "current/all,weight1/.../weightN", 1-based, for example "3/5,18/19/20/21/22".`,
+    );
+  }
 
-  if (shardPair.length !== 2) {
+  const indexPair = shardPair[0].split('/');
+  if (indexPair.length !== 2) {
     throw new Error(
         `--shard "${shard}", expected format is "current/all", 1-based, for example "3/5".`,
     );
   }
 
-  const current = parseInt(shardPair[0], 10);
-  const total = parseInt(shardPair[1], 10);
+  const current = parseInt(indexPair[0], 10);
+  const total = parseInt(indexPair[1], 10);
 
   if (isNaN(total) || total < 1)
     throw new Error(`--shard "${shard}" total must be a positive number`);
@@ -369,7 +375,23 @@ function resolveShardOption(shard?: string): ConfigCLIOverrides['shard'] {
     );
   }
 
-  return { current, total };
+  if (!shardPair[1])
+    return { current, total };
+
+
+  const weights = shardPair[1].split('/').map(w => parseInt(w, 10));
+  if (weights.length !== total) {
+    throw new Error(
+        `--shard "${shard}" weights count must match the shard total of ${total}`,
+    );
+  }
+  if (weights.some(w => isNaN(w) || w < 0)) {
+    throw new Error(
+        `--shard "${shard}" weights must be non-negative numbers`,
+    );
+  }
+
+  return { current, total, weights };
 }
 
 function resolveReporter(id: string) {

packages/playwright/src/runner/testGroups.ts

@@ -130,7 +130,9 @@ export function createTestGroups(projectSuite: Suite, expectedParallelism: numbe
   return result;
 }
 
-export function filterForShard(shard: { total: number, current: number }, testGroups: TestGroup[]): Set<TestGroup> {
+export function filterForShard(shard: { total: number, current: number, weights?: number[] }, testGroups: TestGroup[]): Set<TestGroup> {
+  const weights = shard.weights ?? Array.from({ length: shard.total }, () => 1);
+  const totalWeight = weights.reduce((a, b) => a + b, 0);
   // Note that sharding works based on test groups.
   // This means parallel files will be sharded by single tests,
   // while non-parallel files will be sharded by the whole file.
@@ -143,13 +145,17 @@ export function filterForShard(shard: { total: number, current: number }, testGr
     shardableTotal += group.tests.length;
 
   // Each shard gets some tests.
-  const shardSize = Math.floor(shardableTotal / shard.total);
-  // First few shards get one more test each.
-  const extraOne = shardableTotal - shardSize * shard.total;
+  const shardSizes = weights.map(w => Math.floor(w * shardableTotal / totalWeight));
+  const remainder = shardableTotal - shardSizes.reduce((a, b) => a + b, 0);
+  for (let i = 0; i < remainder; i++) {
+    // First few shards get one more test each.
+    shardSizes[i % shardSizes.length]++;
+  }
 
-  const currentShard = shard.current - 1; // Make it zero-based for calculations.
-  const from = shardSize * currentShard + Math.min(extraOne, currentShard);
-  const to = from + shardSize + (currentShard < extraOne ? 1 : 0);
+  let from = 0;
+  for (let i = 0; i < shard.current - 1; i++)
+    from += shardSizes[i];
+  const to = from + shardSizes[shard.current - 1];
 
   let current = 0;
   const result = new Set<TestGroup>();

packages/playwright/types/test.d.ts

@@ -1644,6 +1644,11 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
      * The total number of shards.
      */
     total: number;
+
+    /**
+     * The shard weights.
+     */
+    weights?: Array<number>;
   };
 
   /**
@@ -2092,6 +2097,11 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
      * The index of the shard to execute, one-based.
      */
     current: number;
+
+    /**
+     * The shard weights.
+     */
+    weights?: Array<number>;
   };
 
   /**

tests/playwright-test/shard.spec.ts

@@ -16,6 +16,8 @@
 
 import { test, expect } from './playwright-test-fixtures';
 
+test.describe.configure({ mode: 'parallel' });
+
 const tests = {
   'a1.spec.ts': `
     import { test } from '@playwright/test';
@@ -330,3 +332,28 @@ test('should shard tests with beforeAll based on shards total instead of workers
     expect(result.outputLines).toEqual(['beforeAll', 'test7']);
   }
 });
+
+test('should respect custom shard weights', async ({ runInlineTest }) => {
+  await test.step('shard 1', async () => {
+    const result = await runInlineTest(tests, { shard: '1/2,40/60', workers: 1 });
+    expect.soft(result.exitCode).toBe(0);
+    expect.soft(result.outputLines).toEqual([
+      'a1-test1-done',
+      'a1-test2-done',
+      'a1-test3-done',
+      'a1-test4-done',
+    ]);
+  });
+  await test.step('shard 2', async () => {
+    const result = await runInlineTest(tests, { shard: '2/2,40/60', workers: 1 });
+    expect.soft(result.exitCode).toBe(0);
+    expect.soft(result.outputLines).toEqual([
+      'a2-test1-done',
+      'a2-test2-done',
+      'a3-test1-done',
+      'a3-test2-done',
+      'a4-test1-done',
+      'a4-test2-done',
+    ]);
+  });
+});