Add walltime support for vitest plugin

.github/workflows/codspeed.yml

@@ -61,5 +61,6 @@ jobs:
         with:
           # Only tinybench supports walltime for now
           run: |
-            pnpm moon run tinybench-plugin:bench
+            pnpm moon run --concurrency 1 :bench
             pnpm --workspace-concurrency 1 -r bench-tinybench
+            pnpm --workspace-concurrency 1 -r bench-vitest

examples/with-typescript-esm/package.json

@@ -16,6 +16,6 @@
     "esbuild-register": "^3.4.2",
     "tinybench": "^4.0.1",
     "typescript": "^5.1.3",
-    "vitest": "^1.2.2"
+    "vitest": "^3.2.4"
   }
 }

examples/with-typescript-esm/src/fibonacci.bench.ts

@@ -1,8 +1,8 @@
 import { bench, describe } from "vitest";
-import { iterativeFibonacci } from "./fibonacci";
+import { recursiveFibonacci } from "./fibonacci";
 
-describe("iterativeFibonacci", () => {
-  bench("fibo 10", () => {
-    iterativeFibonacci(10);
+describe("recursiveFibonacci", () => {
+  bench("fibo 30", () => {
+    recursiveFibonacci(30);
   });
 });

packages/tinybench-plugin/package.json

@@ -28,7 +28,7 @@
     "@types/stack-trace": "^0.0.30",
     "esbuild-register": "^3.4.2",
     "tinybench": "^4.0.1",
-    "vitest": "^1.2.2"
+    "vitest": "^3.2.4"
   },
   "dependencies": {
     "@codspeed/core": "workspace:^4.0.1",

packages/vitest-plugin/benches/hooks.bench.ts

@@ -8,85 +8,30 @@ import {
   expect,
 } from "vitest";
 
-let count = -1;
-
-beforeAll(() => {
-  count += 1;
-});
-
-beforeEach(() => {
-  count += 1;
-});
-
-// the count is multiplied by 2 because the bench function is called twice with codspeed (once for the optimization and once for the actual measurement)
-bench("one", () => {
-  expect(count).toBe(1 * 2);
-});
-
-describe("level1", () => {
-  bench("two", () => {
-    expect(count).toBe(2 * 2);
-  });
-
-  bench("three", () => {
-    expect(count).toBe(3 * 2);
-  });
-
-  describe("level 2", () => {
-    beforeEach(() => {
-      count += 1;
-    });
-
-    bench("five", () => {
-      expect(count).toBe(5 * 2);
-    });
-
-    describe("level 3", () => {
-      bench("seven", () => {
-        expect(count).toBe(7 * 2);
-      });
-    });
-  });
-
-  describe("level 2 bench nested beforeAll", () => {
-    beforeAll(() => {
-      count = 0;
-    });
-
-    bench("one", () => {
-      expect(count).toBe(1 * 2);
-    });
-  });
-
-  bench("two", () => {
-    expect(count).toBe(2 * 2);
-  });
-});
-
-describe("hooks cleanup", () => {
-  let cleanUpCount = 0;
+describe("hooks", () => {
+  let count = 0;
   describe("run", () => {
     beforeAll(() => {
-      cleanUpCount += 10;
+      count += 10;
     });
     beforeEach(() => {
-      cleanUpCount += 1;
+      count += 1;
     });
     afterEach(() => {
-      cleanUpCount -= 1;
+      count -= 1;
     });
     afterAll(() => {
-      cleanUpCount -= 10;
+      count -= 10;
     });
 
     bench("one", () => {
-      expect(cleanUpCount).toBe(11);
+      expect(count).toBe(11);
     });
     bench("two", () => {
-      expect(cleanUpCount).toBe(11);
+      expect(count).toBe(11);
     });
   });
   bench("end", () => {
-    expect(cleanUpCount).toBe(0);
+    expect(count).toBe(0);
   });
 });

packages/vitest-plugin/benches/timing.bench.ts

@@ -0,0 +1,39 @@
+import { bench, describe, type BenchOptions } from "vitest";
+
+const busySleep = (ms: number): void => {
+  const end = performance.now() + ms;
+  while (performance.now() < end) {
+    // Busy wait
+  }
+};
+
+const timingBenchOptions: BenchOptions = {
+  iterations: 5,
+  warmupIterations: 0,
+};
+
+describe("timing tests", () => {
+  bench(
+    "wait 1ms",
+    async () => {
+      busySleep(1);
+    },
+    timingBenchOptions
+  );
+
+  bench(
+    "wait 500ms",
+    async () => {
+      busySleep(500);
+    },
+    timingBenchOptions
+  );
+
+  bench(
+    "wait 1sec",
+    async () => {
+      busySleep(1_000);
+    },
+    timingBenchOptions
+  );
+});

packages/vitest-plugin/moon.yml

@@ -6,6 +6,8 @@ tasks:
     local: true
     options:
       cache: false
+    deps:
+      - build
 
   test:
     command: vitest --run

packages/vitest-plugin/package.json

@@ -31,13 +31,13 @@
     "@codspeed/core": "workspace:^4.0.1"
   },
   "peerDependencies": {
-    "vite": "^4.2.0 || ^5.0.0 || ^6.0.0",
-    "vitest": ">=1.2.2"
+    "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0",
+    "vitest": ">=3.2"
   },
   "devDependencies": {
     "@total-typescript/shoehorn": "^0.1.1",
     "execa": "^8.0.1",
     "vite": "^5.0.0",
-    "vitest": "^1.2.2"
+    "vitest": "^3.2.4"
   }
 }

packages/vitest-plugin/rollup.config.ts

@@ -21,8 +21,14 @@ export default defineConfig([
     external: ["@codspeed/core", /^vitest/],
   },
   {
-    input: "src/runner.ts",
-    output: { file: "dist/runner.mjs", format: "es" },
+    input: "src/instrumented.ts",
+    output: { file: "dist/instrumented.mjs", format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/],
+  },
+  {
+    input: "src/walltime.ts",
+    output: { file: "dist/walltime.mjs", format: "es" },
     plugins: jsPlugins(pkg.version),
     external: ["@codspeed/core", /^vitest/],
   },

packages/vitest-plugin/src/__tests__/index.test.ts

@@ -1,5 +1,5 @@
 import { fromPartial } from "@total-typescript/shoehorn";
-import { describe, expect, it, vi } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import codspeedPlugin from "../index";
 
 const coreMocks = vi.hoisted(() => {
@@ -25,6 +25,18 @@ vi.mock("@codspeed/core", async (importOriginal) => {
 console.warn = vi.fn();
 
 describe("codSpeedPlugin", () => {
+  beforeAll(() => {
+    // Set environment variables to trigger instrumented mode
+    process.env.CODSPEED_ENV = "1";
+    process.env.CODSPEED_RUNNER_MODE = "instrumentation";
+  });
+
+  afterAll(() => {
+    // Clean up environment variables
+    delete process.env.CODSPEED_ENV;
+    delete process.env.CODSPEED_RUNNER_MODE;
+  });
+
   it("should have a name", async () => {
     expect(resolvedCodSpeedPlugin.name).toBe("codspeed:vitest");
   });
@@ -96,7 +108,9 @@ describe("codSpeedPlugin", () => {
             ],
           },
         },
-        runner: expect.stringContaining("packages/vitest-plugin/src/runner.ts"),
+        runner: expect.stringContaining(
+          "packages/vitest-plugin/src/instrumented.ts"
+        ),
       },
     });
   });

packages/vitest-plugin/src/__tests__/runner.test.ts → packages/vitest-plugin/src/__tests__/instrumented.test.ts

@@ -1,7 +1,7 @@
 import { fromPartial } from "@total-typescript/shoehorn";
-import { describe, expect, it, Suite, vi } from "vitest";
+import { describe, expect, it, vi, type RunnerTestSuite } from "vitest";
 import { getBenchFn } from "vitest/suite";
-import CodSpeedRunner from "../runner";
+import { InstrumentedRunner as CodSpeedRunner } from "../instrumented";
 
 const coreMocks = vi.hoisted(() => {
   return {
@@ -37,40 +37,46 @@ vi.mock("vitest/suite", async (importOriginal) => {
 const mockedGetBenchFn = vi.mocked(getBenchFn);
 
 describe("CodSpeedRunner", () => {
-  it("should run the bench functions only twice", async () => {
+  it("should run the bench function", async () => {
     const benchFn = vi.fn();
     mockedGetBenchFn.mockReturnValue(benchFn);
 
     const runner = new CodSpeedRunner(fromPartial({}));
-    const suite = fromPartial<Suite>({
-      filepath: __filename,
+    const suite = fromPartial<RunnerTestSuite>({
+      file: { filepath: __filename },
       name: "test suite",
-      tasks: [{ mode: "run", meta: { benchmark: true }, name: "test bench" }],
+      tasks: [
+        {
+          type: "test",
+          mode: "run",
+          meta: { benchmark: true },
+          name: "test bench",
+        },
+      ],
     });
-    suite.tasks[0].suite = suite;
     await runner.runSuite(suite);
 
     // setup
     expect(coreMocks.setupCore).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts"
     );
 
     // run
     expect(coreMocks.mongoMeasurement.start).toHaveBeenCalledWith(
-      "packages/vitest-plugin/src/__tests__/runner.test.ts::test bench"
+      "packages/vitest-plugin/src/__tests__/instrumented.test.ts::test bench"
     );
     expect(coreMocks.Measurement.startInstrumentation).toHaveBeenCalledTimes(1);
     expect(benchFn).toHaveBeenCalledTimes(8);
     expect(coreMocks.Measurement.stopInstrumentation).toHaveBeenCalledTimes(1);
     expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] packages/vitest-plugin/src/__tests__/runner.test.ts::test bench done"
+      "[CodSpeed] packages/vitest-plugin/src/__tests__/instrumented.test.ts::test bench done"
     );
 
     // teardown
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts done"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts done"
     );
     expect(coreMocks.teardownCore).toHaveBeenCalledTimes(1);
   });
@@ -80,8 +86,8 @@ describe("CodSpeedRunner", () => {
     mockedGetBenchFn.mockReturnValue(benchFn);
 
     const runner = new CodSpeedRunner(fromPartial({}));
-    const rootSuite = fromPartial<Suite>({
-      filepath: __filename,
+    const rootSuite = fromPartial<RunnerTestSuite>({
+      file: { filepath: __filename },
       name: "test suite",
       tasks: [
         {
@@ -90,6 +96,7 @@ describe("CodSpeedRunner", () => {
           mode: "run",
           tasks: [
             {
+              type: "test",
               mode: "run",
               meta: { benchmark: true },
               name: "test bench",
@@ -98,33 +105,30 @@ describe("CodSpeedRunner", () => {
         },
       ],
     });
-    rootSuite.tasks[0].suite = rootSuite;
-    // @ts-expect-error type is not narrow enough, but it is fine
-    rootSuite.tasks[0].tasks[0].suite = rootSuite.tasks[0];
 
     await runner.runSuite(rootSuite);
 
     // setup
     expect(coreMocks.setupCore).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts"
     );
 
     // run
     expect(coreMocks.mongoMeasurement.start).toHaveBeenCalledWith(
-      "packages/vitest-plugin/src/__tests__/runner.test.ts::nested suite::test bench"
+      "packages/vitest-plugin/src/__tests__/instrumented.test.ts::nested suite::test bench"
     );
     expect(coreMocks.Measurement.startInstrumentation).toHaveBeenCalledTimes(1);
     expect(benchFn).toHaveBeenCalledTimes(8);
     expect(coreMocks.Measurement.stopInstrumentation).toHaveBeenCalledTimes(1);
     expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] packages/vitest-plugin/src/__tests__/runner.test.ts::nested suite::test bench done"
+      "[CodSpeed] packages/vitest-plugin/src/__tests__/instrumented.test.ts::nested suite::test bench done"
     );
 
     // teardown
     expect(console.log).toHaveBeenCalledWith(
-      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/runner.test.ts done"
+      "[CodSpeed] running suite packages/vitest-plugin/src/__tests__/instrumented.test.ts done"
     );
     expect(coreMocks.teardownCore).toHaveBeenCalledTimes(1);
   });