feat: implement an benchmark script (#118)

* feat: implement an benchmark script

* fix some import paths

* switch to git url

* format & and end to end test

* update comment

* switch to npm-release and add override test

* udpate comments

* remove packages

* remove un needed

* types

Author: 0hmX

benchmark.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+LIMIT=""
+
+default_concurrency() {
+  getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 4
+}
+
+CONCURRENT="${BENCHMARK_CONCURRENCY:-$(default_concurrency)}"
+
+print_help() {
+  cat <<'EOF'
+Usage:
+  ./benchmark.sh [--concurrent N] [--limit N]
+
+Options:
+  --concurrent N  Number of Bun worker processes to run, or "auto"
+  --limit N       Run only the first N dataset scenarios
+  -h, --help      Show this help
+
+Examples:
+  ./benchmark.sh
+  ./benchmark.sh --concurrent 8
+  ./benchmark.sh --limit 25
+  ./benchmark.sh --concurrent auto --limit 50
+EOF
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -h|--help)
+      print_help
+      exit 0
+      ;;
+    --concurrent)
+      CONCURRENT="${2:-}"
+      if [ "$CONCURRENT" = "auto" ]; then
+        CONCURRENT="$(default_concurrency)"
+      fi
+      shift 2
+      ;;
+    --limit)
+      LIMIT="${2:-}"
+      shift 2
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Run ./benchmark.sh --help for usage"
+      exit 1
+      ;;
+  esac
+done
+
+CMD=(bun "scripts/benchmark/index.ts" "--concurrent" "$CONCURRENT")
+
+if [ -n "$LIMIT" ]; then
+  CMD+=("--limit" "$LIMIT")
+fi
+
+"${CMD[@]}"

global.d.ts

@@ -2,3 +2,10 @@ declare module "*.json" {
   const value: any
   export default value
 }
+
+declare module "@tscircuit/autorouting-dataset-01" {
+  import type { SimpleRouteJson } from "./lib/types/srj-types"
+
+  const dataset: Record<string, SimpleRouteJson>
+  export = dataset
+}

package.json

@@ -18,6 +18,8 @@
   "devDependencies": {
     "@biomejs/biome": "^2.3.5",
     "@react-hook/resize-observer": "^2.0.2",
+    "@tscircuit/autorouting-dataset-01": "^1.0.32",
+    "@tscircuit/capacity-autorouter": "https://npm-releases.tscircuit.com/npm-tarballs/25314012755-1/tscircuit-capacity-autorouter-0.0.489.tgz",
     "@tscircuit/math-utils": "^0.0.29",
     "@tscircuit/solver-utils": "^0.0.16",
     "@types/bun": "latest",

scripts/benchmark/benchmark-child/constants.ts

@@ -0,0 +1,6 @@
+/**
+ * Marks the first phase after port-point pathing completes.
+ * The benchmark uses this as the stop condition for partial solver runs.
+ */
+export const NEXT_PHASE_AFTER_PORT_POINT_PATHING =
+  "uniformPortDistributionSolver"

scripts/benchmark/benchmark-child/createSolver.ts

@@ -0,0 +1,34 @@
+import { RectDiffPipeline } from "../../../lib/RectDiffPipeline"
+import type { WorkerTaskMessage } from "../benchmark-types"
+import { getAutorouterPipeline4ModulePath } from "./getAutorouterPipeline4ModulePath"
+import { importRuntimeModule } from "./importRuntimeModule"
+import type { Pipeline4Constructor } from "./types"
+
+/**
+ * Creates the benchmark solver with the repo-local RectDiff pipeline override.
+ * The benchmark imports Pipeline 4 from the published autorouter bundle while
+ * forcing it to instantiate this repo's RectDiff implementation by default.
+ */
+export const createSolver = async (
+  scenario: WorkerTaskMessage["task"]["scenario"],
+  RectDiffPipelineClass: typeof RectDiffPipeline = RectDiffPipeline,
+) => {
+  // Load Pipeline 4 from the installed capacity-autorouter dist bundle, then
+  // inject the RectDiff class explicitly through its runtime override hook.
+  const solverModule = await importRuntimeModule(
+    getAutorouterPipeline4ModulePath(),
+  )
+
+  // TODO: Replace this cast once capacity-autorouter exposes a typed
+  // Pipeline 4 override interface. As of this repo state, the installed
+  // Pipeline 4 constructor only exposes CapacityMeshSolverOptions and does
+  // not type the benchmark's overrides.RectDiffPipelineClass injection.
+  const AutoroutingPipelineSolver4 = (solverModule as any)
+    .AutoroutingPipelineSolver4 as Pipeline4Constructor
+
+  return new AutoroutingPipelineSolver4(scenario, {
+    overrides: {
+      RectDiffPipelineClass,
+    },
+  })
+}

scripts/benchmark/benchmark-child/getAutorouterPipeline4ModulePath.ts

@@ -0,0 +1,7 @@
+/**
+ * Resolves the published capacity-autorouter dist entry for benchmarking.
+ * The benchmark relies on the bundled runtime export instead of package-
+ * internal source paths.
+ */
+export const getAutorouterPipeline4ModulePath = () =>
+  import.meta.resolve("@tscircuit/capacity-autorouter")

scripts/benchmark/benchmark-child/handleWorkerTaskLine.ts

@@ -0,0 +1,64 @@
+import type { WorkerResult, WorkerTaskMessage } from "../benchmark-types"
+import { NEXT_PHASE_AFTER_PORT_POINT_PATHING } from "./constants"
+import { createSolver } from "./createSolver"
+import { writeWorkerResultMessage } from "./writeWorkerResultMessage"
+
+/**
+ * Handles one benchmark request line from the parent process.
+ * Each line must contain a single JSON-encoded worker task message.
+ */
+export const handleWorkerTaskLine = async (line: string) => {
+  if (!line.trim()) {
+    return
+  }
+
+  let message: WorkerTaskMessage
+  try {
+    message = JSON.parse(line) as WorkerTaskMessage
+  } catch (error) {
+    const result: WorkerResult = {
+      scenarioName: "unknown",
+      elapsedTimeMs: 0,
+      didSolve: false,
+      didTimeout: false,
+      error: `Invalid worker message: ${String(error)}`,
+    }
+    writeWorkerResultMessage(result)
+    return
+  }
+
+  const startedAt = performance.now()
+
+  try {
+    const solver = await createSolver(message.task.scenario)
+
+    while (
+      !solver.failed &&
+      solver.getCurrentPhase() !== NEXT_PHASE_AFTER_PORT_POINT_PATHING
+    ) {
+      solver.step()
+    }
+
+    const result: WorkerResult = {
+      scenarioName: message.task.scenarioName,
+      elapsedTimeMs: Math.max(0, Math.round(performance.now() - startedAt)),
+      didSolve:
+        !solver.failed &&
+        solver.getCurrentPhase() === NEXT_PHASE_AFTER_PORT_POINT_PATHING &&
+        solver.portPointPathingSolver !== undefined,
+      didTimeout: false,
+      error: solver.failed ? solver.error : null,
+    }
+
+    writeWorkerResultMessage(result)
+  } catch (error) {
+    const result: WorkerResult = {
+      scenarioName: message.task.scenarioName,
+      elapsedTimeMs: Math.max(0, Math.round(performance.now() - startedAt)),
+      didSolve: false,
+      didTimeout: false,
+      error: String(error),
+    }
+    writeWorkerResultMessage(result)
+  }
+}

scripts/benchmark/benchmark-child/importRuntimeModule.ts

@@ -0,0 +1,8 @@
+/**
+ * Imports a runtime-resolved module path.
+ * This keeps the benchmark loader on a runtime-only import path.
+ */
+export const importRuntimeModule = (modulePath: string) =>
+  new Function("modulePath", "return import(modulePath)")(
+    modulePath,
+  ) as Promise<unknown>

scripts/benchmark/benchmark-child/types.ts

@@ -0,0 +1,20 @@
+/**
+ * Describes the subset of solver state the benchmark worker needs.
+ * This keeps the child process decoupled from internal package typings.
+ */
+export type BenchmarkWorkerSolver = {
+  failed: boolean
+  error: string | null
+  portPointPathingSolver?: unknown
+  getCurrentPhase(): string
+  step(): void
+}
+
+/**
+ * Represents the Pipeline 4 constructor shape used by the benchmark.
+ * It allows the worker to inject the local RectDiff override.
+ */
+export type Pipeline4Constructor = new (
+  srj: unknown,
+  opts?: Record<string, unknown>,
+) => BenchmarkWorkerSolver

scripts/benchmark/benchmark-child/writeWorkerResultMessage.ts

@@ -0,0 +1,10 @@
+import type { WorkerResult, WorkerResultMessage } from "../benchmark-types"
+
+/**
+ * Writes a worker result message as a single JSON line.
+ * The parent benchmark process reads these messages over stdout.
+ */
+export const writeWorkerResultMessage = (result: WorkerResult) => {
+  const response: WorkerResultMessage = { result }
+  process.stdout.write(`${JSON.stringify(response)}\n`)
+}