tscircuit · AnasSarkiz · Aug 5, 2026 · Aug 5, 2026 · Aug 6, 2026 · Aug 6, 2026
diff --git a/.github/scripts/benchmark-comment-diagnostics.cjs b/.github/scripts/benchmark-comment-diagnostics.cjs
@@ -0,0 +1,201 @@
+const refinementCountStatNames = [
+  "physicalPortalGroupCount",
+  "eligibleRouteCount",
+  "routesConsidered",
+  "candidateCount",
+  "acceptedCandidateCount",
+  "routesImproved",
+  "predictedViaDemandBefore",
+  "predictedViaDemandAfter",
+  "entryExitLayerChangesBefore",
+  "entryExitLayerChangesAfter",
+  "rejectedForRegionCostCount",
+  "rejectedForIntersectionRegressionCount",
+  "rejectedForPortConflictCount",
+  "rejectedForLockedAssignmentCount",
+  "rejectedForNoViaDemandImprovementCount",
+  "rejectedForNoEntryExitImprovementCount",
+  "touchedRegionCount",
+]
+
+const refinementTimingStatNames = [
+  "tinyHypergraphSolveMs",
+  "tinyHypergraphSectionOptimizationMs",
+  "portalLayerRefinementMs",
+  "uniformPortDistributionMs",
+  "highDensityRouteMs",
+  "highDensityForceImproveMs",
+  "highDensityRepairMs",
+  "stitchingMs",
+  "traceSimplificationMs",
+  "traceWidthMs",
+  "globalDrcMs",
+  "exactDrcMs",
+  "totalMs",
+]
+
+const getFiniteValues = (tests, statName) =>
+  tests.flatMap((test) => {
+    const value = test?.benchmarkStats?.[statName]
+    return typeof value === "number" && Number.isFinite(value) ? [value] : []
+  })
+
+const sumValues = (values) => values.reduce((total, value) => total + value, 0)
+
+const averageValues = (values) =>
+  values.length > 0 ? sumValues(values) / values.length : null
+
+const getRefinementCell = (test) => {
+  const stats = test?.benchmarkStats
+  if (!stats) return ""
+  return `eligible=${stats.eligibleRouteCount ?? 0}; accepted=${stats.acceptedCandidateCount ?? 0}; predicted=${stats.predictedViaDemandBefore ?? 0}→${stats.predictedViaDemandAfter ?? 0}`
+}
+
+const renderRefinementSummary = ({ report, formatAverage, formatTime }) => {
+  const tests = Array.isArray(report?.tests) ? report.tests : []
+  if (!tests.some((test) => test?.benchmarkStats)) return []
+  return [
+    "### Portal-layer refinement diagnostics",
+    "",
+    "| Field | Total |",
+    "| --- | ---: |",
+    ...refinementCountStatNames.map(
+      (statName) =>
+        `| ${statName} | ${formatAverage(sumValues(getFiniteValues(tests, statName)))} |`,
+    ),
+    "",
+    "| Timing field | Average |",
+    "| --- | ---: |",
+    ...refinementTimingStatNames.map(
+      (statName) =>
+        `| ${statName} | ${formatTime(averageValues(getFiniteValues(tests, statName)))} |`,
+    ),
+  ]
+}
+
+const renderPairedComparison = ({
+  mainReport,
+  prReport,
+  buildMainIndex,
+  formatAverage,
+  formatTime,
+}) => {
+  if (!Array.isArray(mainReport?.tests) || !Array.isArray(prReport?.tests)) {
+    return []
+  }
+
+  const mainIndex = buildMainIndex(mainReport)
+  const classifications = {
+    "baseline solved + PR solved": 0,
+    "baseline solved + PR failed": 0,
+    "baseline solved + PR timeout": 0,
+    "baseline failed + PR solved": 0,
+    "baseline timeout + PR solved": 0,
+    "both failed": 0,
+    "both timeout": 0,
+    "other status transitions": 0,
+    "unmatched PR samples": 0,
+  }
+  const pairedSolved = []
+
+  for (const prTest of prReport.tests) {
+    const mainTest = mainIndex.get(
+      `${prTest.solverName}::${prTest.scenarioName}`,
+    )
+    if (!mainTest) {
+      classifications["unmatched PR samples"] += 1
+      continue
+    }
+    if (mainTest.didSolve && prTest.didSolve) {
+      classifications["baseline solved + PR solved"] += 1
+      pairedSolved.push({ mainTest, prTest })
+    } else if (mainTest.didSolve && prTest.didTimeout) {
+      classifications["baseline solved + PR timeout"] += 1
+    } else if (mainTest.didSolve) {
+      classifications["baseline solved + PR failed"] += 1
+    } else if (mainTest.didTimeout && prTest.didSolve) {
+      classifications["baseline timeout + PR solved"] += 1
+    } else if (prTest.didSolve) {
+      classifications["baseline failed + PR solved"] += 1
+    } else if (mainTest.didTimeout && prTest.didTimeout) {
+      classifications["both timeout"] += 1
+    } else if (!mainTest.didTimeout && !prTest.didTimeout) {
+      classifications["both failed"] += 1
+    } else {
+      classifications["other status transitions"] += 1
+    }
+  }
+
+  const pairedMetric = (mainSelector, prSelector) => {
+    const values = pairedSolved.flatMap(({ mainTest, prTest }) => {
+      const mainValue = mainSelector(mainTest)
+      const prValue = prSelector(prTest)
+      return typeof mainValue === "number" &&
+        Number.isFinite(mainValue) &&
+        typeof prValue === "number" &&
+        Number.isFinite(prValue)
+        ? [{ mainValue, prValue }]
+        : []
+    })
+    if (values.length === 0) return { main: null, pr: null, delta: null }
+    const main = averageValues(values.map(({ mainValue }) => mainValue))
+    const pr = averageValues(values.map(({ prValue }) => prValue))
+    return { main, pr, delta: pr - main }
+  }
+
+  const finalVia = pairedMetric(
+    (test) => test.viaCount,
+    (test) => test.viaCount,
+  )
+  const drcErrors = pairedMetric(
+    (test) => test.drcErrorCount,
+    (test) => test.drcErrorCount,
+  )
+  const runtime = pairedMetric(
+    (test) => test.elapsedTimeMs,
+    (test) => test.elapsedTimeMs,
+  )
+  const highDensityVia = pairedMetric(
+    (test) => test.highDensityViaCount,
+    (test) => test.highDensityViaCount,
+  )
+  const acceptedCandidateCount = sumValues(
+    getFiniteValues(prReport.tests, "acceptedCandidateCount"),
+  )
+  const predictedBefore = sumValues(
+    getFiniteValues(prReport.tests, "predictedViaDemandBefore"),
+  )
+  const predictedAfter = sumValues(
+    getFiniteValues(prReport.tests, "predictedViaDemandAfter"),
+  )
+  const formatDelta = (value) =>
+    typeof value === "number" && Number.isFinite(value)
+      ? `${value > 0 ? "+" : ""}${value.toFixed(2)}`
+      : "n/a"
+
+  return [
+    "### Paired comparison",
+    "",
+    "| Classification | Count |",
+    "| --- | ---: |",
+    ...Object.entries(classifications).map(
+      ([label, count]) => `| ${label} | ${count} |`,
+    ),
+    "",
+    "| Paired solved metric | Baseline | PR | Delta |",
+    "| --- | ---: | ---: | ---: |",
+    `| final via count | ${formatAverage(finalVia.main)} | ${formatAverage(finalVia.pr)} | ${formatDelta(finalVia.delta)} |`,
+    `| final relaxed DRC error count | ${formatAverage(drcErrors.main)} | ${formatAverage(drcErrors.pr)} | ${formatDelta(drcErrors.delta)} |`,
+    `| runtime | ${formatTime(runtime.main)} | ${formatTime(runtime.pr)} | ${formatTime(runtime.delta)} |`,
+    `| high-density via count | ${formatAverage(highDensityVia.main)} | ${formatAverage(highDensityVia.pr)} | ${formatDelta(highDensityVia.delta)} |`,
+    "",
+    `PR refinement acceptedCandidateCount: ${formatAverage(acceptedCandidateCount)}`,
+    `PR predictedViaDemandBefore → predictedViaDemandAfter: ${formatAverage(predictedBefore)} → ${formatAverage(predictedAfter)} (${formatDelta(predictedAfter - predictedBefore)})`,
+  ]
+}
+
+module.exports = {
+  getRefinementCell,
+  renderPairedComparison,
+  renderRefinementSummary,
+}
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -479,6 +479,11 @@ jobs:
           github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
           script: |
             const fs = require('node:fs')
+            const {
+              getRefinementCell,
+              renderPairedComparison,
+              renderRefinementSummary,
+            } = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/benchmark-comment-diagnostics.cjs`)
             const maxLength = 60000
             const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s
             const formatTime = (timeMs) => {
@@ -610,7 +615,8 @@ jobs:
                 test.relaxedDrcPassed &&
                 mainTest.didSolve &&
                 mainTest.relaxedDrcPassed &&
-                sameSolverViaUnchanged
+                sameSolverViaUnchanged &&
+                Number(test?.benchmarkStats?.acceptedCandidateCount ?? 0) === 0
               )
             }
             const renderTests = (report, options = {}) => {
@@ -628,12 +634,12 @@ jobs:
               const omittedCount = report.tests.length - tests.length
               const header = includeDelta
                 ? [
-                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Error | Delta |',
-                    '| --- | --- | --- | ---: | --- | --- | --- | --- |',
+                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Refinement | Error | Delta |',
+                    '| --- | --- | --- | ---: | --- | --- | --- | --- | --- |',
                   ]
                 : [
-                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Error |',
-                    '| --- | --- | --- | ---: | --- | --- | --- |',
+                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Refinement | Error |',
+                    '| --- | --- | --- | ---: | --- | --- | --- | --- |',
                   ]
               return [
                 ...(omittedCount > 0
@@ -642,8 +648,8 @@ jobs:
                 ...header,
                 ...tests.map((test) =>
                   includeDelta
-                    ? `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getErrorCell(test)} | ${getDelta(test, mainIndex)} |`
-                    : `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getErrorCell(test)} |`,
+                    ? `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getRefinementCell(test)} | ${getErrorCell(test)} | ${getDelta(test, mainIndex)} |`
+                    : `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getRefinementCell(test)} | ${getErrorCell(test)} |`,
                 ),
               ]
             }
@@ -725,6 +731,20 @@ jobs:
                 omitSummary: true,
               }),
               '',
+              ...renderPairedComparison({
+                mainReport: mainDatasetReport,
+                prReport,
+                buildMainIndex,
+                formatAverage,
+                formatTime,
+              }),
+              '',
+              ...renderRefinementSummary({
+                report: prReport,
+                formatAverage,
+                formatTime,
+              }),
+              '',
               ...renderReport(prReport, prText ?? '(benchmark results were not produced)', {
                 detailsLabel: 'PR run details',
                 includeDelta: true,
@@ -962,6 +982,11 @@ jobs:
           github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
           script: |
             const fs = require('node:fs')
+            const {
+              getRefinementCell,
+              renderPairedComparison,
+              renderRefinementSummary,
+            } = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/benchmark-comment-diagnostics.cjs`)
             const maxLength = 60000
             const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s
             const formatTime = (timeMs) => {
@@ -1093,7 +1118,8 @@ jobs:
                 test.relaxedDrcPassed &&
                 mainTest.didSolve &&
                 mainTest.relaxedDrcPassed &&
-                sameSolverViaUnchanged
+                sameSolverViaUnchanged &&
+                Number(test?.benchmarkStats?.acceptedCandidateCount ?? 0) === 0
               )
             }
             const renderTests = (report, options = {}) => {
@@ -1111,12 +1137,12 @@ jobs:
               const omittedCount = report.tests.length - tests.length
               const header = includeDelta
                 ? [
-                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Error | Delta |',
-                    '| --- | --- | --- | ---: | --- | --- | --- | --- |',
+                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Refinement | Error | Delta |',
+                    '| --- | --- | --- | ---: | --- | --- | --- | --- | --- |',
                   ]
                 : [
-                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Error |',
-                    '| --- | --- | --- | ---: | --- | --- | --- |',
+                    '| Solver | Sample | Status | Via | Time | Relaxed DRC | Refinement | Error |',
+                    '| --- | --- | --- | ---: | --- | --- | --- | --- |',
                   ]
               return [
                 ...(omittedCount > 0
@@ -1125,8 +1151,8 @@ jobs:
                 ...header,
                 ...tests.map((test) =>
                   includeDelta
-                    ? `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getErrorCell(test)} | ${getDelta(test, mainIndex)} |`
-                    : `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getErrorCell(test)} |`,
+                    ? `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getRefinementCell(test)} | ${getErrorCell(test)} | ${getDelta(test, mainIndex)} |`
+                    : `| ${formatSolverName(test.solverName)} | ${getSampleLabel(test)} | ${getStatus(test)} | ${getViaCell(test, viaCellIndex, report.datasetName)} | ${formatTime(test.elapsedTimeMs)} | ${getDrc(test)} | ${getRefinementCell(test)} | ${getErrorCell(test)} |`,
                 ),
               ]
             }
@@ -1335,6 +1361,20 @@ jobs:
                   )
                 : []),
               '',
+              ...renderPairedComparison({
+                mainReport: mainDatasetReport,
+                prReport,
+                buildMainIndex,
+                formatAverage,
+                formatTime,
+              }),
+              '',
+              ...renderRefinementSummary({
+                report: prReport,
+                formatAverage,
+                formatTime,
+              }),
+              '',
               ...renderReport(prReport, prText ?? '(benchmark results were not produced)', {
                 detailsLabel: 'PR run details',
                 includeDelta: true,

diff --git a/...ipelines/AutoroutingPipeline4_TinyHypergraph/AutoroutingPipelineSolver4_TinyHypergraph.ts b/...ipelines/AutoroutingPipeline4_TinyHypergraph/AutoroutingPipelineSolver4_TinyHypergraph.ts
@@ -35,6 +35,7 @@ import {
 import { getPresuppliedTraceVisualization } from "lib/utils/getPresuppliedTraceVisualization"
 import { calculateOptimalCapacityDepth } from "lib/utils/getTunedTotalCapacity1"
 import { getViaDimensions } from "lib/utils/getViaDimensions"
+import { getAutoroutingPipelineBenchmarkStats } from "lib/utils/getAutoroutingPipelineBenchmarkStats"
 import { AvailableSegmentPointSolver } from "../../solvers/AvailableSegmentPointSolver/AvailableSegmentPointSolver"
 import { BaseSolver } from "../../solvers/BaseSolver"
 import { CapacityMeshEdgeSolver } from "../../solvers/CapacityMeshSolver/CapacityMeshEdgeSolver"
@@ -505,6 +506,10 @@ export class AutoroutingPipelineSolver4_TinyHypergraph extends BaseSolver {
         this.timeSpentOnPhase[pipelineStepDef.solverName] =
           this.endTimeOfPhase[pipelineStepDef.solverName] -
           this.startTimeOfPhase[pipelineStepDef.solverName]
+        this.stats = getAutoroutingPipelineBenchmarkStats({
+          timeSpentOnPhase: this.timeSpentOnPhase,
+          portalLayerRefinementStats: this.portPointPathingSolver?.stats,
+        })
         pipelineStepDef.onSolved?.(this)
         this.activeSubSolver = null
         this.currentPipelineStepIndex++

diff --git a/...router-pipelines/AutoroutingPipeline7_MultiGraph/AutoroutingPipelineSolver7_MultiGraph.ts b/...router-pipelines/AutoroutingPipeline7_MultiGraph/AutoroutingPipelineSolver7_MultiGraph.ts
@@ -45,6 +45,7 @@ import {
 import { getPresuppliedTraceVisualization } from "lib/utils/getPresuppliedTraceVisualization"
 import { calculateOptimalCapacityDepth } from "lib/utils/getTunedTotalCapacity1"
 import { getViaDimensions } from "lib/utils/getViaDimensions"
+import { getAutoroutingPipelineBenchmarkStats } from "lib/utils/getAutoroutingPipelineBenchmarkStats"
 import {
   AvailableSegmentPointSolver,
   type SharedEdgeSegment,
@@ -873,6 +874,10 @@ export class AutoroutingPipelineSolver7_MultiGraph extends BaseSolver {
         this.timeSpentOnPhase[pipelineStepDef.solverName] =
           this.endTimeOfPhase[pipelineStepDef.solverName] -
           this.startTimeOfPhase[pipelineStepDef.solverName]
+        this.stats = getAutoroutingPipelineBenchmarkStats({
+          timeSpentOnPhase: this.timeSpentOnPhase,
+          portalLayerRefinementStats: this.portPointPathingSolver?.stats,
+        })
         pipelineStepDef.onSolved?.(this)
         this.activeSubSolver = null
         this.currentPipelineStepIndex++