Skip to content

Repair trace overlaps using geometry-sized clearance detours #936

Repair trace overlaps using geometry-sized clearance detours

Repair trace overlaps using geometry-sized clearance detours #936

Workflow file for this run

name: Benchmark
on:
issue_comment:
types: [created]
pull_request:
types: [opened, reopened, synchronize, edited]
push:
branches:
- main
workflow_dispatch:
inputs:
dataset:
description: Dataset to benchmark
required: false
default: srj18
type: choice
options:
- drc14
- srj18
scenario_limit:
description: 'Scenario limit (number or "all"). Default: all.'
required: false
type: string
concurrency:
description: Number of workers, or "auto" (optional)
required: false
type: string
effort:
description: Solver effort value (optional)
required: false
type: string
max_iterations:
description: Solver max iterations override (optional)
required: false
type: string
ref:
description: Git ref (branch, tag, or SHA) to benchmark
required: false
type: string
permissions:
contents: read
issues: write
pull-requests: write
actions: read
jobs:
benchmark:
name: Run benchmark
if: |
github.event_name == 'workflow_dispatch' || (
github.event_name == 'push' &&
github.ref_name == 'main'
) || (
github.event_name == 'pull_request' &&
contains(github.event.pull_request.title, '[BENCHMARK TEST]')
) || (
github.event_name == 'issue_comment' &&
github.event.issue.pull_request &&
github.event.comment.user.type != 'Bot' &&
startsWith(github.event.comment.body, '/benchmark') &&
(
github.event.comment.author_association == 'OWNER' ||
github.event.comment.author_association == 'MEMBER' ||
github.event.comment.author_association == 'CONTRIBUTOR' ||
github.event.comment.author_association == 'COLLABORATOR'
)
)
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Parse benchmark command
id: parse
uses: actions/github-script@v7
with:
github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
script: |
const isComment = context.eventName === 'issue_comment'
const splitShellArgs = (input) => {
const args = []
let current = ''
let quote = null
let escaping = false
let tokenStarted = false
const pushCurrent = () => {
if (!tokenStarted) return
args.push(current)
current = ''
tokenStarted = false
}
for (const char of input) {
if (escaping) {
if (quote === '"' && char === '\n') {
escaping = false
continue
}
if (quote === '"' && !['"', '\\', '$', '`'].includes(char)) {
current += '\\'
}
current += char
tokenStarted = true
escaping = false
continue
}
if (quote === "'") {
if (char === "'") {
quote = null
} else {
current += char
}
tokenStarted = true
continue
}
if (quote === '"') {
if (char === '"') {
quote = null
} else if (char === '\\') {
escaping = true
} else {
current += char
}
tokenStarted = true
continue
}
if (/\s/.test(char)) {
pushCurrent()
continue
}
if (char === "'" || char === '"') {
quote = char
tokenStarted = true
continue
}
if (char === '\\') {
escaping = true
tokenStarted = true
continue
}
current += char
tokenStarted = true
}
if (escaping) {
current += '\\'
}
if (quote !== null) {
throw new Error('Unterminated quote in /benchmark command')
}
pushCurrent()
return args
}
let benchmarkArgs = []
let ref = context.sha
let baseRef = context.sha
let statusCommentId = ''
if (isComment) {
const body = context.payload.comment.body.trim()
const commentArgs = body.replace(/^\/benchmark\b/, '').trim()
benchmarkArgs = splitShellArgs(commentArgs)
const pr = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number,
})
ref = pr.data.head.sha
baseRef = pr.data.base.sha
const statusComment = await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `## Benchmark\n\nRunning benchmark on \`${ref.slice(0, 7)}\`...\n\nWorkflow: [View run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})`,
})
statusCommentId = String(statusComment.data.id)
}
if (context.eventName === 'workflow_dispatch') {
const inputs = context.payload.inputs || {}
const scenarioLimit = (inputs.scenario_limit || '').trim()
const dataset = (inputs.dataset || 'srj18').trim()
const concurrency = (inputs.concurrency || '').trim()
const effort = (inputs.effort || '').trim()
const maxIterations = (inputs.max_iterations || '').trim()
benchmarkArgs.push('--dataset', dataset)
if (scenarioLimit) {
benchmarkArgs.push('--scenario-limit', scenarioLimit)
}
if (concurrency) {
benchmarkArgs.push('--concurrency', concurrency)
}
if (effort) {
benchmarkArgs.push('--effort', effort)
}
if (maxIterations) {
benchmarkArgs.push('--max-iterations', maxIterations)
}
const requestedRef = (inputs.ref || '').trim() || ref
const candidateCommit = await github.rest.repos.getCommit({
owner: context.repo.owner,
repo: context.repo.repo,
ref: requestedRef,
})
ref = candidateCommit.data.sha
const repository = await github.rest.repos.get({
owner: context.repo.owner,
repo: context.repo.repo,
})
const baseCommit = await github.rest.repos.getCommit({
owner: context.repo.owner,
repo: context.repo.repo,
ref: repository.data.default_branch,
})
baseRef = baseCommit.data.sha
}
if (context.eventName === 'pull_request') {
ref = context.payload.pull_request.head.sha
baseRef = context.payload.pull_request.base.sha
}
core.setOutput('benchmark_args_json', JSON.stringify(benchmarkArgs))
core.setOutput('ref', ref)
core.setOutput('base_ref', baseRef)
core.setOutput('status_comment_id', statusCommentId)
- name: Checkout code
if: github.event_name == 'push'
uses: actions/checkout@v4
with:
ref: ${{ steps.parse.outputs.ref }}
- name: Checkout PR code
if: github.event_name != 'push'
uses: actions/checkout@v4
with:
ref: ${{ steps.parse.outputs.ref }}
path: candidate
- name: Checkout base code
if: github.event_name != 'push'
uses: actions/checkout@v4
with:
ref: ${{ steps.parse.outputs.base_ref }}
path: baseline
- name: Setup bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install dependencies
if: github.event_name == 'push'
run: bun install --no-save
- name: Install PR dependencies
if: github.event_name != 'push'
working-directory: candidate
run: bun install --no-save
- name: Install base dependencies
if: github.event_name != 'push'
working-directory: baseline
run: bun install --no-save
- name: Run paired benchmark on the same runner
if: github.event_name != 'push'
env:
BENCHMARK_ARGS_JSON: ${{ steps.parse.outputs.benchmark_args_json }}
CANDIDATE_SHA: ${{ steps.parse.outputs.ref }}
BASE_SHA: ${{ steps.parse.outputs.base_ref }}
run: |
node <<'NODE'
const fs = require('node:fs')
const path = require('node:path')
const { spawnSync } = require('node:child_process')
const requestedArgs = JSON.parse(process.env.BENCHMARK_ARGS_JSON || '[]')
const args = []
for (let index = 0; index < requestedArgs.length; index += 1) {
const arg = requestedArgs[index]
if (arg === '--out') {
index += 1
continue
}
if (arg === '--no-out' || arg.startsWith('--out=')) continue
args.push(arg)
}
const runBenchmark = ({ label, directory, sha, outputStem }) => {
const benchmarkPath = path.join(directory, 'benchmark.sh')
fs.chmodSync(benchmarkPath, 0o755)
const outputJson = path.resolve(`${outputStem}.json`)
const commandArgs = [...args, '--out', outputJson]
const renderedArgs = commandArgs.map((arg) => JSON.stringify(arg)).join(' ')
console.log(`\n## ${label} (${sha.slice(0, 7)})`)
console.log(`Running: ./benchmark.sh ${renderedArgs}`)
const result = spawnSync('./benchmark.sh', commandArgs, {
cwd: directory,
encoding: 'utf8',
env: process.env,
maxBuffer: 50 * 1024 * 1024,
})
const output = `${result.stdout || ''}${result.stderr || ''}`
process.stdout.write(output)
fs.writeFileSync(`${outputStem}.txt`, output)
if (result.error) throw result.error
if (result.status !== 0) process.exit(result.status ?? 1)
return {
output,
report: JSON.parse(fs.readFileSync(outputJson, 'utf8')),
}
}
const averageReports = (runs, outputStem) => {
const reference = runs[0].report
const sampleResults = reference.sampleResults.map((sample, sampleIndex) => {
const elapsedMs = runs.reduce(
(sum, run) => sum + run.report.sampleResults[sampleIndex].elapsedMs,
0,
) / runs.length
return { ...sample, elapsedMs }
})
const totalSolveTimeMs = sampleResults.reduce(
(sum, sample) => sum + sample.elapsedMs,
0,
)
const report = {
...reference,
totalSolveTimeMs,
averageSolveTimeMs: totalSolveTimeMs / sampleResults.length,
sampleResults,
metadata: {
...reference.metadata,
pairedRuns: runs.length,
},
}
fs.writeFileSync(`${outputStem}.json`, `${JSON.stringify(report, null, 2)}\n`)
fs.writeFileSync(
`${outputStem}.txt`,
runs.map((run, index) => `## Run ${index + 1}\n\n${run.output}`).join('\n'),
)
}
// ABBA ordering gives each revision one early and one late run,
// reducing cold-cache, warm-up, and thermal-order bias.
const baseA = runBenchmark({
label: 'Base benchmark A',
directory: 'baseline',
sha: process.env.BASE_SHA,
outputStem: 'benchmark-result-main-a',
})
const candidateA = runBenchmark({
label: 'PR benchmark A',
directory: 'candidate',
sha: process.env.CANDIDATE_SHA,
outputStem: 'benchmark-result-pr-a',
})
const candidateB = runBenchmark({
label: 'PR benchmark B',
directory: 'candidate',
sha: process.env.CANDIDATE_SHA,
outputStem: 'benchmark-result-pr-b',
})
const baseB = runBenchmark({
label: 'Base benchmark B',
directory: 'baseline',
sha: process.env.BASE_SHA,
outputStem: 'benchmark-result-main-b',
})
averageReports([baseA, baseB], 'benchmark-result-main')
averageReports([candidateA, candidateB], 'benchmark-result-pr')
fs.copyFileSync('benchmark-result-pr.txt', 'benchmark-result.txt')
fs.copyFileSync('benchmark-result-pr.json', 'benchmark-result.json')
NODE
- name: Run SRJ18 main benchmark
if: github.event_name == 'push'
run: |
set -o pipefail
chmod +x ./benchmark.sh
./benchmark.sh --dataset srj18 --out benchmark-result-srj18.json 2>&1 | tee benchmark-result-srj18.txt
- name: Run DRC14 main benchmark
if: github.event_name == 'push'
run: |
set -o pipefail
./benchmark.sh --dataset drc14 --out benchmark-result-drc14.json 2>&1 | tee benchmark-result-drc14.txt
- name: Upload SRJ18 main benchmark result
if: always() && github.event_name == 'push'
uses: actions/upload-artifact@v4
with:
name: benchmark-result-srj18
path: |
./benchmark-result-srj18.txt
./benchmark-result-srj18.json
overwrite: true
if-no-files-found: ignore
- name: Upload DRC14 main benchmark result
if: always() && github.event_name == 'push'
uses: actions/upload-artifact@v4
with:
name: benchmark-result-drc14
path: |
./benchmark-result-drc14.txt
./benchmark-result-drc14.json
overwrite: true
if-no-files-found: ignore
- name: Upload benchmark result
if: always() && github.event_name != 'push'
uses: actions/upload-artifact@v4
with:
name: benchmark-result
path: |
./benchmark-result.txt
./benchmark-result.json
./benchmark-result-pr.txt
./benchmark-result-pr.json
./benchmark-result-main.txt
./benchmark-result-main.json
overwrite: true
if-no-files-found: ignore
- name: Summarize paired benchmark
if: always() && github.event_name != 'push'
env:
CANDIDATE_SHA: ${{ steps.parse.outputs.ref }}
BASE_SHA: ${{ steps.parse.outputs.base_ref }}
run: |
node <<'NODE'
const fs = require('node:fs')
const readReport = (filePath) => {
if (!fs.existsSync(filePath)) return null
try {
return JSON.parse(fs.readFileSync(filePath, 'utf8'))
} catch {
return null
}
}
const formatDuration = (milliseconds) =>
Number.isFinite(milliseconds) ? `${(milliseconds / 1000).toFixed(2)}s` : 'n/a'
const formatDelta = (candidate, baseline) => {
if (!Number.isFinite(candidate) || !Number.isFinite(baseline) || baseline === 0) return 'n/a'
const percent = ((candidate - baseline) / baseline) * 100
return `${percent >= 0 ? '+' : ''}${percent.toFixed(2)}%`
}
const baseline = readReport('benchmark-result-main.json')
const candidate = readReport('benchmark-result-pr.json')
const lines = [
'# Paired benchmark',
'',
`Base \`${process.env.BASE_SHA.slice(0, 7)}\` and PR \`${process.env.CANDIDATE_SHA.slice(0, 7)}\` each ran twice in ABBA order on the same GitHub Actions runner.`,
'',
]
if (baseline && candidate) {
lines.push(
'| Metric | Base | PR | Delta |',
'| --- | ---: | ---: | ---: |',
`| Total solve time | ${formatDuration(baseline.totalSolveTimeMs)} | ${formatDuration(candidate.totalSolveTimeMs)} | ${formatDelta(candidate.totalSolveTimeMs, baseline.totalSolveTimeMs)} |`,
`| Average solve time | ${formatDuration(baseline.averageSolveTimeMs)} | ${formatDuration(candidate.averageSolveTimeMs)} | ${formatDelta(candidate.averageSolveTimeMs, baseline.averageSolveTimeMs)} |`,
`| Final DRC | ${baseline.totalFinalDrcCount} | ${candidate.totalFinalDrcCount} | ${candidate.totalFinalDrcCount - baseline.totalFinalDrcCount} |`,
)
} else {
lines.push('One or both benchmark reports were unavailable.')
}
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`)
NODE
- name: Post benchmark result comment
if: always() && github.event_name == 'issue_comment' && steps.parse.outputs.status_comment_id != ''
uses: actions/github-script@v7
with:
github-token: ${{ secrets.TSCIRCUIT_BOT_GITHUB_TOKEN }}
script: |
const fs = require('node:fs')
const maxLength = 60000
const truncate = (s, max) => s.length > max ? `${s.slice(0, max)}\n\n...truncated...` : s
const formatDuration = (value) => {
if (typeof value !== 'number' || !Number.isFinite(value)) return 'n/a'
return value < 1000 ? `${Math.round(value)}ms` : `${(value / 1000).toFixed(2)}s`
}
const formatSignedInteger = (value) => {
if (!Number.isFinite(value) || value === 0) return value === 0 ? '0' : 'n/a'
return value > 0 ? `+${Math.round(value)}` : `${Math.round(value)}`
}
const formatSignedDuration = (value) => {
if (!Number.isFinite(value) || value === 0) return value === 0 ? '0ms' : 'n/a'
const absolute = Math.abs(value)
const rendered = absolute < 1000 ? `${Math.round(absolute)}ms` : `${(absolute / 1000).toFixed(2)}s`
return value > 0 ? `+${rendered}` : `-${rendered}`
}
const readJson = (path) => {
if (!fs.existsSync(path)) return null
const raw = fs.readFileSync(path, 'utf8').trim()
if (!raw) return null
try {
return JSON.parse(raw)
} catch {
return null
}
}
const readText = (path) => {
if (!fs.existsSync(path)) return null
return fs.readFileSync(path, 'utf8').trim()
}
const deltaValue = (report, baseline, field) => {
if (!report || !baseline) return null
const reportValue = Number(report[field])
const baselineValue = Number(baseline[field])
if (!Number.isFinite(reportValue) || !Number.isFinite(baselineValue)) return null
return reportValue - baselineValue
}
const renderMetadata = (report) => {
if (!report?.metadata) return []
const metadata = report.metadata
return [
`Dataset: ${String(report.dataset ?? 'n/a').toUpperCase()}`,
`Concurrency: ${metadata.concurrency ?? 'n/a'}`,
`Effort: ${metadata.effort ?? 'n/a'}`,
...(metadata.maxIterations !== undefined ? [`Max iterations: ${metadata.maxIterations}`] : []),
`Scenario limit used: ${metadata.scenarioLimitUsed ?? 'n/a'}`,
'',
]
}
const renderSummaryTable = (report, options = {}) => {
if (!report) return ['Summary table unavailable.']
const includeDelta = Boolean(options.includeDelta)
const baseline = options.baseline ?? null
const rows = [
['Samples', 'sampleCount', 'integer'],
['Succeeded', 'succeeded', 'integer'],
['Failed', 'failed', 'integer'],
['Improved', 'improved', 'integer'],
['Clean', 'clean', 'integer'],
['Initial DRC', 'totalInitialDrcCount', 'integer'],
['Final DRC', 'totalFinalDrcCount', 'integer'],
['DRC improvement', 'totalImprovement', 'integer'],
['Total solve time', 'totalSolveTimeMs', 'duration'],
['Average solve time', 'averageSolveTimeMs', 'duration'],
]
const renderValue = (value, type) =>
type === 'duration' ? formatDuration(Number(value)) : String(value ?? 'n/a')
const renderDelta = (field, type) => {
const delta = deltaValue(report, baseline, field)
if (delta === null) return 'n/a'
return type === 'duration' ? formatSignedDuration(delta) : formatSignedInteger(delta)
}
const table = includeDelta
? [
'| Metric | PR Value | Delta vs Main |',
'| --- | --- | --- |',
...rows.map(([label, field, type]) => `| ${label} | ${renderValue(report[field], type)} | ${renderDelta(field, type)} |`),
]
: [
'| Metric | Value |',
'| --- | --- |',
...rows.map(([label, field, type]) => `| ${label} | ${renderValue(report[field], type)} |`),
]
return [
...renderMetadata(report),
...table,
]
}
const prText = readText('benchmark-result-pr.txt') ?? readText('benchmark-result.txt') ?? '(benchmark output unavailable)'
const mainText = readText('benchmark-result-main.txt') ?? '(main benchmark output unavailable)'
const prReport = readJson('benchmark-result-pr.json') ?? readJson('benchmark-result.json')
const mainReport = readJson('benchmark-result-main.json')
const reportsAreComparable =
Boolean(prReport && mainReport) &&
prReport.dataset === mainReport.dataset
const comparableMainReport = reportsAreComparable ? mainReport : null
const mainSummary = reportsAreComparable
? renderSummaryTable(mainReport, { includeDelta: false })
: [
prReport && mainReport
? `No matching main-branch baseline is available for ${String(prReport.dataset).toUpperCase()} (latest main artifact uses ${String(mainReport.dataset).toUpperCase()}).`
: 'Summary table unavailable.',
]
const runUrl = `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`
const jobStatus = '${{ job.status }}'
const benchmarkFailed = jobStatus !== 'success'
const body = [
benchmarkFailed ? '## Benchmark Failed' : '## Benchmark Results',
'',
`Base \`${String('${{ steps.parse.outputs.base_ref }}').slice(0, 7)}\` and PR \`${String('${{ steps.parse.outputs.ref }}').slice(0, 7)}\` each ran twice in ABBA order on the same GitHub Actions runner.`,
'',
...(benchmarkFailed
? [
`Benchmark workflow ended with **${jobStatus}** before completion.`,
'',
]
: []),
'<details>',
'<summary>Main Branch Results</summary>',
'',
...mainSummary,
'',
'<details>',
'<summary>Raw output</summary>',
'',
'```',
truncate(mainText, 25000),
'```',
'</details>',
'</details>',
'',
'<details open>',
'<summary>PR Results</summary>',
'',
...renderSummaryTable(prReport, { includeDelta: true, baseline: comparableMainReport }),
'',
'<details>',
'<summary>Raw output</summary>',
'',
'```',
truncate(prText, 25000),
'```',
'</details>',
'</details>',
'',
`Workflow: [View run](${runUrl})`,
`Artifact: ${runUrl}`,
].join('\n')
const finalBody = body.length > maxLength ? truncate(body, maxLength) : body
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: Number('${{ steps.parse.outputs.status_comment_id }}'),
body: finalBody,
})