Skip to content
ATOM Benchmark

ATOM Benchmark #457

Sign in to view logs

ATOM Benchmark

ATOM Benchmark #457

Workflow file for this run

.github/workflows/atom-benchmark.yaml at 945b312
name: ATOM Benchmark
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
on:
schedule:
# Nightly at 00:12 Beijing time (16:12 UTC)
- cron: '12 16 * * *'
workflow_dispatch:
inputs:
deepseek-v4-pro:
description: "Benchmark DeepSeek-V4-Pro"
type: boolean
default: true
glm-5-2-fp8:
description: "Benchmark GLM-5.2-FP8"
type: boolean
default: true
glm-5-2-mxfp4:
description: "Benchmark GLM-5.2-MXFP4 (+ MTP3)"
type: boolean
default: true
deepseek-r1-0528-mxfp4:
description: "Benchmark DeepSeek-R1-0528 MXFP4 (+ MXFP4-MTP3)"
type: boolean
default: true
gpt-oss-120b:
description: "Benchmark gpt-oss-120b"
type: boolean
default: true
kimi-k27-code-mxfp4:
description: "Benchmark Kimi-K2.7-Code-MXFP4"
type: boolean
default: true
m3-mxfp8:
description: "Benchmark MiniMax-M3-MXFP8 (+ EAGLE3)"
type: boolean
default: true
m3-mxfp4:
description: "Benchmark MiniMax-M3-MXFP4 (+ EAGLE3)"
type: boolean
default: true
qwen35-397b-fp8:
description: "Benchmark Qwen3.5-397B-A17B-FP8"
type: boolean
default: true
qwen35-397b-mxfp4:
description: "Benchmark Qwen3.5-397B-A17B-MXFP4"
type: boolean
default: true
llama-3-3-70b-instruct-mxfp4:
description: "Benchmark Llama-3.3-70B-Instruct-MXFP4"
type: boolean
default: true
extra_args:
description: "Extra arguments to pass to the ATOM server"
type: string
image:
description: "Image to use for the benchmark"
type: string
default: "rocm/atom-dev:latest"
runner:
description: "Optional runner label override for manual runs; leave empty to use model defaults"
type: string
default: ""
enable_profiler:
description: "Enable torch profiler to collect trace for performance debugging"
type: boolean
default: false
enable_rtl:
description: "Enable RTL (rocm-trace-lite) GPU kernel tracing for prefill/decode analysis"
type: boolean
default: false
param_lists:
description: |
"Benchmark parameter lists.
Input as a single or multiple sets (comma-separated, semicolon between sets),
format: input_length,output_length,concurrency,random_range_ratio.
Example (single set): 1024,1024,128,0.8
Example (multiple sets): 1024,1024,128,0.8;2048,1024,256,0.7"
type: string
default: "1024,1024,128,0.8"
atom_commit:
description: "ATOM commit SHA to checkout; leave empty to use the workflow ref"
type: string
default: ""
permissions:
actions: read
contents: write
issues: write
jobs:
# Single source of the benchmark matrix. Expands the structured catalog
# (.github/benchmark/models.json) into fully-resolved per-run "cells" — one
# matrix dimension — applying model selection and per-variant concurrency
# bands in one place. See .github/scripts/{catalog,build_benchmark_matrix}.py
# and .github/benchmark/README.md.
build-matrix:
name: Build benchmark matrix
runs-on: ubuntu-latest
outputs:
configs_json: ${{ steps.build.outputs.configs_json }}
has_cells: ${{ steps.build.outputs.has_cells }}
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.atom_commit || github.ref }}
- id: build
env:
EVENT_NAME: ${{ github.event_name }}
INPUTS_JSON: ${{ toJson(inputs) }}
run: python3 .github/scripts/build_benchmark_matrix.py
# Top-level fan-out: one matrix entry per (model variant × scenario) config.
# Each invokes benchmark-tmpl.yml, which fans out over the config's
# `concurrency` list. Two bounded matrices replace the former flat per-cell
# matrix that overflowed GitHub's 256-job-per-matrix limit (278 cells). Every
# (config × conc) cell still runs as its own parallel job; the caller job name
# stays `benchmark` so downstream `needs:` are unchanged.
benchmark:
name: ${{ matrix.config.display }} ${{ matrix.config.scenario }}
needs: [build-matrix]
if: >-
!cancelled()
&& needs.build-matrix.result == 'success'
&& needs.build-matrix.outputs.has_cells == 'true'
strategy:
fail-fast: false
matrix:
config: ${{ fromJson(needs.build-matrix.outputs.configs_json) }}
uses: ./.github/workflows/benchmark-tmpl.yml
secrets: inherit
with:
display: ${{ matrix.config.display }}
prefix: ${{ matrix.config.prefix }}
suffix: ${{ matrix.config.suffix }}
model_path: ${{ matrix.config.model_path }}
server_args: ${{ matrix.config.server_args }}
bench_args: ${{ matrix.config.bench_args }}
env_vars: ${{ matrix.config.env_vars }}
runner: ${{ (github.event_name == 'workflow_dispatch' && inputs.runner != '' && inputs.runner) || matrix.config.runner }}
isl: ${{ matrix.config.isl }}
osl: ${{ matrix.config.osl }}
ratio: ${{ matrix.config.ratio }}
ratio_str: ${{ matrix.config.ratio_str }}
concurrency: ${{ matrix.config.concurrency }}
image: ${{ inputs.image || 'rocm/atom-dev:latest' }}
enable_profiler: ${{ inputs.enable_profiler || false }}
enable_rtl: ${{ inputs.enable_rtl || false }}
extra_args: ${{ inputs.extra_args || '' }}
atom_commit: ${{ inputs.atom_commit || '' }}
summarize-benchmark-result:
concurrency:
group: gh-pages-deploy
cancel-in-progress: false
if: always()
name: Summarize benchmark result
needs: [benchmark]
runs-on: ubuntu-latest
outputs:
has_regression: ${{ steps.check-regression.outputs.has_regression }}
steps:
- name: Checkout ATOM repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.atom_commit || github.ref }}
- name: Download all benchmark results
uses: actions/download-artifact@v8
with:
pattern: 'benchmark-*'
merge-multiple: true
path: .
- name: Download baseline from previous nightly run
id: baseline
run: |
# Use the most recent completed nightly run (regardless of success/failure)
# because even failed runs have valid per-model benchmark artifacts.
# Only requiring --status=success misses baselines when any single job fails.
PREV_RUN_ID=$(gh run list \
--workflow="ATOM Benchmark" \
--branch=main \
--event=schedule \
--limit=5 \
--json databaseId,status \
--jq '[.[] | select(.status == "completed")][0].databaseId // empty')
if [ -n "$PREV_RUN_ID" ] && [ "$PREV_RUN_ID" != "${{ github.run_id }}" ]; then
echo "Downloading baseline from run #$PREV_RUN_ID"
mkdir -p /tmp/baseline
# Only download benchmark result artifacts (prefix: benchmark-),
# not profiler/regression traces which can be multi-GB.
gh run download "$PREV_RUN_ID" --dir /tmp/baseline --pattern 'benchmark-*' || echo "::warning::Failed to download baseline artifacts"
BASELINE_COUNT=$(find /tmp/baseline -name '*.json' | wc -l)
echo "Downloaded $BASELINE_COUNT baseline files"
echo "baseline_dir=/tmp/baseline" >> $GITHUB_OUTPUT
else
echo "No previous completed nightly run found"
echo "baseline_dir=" >> $GITHUB_OUTPUT
fi
env:
GH_TOKEN: ${{ github.token }}
- name: List all benchmark results
run: |
echo "=== Current results ==="
ls -la *.json 2>/dev/null || echo "No JSON files in current dir"
if [ -d "/tmp/baseline" ]; then
echo "=== Baseline results ==="
find /tmp/baseline -name '*.json' | head -20
fi
- name: Summarize benchmark result
id: check-regression
run: |
BASELINE_ARG=""
if [ -n "${{ steps.baseline.outputs.baseline_dir }}" ]; then
BASELINE_ARG="--baseline-dir ${{ steps.baseline.outputs.baseline_dir }}"
fi
.github/scripts/summarize.py . $BASELINE_ARG \
--output-json regression_report.json \
>> $GITHUB_STEP_SUMMARY || true
# Check if regression was detected (exit code 2)
if [ -f regression_report.json ]; then
REG_COUNT=$(python3 -c "import json; print(json.load(open('regression_report.json'))['regression_count'])")
if [ "$REG_COUNT" -gt 0 ]; then
echo "has_regression=true" >> $GITHUB_OUTPUT
else
echo "has_regression=false" >> $GITHUB_OUTPUT
fi
else
echo "has_regression=false" >> $GITHUB_OUTPUT
fi
- name: Upload regression report
if: steps.check-regression.outputs.has_regression == 'true'
uses: actions/upload-artifact@v7
with:
name: regression-report
path: regression_report.json
- name: Transform results for benchmark dashboard
run: |
python3 .github/scripts/plugin_benchmark_to_dashboard.py \
. \
--output benchmark-action-input.json \
--default-backend ATOM \
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
- name: Store benchmark result to dashboard
uses: benchmark-action/github-action-benchmark@v1
with:
tool: customBiggerIsBetter
output-file-path: benchmark-action-input.json
gh-pages-branch: gh-pages
benchmark-data-dir-path: benchmark-dashboard
auto-push: false
alert-threshold: "80%"
comment-on-alert: true
fail-on-alert: false
max-items-in-chart: 300
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Deploy custom dashboard to gh-pages
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
CURRENT_SHA=$(git rev-parse HEAD)
# Generate prefix→display name mapping from the catalog
python3 .github/scripts/dashboard_models_map.py \
.github/benchmark/models.json /tmp/dashboard_models_map.js
# Save dashboard assets before switching branches
cp .github/dashboard/index.html /tmp/dashboard_index.html
cp docs/assets/atom_logo.png /tmp/dashboard_logo.png
cp docs/assets/atom_logo_mini.png /tmp/dashboard_logo_mini.png
git fetch origin gh-pages
git checkout gh-pages
# Replace auto-generated index.html with custom dashboard + logo
cp /tmp/dashboard_index.html benchmark-dashboard/index.html
cp /tmp/dashboard_models_map.js benchmark-dashboard/models_map.js
cp /tmp/dashboard_logo.png benchmark-dashboard/atom_logo.png
cp /tmp/dashboard_logo_mini.png benchmark-dashboard/atom_logo_mini.png
git add benchmark-dashboard/
git diff --cached --quiet || git commit -m "Update benchmark data and dashboard"
git push origin gh-pages
git checkout "$CURRENT_SHA"
# ---------- Generate regression matrix (lightweight, ubuntu) ----------
generate-regression-matrix:
if: always() && needs.summarize-benchmark-result.outputs.has_regression == 'true'
name: Generate regression rerun matrix
needs: [summarize-benchmark-result]
runs-on: ubuntu-latest
outputs:
matrix_json: ${{ steps.gen.outputs.matrix_json }}
has_matrix: ${{ steps.gen.outputs.has_matrix }}
steps:
- uses: actions/checkout@v6
with:
ref: ${{ inputs.atom_commit || github.ref }}
- uses: actions/download-artifact@v8
with:
name: regression-report
- name: Generate matrix from regression report
id: gen
run: |
python3 .github/scripts/regression_rerun.py \
regression_report.json \
.github/benchmark/models.json \
--output-matrix /tmp/matrix.json
if [ -s /tmp/matrix.json ] && [ "$(cat /tmp/matrix.json)" != "[]" ]; then
echo "matrix_json=$(cat /tmp/matrix.json)" >> $GITHUB_OUTPUT
echo "has_matrix=true" >> $GITHUB_OUTPUT
echo "=== Matrix cells ==="
python3 -c "
import json
cells = json.load(open('/tmp/matrix.json'))
for c in cells:
configs = json.loads(c['configs'])
print(f\" {c['prefix']}: {len(configs)} config(s) on {c['runner']}\")
"
else
echo "matrix_json=[]" >> $GITHUB_OUTPUT
echo "has_matrix=false" >> $GITHUB_OUTPUT
echo "No regression configs generated"
fi
# ---------- Regression re-run (GPU, matrix by model) ----------
regression-rerun:
if: >-
!cancelled()
&& needs.generate-regression-matrix.outputs.has_matrix == 'true'
name: Rerun ${{ matrix.cell.prefix }}
needs: [generate-regression-matrix]
strategy:
fail-fast: false
matrix:
cell: ${{ fromJson(needs.generate-regression-matrix.outputs.matrix_json) }}
runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.runner != '' && inputs.runner) || matrix.cell.runner }}
timeout-minutes: 60
steps:
- name: Kill all Docker containers
run: |
echo "=== Cleaning up containers on $(hostname) ==="
containers=$(docker ps -q)
if [ -n "$containers" ]; then
docker kill $containers || true
fi
docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "find /workspace -mindepth 1 -delete" || true
- name: Checkout ATOM repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.atom_commit || github.ref }}
- name: Docker Login
uses: ./.github/actions/docker-auth
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Start container + download model
uses: ./.github/actions/atom-bench-container
with:
image: ${{ inputs.image || 'rocm/atom-dev:latest' }}
container-name: atom-regression
model-path: ${{ matrix.cell.model_path }}
env-vars: ${{ matrix.cell.env_vars }}
hf-token: ${{ secrets.AMD_HF_TOKEN }}
download-required: "false"
- name: Launch server
env:
SERVER_ARGS: ${{ matrix.cell.server_args }}
run: |
if [ -d "/models" ]; then MODEL_DIR="/models/${{ matrix.cell.model_path }}"
else MODEL_DIR="${{ matrix.cell.model_path }}"; fi
# cell.env_vars are already injected into the container env via
# --env-file in "Start CI container" — `docker exec` inherits them,
# so no need to re-prefix them on the command line (the previous
# inline form silently dropped the first var when env_vars contained
# a literal newline).
#
# Launch via stdin so the container's bash parses the shell quoting in
# SERVER_ARGS exactly once -- single-quoted JSON values survive intact
# (e.g. --hf-overrides '{"use_index_cache": true}'). See "Run benchmark".
echo "ENABLE_TORCH_PROFILER=1 \
.github/scripts/atom_test.sh launch $MODEL_DIR $SERVER_ARGS" \
| docker exec -i atom-regression bash -l
- name: Run regression configs with profiler
env:
SERVER_ARGS: ${{ matrix.cell.server_args }}
run: |
set -uo pipefail # no -e: allow per-config failure without aborting loop
if [ -d "/models" ]; then MODEL_DIR="/models/${{ matrix.cell.model_path }}"
else MODEL_DIR="${{ matrix.cell.model_path }}"; fi
FAIL_COUNT=0
TOTAL_COUNT=0
# Parse configs JSON into pipe-delimited lines for shell loop
echo '${{ matrix.cell.configs }}' | python3 -c "
import json, sys
for c in json.load(sys.stdin):
print('{isl}|{osl}|{conc}|{bench_args}|{prefix}'.format(**c))
" > /tmp/regression_configs.txt
while IFS='|' read -r ISL OSL CONC BENCH_ARGS PREFIX; do
TOTAL_COUNT=$((TOTAL_COUNT + 1))
TRACE_SUBDIR="${PREFIX}-${ISL}-${OSL}-${CONC}"
docker exec atom-regression bash -lc "mkdir -p /app/trace/$TRACE_SUBDIR"
# Record profiler stop count before benchmark (to detect new completions)
STOP_COUNT_BEFORE=$(docker exec atom-regression grep -c "Profiler stopped." /tmp/atom_server.log 2>/dev/null) || STOP_COUNT_BEFORE=0
echo "=== Profiling: $PREFIX ISL=$ISL OSL=$OSL CONC=$CONC ==="
# cell.env_vars are already in the container env via --env-file
# (see "Launch server"); docker exec inherits them, so do NOT
# re-prefix them inline -- V4-Pro's env_vars contains a literal
# newline that would break the bash -lc string / drop a var.
if ! docker exec \
-e SERVER_ARGS="$SERVER_ARGS" \
atom-regression bash -lc "
ENABLE_TORCH_PROFILER=1 \
ISL=$ISL OSL=$OSL CONC=$CONC \
RANDOM_RANGE_RATIO=0.8 \
NUM_PROMPTS_OVERRIDE=$((CONC*2)) \
RESULT_FILENAME=regression-${PREFIX}-${ISL}-${OSL}-${CONC} \
BENCH_EXTRA_ARGS='$BENCH_ARGS' \
.github/scripts/atom_test.sh benchmark $MODEL_DIR"; then
echo "::warning::Benchmark failed for $PREFIX ISL=$ISL OSL=$OSL CONC=$CONC"
FAIL_COUNT=$((FAIL_COUNT + 1))
continue
fi
# Wait for engine core to finish exporting profiler trace (blocks inference)
# See .claude/plan/profiler-async-refactor.md for long-term fix
for i in $(seq 1 300); do
STOP_COUNT=$(docker exec atom-regression grep -c "Profiler stopped." /tmp/atom_server.log 2>/dev/null) || STOP_COUNT=0
[ "$STOP_COUNT" -gt "$STOP_COUNT_BEFORE" ] && echo "Profiler trace export done after ${i}s" && break
[ "$i" -eq 1 ] && echo "Waiting for profiler trace export..."
[ "$i" -eq 300 ] && echo "WARNING: Profiler did not finish after 300s, proceeding anyway"
sleep 1
done
# Move trace files into per-config subdir
docker exec atom-regression bash -lc "
for d in /app/trace/rank_*; do
[ -d \"\$d\" ] && mv \"\$d\" /app/trace/$TRACE_SUBDIR/ 2>/dev/null || true
done"
done < /tmp/regression_configs.txt
echo "=== Regression re-run complete: $((TOTAL_COUNT - FAIL_COUNT))/$TOTAL_COUNT succeeded ==="
if [ "$FAIL_COUNT" -gt 0 ]; then
echo "::warning::$FAIL_COUNT/$TOTAL_COUNT configs failed"
fi
- name: Stop server
if: always()
run: |
docker exec atom-regression bash -lc \
"TORCH_PROFILER_DIR=/app/trace .github/scripts/atom_test.sh stop" || true
- name: Copy profiler traces from container
id: copy-traces
if: always()
run: |
docker cp atom-regression:/app/trace ./regression-traces 2>/dev/null || echo "No traces found"
TRACE_COUNT=$(find ./regression-traces -type f \( -name "*.json.gz" -o -name "*.pt.trace.*" \) 2>/dev/null | wc -l | tr -d ' ')
echo "trace_count=$TRACE_COUNT" >> $GITHUB_OUTPUT
if [ "$TRACE_COUNT" -gt 0 ]; then
echo "Found $TRACE_COUNT trace files"
ls -lhR ./regression-traces/
else
echo "::warning::No trace files found in container — profiler may not have produced output"
fi
- name: Upload regression traces
if: always() && fromJSON(steps.copy-traces.outputs.trace_count) > 0
uses: actions/upload-artifact@v7
with:
name: regression-traces-${{ github.run_id }}-${{ matrix.cell.prefix }}
path: regression-traces/
- name: Upload regression results
if: always()
uses: actions/upload-artifact@v7
with:
name: regression-results-${{ github.run_id }}-${{ matrix.cell.prefix }}
path: regression-*.json
if-no-files-found: ignore
- name: Clean Up
if: always()
run: |
docker stop atom-regression || true
docker rm atom-regression || true
# ---------- Collect regression traces from all matrix cells ----------
collect-regression-traces:
if: always() && needs.regression-rerun.result != 'skipped'
name: Collect regression traces
needs: [regression-rerun]
runs-on: ubuntu-latest
steps:
- name: Download all regression trace artifacts
uses: actions/download-artifact@v8
with:
pattern: regression-traces-${{ github.run_id }}-*
merge-multiple: true
path: regression-traces/
continue-on-error: true
- name: Upload combined regression traces
uses: actions/upload-artifact@v7
with:
name: regression-traces-${{ github.run_id }}
path: regression-traces/
if-no-files-found: ignore
# ---------- Profiler analysis (lightweight, runs on regression OR manual profiler) ----------
profiler-analysis:
if: >-
always()
&& (needs.collect-regression-traces.result != 'cancelled'
&& needs.collect-regression-traces.result != 'skipped'
|| inputs.enable_profiler == true)
name: Profiler trace analysis
needs: [benchmark, summarize-benchmark-result, generate-regression-matrix, regression-rerun, collect-regression-traces]
runs-on: ubuntu-latest
steps:
- name: Checkout ATOM repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.atom_commit || github.ref }}
- name: Download regression traces
if: needs.collect-regression-traces.result == 'success'
uses: actions/download-artifact@v8
continue-on-error: true
with:
name: regression-traces-${{ github.run_id }}
path: profiler-traces/
- name: Download benchmark profiler traces
if: needs.collect-regression-traces.result != 'success'
uses: actions/download-artifact@v8
continue-on-error: true
with:
pattern: profiler-traces-*
merge-multiple: true
path: profiler-traces/
- name: Analyze traces
run: |
mkdir -p profiler-analysis
pip install -q openpyxl 2>/dev/null || true
# Find all config subdirectories containing rank_* dirs
# Structure: profiler-traces/<config-name>/rank_0/*.gz
CONFIG_DIRS=$(find profiler-traces -mindepth 1 -maxdepth 1 -type d 2>/dev/null || true)
if [ -z "$CONFIG_DIRS" ]; then
echo "No config directories found in profiler-traces/"
ls -lhR profiler-traces/ 2>/dev/null || true
exit 0
fi
for config_dir in $CONFIG_DIRS; do
config=$(basename "$config_dir")
echo ""
echo "========== Config: $config =========="
mkdir -p "profiler-analysis/$config"
# Analyze rank_0 trace (representative for kernel breakdown + summary)
RANK0_TRACE=$(find "$config_dir/rank_0" -name "*.pt.trace.json.gz" ! -name "capture_graph_*" 2>/dev/null | sort | tail -1)
if [ -z "$RANK0_TRACE" ]; then
echo "No rank_0 trace found for $config, skipping"
continue
fi
echo "=== Analyzing: $RANK0_TRACE ==="
cd "profiler-analysis/$config"
python ../../tools/parse_trace.py "../../$RANK0_TRACE" --layer 3 2>&1 | tee analysis.log || true
python ../../tools/analyze_trace_summary.py "../../$RANK0_TRACE" \
--output performance_summary.md 2>&1 || true
if [ -f performance_summary.md ]; then
echo "=== Performance Summary ($config) ==="
cat performance_summary.md
fi
cd ../..
done
ls -laR profiler-analysis/ 2>/dev/null || true
- name: Upload analysis results
if: always()
uses: actions/upload-artifact@v7
with:
name: profiler-analysis-${{ github.run_id }}
path: profiler-analysis/
- name: Download regression report
if: needs.summarize-benchmark-result.outputs.has_regression == 'true'
uses: actions/download-artifact@v8
with:
name: regression-report
continue-on-error: true
- name: Create GitHub Issue for regression
if: needs.summarize-benchmark-result.outputs.has_regression == 'true'
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
let report;
try {
report = JSON.parse(fs.readFileSync('regression_report.json', 'utf8'));
} catch (e) {
console.log('No regression report found, skipping issue creation');
return;
}
if (!report.regressions || report.regressions.length === 0) return;
const regs = report.regressions.map(r => {
const throughput = r.metrics.output_throughput || {};
const tpot = r.metrics.mean_tpot_ms || {};
return `| ${r.model} | ${r.isl}/${r.osl} | ${r.conc} | ${throughput.current?.toFixed(1) || 'N/A'} | ${throughput.baseline?.toFixed(1) || 'N/A'} | ${throughput.pct?.toFixed(1) || 'N/A'}% | ${tpot.current?.toFixed(2) || 'N/A'} | ${tpot.baseline?.toFixed(2) || 'N/A'} | ${tpot.pct?.toFixed(1) || 'N/A'}% |`;
}).join('\n');
let summary = 'Summary not available';
try {
const dirs = fs.readdirSync('profiler-analysis').filter(d =>
fs.statSync(`profiler-analysis/${d}`).isDirectory());
for (const d of dirs) {
const p = `profiler-analysis/${d}/performance_summary.md`;
if (fs.existsSync(p)) { summary = fs.readFileSync(p, 'utf8'); break; }
}
} catch(e) { /* ignore */ }
const body = `## Performance Regression Detected
**Commit:** \`${context.sha.substring(0, 8)}\`
**Run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}
**Date:** ${report.timestamp}
### Regressed Configurations
| Model | ISL/OSL | Conc | Tput (cur) | Tput (base) | Δ% | TPOT (cur) | TPOT (base) | Δ% |
|-------|---------|------|-----------|------------|-----|-----------|------------|-----|
${regs}
### Performance Summary
\`\`\`
${summary}
\`\`\`
### Profiler Traces
Download from [workflow artifacts](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).
Open in [Perfetto UI](https://ui.perfetto.dev/) or Chrome \`chrome://tracing\` for analysis.
### Next Steps
1. Download \`profiler-analysis-${context.runId}\` artifact
2. Open trace files in Perfetto UI
3. Compare kernel durations against previous traces
4. Identify bottleneck changes
`.replace(/^ /gm, '');
await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title: `[Perf Regression] ${report.regressions.length} config(s) regressed @ ${context.sha.substring(0, 8)}`,
body: body,
labels: ['performance', 'regression']
});