Skip to content

Commit 8de36aa

Browse files
wanzhenchnwanzhenchn
authored
publish ATOMesh benchmark summaries with MI350X/MI355X hardware tag on dashboard (#1519)
1 parent 7211a4a commit 8de36aa

3 files changed

Lines changed: 61 additions & 12 deletions

File tree

‎.github/dashboard/index.html‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,7 @@ <h1>Benchmark Dashboard</h1>
516516
const ATOMESH_DATA = {
517517
updatedAt: Date.UTC(2026, 6, 3, 7, 9, 0),
518518
backend: 'ATOMesh',
519-
hardware: 'MI355X',
519+
hardware: window.ATOMESH_HARDWARE || 'MI355X',
520520
gpuCount: 16,
521521
rocmVersion: '7.2.4',
522522
dockerImage: 'rocm/atom-dev:nightly_202607030204',
@@ -536,6 +536,13 @@ <h1>Benchmark Dashboard</h1>
536536
return String(header || '').toLowerCase().replace(/[^a-z0-9]+/g, '');
537537
}
538538

539+
function atomeshHardwareKey(value) {
540+
const text = String(value || ATOMESH_DATA.hardware || '').toLowerCase();
541+
if (text.includes('mi350')) return 'mi350x';
542+
if (text.includes('mi355')) return 'mi355x';
543+
return text || 'unknown';
544+
}
545+
539546
function parseAtomeshSummaryRows(markdown) {
540547
const lines = String(markdown || '').split(/\r?\n/).filter(line => line.trim().startsWith('|'));
541548
if (lines.length < 3) return [];
@@ -551,7 +558,7 @@ <h1>Benchmark Dashboard</h1>
551558
return {
552559
id: `atomesh-summary-${idx}`,
553560
backend: ATOMESH_DATA.backend,
554-
hardware: ATOMESH_DATA.hardware.toLowerCase(),
561+
hardware: atomeshHardwareKey(raw.hardware),
555562
model: raw.model || 'unknown',
556563
topology: raw.topology || 'unknown',
557564
islOsl: raw.islosl || '--',
@@ -626,6 +633,7 @@ <h1>Benchmark Dashboard</h1>
626633

627634
function hardwareFromGpuName(name) {
628635
const text = String(name || '').toLowerCase();
636+
if (text.includes('mi350')) return 'mi350x';
629637
if (text.includes('mi355') || text.includes('mi35x')) return 'mi355x';
630638
if (text.includes('mi325')) return 'mi325x';
631639
if (text.includes('mi308')) return 'mi308x';
@@ -1209,7 +1217,7 @@ <h1>Benchmark Dashboard</h1>
12091217
const span = document.getElementById('header-gpu-meta');
12101218
if (!span) return;
12111219
if (state.dashboardMode === 'Disaggregated') {
1212-
span.textContent = ` · ${ATOMESH_DATA.hardware} · ROCm ${ATOMESH_DATA.rocmVersion}`;
1220+
span.textContent = ` · ${fmtHardware(selectedHardware())} · ROCm ${ATOMESH_DATA.rocmVersion}`;
12131221
return;
12141222
}
12151223
const entries = Object.values(filteredConfigs());
@@ -3279,7 +3287,7 @@ <h3>${escHTML(model)} MTP Acceptance</h3>
32793287
<div class="kpi-card">
32803288
<div class="kpi-label">Hardware</div>
32813289
<div class="kpi-value">${ATOMESH_DATA.gpuCount} <span class="kpi-unit">GPUs</span></div>
3282-
<div class="kpi-sub">${ATOMESH_DATA.hardware} · ROCm ${ATOMESH_DATA.rocmVersion}</div>
3290+
<div class="kpi-sub">${fmtHardware(selectedHardware())} · ROCm ${ATOMESH_DATA.rocmVersion}</div>
32833291
</div>
32843292
<div class="kpi-card clickable" id="atomesh-acc-card" title="Click to view accuracy">
32853293
<div class="kpi-label">Accuracy</div>

‎.github/scripts/atomesh/process_result.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ def derive_fields(path: Path, payload: dict[str, Any]) -> dict[str, Any] | None:
243243

244244

245245
def enrich_payload(
246-
path: Path, payload: dict[str, Any], fields: dict[str, Any]
246+
path: Path, payload: dict[str, Any], fields: dict[str, Any], hardware: str | None
247247
) -> dict[str, Any]:
248248
env = slurm_job_env(path)
249249
enriched = dict(payload)
@@ -263,6 +263,13 @@ def enrich_payload(
263263
enriched.setdefault("decode_workers", env.get("DECODE_WORKERS"))
264264
enriched.setdefault("prefill_tp", env.get("PREFILL_TP"))
265265
enriched.setdefault("decode_tp", env.get("DECODE_TP"))
266+
runner = env.get("SLURM_SUBMIT_RUNNER", "")
267+
if hardware:
268+
enriched["hardware"] = hardware
269+
elif runner == "atomesh-cicd-mi350":
270+
enriched["hardware"] = "MI350X"
271+
elif runner == "atomesh-cicd":
272+
enriched["hardware"] = "MI355X"
266273

267274
if "total_token_throughput" not in enriched:
268275
enriched["total_token_throughput"] = number(
@@ -330,7 +337,9 @@ def perf_point(
330337
hardware = string_value(
331338
payload.get("hardware"), payload.get("gpu_name"), default="mi355x"
332339
).lower()
333-
if "mi355" in hardware:
340+
if "mi350" in hardware:
341+
hardware = "mi350x"
342+
elif "mi355" in hardware:
334343
hardware = "mi355x"
335344
backend = string_value(
336345
payload.get("backend"), fields.get("backend"), default="atom"
@@ -457,6 +466,7 @@ def collect_dashboard_entries(
457466
paths: list[Path],
458467
run_url: str | None,
459468
gsm8k_scores: dict[tuple[str, int], dict[str, Any]],
469+
hardware: str | None,
460470
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
461471
entries: list[dict[str, Any]] = []
462472
rows: list[dict[str, Any]] = []
@@ -469,7 +479,7 @@ def collect_dashboard_entries(
469479
fields = derive_fields(path, payload)
470480
if not fields:
471481
continue
472-
payload = enrich_payload(path, payload, fields)
482+
payload = enrich_payload(path, payload, fields, hardware)
473483
conc = int(payload.get("max_concurrency", fields["conc"]))
474484
gsm8k_score = gsm8k_scores.get((topology_key(fields["topology"]), conc))
475485
if gsm8k_score is None:
@@ -538,12 +548,13 @@ def write_summary(rows: list[dict[str, Any]], summary_path: Path) -> None:
538548
lines = [
539549
"### ATOMesh Model Performance Benchmark Summary",
540550
"",
541-
"| Model | Topology | ISL/OSL | Concurrency | Interactivity | Total tok/s | Input tok/s | Output tok/s | Total tok/s/GPU | Input tok/s/GPU | Output tok/s/GPU | TTFT ms | TPOT ms | E2E ms | GSM8K |",
542-
"| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
551+
"| Hardware | Model | Topology | ISL/OSL | Concurrency | Interactivity | Total tok/s | Input tok/s | Output tok/s | Total tok/s/GPU | Input tok/s/GPU | Output tok/s/GPU | TTFT ms | TPOT ms | E2E ms | GSM8K |",
552+
"| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
543553
]
544554
for row in rows:
545555
lines.append(
546-
"| {model} | {topology} | {isl}/{osl} | {conc} | {interactivity} | {total} | {input_} | {output} | {total_per_gpu} | {input_per_gpu} | {output_per_gpu} | {ttft} | {tpot} | {e2e} | {gsm8k} |".format(
556+
"| {hardware} | {model} | {topology} | {isl}/{osl} | {conc} | {interactivity} | {total} | {input_} | {output} | {total_per_gpu} | {input_per_gpu} | {output_per_gpu} | {ttft} | {tpot} | {e2e} | {gsm8k} |".format(
557+
hardware=row.get("hardware", "--"),
547558
model=row.get("benchmark_model_name", "--"),
548559
topology=row.get("display_topology") or row.get("topology", "--"),
549560
isl=row.get("random_input_len", "--"),
@@ -578,12 +589,15 @@ def main() -> None:
578589
)
579590
parser.add_argument("--summary", default="benchmark-summary.md")
580591
parser.add_argument("--run-url", default=None)
592+
parser.add_argument("--hardware", default=None)
581593
args = parser.parse_args()
582594

583595
root = Path(args.result_dir)
584596
bench_paths = list(root.rglob("pd-*.json"))
585597
gsm8k_scores = find_eval_scores(root)
586-
entries, rows = collect_dashboard_entries(bench_paths, args.run_url, gsm8k_scores)
598+
entries, rows = collect_dashboard_entries(
599+
bench_paths, args.run_url, gsm8k_scores, args.hardware
600+
)
587601
Path(args.output).write_text(json.dumps(entries, indent=2), encoding="utf-8")
588602
write_summary(rows, Path(args.summary))
589603
print(

‎.github/workflows/atomesh-benchmark.yaml‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ env:
117117
ATOMESH_SLURM_ACCOUNT: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_account != '' && inputs.atomesh_slurm_account || ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '' || 'amd-frameworks') }}
118118
ATOMESH_SLURM_PARTITION: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_partition != '' && inputs.atomesh_slurm_partition || ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '' || 'amd-frameworks') }}
119119
ATOMESH_SLURM_SUBMIT_RUNNER: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd') }}
120+
ATOMESH_DASHBOARD_HARDWARE: ${{ (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && 'MI350X' || 'MI355X' }}
120121
ATOMESH_LOG_ROOT: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_log_root != '' && inputs.atomesh_log_root || ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '/data/${USER}/logs/ATOMESH_LOG/' || '/it-share/ATOMESH_LOG/') }}
121122
ATOMESH_MODEL_ROOT: ${{ (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '/data/models2' || '/mnt/models' }}
122123
ATOMESH_1P1D_NODES: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_1p1d_nodes != '' && inputs.atomesh_1p1d_nodes || ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '' || 'mia1-p02-g42,mia1-p02-g44') }}
@@ -289,6 +290,7 @@ jobs:
289290
"atomesh-results/${MATRIX_ID}" \
290291
--output "${BENCHMARK_ACTION_INPUT}" \
291292
--summary "${SUMMARY_PATH}" \
293+
--hardware "${ATOMESH_DASHBOARD_HARDWARE}" \
292294
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
293295
294296
{
@@ -315,8 +317,10 @@ jobs:
315317
summarize-results:
316318
name: Summarize ATOMesh benchmark results
317319
needs: [run-model-benchmark]
318-
if: ${{ !cancelled() && needs.run-model-benchmark.result != 'skipped' }}
320+
if: ${{ always() && !cancelled() && needs.run-model-benchmark.result != 'skipped' }}
319321
runs-on: ubuntu-latest
322+
outputs:
323+
has_results: ${{ steps.summary.outputs.has_results }}
320324
steps:
321325
- name: Checkout ATOM repo
322326
uses: actions/checkout@v6
@@ -329,16 +333,35 @@ jobs:
329333
path: atomesh-results
330334

331335
- name: Build benchmark summary
336+
id: summary
332337
run: |
333338
set -euo pipefail
339+
mkdir -p atomesh-results
334340
python3 .github/scripts/atomesh/process_result.py \
335341
atomesh-results \
336342
--output atomesh-benchmark-action.json \
337343
--summary atomesh-benchmark-summary.md \
344+
--hardware "${ATOMESH_DASHBOARD_HARDWARE}" \
338345
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
339346
cat atomesh-benchmark-summary.md >> "$GITHUB_STEP_SUMMARY"
347+
ENTRY_COUNT="$(python3 - <<'PY'
348+
import json
349+
from pathlib import Path
350+
path = Path("atomesh-benchmark-action.json")
351+
entries = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else []
352+
print(len(entries))
353+
PY
354+
)"
355+
echo "entry_count=${ENTRY_COUNT}" >> "$GITHUB_OUTPUT"
356+
if [[ "${ENTRY_COUNT}" -gt 0 ]]; then
357+
echo "has_results=true" >> "$GITHUB_OUTPUT"
358+
else
359+
echo "has_results=false" >> "$GITHUB_OUTPUT"
360+
echo "No successful ATOMesh benchmark result JSONs were produced; dashboard publish will be skipped." >> "$GITHUB_STEP_SUMMARY"
361+
fi
340362
341363
- name: Upload benchmark summary
364+
if: ${{ steps.summary.outputs.has_results == 'true' }}
342365
uses: actions/upload-artifact@v7
343366
with:
344367
name: atomesh-benchmark-summary
@@ -355,6 +378,7 @@ jobs:
355378
if: >-
356379
!cancelled()
357380
&& needs.summarize-results.result == 'success'
381+
&& needs.summarize-results.outputs.has_results == 'true'
358382
&& (
359383
github.event_name == 'schedule'
360384
|| github.event_name == 'push'
@@ -385,6 +409,9 @@ jobs:
385409
"window.ATOMESH_RUN_URL = "
386410
+ json.dumps("https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}")
387411
+ ";\n"
412+
+ "window.ATOMESH_HARDWARE = "
413+
+ json.dumps("${{ env.ATOMESH_DASHBOARD_HARDWARE }}")
414+
+ ";\n"
388415
+ "window.ATOMESH_SUMMARY_MARKDOWN = "
389416
+ json.dumps(summary, ensure_ascii=False)
390417
+ ";\n",

0 commit comments

Comments
 (0)