[ci][mesh] improve ATOMesh benchmark configuration and dashboard reporting #6533

Workflow file for this run

.github/workflows/atom-test.yaml at dd27b69

	name: ATOM Test

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main] # Triggers on PRs targeting `main`
	types: [opened, synchronize, reopened, ready_for_review, labeled, closed]
	paths-ignore:
	- '*/.md'
	- 'docs/**'
	- 'atom/plugin/**'
	- 'atom/mesh/**'
	- '.github/benchmark/models_atomesh.yaml'
	- '.github/scripts/atomesh/**'
	- '.github/workflows/atomesh-*.yaml'
	- '.github/workflows/atom-vllm-*.yaml'
	- '.github/workflows/atom-sglang-*.yaml'
	- '.github/benchmark/oot_models_accuracy.json'
	- '.github/benchmark/sglang_models_accuracy.json'
	- 'LICENSE'
	- '.gitignore'
	pull_request_review:
	types: [submitted]
	schedule:
	# Nightly at 00:00 Beijing time (16:00 UTC)
	- cron: '0 16 * * *'
	workflow_dispatch:
	inputs:
	aiter_branch:
	description: 'ROCm/aiter branch to build inside the CI image'
	required: false
	default: 'main'
	type: string
	atom_base_image:
	description: 'Docker image used as the ATOM test base image'
	required: false
	default: 'rocm/atom-dev:latest'
	type: string

	concurrency:
	# Keep scheduled main runs from blocking push-triggered validation.
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}-${{ github.event.pull_request.number && 'pr' \|\| github.event_name }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' && (github.event_name != 'pull_request_review' \|\| github.event.review.state == 'approved') }}

	env:
	ATOM_BASE_IMAGE: ${{ github.event_name == 'workflow_dispatch' && inputs.atom_base_image \|\| 'rocm/atom-dev:latest' }}
	ATOM_PYTHON_TAG: "cp312"
	GITHUB_REPO_URL: ${{ github.event.pull_request.head.repo.clone_url \|\| 'https://github.com/ROCm/ATOM.git' }}
	GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id \|\| github.sha }}
	# workflow_dispatch: inputs.aiter_branch; otherwise main (matches previous default-branch shallow clone)
	AITER_GIT_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.aiter_branch \|\| 'main' }}

	jobs:
	ci-gate:
	name: Check Heavy CI Gate
	runs-on: ubuntu-latest
	permissions:
	contents: read
	pull-requests: read
	outputs:
	should_run: ${{ steps.gate.outputs.should_run }}
	reason: ${{ steps.gate.outputs.reason }}
	steps:
	- name: Checkout ATOM repo
	uses: actions/checkout@v6

	- name: Check approval and label gate
	id: gate
	env:
	CI_GATE_LABELS: ci:full,ci:atom
	CI_GATE_PATHS_IGNORE: \|
	*/.md
	docs/**
	atom/plugin/**
	atom/mesh/**
	.github/workflows/atomesh-*.yaml
	.github/scripts/atomesh_*.sh
	.github/dashboard/atomesh_*.html
	.github/workflows/atom-vllm-*.yaml
	.github/workflows/atom-sglang-*.yaml
	.github/benchmark/oot_models_accuracy.json
	.github/benchmark/sglang_models_accuracy.json
	LICENSE
	.gitignore
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: bash ./.github/scripts/check_heavy_ci_gate.sh

	check-signal:
	if: ${{ needs.ci-gate.outputs.should_run == 'true' }}
	needs: [ci-gate]
	name: Check Pre Checkin Signal
	runs-on: ubuntu-latest
	permissions:
	actions: read
	contents: read
	steps:
	- name: Checkout ATOM repo
	if: ${{ github.event_name != 'workflow_dispatch' }}
	uses: actions/checkout@v6

	- name: Wait for Pre Checkin workflow
	if: ${{ github.event_name != 'workflow_dispatch' }}
	run: bash ./.github/scripts/check_signal.sh
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	GITHUB_SHA: ${{ github.sha }}
	CHECK_SIGNAL_EVENT_NAME: ${{ github.event.pull_request && 'pull_request' \|\| github.event_name }}
	CHECK_SIGNAL_HEAD_REF: ${{ github.event.pull_request.head.ref \|\| github.head_ref \|\| github.ref_name }}
	CHECK_SIGNAL_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}

	download_aiter_wheel:
	if: ${{ needs.ci-gate.outputs.should_run == 'true' && needs.check-signal.result == 'success' }}
	needs: [ci-gate, check-signal]
	name: Download aiter wheel
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@v6
	- name: Prefer latest main aiter wheel manifest and fallback to artifact
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: bash .github/scripts/download_aiter_wheel.sh

	- name: Upload aiter wheel
	uses: actions/upload-artifact@v7
	with:
	name: aiter-whl
	path: aiter-whl/amd_aiter*.whl
	retention-days: 7

	load-test-models:
	if: ${{ needs.ci-gate.outputs.should_run == 'true' }}
	needs: [ci-gate]
	name: Load test model configs
	runs-on: ubuntu-latest
	outputs:
	models_json: ${{ steps.load.outputs.models_json }}
	steps:
	- uses: actions/checkout@v6
	- id: load
	env:
	EVENT_NAME: ${{ github.event_name }}
	run: \|
	python3 << 'PY'
	import json, os
	event = os.environ["EVENT_NAME"]
	# pr → pr models only; push to main → pr+main; schedule/dispatch → all
	level_map = {"schedule": "nightly", "workflow_dispatch": "nightly", "push": "main"}
	current = level_map.get(event, "pr")
	allowed = {"pr": {"pr"}, "main": {"pr", "main"}, "nightly": {"pr", "main", "nightly"}}[current]
	models = json.load(open(".github/benchmark/models_accuracy.json", encoding="utf-8"))
	filtered = [m for m in models if m.get("test_level", "nightly") in allowed]
	with open(os.environ["GITHUB_OUTPUT"], "a") as f:
	f.write(f"models_json={json.dumps(filtered)}\n")
	print(f"Event={event} level={current}: {len(filtered)}/{len(models)} models")
	print(f"{'Model':<45} {'Level':<10} {'Runner'}")
	print("-" * 80)
	for m in models:
	enabled = "✓" if m in filtered else "·"
	print(f" {enabled} {m['model_name']:<43} {m.get('test_level','?'):<10} {m['runner']}")
	PY

	atom-test:
	needs: [ci-gate, download_aiter_wheel, load-test-models]
	name: Accuracy
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJson(needs.load-test-models.outputs.models_json \|\| '[{"runner":"ubuntu-latest","model_name":"skip"}]') }}
	if: ${{ needs.ci-gate.outputs.should_run == 'true' && needs.download_aiter_wheel.result == 'success' && needs.load-test-models.result == 'success' }}
	runs-on: ${{ matrix.runner }}

	env:
	CONTAINER_NAME: atom_test_${{ strategy.job-index }}

	steps:
	- name: Kill all Docker containers and clean up workspace
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'linux-atom-do-mi350x-8'
	run: \|
	echo "=== Cleaning up containers on $(hostname) ==="
	containers=$(docker ps -q)
	if [ -n "$containers" ]; then
	docker kill $containers \|\| true
	fi
	docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "ls -la /workspace/ && find /workspace -mindepth 1 -delete" \|\| true

	- name: Show Docker containers
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'linux-atom-do-mi350x-8'
	run: docker ps -a

	- name: Show ROCm memory usage
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'linux-atom-do-mi350x-8'
	run: rocm-smi --showmemuse

	- name: Show ROCm GPU processes
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'linux-atom-do-mi350x-8'
	run: rocm-smi --showpidgpus

	- name: Checkout ATOM repo
	uses: actions/checkout@v6

	- name: Docker Login
	if: ${{ !github.event.pull_request.head.repo.fork }}
	uses: ./.github/actions/docker-auth
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Resolve immutable native dashboard image
	if: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'push' \|\| github.event_name == 'schedule') }}
	env:
	DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
	DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
	run: \|
	set -euo pipefail
	if RESOLUTION_JSON="$(
	python3 .github/scripts/resolve_atom_image.py \
	--repository rocm/atom-dev \
	--reference-tag latest \
	--image-family native
	)"; then
	RESOLVED_ATOM_IMAGE="$(
	RESOLUTION_JSON="${RESOLUTION_JSON}" python3 - <<'PY'
	import json
	import os

	resolution = json.loads(os.environ["RESOLUTION_JSON"])
	print(resolution["resolved_image"])
	PY
	)"
	echo "Resolved native dashboard image: ${RESOLVED_ATOM_IMAGE}"
	else
	echo "::error::Failed to resolve ${ATOM_BASE_IMAGE} to an immutable reference for dashboard-uploading native runs."
	exit 1
	fi
	echo "RESOLVED_ATOM_BASE_IMAGE=${RESOLVED_ATOM_IMAGE}" >> "$GITHUB_ENV"
	echo "ATOM_DASHBOARD_DOCKER_IMAGE=${RESOLVED_ATOM_IMAGE}" >> "$GITHUB_ENV"

	- name: Pull immutable native dashboard image
	if: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'push' \|\| github.event_name == 'schedule') }}
	run: \|
	echo "Pulling immutable native dashboard image: ${RESOLVED_ATOM_BASE_IMAGE}"
	docker pull "${RESOLVED_ATOM_BASE_IMAGE}"

	- name: Generate Dockerfile for forked repo
	if: ${{ github.event.pull_request.head.repo.fork }}
	run: \|
	cat <<EOF > Dockerfile.mod
	FROM ${{ env.ATOM_BASE_IMAGE }}
	RUN pip install -U lm-eval[api]
	RUN pip show lm-eval \|\| true
	RUN pip install hf_transfer
	RUN pip show hf_transfer \|\| true
	RUN echo "=== Aiter version BEFORE uninstall ===" && pip show amd-aiter \|\| true
	RUN pip uninstall -y amd-aiter
	RUN pip install --upgrade "pybind11>=3.0.1"
	RUN pip show pybind11
	RUN rm -rf /app/aiter-test
	RUN git clone --filter=blob:none -b ${{ env.AITER_GIT_REF }} https://github.com/ROCm/aiter.git /app/aiter-test && \\
	cd /app/aiter-test && \\
	git submodule sync && git submodule update --init --recursive && \\
	MAX_JOBS=64 PREBUILD_KERNELS=0 GPU_ARCHS=gfx950 python3 setup.py develop
	RUN echo "=== Aiter version AFTER installation ===" && pip show amd-aiter \|\| true

	RUN echo "=== ATOM version BEFORE uninstall ===" && pip show atom \|\| true
	RUN pip uninstall -y atom
	RUN rm -rf /app/ATOM
	RUN git clone ${{ env.GITHUB_REPO_URL }} /app/ATOM && \\
	cd /app/ATOM && \\
	git checkout ${{ env.GITHUB_COMMIT_SHA }} && \\
	pip install -e .

	RUN echo "=== ATOM version AFTER installation ===" && pip show atom \|\| true
	EOF

	- name: Download aiter wheel
	uses: actions/download-artifact@v8
	with:
	name: aiter-whl
	path: /tmp/aiter-whl

	- name: Set HF token for predownload runner
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'linux-atom-do-mi350x-8'
	run: echo "HF_TOKEN=${HF_TOKEN:-${{ secrets.AMD_HF_TOKEN }}}" >> "$GITHUB_ENV"

	- name: Start CI container
	uses: ./.github/actions/setup-gpu-container
	with:
	container-name: ${{ env.CONTAINER_NAME }}
	base-image: ${{ env.ATOM_BASE_IMAGE }}
	resolved-image: ${{ env.RESOLVED_ATOM_BASE_IMAGE }}
	runner: ${{ matrix.runner }}
	env-vars: ${{ matrix.env_vars }}
	hf-token: ${{ env.HF_TOKEN }}
	dashboard-image: ${{ env.ATOM_DASHBOARD_DOCKER_IMAGE }}
	network-host: "true"

	- name: Check shm size
	run: \|
	docker exec "$CONTAINER_NAME" df -h /dev/shm

	- name: GPU preflight check
	if: matrix.runner == 'linux-atom-do-mi350x-8'
	timeout-minutes: 5
	env:
	GPU_PREFLIGHT_ALLOCATION_MB: "8"
	run: bash .github/scripts/gpu_preflight_check.sh "$CONTAINER_NAME" docker

	- name: Collect GPU info (inside container)
	id: gpu-info
	env:
	COLLECT_VERBOSE_GPU_INFO: ${{ matrix.runner == 'linux-atom-do-mi350x-8' && '1' \|\| '0' }}
	run: bash .github/scripts/collect_gpu_info.sh "$CONTAINER_NAME" docker "${{ matrix.runner }}"

	- name: Install aiter from wheel
	run: bash .github/scripts/install_aiter_wheel.sh

	- name: Install ATOM and dependencies
	run: \|
	docker exec "$CONTAINER_NAME" bash -lc "
	set -euo pipefail
	pip install --timeout 60 --retries 10 -U 'lm-eval[api]'
	pip install --timeout 60 --retries 10 hf_transfer
	pip install --timeout 60 --retries 10 --upgrade 'pybind11>=3.0.1'

	echo '=== Installing ATOM ==='
	cd /workspace
	git config --global --add safe.directory /workspace
	pip install -e .

	echo '=== Installed package versions ==='
	pip show amd-aiter \| grep -E '^(Name\|Version):'
	pip show atom \| grep -E '^(Name\|Version):'
	pip show triton \| grep -E '^(Name\|Version):'
	pip show torch \| grep -E '^(Name\|Version):'
	"

	- name: Download models
	timeout-minutes: 150
	run: \|
	set -euo pipefail
	if [ -d "/models" ]; then
	model_dir="/models/${{ matrix.model_path }}"
	echo "/models directory found, checking cache and lock-protected download for ${model_dir}"
	if ! docker exec \
	-e HF_TOKEN="${HF_TOKEN:-}" \
	-e MODEL_ID="${{ matrix.model_path }}" \
	-e TARGET_DIR="${model_dir}" \
	-e MODEL_DOWNLOAD_TIMEOUT="${MODEL_DOWNLOAD_TIMEOUT}" \
	-e MODEL_LOCK_WAIT_SECONDS="${MODEL_LOCK_WAIT_SECONDS}" \
	-e MODEL_LOCK_POLL_INTERVAL="${MODEL_LOCK_POLL_INTERVAL}" \
	-e MODEL_PROGRESS_INTERVAL="${MODEL_PROGRESS_INTERVAL}" \
	"$CONTAINER_NAME" bash -lc 'bash /workspace/.github/scripts/download_model_with_lock.sh "$MODEL_ID" "$TARGET_DIR"'; then
	echo "Model download failed for '${{ matrix.model_path }}'. Aborting."
	exit 1
	fi
	else
	echo "/models directory not found, skipping model download"
	fi
	env:
	MODEL_DOWNLOAD_TIMEOUT: "30m"
	MODEL_LOCK_WAIT_SECONDS: "1800"
	MODEL_LOCK_POLL_INTERVAL: "30"
	MODEL_PROGRESS_INTERVAL: "60"

	- name: Run ATOM simple inference
	# Skip simple inference; accuracy test already validates correctness
	if: false
	timeout-minutes: 30
	run: \|
	# Run the inference and capture output
	set -euo pipefail

	echo ""
	echo "========== Running test =========="

	if [ -d "/models" ]; then
	model_path="/models/${{ matrix.model_path }}"
	else
	model_path="${{ matrix.model_path }}"
	fi
	echo "Model path: $model_path"
	ls -la $model_path \|\| true
	# Print debug logs
	echo "========= Runner debug logs ==============="
	ps aux
	rocm-smi --showmemuse
	rocm-smi --showpids
	docker ps -a
	echo "========= End runner debug logs ==============="
	docker exec "$CONTAINER_NAME" bash -lc "
	set -euo pipefail
	python3 -m atom.examples.simple_inference \
	--model \"$model_path\" \
	${{ matrix.extraArgs }} \
	--temperature 0 \
	\| grep -E '^Prompt: \|^Completion:'
	" > atom_test_output.txt

	echo ""
	echo "========== Showing test output below =========="
	cat atom_test_output.txt

	- name: Compare output with golden outputs
	if: false
	timeout-minutes: 30
	# TODO: skip for all test until it's fixed
	run: \|
	echo "========== Comparing output with golden outputs =========="
	if ! diff -u -B -w --strip-trailing-cr \
	atom_test_output.txt \
	".github/workflows/golden_outputs/${{ matrix.model_name }}_golden_output.txt"; then
	echo "Failed: Output does not match golden outputs."
	exit 1
	else
	echo "Success: Output matches golden outputs."
	fi

	- name: Run ATOM accuracy test
	timeout-minutes: 30
	env:
	MODEL_EXTRA_ARGS: ${{ matrix.extraArgs }}
	CLIENT_COMMAND: ${{ matrix.client_command \|\| '' }}
	ATOM_DEBUG_DEVICE_MAPPING: ${{ matrix.runner == 'linux-atom-do-mi350x-8' && '1' \|\| '0' }}
	run: \|
	set -euo pipefail
	echo ""
	echo "========== Launching ATOM server =========="
	if [ -d "/models" ]; then
	model_path="/models/${{ matrix.model_path }}"
	else
	model_path="${{ matrix.model_path }}"
	fi
	# Pipe via stdin so container bash parses shell quoting in extraArgs
	# (e.g. single-quoted JSON in --default-chat-template-kwargs) naturally.
	echo ".github/scripts/atom_test.sh launch $model_path $MODEL_EXTRA_ARGS" \| \
	docker exec \
	-e ATOM_DEBUG_DEVICE_MAPPING="${ATOM_DEBUG_DEVICE_MAPPING}" \
	-i "$CONTAINER_NAME" bash -l
	echo ""
	echo "========== Running accuracy test =========="
	docker exec \
	-e CLIENT_COMMAND="${CLIENT_COMMAND}" \
	-e GPU_NAME="${{ steps.gpu-info.outputs.gpu_name }}" \
	-e GPU_VRAM_GB="${{ steps.gpu-info.outputs.gpu_vram_gb }}" \
	-e ROCM_VERSION="${{ steps.gpu-info.outputs.rocm_version }}" \
	"$CONTAINER_NAME" bash -lc "
	.github/scripts/atom_test.sh accuracy $model_path
	" 2>&1 \| tee atom_accuracy_output.txt

	- name: Dump server log
	if: always()
	run: \|
	docker exec "$CONTAINER_NAME" cat /tmp/atom_server.log 2>/dev/null \|\| true

	- name: Dump client log
	if: always()
	run: \|
	docker exec "$CONTAINER_NAME" cat /tmp/atom_client.log 2>/dev/null \|\| true

	- name: Check accuracy test results
	if: success()
	env:
	MODEL_NAME: ${{ matrix.model_name }}
	run: \|
	result_file=$(ls -1t accuracy_test_results/*.json 2>/dev/null \| head -n 1)
	if [ -z "$result_file" ] \|\| [ ! -f "$result_file" ]; then
	echo "ERROR: No results JSON file found in accuracy_test_results/"
	exit 2
	else
	echo "RESULT_FILE: $result_file"
	fi
	flexible_extract_value=$(jq '.results.gsm8k["exact_match,flexible-extract"]' "$result_file")
	echo "Flexible extract value: $flexible_extract_value"

	# Read threshold from models_accuracy.json (via env var to avoid shell injection)
	threshold=$(python3 -c "
	import json, os
	models = json.load(open('.github/benchmark/models_accuracy.json', encoding='utf-8'))
	name = os.environ['MODEL_NAME']
	t = next((m.get('accuracy_threshold', 0) for m in models if m['model_name'] == name), 0)
	print(t)
	")
	echo "Accuracy test threshold: $threshold"

	result=$(awk -v val="$flexible_extract_value" -v threshold="$threshold" 'BEGIN {print (val < threshold) ? 1 : 0}')
	if [ "$result" -eq 1 ]; then
	echo "Accuracy test failed: $flexible_extract_value < $threshold"
	exit 1
	else
	echo "Accuracy test passed: $flexible_extract_value >= $threshold"
	fi

	- name: Check MTP acceptance rate
	# Only gates models that declare mtp_accept_threshold in the catalog
	# (MTP/spec-decode entries); all other models skip this step.
	if: ${{ success() && matrix.mtp_accept_threshold != '' }}
	env:
	MTP_ACCEPT_THRESHOLD: ${{ matrix.mtp_accept_threshold }}
	run: \|
	# gsm8k accuracy alone CANNOT guard MTP: speculative decoding is
	# lossless w.r.t. the target model, so a broken draft head leaves
	# accuracy unchanged and only craters acceptance/throughput. This
	# step is the only gate that catches such an MTP regression.
	# atom_test.sh scrapes the cumulative "[MTP Stats ]" line from the
	# server log during the gsm8k run and records mtp_acceptance_rate
	# (a PERCENT, e.g. 62.5) into atom_ci_metadata. The catalog
	# threshold is a FRACTION (0-1, same convention as accuracy_threshold),
	# so we divide the recorded percent by 100 before comparing.
	result_file=$(ls -1t accuracy_test_results/*.json 2>/dev/null \| head -n 1)
	if [ -z "$result_file" ] \|\| [ ! -f "$result_file" ]; then
	echo "ERROR: No results JSON file found for MTP acceptance check."
	exit 2
	fi
	echo "RESULT_FILE: $result_file"

	rate=$(jq -r '.atom_ci_metadata.mtp_acceptance_rate // empty' "$result_file")
	if [ -z "$rate" ]; then
	echo "FAIL: mtp_acceptance_rate missing — MTP stats were not emitted"
	echo " during the accuracy run. Treating as an MTP regression."
	exit 1
	fi

	echo "MTP acceptance: ${rate}% (threshold ${MTP_ACCEPT_THRESHOLD} = $(awk -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN{printf "%.1f", t*100}')%)"
	if awk -v v="$rate" -v t="$MTP_ACCEPT_THRESHOLD" 'BEGIN {exit !((v/100) < t)}'; then
	echo "MTP acceptance gate FAILED: ${rate}% < threshold"
	exit 1
	fi
	echo "MTP acceptance gate PASSED."

	- name: Collect Test Summary
	if: success()
	env:
	MODEL_NAME: ${{ matrix.model_name }}
	run: \|
	# Read threshold and score for summary
	threshold=$(python3 -c "
	import json, os
	models = json.load(open('.github/benchmark/models_accuracy.json', encoding='utf-8'))
	name = os.environ['MODEL_NAME']
	print(next((m.get('accuracy_threshold', 0) for m in models if m['model_name'] == name), 0))
	")
	result_file=$(ls -1t accuracy_test_results/*.json 2>/dev/null \| head -n 1)
	score=$(jq '.results.gsm8k["exact_match,flexible-extract"]' "$result_file" 2>/dev/null \|\| echo "N/A")

	echo "Accuracy Test Summary for ${{ matrix.model_name }} (threshold: ${threshold}, score: ${score}):" >> $GITHUB_STEP_SUMMARY
	awk '/\\|Tasks\\|Version\\|/,/^$/ { if (NF > 0) print }' atom_accuracy_output.txt >> $GITHUB_STEP_SUMMARY

	- name: Upload output
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: ${{ matrix.model_name }}_atom_test_output.txt
	path: atom_test_output.txt

	- name: Upload accuracy results
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: accuracy-${{ matrix.model_name }}
	path: accuracy_test_results/*.json
	if-no-files-found: ignore

	- name: Clean Up
	if: always()
	run: \|
	# TODO: run a separate container for cleanup of the workspace due to permission issue to remove some pyc files under __pycache__ whose owners are root.
	# We should use non-root user to run the test to avoid this issue.
	set -x
	echo "========== Cleaning up workspace =========="
	if [[ ${{ matrix.runner }} == atom-mi355-8gpu.predownload ]]; then
	docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "ls -la /workspace/ && find /workspace -mindepth 1 -delete" \|\| true
	fi
	docker stop "$CONTAINER_NAME" \|\| true
	docker rm "$CONTAINER_NAME" \|\| true
	# Remove the pre-built image to free disk space on the runner
	docker rmi "rocm/atom-dev:pre-build-${{ env.GITHUB_COMMIT_SHA }}" \|\| true

	# ---------- Push accuracy data to benchmark dashboard ----------
	accuracy-dashboard:
	name: Update accuracy dashboard
	needs: [atom-test]
	if: always() && github.ref == 'refs/heads/main' && (github.event_name == 'push' \|\| github.event_name == 'schedule')
	# Serialize with every other gh-pages push so the auto-push below does not
	# race concurrent deploys (docs / benchmark dashboards) on the gh-pages branch.
	concurrency:
	group: gh-pages-deploy
	cancel-in-progress: false
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v6

	- uses: actions/setup-python@v6
	with:
	python-version: '3.12'

	- name: Download accuracy artifacts
	uses: actions/download-artifact@v8
	with:
	path: /tmp/accuracy-results
	pattern: accuracy-*

	- name: List downloaded artifacts
	run: \|
	echo "=== Downloaded accuracy artifacts ==="
	find /tmp/accuracy-results -type f -name '*.json' \| head -20 \|\| echo "No JSON files found"

	- name: Transform accuracy results for dashboard
	run: \|
	python3 .github/scripts/accuracy_to_dashboard.py \
	/tmp/accuracy-results \
	--output accuracy-benchmark-input.json \
	--models .github/benchmark/models_accuracy.json \
	--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
	echo "=== Generated entries ==="
	cat accuracy-benchmark-input.json

	- name: Store accuracy result to dashboard
	if: hashFiles('accuracy-benchmark-input.json') != ''
	uses: benchmark-action/github-action-benchmark@v1
	with:
	tool: customBiggerIsBetter
	output-file-path: accuracy-benchmark-input.json
	gh-pages-branch: gh-pages
	benchmark-data-dir-path: benchmark-dashboard
	auto-push: true
	max-items-in-chart: 300
	github-token: ${{ secrets.GITHUB_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[ci][mesh] improve ATOMesh benchmark configuration and dashboard reporting #6533

Workflow file

[ci][mesh] improve ATOMesh benchmark configuration and dashboard reporting #6533

Uh oh!

Workflow file for this run