[ci][mesh] improve ATOMesh benchmark configuration and dashboard reporting #3942

Workflow file for this run

.github/workflows/atom-vllm-test.yaml at 5b7c945

	name: ATOM vLLM Test

	on:
	pull_request:
	branches: [main] # Triggers on PRs targeting `main`
	types: [opened, synchronize, reopened, ready_for_review, labeled, closed]
	paths-ignore:
	- '*/.md'
	- 'docs/**'
	- 'atom/plugin/sglang/**'
	- 'atom/mesh/**'
	- '.github/benchmark/models_atomesh.yaml'
	- '.github/scripts/atomesh/**'
	- '.github/workflows/atomesh-*.yaml'
	- '.github/workflows/atom-sglang-*.yaml'
	- '.github/benchmark/sglang_models_accuracy.json'
	- 'LICENSE'
	- '.gitignore'
	pull_request_review:
	types: [submitted]

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: ${{ github.event_name != 'pull_request_review' \|\| github.event.review.state == 'approved' }}

	env:
	ATOM_BASE_NIGHTLY_IMAGE: rocm/atom-dev:latest
	ATOM_PYTHON_TAG: "cp312"
	OOT_IMAGE_CACHE_KEEP: "3"
	GITHUB_REPO_URL: ${{ github.event.pull_request.head.repo.clone_url \|\| 'https://github.com/ROCm/ATOM.git' }}
	GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha \|\| github.event.head_commit.id \|\| github.sha }}
	NIGHTLY_OOT_IMAGE_TAG: rocm/atom-dev:vllm-latest
	VLLM_COMMIT: "0b3ba88f165976e77ca5e6a7a3f5bba4562b80af"

	jobs:
	# ---------- OOT CI path (pull_request only) ----------
	ci-gate:
	name: Check Heavy CI Gate
	runs-on: ubuntu-latest
	permissions:
	contents: read
	pull-requests: read
	outputs:
	should_run: ${{ steps.gate.outputs.should_run }}
	reason: ${{ steps.gate.outputs.reason }}
	steps:
	- name: Checkout ATOM repo
	uses: actions/checkout@v6

	- name: Check approval and label gate
	id: gate
	env:
	CI_GATE_LABELS: ci:full,ci:vllm
	CI_GATE_PATHS_IGNORE: \|
	*/.md
	docs/**
	atom/plugin/sglang/**
	atom/mesh/**
	.github/workflows/atomesh-*.yaml
	.github/scripts/atomesh_*.sh
	.github/dashboard/atomesh_*.html
	.github/workflows/atom-sglang-*.yaml
	.github/benchmark/sglang_models_accuracy.json
	LICENSE
	.gitignore
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: bash ./.github/scripts/check_heavy_ci_gate.sh

	check-signal:
	if: ${{ needs.ci-gate.outputs.should_run == 'true' }}
	needs: [ci-gate]
	name: Check Pre Checkin Signal
	runs-on: ubuntu-latest
	permissions:
	actions: read
	contents: read
	steps:
	- name: Checkout ATOM repo
	uses: actions/checkout@v6

	- name: Wait for Pre Checkin workflow
	run: bash ./.github/scripts/check_signal.sh
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	GITHUB_SHA: ${{ github.sha }}
	CHECK_SIGNAL_EVENT_NAME: pull_request
	CHECK_SIGNAL_HEAD_REF: ${{ github.event.pull_request.head.ref }}
	CHECK_SIGNAL_SHA: ${{ github.event.pull_request.head.sha }}

	download_aiter_wheel:
	if: ${{ needs.ci-gate.outputs.should_run == 'true' && needs.check-signal.result == 'success' }}
	needs: [ci-gate, check-signal]
	name: Download aiter wheel
	runs-on: ubuntu-latest
	outputs:
	aiter_artifact_id: ${{ steps.download.outputs.aiter_artifact_id }}
	aiter_wheel_name: ${{ steps.download.outputs.aiter_wheel_name }}
	steps:
	- name: Prefer latest main aiter wheel manifest and fallback to artifact
	id: download
	run: \|
	set -euo pipefail
	echo "=== Trying latest main aiter wheel manifest from S3 first ==="

	S3_MAIN_MANIFEST_URL="https://rocm.frameworks-nightlies.amd.com/whl-staging/gfx942-gfx950/main/latest.json"
	API_URL="https://api.github.com"
	AUTH_HEADER="Authorization: token ${{ secrets.GITHUB_TOKEN }}"
	AITER_TEST_WORKFLOW_ID=179476100

	ARTIFACT_ID=""
	ARTIFACT_NAME=""
	ARTIFACT_RUN_ID=""
	ARTIFACT_RUN_SHA=""
	ARTIFACT_RUN_CREATED_AT=""

	resolve_download_url() {
	python3 -c 'import sys
	from urllib.parse import quote, unquote, urlsplit, urlunsplit
	parts = urlsplit(sys.argv[1])
	encoded_path = "/".join(quote(unquote(segment), safe="") for segment in parts.path.split("/"))
	print(urlunsplit((parts.scheme, parts.netloc, encoded_path, parts.query, parts.fragment)))' "$1"
	}

	find_latest_artifact() {
	local runs_json artifact_json run_id python_artifact_suffix

	if [ -n "$ARTIFACT_ID" ] && [ "$ARTIFACT_ID" != "null" ]; then
	return 0
	fi

	python_artifact_suffix="py${ATOM_PYTHON_TAG#cp}"
	python_artifact_suffix="${python_artifact_suffix:0:3}.${python_artifact_suffix:3}"

	echo "=== Finding latest aiter-whl-* artifact for ${python_artifact_suffix} from ROCm/aiter ==="
	runs_json=$(curl -fsSL -H "$AUTH_HEADER" \
	"$API_URL/repos/ROCm/aiter/actions/workflows/$AITER_TEST_WORKFLOW_ID/runs?per_page=100&branch=main&event=push")

	for run_id in $(echo "$runs_json" \| jq -r '.workflow_runs[].id'); do
	artifact_json=$(curl -fsSL -H "$AUTH_HEADER" \
	"$API_URL/repos/ROCm/aiter/actions/runs/$run_id/artifacts" \
	\| jq --arg artifact_suffix "-${python_artifact_suffix}" '[.artifacts[] \| select(.name \| startswith("aiter-whl-") and endswith($artifact_suffix)) \| select(.expired == false)] \| sort_by(.created_at) \| last')

	if [ "$artifact_json" != "null" ] && [ -n "$artifact_json" ]; then
	ARTIFACT_ID=$(echo "$artifact_json" \| jq -r '.id')
	ARTIFACT_NAME=$(echo "$artifact_json" \| jq -r '.name')
	ARTIFACT_RUN_ID="$run_id"
	ARTIFACT_RUN_SHA=$(echo "$runs_json" \| jq -r --arg run_id "$run_id" '.workflow_runs[] \| select((.id \| tostring) == $run_id) \| .head_sha')
	ARTIFACT_RUN_CREATED_AT=$(echo "$runs_json" \| jq -r --arg run_id "$run_id" '.workflow_runs[] \| select((.id \| tostring) == $run_id) \| .created_at')
	echo "Found artifact in run $ARTIFACT_RUN_ID: $ARTIFACT_NAME (ID: $ARTIFACT_ID, SHA: $ARTIFACT_RUN_SHA)"
	return 0
	fi
	done

	return 1
	}

	download_from_s3_manifest() {
	local manifest_file manifest_fetch_url manifest_branch manifest_timestamp manifest_commit wheel_name wheel_url resolved_wheel_url

	mkdir -p aiter-whl
	rm -f aiter-whl/amd_aiter*.whl

	manifest_file=$(mktemp)
	trap 'rm -f "$manifest_file"' RETURN
	manifest_fetch_url="${S3_MAIN_MANIFEST_URL}?ts=$(date +%s)"
	curl -fsSL -H "Cache-Control: no-cache" "$manifest_fetch_url" -o "$manifest_file" \|\| return 1

	manifest_branch=$(jq -r '.branch // empty' "$manifest_file")
	manifest_timestamp=$(jq -r '.timestamp // empty' "$manifest_file")
	manifest_commit=$(jq -r '.commit // empty' "$manifest_file")

	wheel_name=$(jq -r ".wheels.${ATOM_PYTHON_TAG}.wheel_name // empty" "$manifest_file")
	wheel_url=$(jq -r ".wheels.${ATOM_PYTHON_TAG}.wheel_url // empty" "$manifest_file")
	if [ -n "$wheel_name" ] && [ -n "$wheel_url" ]; then
	echo "Selected ${ATOM_PYTHON_TAG} wheel from versioned manifest"
	else
	wheel_name=$(jq -r '.wheel_name // empty' "$manifest_file")
	wheel_url=$(jq -r '.wheel_url // empty' "$manifest_file")
	echo "Versioned manifest not available, using top-level wheel fields"
	fi

	if [ "$manifest_branch" != "main" ] \|\| [ -z "$manifest_timestamp" ] \|\| [ -z "$manifest_commit" ] \|\| [ -z "$wheel_name" ] \|\| [ -z "$wheel_url" ]; then
	echo "Invalid latest main wheel manifest"
	return 1
	fi

	if [[ "$wheel_name" == cp ]] && [[ "$wheel_name" != ${ATOM_PYTHON_TAG} ]]; then
	echo "WARNING: wheel $wheel_name does not match target Python ${ATOM_PYTHON_TAG}"
	return 1
	fi

	if find_latest_artifact; then
	if [ -n "$ARTIFACT_RUN_SHA" ] && [ "$manifest_commit" != "$ARTIFACT_RUN_SHA" ]; then
	if [ -n "$ARTIFACT_RUN_CREATED_AT" ] && [[ "$manifest_timestamp" < "$ARTIFACT_RUN_CREATED_AT" ]]; then
	echo "Manifest commit $manifest_commit is older than latest artifact run $ARTIFACT_RUN_ID ($ARTIFACT_RUN_SHA); treating manifest as stale"
	return 1
	fi
	echo "Manifest commit $manifest_commit differs from latest artifact run $ARTIFACT_RUN_ID ($ARTIFACT_RUN_SHA), but manifest timestamp is not older"
	fi
	else
	echo "No GitHub fallback artifact found while checking manifest freshness"
	fi

	resolved_wheel_url=$(resolve_download_url "$wheel_url")

	echo "Selected latest main wheel manifest: $S3_MAIN_MANIFEST_URL"
	echo "Manifest timestamp: $manifest_timestamp"
	echo "Manifest commit: $manifest_commit"
	echo "Manifest wheel: $wheel_name"
	echo "Downloading manifest-selected wheel: $resolved_wheel_url"
	curl -fsSL "$resolved_wheel_url" -o "aiter-whl/$wheel_name" \|\| return 1
	echo "Downloaded wheel from manifest: aiter-whl/$wheel_name"

	rm -f "$manifest_file"
	trap - RETURN
	}

	download_from_artifact() {
	local fallback_wheel fallback_wheel_name

	echo "=== Falling back to latest ${ATOM_PYTHON_TAG} aiter-whl-* artifact from ROCm/aiter ==="
	find_latest_artifact \|\| {
	echo "ERROR: No ${ATOM_PYTHON_TAG} aiter-whl-* artifact found in recent Aiter Test runs"
	return 1
	}

	mkdir -p aiter-whl
	rm -f aiter-whl/amd_aiter*.whl
	curl -fsSL -H "$AUTH_HEADER" \
	"$API_URL/repos/ROCm/aiter/actions/artifacts/$ARTIFACT_ID/zip" \
	-o aiter-whl.zip
	unzip -o aiter-whl.zip -d aiter-whl
	rm -f aiter-whl.zip

	fallback_wheel=$(ls -t aiter-whl/amd_aiter*.whl 2>/dev/null \| head -1)
	fallback_wheel_name=$(basename "${fallback_wheel:-}")
	if [ -z "$fallback_wheel" ] \|\| [[ "$fallback_wheel_name" != ${ATOM_PYTHON_TAG} ]]; then
	echo "ERROR: artifact fallback did not produce a ${ATOM_PYTHON_TAG} wheel"
	ls -la aiter-whl/ \|\| true
	return 1
	fi
	echo "Downloaded artifact-selected wheel: $fallback_wheel"
	}

	if download_from_s3_manifest; then
	echo "Using wheel from S3 main manifest"
	else
	echo "Main wheel manifest download failed, falling back to GitHub artifact"
	download_from_artifact
	fi

	AITER_WHL=$(ls -t aiter-whl/amd_aiter*.whl 2>/dev/null \| head -1)
	if [ -z "$AITER_WHL" ]; then
	echo "ERROR: No amd_aiter wheel available after S3/artifact attempts"
	ls -la aiter-whl/ \|\| true
	exit 1
	fi
	if [[ "$(basename "$AITER_WHL")" != ${ATOM_PYTHON_TAG} ]]; then
	echo "ERROR: selected wheel $AITER_WHL does not match target Python ${ATOM_PYTHON_TAG}"
	exit 1
	fi

	echo "Selected wheel: $AITER_WHL"
	echo "aiter_artifact_id=${ARTIFACT_ID}" >> "$GITHUB_OUTPUT"
	echo "aiter_wheel_name=$(basename "$AITER_WHL")" >> "$GITHUB_OUTPUT"

	- name: Upload aiter wheel
	uses: actions/upload-artifact@v7
	with:
	name: aiter-whl
	path: aiter-whl/amd_aiter*.whl
	retention-days: 7

	atom-vllm-oot:
	needs: [ci-gate, check-signal, download_aiter_wheel]
	if: ${{ needs.ci-gate.outputs.should_run == 'true' && needs.check-signal.result == 'success' && needs.download_aiter_wheel.result == 'success' }}
	name: Accuracy (${{ matrix.display_name }})
	strategy:
	fail-fast: false
	matrix:
	include:
	- display_name: "DeepSeek-V4-Flash TP4"
	model_name: "DeepSeek-V4-Flash"
	model_path: "deepseek-ai/DeepSeek-V4-Flash"
	extra_args: "--tensor-parallel-size 4 --gpu-memory-utilization 0.9 --max-num-seqs 512 --tokenizer-mode deepseek_v4"
	env_vars: \|
	AITER_BF16_FP8_MOE_BOUND=0
	ATOM_MOE_GU_ITLV=1
	lm_eval_num_fewshot: 20
	accuracy_test_threshold: 0.93
	runner: atom-mi355-8gpu-vllm-sgl-ci
	- display_name: "gpt-oss-120b TP1"
	model_name: "gpt-oss-120b"
	model_path: "openai/gpt-oss-120b"
	extra_args: "--tensor-parallel-size 1 --gpu-memory-utilization 0.5"
	client_command: "lm_eval --model local-chat-completions --apply_chat_template --model_args model=${MODEL_PATH},base_url=http://127.0.0.1:${VLLM_PORT}/v1/chat/completions,num_concurrent=65,max_retries=3,max_gen_toks=2048,tokenized_requests=False,trust_remote_code=True --tasks gsm8k --num_fewshot ${LM_EVAL_NUM_FEWSHOT} --output_path ${OUTPUT_PATH}"
	env_vars: "ATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=1\nVLLM_USE_V2_MODEL_RUNNER=1"
	lm_eval_num_fewshot: 3
	accuracy_test_threshold: 0.88
	runner: atom-mi355-8gpu-vllm-sgl-ci
	- display_name: "Kimi-K2.5-MXFP4 TP4"
	model_name: "Kimi-K2.5-MXFP4"
	model_path: "amd/Kimi-K2.5-MXFP4-AttnFP8"
	extra_args: "--trust-remote-code --tensor-parallel-size 4 --max-num-batched-tokens 16384 --max-model-len 16384"
	env_vars: "AITER_QUICK_REDUCE_QUANTIZATION=INT4"
	lm_eval_num_fewshot: 3
	accuracy_test_threshold: 0.92
	runner: atom-mi355-8gpu-vllm-sgl-ci
	- display_name: "Qwen3.5-35B-A3B-FP8 TP2"
	model_name: "Qwen3.5-35B-A3B-FP8"
	model_path: "Qwen/Qwen3.5-35B-A3B-FP8"
	extra_args: "--tensor-parallel-size 2 --attention-backend ROCM_AITER_FA"
	env_vars: \|
	ATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=1
	ATOM_USE_CUSTOM_ALL_GATHER=0
	lm_eval_num_fewshot: 3
	accuracy_test_threshold: 0.77
	runner: atom-mi355-8gpu-vllm-sgl-ci
	runs-on: ${{ matrix.runner }}
	timeout-minutes: 180
	env:
	CONTAINER_NAME: atom_vllm_oot_${{ strategy.job-index }}
	VLLM_COMMIT: 0b3ba88f165976e77ca5e6a7a3f5bba4562b80af
	VLLM_VERSION: "0.22.0"
	AITER_ARTIFACT_ID: ${{ needs.download_aiter_wheel.outputs.aiter_artifact_id }}

	steps:
	- name: Configure Docker client config path
	run: \|
	if [ "${{ matrix.runner }}" = "atom-mi355-8gpu.predownload" ]; then
	echo "DOCKER_CONFIG=${GITHUB_WORKSPACE}/.atom-docker-client" >> "$GITHUB_ENV"
	echo "Set DOCKER_CONFIG to workspace .atom-docker-client (runner-owned, avoids /it-share chmod)."
	else
	echo "Runner ${{ matrix.runner }}: leave DOCKER_CONFIG unset (default ~/.docker)."
	fi

	- name: Clean up containers and workspace
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'atom-mi35x-8gpu-oot-acc'
	run: \|
	echo "=== Cleaning up containers on $(hostname) ==="
	if ! docker ps >/tmp/docker-ps.out 2>/tmp/docker-ps.err; then
	echo "::warning::Docker is unavailable on this runner. Skipping pre-cleanup."
	cat /tmp/docker-ps.err \|\| true
	exit 0
	fi
	containers=$(cat /tmp/docker-ps.out)
	if [ -n "$containers" ]; then
	docker kill $containers \|\| true
	fi
	docker rm -f "$CONTAINER_NAME" 2>/dev/null \|\| true
	docker run --rm -v "${GITHUB_WORKSPACE:-$PWD}":/workspace -w /workspace --privileged rocm/pytorch:latest bash -lc "ls -la /workspace/ && find /workspace -mindepth 1 -delete" \|\| true

	- name: Checkout ATOM repo
	uses: actions/checkout@v4

	- name: Ensure Docker client config directory
	if: matrix.runner == 'atom-mi355-8gpu.predownload' \|\| matrix.runner == 'atom-mi35x-8gpu-oot-acc'
	run: \|
	set -euo pipefail
	if [ -z "${DOCKER_CONFIG:-}" ]; then
	echo "DOCKER_CONFIG is unset for runner ${{ matrix.runner }}; using Docker's default config path."
	exit 0
	fi
	mkdir -p "$DOCKER_CONFIG"
	chmod 700 "$DOCKER_CONFIG"

	- name: Set HF_TOKEN
	run: echo "HF_TOKEN=${HF_TOKEN:-${{ secrets.AMD_HF_TOKEN }}}" >> $GITHUB_ENV

	- name: Docker Login
	if: ${{ github.event_name != 'pull_request' \|\| !github.event.pull_request.head.repo.fork }}
	run: \|
	echo "${{ secrets.DOCKER_PASSWORD }}" \| docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin

	- name: Print runner user
	run: \|
	echo "=== Container engine diagnostics ==="
	echo "PATH=${PATH}"
	echo "whoami=$(whoami)"
	echo "id=$(id)"
	echo "docker path: $(command -v docker \|\| true)"
	echo "podman path: $(command -v podman \|\| true)"
	echo "docker version:"
	docker version \|\| true
	echo "docker info:"
	docker info \|\| true
	echo "podman version:"
	podman version \|\| true
	echo "podman info:"
	podman info \|\| true
	echo "=== End container engine diagnostics ==="

	- name: Download aiter wheel
	uses: actions/download-artifact@v8
	with:
	name: aiter-whl
	path: aiter-whl

	- name: Generate OOT overlay Dockerfile
	run: \|
	cat <<'EOF' > Dockerfile.mod
	ARG OOT_BASE_IMAGE
	FROM ${OOT_BASE_IMAGE}
	ARG GITHUB_REPO_URL
	ARG GITHUB_COMMIT_SHA
	ARG VLLM_COMMIT
	ARG AITER_ARTIFACT_ID
	ARG INSTALL_LM_EVAL=1
	LABEL com.rocm.atom.vllm_commit="${VLLM_COMMIT}"
	LABEL com.rocm.atom.aiter_artifact_id="${AITER_ARTIFACT_ID}"
	COPY aiter-whl/ /tmp/aiter-whl/
	RUN if [ "${INSTALL_LM_EVAL}" = "1" ]; then pip install -U "lm-eval[api]"; else echo "Skip lm-eval install"; fi
	RUN pip show lm-eval \|\| true
	RUN pip install hf_transfer
	RUN pip show hf_transfer \|\| true
	RUN pip install --upgrade "pybind11>=3.0.1"
	RUN pip show pybind11
	RUN echo "=== Aiter version BEFORE uninstall ===" && pip show amd-aiter \|\| true && \
	pip uninstall -y amd-aiter \|\| true && \
	rm -rf /app/aiter-test && \
	AITER_WHL="$(ls -t /tmp/aiter-whl/amd_aiter*.whl 2>/dev/null \| head -1)" && \
	if [ -z "${AITER_WHL}" ]; then echo "ERROR: No amd_aiter wheel found under /tmp/aiter-whl"; ls -la /tmp/aiter-whl \|\| true; exit 1; fi && \
	pip install "${AITER_WHL}" && \
	echo "=== Aiter version AFTER installation ===" && pip show amd-aiter \|\| true && \
	rm -rf /tmp/aiter-whl
	RUN echo "=== ATOM version BEFORE uninstall ===" && pip show atom \|\| true && \
	pip uninstall -y atom \|\| true && \
	rm -rf /app/ATOM && \
	git clone "${GITHUB_REPO_URL}" /app/ATOM && \
	cd /app/ATOM && \
	git checkout "${GITHUB_COMMIT_SHA}" && \
	pip install -e . --no-deps && \
	echo "=== ATOM version AFTER installation ===" && pip show atom \|\| true
	RUN pip show amd-aiter atom vllm torch triton mori \|\| true
	EOF

	- name: Build OOT image locally
	run: \|
	set -euo pipefail
	docker rmi "atom_oot_base:ci" \|\| true
	docker rmi "atom_oot:ci" \|\| true

	BUILD_MODE="full"
	if ! docker pull "${NIGHTLY_OOT_IMAGE_TAG}"; then
	echo "Unable to pull ${NIGHTLY_OOT_IMAGE_TAG}; rebuilding vLLM locally."
	else
	echo "Inspecting ${NIGHTLY_OOT_IMAGE_TAG} labels before deciding whether to reuse it locally."
	docker inspect --format 'Image ID: {{.Id}}' "${NIGHTLY_OOT_IMAGE_TAG}" \|\| true
	docker inspect --format 'Image labels: {{json .Config.Labels}}' "${NIGHTLY_OOT_IMAGE_TAG}" \|\| true
	LATEST_VLLM_COMMIT="$(docker inspect --format '{{ index .Config.Labels "com.rocm.atom.vllm_commit" }}' "${NIGHTLY_OOT_IMAGE_TAG}" 2>/dev/null \|\| true)"
	echo "Expected vLLM commit label: ${VLLM_COMMIT}"
	echo "Resolved ${NIGHTLY_OOT_IMAGE_TAG} label com.rocm.atom.vllm_commit: ${LATEST_VLLM_COMMIT:-<missing>}"
	if [ -n "${LATEST_VLLM_COMMIT}" ] && [ "${LATEST_VLLM_COMMIT}" = "${VLLM_COMMIT}" ]; then
	BUILD_MODE="fast"
	echo "Reusing ${NIGHTLY_OOT_IMAGE_TAG} locally for OOT CI."
	else
	echo "OOT CI latest image label mismatch or missing. Expected ${VLLM_COMMIT}, got ${LATEST_VLLM_COMMIT:-<missing>}; rebuilding vLLM locally."
	fi
	fi

	if [ "${BUILD_MODE}" = "fast" ]; then
	DOCKER_BUILDKIT=1 docker build --network=host \
	-t atom_oot:ci \
	--build-arg OOT_BASE_IMAGE="${NIGHTLY_OOT_IMAGE_TAG}" \
	--build-arg GITHUB_REPO_URL="${GITHUB_REPO_URL}" \
	--build-arg GITHUB_COMMIT_SHA="${GITHUB_COMMIT_SHA}" \
	--build-arg VLLM_COMMIT="${VLLM_COMMIT}" \
	--build-arg AITER_ARTIFACT_ID="${AITER_ARTIFACT_ID}" \
	--build-arg INSTALL_LM_EVAL=1 \
	-f Dockerfile.mod .
	exit 0
	fi

	DOCKER_BUILDKIT=1 docker build --pull --network=host \
	--no-cache \
	-t atom_oot_base:ci \
	--build-arg OOT_BASE_IMAGE="${ATOM_BASE_NIGHTLY_IMAGE}" \
	--build-arg GITHUB_REPO_URL="${GITHUB_REPO_URL}" \
	--build-arg GITHUB_COMMIT_SHA="${GITHUB_COMMIT_SHA}" \
	--build-arg VLLM_COMMIT="${VLLM_COMMIT}" \
	--build-arg AITER_ARTIFACT_ID="${AITER_ARTIFACT_ID}" \
	--build-arg INSTALL_LM_EVAL=1 \
	-f Dockerfile.mod .

	DOCKER_BUILDKIT=1 docker build --network=host \
	--no-cache \
	-t atom_oot:ci \
	--target atom_oot \
	--build-arg OOT_BASE_IMAGE="atom_oot_base:ci" \
	--build-arg MAX_JOBS=64 \
	--build-arg VLLM_COMMIT="${VLLM_COMMIT}" \
	--build-arg INSTALL_LM_EVAL=1 \
	--build-arg INSTALL_FASTSAFETENSORS=1 \
	-f docker/Dockerfile .

	- name: Set OOT image tag
	run: \|
	echo "OOT_IMAGE_TAG=atom_oot:ci" >> "$GITHUB_ENV"

	- name: Prepare model cache mount
	run: \|
	MODEL_CACHE_MOUNT=""
	MODEL_CACHE_DESC="container-local /models (no host cache mount)"

	if [ -d "/models" ]; then
	MODEL_CACHE_MOUNT="-v /models:/models"
	MODEL_CACHE_DESC="/models (host mount)"
	elif [ -d "/it-share/models" ]; then
	MODEL_CACHE_MOUNT="-v /it-share/models:/models"
	MODEL_CACHE_DESC="/it-share/models (host path)"
	elif [ -d "/mnt/dcgpuval/models" ]; then
	MODEL_CACHE_MOUNT="-v /mnt/dcgpuval/models:/models"
	MODEL_CACHE_DESC="/mnt/dcgpuval/models (host path)"
	elif [ -d "/shareddata/models" ]; then
	MODEL_CACHE_MOUNT="-v /shareddata/models:/models"
	MODEL_CACHE_DESC="/shareddata/models (host path)"
	elif [ -d "/data/models" ]; then
	MODEL_CACHE_MOUNT="-v /data/models:/models"
	MODEL_CACHE_DESC="/data/models (host path)"
	else
	echo "Warning: /models and /it-share/models and /mnt/dcgpuval/models and /shareddata/models and /data/models directory not found on runner; using container-local /models."
	fi

	echo "Using model cache backend: ${MODEL_CACHE_DESC}"
	echo "MODEL_CACHE_MOUNT=${MODEL_CACHE_MOUNT}" >> "$GITHUB_ENV"
	echo "MODEL_CACHE_DESC=${MODEL_CACHE_DESC}" >> "$GITHUB_ENV"

	- name: Start OOT test container
	run: \|
	if [ -f "/etc/podinfo/gha-render-devices" ]; then
	DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
	else
	DEVICE_FLAG="--device /dev/dri"
	fi

	MODEL_MOUNT="${MODEL_CACHE_MOUNT}"
	echo "Using model cache backend: ${MODEL_CACHE_DESC}"

	docker run -dt --device=/dev/kfd $DEVICE_FLAG \
	-v "${GITHUB_WORKSPACE:-$PWD}":/workspace \
	$MODEL_MOUNT \
	-w /workspace \
	--ipc=host --network=host --group-add video \
	--shm-size=16G \
	--privileged \
	--cap-add=SYS_PTRACE \
	-e HF_TOKEN="${HF_TOKEN:-}" \
	--security-opt seccomp=unconfined \
	--ulimit memlock=-1 \
	--ulimit stack=67108864 \
	--name "$CONTAINER_NAME" \
	"$OOT_IMAGE_TAG"
	env:
	GITHUB_WORKSPACE: ${{ github.workspace }}

	- name: GPU preflight check
	if: success()
	timeout-minutes: 5
	env:
	GPU_PREFLIGHT_ALLOCATION_MB: "8"
	run: bash .github/scripts/gpu_preflight_check.sh "$CONTAINER_NAME" docker

	- name: Download or refresh model if needed
	if: success()
	run: \|
	set -euo pipefail
	model_dir="/models/${{ matrix.model_path }}"
	if [ -n "${MODEL_CACHE_MOUNT}" ]; then
	model_use_lock="true"
	if [ "${{ matrix.runner }}" = "atom-mi355-8gpu.predownload" ] \|\| [ "${{ matrix.runner }}" = "atom-mi35x-8gpu-oot-acc" ]; then
	model_use_lock="false"
	fi
	echo "Using shared model download script for ${model_dir} (MODEL_USE_LOCK=${model_use_lock})"
	if ! docker exec \
	-e HF_TOKEN="${HF_TOKEN:-}" \
	-e MODEL_ID="${{ matrix.model_path }}" \
	-e TARGET_DIR="${model_dir}" \
	-e MODEL_USE_LOCK="${model_use_lock}" \
	-e MODEL_DOWNLOAD_TIMEOUT="${MODEL_DOWNLOAD_TIMEOUT}" \
	-e MODEL_LOCK_WAIT_SECONDS="${MODEL_LOCK_WAIT_SECONDS}" \
	-e MODEL_LOCK_POLL_INTERVAL="${MODEL_LOCK_POLL_INTERVAL}" \
	-e MODEL_PROGRESS_INTERVAL="${MODEL_PROGRESS_INTERVAL}" \
	"$CONTAINER_NAME" \
	bash -lc 'bash /workspace/.github/scripts/download_model_with_lock.sh "$MODEL_ID" "$TARGET_DIR"'; then
	echo "Model download failed for '${{ matrix.model_path }}'. Aborting."
	exit 1
	fi
	else
	echo "/models directory not mounted; skipping model download"
	fi
	env:
	MODEL_DOWNLOAD_TIMEOUT: "3h"
	MODEL_LOCK_WAIT_SECONDS: "7200"
	MODEL_LOCK_POLL_INTERVAL: "30"
	MODEL_PROGRESS_INTERVAL: "60"

	- name: Resolve OOT model path
	if: success()
	run: \|
	if [ -n "${MODEL_CACHE_MOUNT}" ]; then
	echo "OOT_RESOLVED_MODEL_PATH=/models/${{ matrix.model_path }}" >> "$GITHUB_ENV"
	echo "Using mounted model path: /models/${{ matrix.model_path }}"
	else
	echo "OOT_RESOLVED_MODEL_PATH=${{ matrix.model_path }}" >> "$GITHUB_ENV"
	echo "Using model id: ${{ matrix.model_path }}"
	fi

	- name: Run OOT launch and gsm8k accuracy via script (ci mode)
	if: success()
	timeout-minutes: 45
	env:
	OOT_MODEL_NAME: ${{ matrix.model_name }}
	OOT_MODEL_PATH: ${{ matrix.model_path }}
	OOT_EXTRA_ARGS: ${{ matrix.extra_args }}
	OOT_CLIENT_COMMAND: ${{ matrix.client_command \|\| '' }}
	OOT_ENV_VARS: ${{ matrix.env_vars }}
	LM_EVAL_NUM_FEWSHOT: ${{ matrix.lm_eval_num_fewshot }}
	MAX_WAIT_RETRIES: "40"
	STREAM_VLLM_LOGS: "1"
	run: \|
	docker exec \
	-e OOT_MODEL_NAME="${OOT_MODEL_NAME}" \
	-e OOT_MODEL_PATH="${OOT_RESOLVED_MODEL_PATH:-$OOT_MODEL_PATH}" \
	-e OOT_EXTRA_ARGS="${OOT_EXTRA_ARGS}" \
	-e OOT_CLIENT_COMMAND="${OOT_CLIENT_COMMAND}" \
	-e OOT_ENV_VARS="${OOT_ENV_VARS}" \
	-e LM_EVAL_NUM_FEWSHOT="${LM_EVAL_NUM_FEWSHOT}" \
	-e MAX_WAIT_RETRIES="${MAX_WAIT_RETRIES}" \
	-e STREAM_VLLM_LOGS="${STREAM_VLLM_LOGS}" \
	"$CONTAINER_NAME" bash -lc "
	set -euo pipefail
	bash .github/scripts/atom_oot_test.sh accuracy ci
	"

	- name: Check OOT accuracy test results
	if: success()
	run: \|
	docker cp "$CONTAINER_NAME":/tmp/oot_accuracy_results ./oot_accuracy_results \|\| true
	result_file=$(ls -1t oot_accuracy_results/*.json 2>/dev/null \| head -n 1)
	if [ -z "$result_file" ] \|\| [ ! -f "$result_file" ]; then
	echo "ERROR: No results JSON file found in oot_accuracy_results/"
	exit 2
	fi

	echo "RESULT_FILE: $result_file"
	flexible_extract_value=$(python3 - "$result_file" <<'PY'
	import json
	import sys

	with open(sys.argv[1], encoding="utf-8") as f:
	data = json.load(f)
	print(data["results"]["gsm8k"]["exact_match,flexible-extract"])
	PY
	)
	echo "Flexible extract value: $flexible_extract_value"
	echo "Accuracy test threshold: ${{ matrix.accuracy_test_threshold }}"

	result=$(awk -v val="$flexible_extract_value" -v threshold="${{ matrix.accuracy_test_threshold }}" 'BEGIN {print (val < threshold) ? 1 : 0}')
	if [ "$result" -eq 1 ]; then
	echo "Accuracy test failed: Flexible extract value $flexible_extract_value is less than threshold ${{ matrix.accuracy_test_threshold }}."
	exit 1
	else
	echo "Accuracy test passed: Flexible extract value $flexible_extract_value is greater than or equal to threshold ${{ matrix.accuracy_test_threshold }}."
	exit 0
	fi

	- name: Collect OOT accuracy summary
	if: success()
	run: \|
	echo "OOT Accuracy Test Summary for ${{ matrix.model_name }}:" >> $GITHUB_STEP_SUMMARY
	docker exec "$CONTAINER_NAME" bash -lc "awk '/\\|Tasks\\|Version\\|/,/^$/ { if (NF > 0) print }' /tmp/oot_accuracy_output.txt" >> $GITHUB_STEP_SUMMARY \|\| true

	- name: Collect OOT logs and results
	if: always()
	run: \|
	docker cp "$CONTAINER_NAME":/tmp/vllm_oot.log ./vllm_oot.log \|\| true
	docker cp "$CONTAINER_NAME":/tmp/oot_accuracy_output.txt ./oot_accuracy_output.txt \|\| true
	docker cp "$CONTAINER_NAME":/tmp/oot_accuracy_results ./oot_accuracy_results \|\| true

	- name: Upload OOT artifacts
	if: always()
	uses: actions/upload-artifact@v7
	with:
	name: oot-${{ matrix.model_name }}-artifacts
	path: \|
	vllm_oot.log
	oot_accuracy_output.txt
	oot_accuracy_results

	- name: Clean up OOT test
	if: always()
	run: \|
	docker exec "$CONTAINER_NAME" bash -lc "if [ -f /tmp/vllm_oot.pid ]; then kill \$(cat /tmp/vllm_oot.pid) \|\| true; fi" \|\| true
	docker stop "$CONTAINER_NAME" \|\| true
	docker rm "$CONTAINER_NAME" \|\| true
	rm -f Dockerfile.mod \|\| true
	docker rmi "atom_oot_base:ci" \|\| true
	docker rmi "atom_oot:ci" \|\| true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[ci][mesh] improve ATOMesh benchmark configuration and dashboard reporting #3942

Workflow file

[ci][mesh] improve ATOMesh benchmark configuration and dashboard reporting #3942

Uh oh!

Workflow file for this run