CI: change SGLang plugin M3 nightly accuracy threshold

.github/benchmark/schema/accuracy_catalog.schema.json

@@ -34,6 +34,7 @@
       "priority": { "type": "string", "enum": ["P0", "P1", "P2"] },
       "lm_eval_num_fewshot": { "type": "integer", "minimum": 0 },
       "lm_eval_num_concurrent": { "type": "integer", "minimum": 1 },
+      "lm_eval_use_chat_completions": { "type": "string", "enum": ["0", "1", "true", "false"] },
       "lm_eval_extra_model_args": { "type": "string" },
       "_baseline_note": { "type": "string" }
     }

.github/benchmark/sglang_models_accuracy.json

@@ -110,6 +110,10 @@
     "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_QUICK_REDUCE_QUANTIZATION=INT4\nSGLANG_USE_AITER=1\nATOM_FORCE_ATTN_TRITON=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models",
     "runner": "atom-mi355-8gpu-conductor-sgl-runner",
     "test_level": "nightly",
+    "lm_eval_num_fewshot": 5,
+    "lm_eval_num_concurrent": 32,
+    "lm_eval_use_chat_completions": "1",
+    "lm_eval_extra_model_args": "max_gen_toks=16384",
     "accuracy_threshold": 0.93,
     "accuracy_baseline": null,
     "accuracy_baseline_model": "amd/MiniMax-M3-MXFP4",

.github/scripts/atom_sglang_test.sh

@@ -28,6 +28,7 @@ set -euo pipefail
 #   LM_EVAL_NUM_FEWSHOT
 #   LM_EVAL_NUM_CONCURRENT
 #   LM_EVAL_EXTRA_MODEL_ARGS
+#   LM_EVAL_USE_CHAT_COMPLETIONS
 
 TYPE=${1:-launch}
 if [[ "${TYPE}" != "start" && "${TYPE}" != "launch" && "${TYPE}" != "accuracy" ]]; then
@@ -49,6 +50,7 @@ LM_EVAL_TASK=${LM_EVAL_TASK:-gsm8k}
 LM_EVAL_NUM_FEWSHOT=${LM_EVAL_NUM_FEWSHOT:-3}
 LM_EVAL_NUM_CONCURRENT=${LM_EVAL_NUM_CONCURRENT:-65}
 LM_EVAL_EXTRA_MODEL_ARGS=${LM_EVAL_EXTRA_MODEL_ARGS:-}
+LM_EVAL_USE_CHAT_COMPLETIONS=${LM_EVAL_USE_CHAT_COMPLETIONS:-0}
 
 MODEL_NAME=${SGLANG_MODEL_NAME:-}
 MODEL_PATH=${SGLANG_MODEL_PATH:-}
@@ -242,16 +244,28 @@ run_accuracy() {
   echo "========== Running SGLang accuracy =========="
   echo "Model name: ${MODEL_NAME}"
 
+  local lm_eval_model="local-completions"
+  local lm_eval_endpoint_path="/v1/completions"
+  local -a lm_eval_extra_args=()
   local lm_eval_model_args
-  lm_eval_model_args="model=${resolved_model_path},base_url=http://127.0.0.1:${SGLANG_PORT}/v1/completions,num_concurrent=${LM_EVAL_NUM_CONCURRENT},max_retries=1,tokenized_requests=False,trust_remote_code=True"
+
+  if [[ "${LM_EVAL_USE_CHAT_COMPLETIONS}" == "1" || "${LM_EVAL_USE_CHAT_COMPLETIONS}" == "true" ]]; then
+    lm_eval_model="local-chat-completions"
+    lm_eval_endpoint_path="/v1/chat/completions"
+    lm_eval_extra_args+=(--batch_size 65 --apply_chat_template --fewshot_as_multiturn)
+    lm_eval_model_args="model=${resolved_model_path},base_url=http://127.0.0.1:${SGLANG_PORT}${lm_eval_endpoint_path},num_concurrent=${LM_EVAL_NUM_CONCURRENT}"
+  else
+    lm_eval_model_args="model=${resolved_model_path},base_url=http://127.0.0.1:${SGLANG_PORT}${lm_eval_endpoint_path},num_concurrent=${LM_EVAL_NUM_CONCURRENT},max_retries=1,tokenized_requests=False,trust_remote_code=True"
+  fi
   if [[ -n "${LM_EVAL_EXTRA_MODEL_ARGS}" ]]; then
     lm_eval_model_args="${lm_eval_model_args},${LM_EVAL_EXTRA_MODEL_ARGS#,}"
   fi
 
-  lm_eval --model local-completions \
+  lm_eval --model "${lm_eval_model}" \
     --model_args "${lm_eval_model_args}" \
     --tasks "${LM_EVAL_TASK}" \
     --num_fewshot "${LM_EVAL_NUM_FEWSHOT}" \
+    "${lm_eval_extra_args[@]}" \
     --output_path "${output_path}" 2>&1 | tee -a "${ACCURACY_LOG_FILE}"
   # Capture lm_eval exit code explicitly; tee always exits 0 so PIPESTATUS is needed.
   lm_eval_exit="${PIPESTATUS[0]}"

.github/workflows/atom-sglang-accuracy-validation-gpu-shard.yaml

@@ -398,6 +398,7 @@ jobs:
           LM_EVAL_EXTRA_MODEL_ARGS: ${{ matrix.lm_eval_extra_model_args || '' }}
           LM_EVAL_NUM_FEWSHOT: ${{ matrix.lm_eval_num_fewshot }}
           LM_EVAL_NUM_CONCURRENT: ${{ matrix.lm_eval_num_concurrent }}
+          LM_EVAL_USE_CHAT_COMPLETIONS: ${{ matrix.lm_eval_use_chat_completions || '0' }}
         run: |
           set -euo pipefail
 
@@ -426,6 +427,7 @@ jobs:
             -e LM_EVAL_EXTRA_MODEL_ARGS="${LM_EVAL_EXTRA_MODEL_ARGS}" \
             -e LM_EVAL_NUM_FEWSHOT="${LM_EVAL_NUM_FEWSHOT}" \
             -e LM_EVAL_NUM_CONCURRENT="${LM_EVAL_NUM_CONCURRENT}" \
+            -e LM_EVAL_USE_CHAT_COMPLETIONS="${LM_EVAL_USE_CHAT_COMPLETIONS}" \
             "$CONTAINER_NAME" bash -lc '
               (
                 set +e

.github/workflows/atom-sglang-accuracy-validation.yaml

@@ -234,6 +234,10 @@ jobs:
                   "model_name": "MiniMax-M3-MXFP4 TP4",
                   "model_path": "amd/MiniMax-M3-MXFP4",
                   "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --page-size 128 --mem-fraction-static 0.8 --max-running-requests 128 --disable-radix-cache",
+                  "lm_eval_num_fewshot": 5,
+                  "lm_eval_num_concurrent": 32,
+                  "lm_eval_use_chat_completions": "1",
+                  "lm_eval_extra_model_args": "max_gen_toks=16384",
                   "accuracy_test_threshold": 0.93,
                   "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_QUICK_REDUCE_QUANTIZATION=INT4\nSGLANG_USE_AITER=1\nATOM_FORCE_ATTN_TRITON=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models",
                   "runner": "atom-plugin-acc-validation-runner",