From 59c0f8a1646d39b3306d2e29bcfe9da466c70401 Mon Sep 17 00:00:00 2001 From: Laxmikant Kale Date: Sun, 13 Sep 2026 12:19:22 -0400 Subject: [PATCH] reconverse-site-run: add SITE=frontier, raise -c defaults, carry abort reasons Frontier (OLCF) case: FI_PROVIDER=cxi and LCI_NETWORK_BACKENDS=ofi as on delta, PMI_MAX_KVS_ENTRIES=1000 for Cray PMI's key-value store, and --network=single_node_vni on every srun step. The flag goes on the multi-node argument set too: a Makefile's +p1 cases run as one process, Slurm shrinks that step to one node, and without the flag the cxi provider aborts in LCI ("Function not implemented"). It is harmless on steps that do span nodes. Module loads stay with the caller, as for delta. Raise the launcher defaults, which were too small for the tier's +p6 cases and made reconverse abort with "Multiple PEs assigned to same core": LAUNCHER_ARGS_SINGLE -N1 -c4 -> -N1 -c8, LAUNCHER_ARGS_MULTI -c2 -> -c4 on one node and -c8 per task across nodes; the header's salloc example now says -c 8. When a run fails, append the runtime's "Reason:" line (or the C++ what() text when the network layer threw) to its RESULT line, so a launcher or cpuset abort is not read as a test failure. Verified on Frontier 2026-09-13, reviewed-with-reconverse 8d21fda with reconverse 58921e9: all 13 CI directories pass as 1 process and as 2 processes on one node (job 5481253) and on two nodes (job 5481302). Co-Authored-By: Claude Fable 5.1 --- tests/reconverse-site-run.sh | 42 +++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/reconverse-site-run.sh b/tests/reconverse-site-run.sh index fa5acc14a8..65a82f955e 100755 --- a/tests/reconverse-site-run.sh +++ b/tests/reconverse-site-run.sh @@ -4,14 +4,14 @@ # runners: build each directory, then `make test` once as a single process # and once as PROCS processes, through the site's launcher. # -# Usage (from a Slurm allocation, e.g. `salloc -N 2 -n 4 -c 2` or an sbatch +# Usage (from a Slurm allocation, e.g. `salloc -N 2 -n 4 -c 8` or an sbatch # script), with CHARM_BUILD pointing at the reconverse build directory: # # SITE=anvil CHARM_BUILD=$PWD/reconverse-linux-x86_64 tests/reconverse-site-run.sh # SITE=delta NODES=2 CHARM_BUILD=... tests/reconverse-site-run.sh tests/charm++/megatest # # Environment: -# SITE anvil | delta | generic (module loads and launcher flags) +# SITE anvil | delta | frontier | generic (module loads and launcher flags) # CHARM_BUILD the build directory (has bin/charmc, bin/testrun) # NODES nodes for the multi-process pass (default 2; 1 = two # processes on one node) @@ -25,7 +25,9 @@ # .github/workflows/reconverse-ci.yaml, so the site run and CI stay in step. # # Output: one "RESULT exit=" line per run and a summary; -# paste them on the pull request being validated. +# paste them on the pull request being validated. When a run aborts, the +# runtime's "Reason:" line is appended, so a launcher or cpuset problem +# (e.g. "Multiple PEs assigned to same core") is not read as a test failure. set -u cd "$(dirname "$0")/.." || exit 1 CHARM_BUILD=${CHARM_BUILD:?set CHARM_BUILD to the reconverse build directory} @@ -33,12 +35,12 @@ SITE=${SITE:-generic} NODES=${NODES:-2} PROCS=${PROCS:-2} LAUNCHER=${LAUNCHER:-"srun --mpi=pmi2"} -LAUNCHER_ARGS_SINGLE=${LAUNCHER_ARGS_SINGLE-"-N1 -c4"} +LAUNCHER_ARGS_SINGLE=${LAUNCHER_ARGS_SINGLE-"-N1 -c8"} if [[ $NODES -ge 2 ]]; then - LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N$NODES --ntasks-per-node=$((PROCS / NODES)) -c2"} + LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N$NODES --ntasks-per-node=$((PROCS / NODES)) -c8"} MULTI_SHAPE="${PROCS}proc-${NODES}nodes" else - LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N1 -c2"} + LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N1 -c4"} MULTI_SHAPE="${PROCS}proc-1node" fi @@ -58,12 +60,36 @@ case "$SITE" in export FI_PROVIDER=cxi export LCI_NETWORK_BACKENDS=ofi ;; + frontier) + # OLCF Frontier: Slingshot-11 through libfabric's cxi provider, as on + # delta. Load the modules before running this (PrgEnv-gnu cmake hwloc + # python; a batch script must `source /opt/cray/pe/lmod/lmod/init/bash` + # first, because `module` is not defined in a non-login shell); they + # are not pinned here for the same reason as delta. Cray PMI's key-value + # store holds 30 entries by default, too few for a multi-process launch + # (_pmi2_add_kvs: "The KVS data segment ... is not large enough"). + # Every step gets --network=single_node_vni: a step confined to one node + # has no VNI otherwise (job_vni only covers multi-node steps) and the cxi + # provider aborts in LCI with "Function not implemented". The multi-node + # pass also needs it, because a Makefile's +p1 cases run as one process + # and Slurm shrinks that step to one node; the flag is harmless on a step + # that does span nodes (verified 2026-09-13, 2 nodes x 1 and 2 x 2 tasks). + export FI_PROVIDER=cxi + export LCI_NETWORK_BACKENDS=ofi + export PMI_MAX_KVS_ENTRIES=${PMI_MAX_KVS_ENTRIES:-1000} + LAUNCHER_ARGS_SINGLE="$LAUNCHER_ARGS_SINGLE --network=single_node_vni" + LAUNCHER_ARGS_MULTI="$LAUNCHER_ARGS_MULTI --network=single_node_vni" + ;; generic) ;; *) echo "unknown SITE=$SITE" >&2; exit 2 ;; esac export LD_LIBRARY_PATH=$CHARM_BUILD/lib:${LD_LIBRARY_PATH:-} # GNU timeout is `timeout` on Linux, `gtimeout` from coreutils on macOS. TIMEOUT=$(command -v timeout || command -v gtimeout || true) +# reason : why a failed run aborted, with a leading space so +# the RESULT line can carry it: the runtime's "Reason:" line (CmiAbort), or the +# C++ exception text when the network layer threw instead. +reason() { [[ $1 -ne 0 ]] && grep -m1 -oE 'Reason:.*|what\(\):.*' "$2" 2>/dev/null | cut -c1-200 | sed 's/^/ /'; return 0; } if [[ $# -gt 0 ]]; then DIRS=("$@") @@ -82,10 +108,10 @@ for d in "${DIRS[@]}"; do # single process: testrun appends -n 1 to the launcher TESTRUN_LAUNCHER="$LAUNCHER $LAUNCHER_ARGS_SINGLE" TESTRUN_PROCS=1 \ ${TIMEOUT:+$TIMEOUT 600} make -C "$d" test > "$d/site-1proc.out" 2>&1; e1=$? - echo "RESULT $d 1proc exit=$e1" + echo "RESULT $d 1proc exit=$e1$(reason "$e1" "$d/site-1proc.out")" TESTRUN_LAUNCHER="$LAUNCHER $LAUNCHER_ARGS_MULTI" TESTRUN_PROCS=$PROCS \ ${TIMEOUT:+$TIMEOUT 600} make -C "$d" test > "$d/site-$MULTI_SHAPE.out" 2>&1; e2=$? - echo "RESULT $d $MULTI_SHAPE exit=$e2" + echo "RESULT $d $MULTI_SHAPE exit=$e2$(reason "$e2" "$d/site-$MULTI_SHAPE.out")" [[ $e1 -eq 0 && $e2 -eq 0 ]] || fail=1 done echo "SUMMARY: $([[ $fail -eq 0 ]] && echo ALL PASSED || echo FAILURES ABOVE) $(date)"