diff --git a/tests/reconverse-site-run.sh b/tests/reconverse-site-run.sh index fa5acc14a8..65a82f955e 100755 --- a/tests/reconverse-site-run.sh +++ b/tests/reconverse-site-run.sh @@ -4,14 +4,14 @@ # runners: build each directory, then `make test` once as a single process # and once as PROCS processes, through the site's launcher. # -# Usage (from a Slurm allocation, e.g. `salloc -N 2 -n 4 -c 2` or an sbatch +# Usage (from a Slurm allocation, e.g. `salloc -N 2 -n 4 -c 8` or an sbatch # script), with CHARM_BUILD pointing at the reconverse build directory: # # SITE=anvil CHARM_BUILD=$PWD/reconverse-linux-x86_64 tests/reconverse-site-run.sh # SITE=delta NODES=2 CHARM_BUILD=... tests/reconverse-site-run.sh tests/charm++/megatest # # Environment: -# SITE anvil | delta | generic (module loads and launcher flags) +# SITE anvil | delta | frontier | generic (module loads and launcher flags) # CHARM_BUILD the build directory (has bin/charmc, bin/testrun) # NODES nodes for the multi-process pass (default 2; 1 = two # processes on one node) @@ -25,7 +25,9 @@ # .github/workflows/reconverse-ci.yaml, so the site run and CI stay in step. # # Output: one "RESULT
" line per run and a summary;
-# paste them on the pull request being validated.
+# paste them on the pull request being validated. When a run aborts, the
+# runtime's "Reason:" line is appended, so a launcher or cpuset problem
+# (e.g. "Multiple PEs assigned to same core") is not read as a test failure.
set -u
cd "$(dirname "$0")/.." || exit 1
CHARM_BUILD=${CHARM_BUILD:?set CHARM_BUILD to the reconverse build directory}
@@ -33,12 +35,12 @@ SITE=${SITE:-generic}
NODES=${NODES:-2}
PROCS=${PROCS:-2}
LAUNCHER=${LAUNCHER:-"srun --mpi=pmi2"}
-LAUNCHER_ARGS_SINGLE=${LAUNCHER_ARGS_SINGLE-"-N1 -c4"}
+LAUNCHER_ARGS_SINGLE=${LAUNCHER_ARGS_SINGLE-"-N1 -c8"}
if [[ $NODES -ge 2 ]]; then
- LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N$NODES --ntasks-per-node=$((PROCS / NODES)) -c2"}
+ LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N$NODES --ntasks-per-node=$((PROCS / NODES)) -c8"}
MULTI_SHAPE="${PROCS}proc-${NODES}nodes"
else
- LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N1 -c2"}
+ LAUNCHER_ARGS_MULTI=${LAUNCHER_ARGS_MULTI-"-N1 -c4"}
MULTI_SHAPE="${PROCS}proc-1node"
fi
@@ -58,12 +60,36 @@ case "$SITE" in
export FI_PROVIDER=cxi
export LCI_NETWORK_BACKENDS=ofi
;;
+ frontier)
+ # OLCF Frontier: Slingshot-11 through libfabric's cxi provider, as on
+ # delta. Load the modules before running this (PrgEnv-gnu cmake hwloc
+ # python; a batch script must `source /opt/cray/pe/lmod/lmod/init/bash`
+ # first, because `module` is not defined in a non-login shell); they
+ # are not pinned here for the same reason as delta. Cray PMI's key-value
+ # store holds 30 entries by default, too few for a multi-process launch
+ # (_pmi2_add_kvs: "The KVS data segment ... is not large enough").
+ # Every step gets --network=single_node_vni: a step confined to one node
+ # has no VNI otherwise (job_vni only covers multi-node steps) and the cxi
+ # provider aborts in LCI with "Function not implemented". The multi-node
+ # pass also needs it, because a Makefile's +p1 cases run as one process
+ # and Slurm shrinks that step to one node; the flag is harmless on a step
+ # that does span nodes (verified 2026-09-13, 2 nodes x 1 and 2 x 2 tasks).
+ export FI_PROVIDER=cxi
+ export LCI_NETWORK_BACKENDS=ofi
+ export PMI_MAX_KVS_ENTRIES=${PMI_MAX_KVS_ENTRIES:-1000}
+ LAUNCHER_ARGS_SINGLE="$LAUNCHER_ARGS_SINGLE --network=single_node_vni"
+ LAUNCHER_ARGS_MULTI="$LAUNCHER_ARGS_MULTI --network=single_node_vni"
+ ;;
generic) ;;
*) echo "unknown SITE=$SITE" >&2; exit 2 ;;
esac
export LD_LIBRARY_PATH=$CHARM_BUILD/lib:${LD_LIBRARY_PATH:-}
# GNU timeout is `timeout` on Linux, `gtimeout` from coreutils on macOS.
TIMEOUT=$(command -v timeout || command -v gtimeout || true)
+# reason : why a failed run aborted, with a leading space so
+# the RESULT line can carry it: the runtime's "Reason:" line (CmiAbort), or the
+# C++ exception text when the network layer threw instead.
+reason() { [[ $1 -ne 0 ]] && grep -m1 -oE 'Reason:.*|what\(\):.*' "$2" 2>/dev/null | cut -c1-200 | sed 's/^/ /'; return 0; }
if [[ $# -gt 0 ]]; then
DIRS=("$@")
@@ -82,10 +108,10 @@ for d in "${DIRS[@]}"; do
# single process: testrun appends -n 1 to the launcher
TESTRUN_LAUNCHER="$LAUNCHER $LAUNCHER_ARGS_SINGLE" TESTRUN_PROCS=1 \
${TIMEOUT:+$TIMEOUT 600} make -C "$d" test > "$d/site-1proc.out" 2>&1; e1=$?
- echo "RESULT $d 1proc exit=$e1"
+ echo "RESULT $d 1proc exit=$e1$(reason "$e1" "$d/site-1proc.out")"
TESTRUN_LAUNCHER="$LAUNCHER $LAUNCHER_ARGS_MULTI" TESTRUN_PROCS=$PROCS \
${TIMEOUT:+$TIMEOUT 600} make -C "$d" test > "$d/site-$MULTI_SHAPE.out" 2>&1; e2=$?
- echo "RESULT $d $MULTI_SHAPE exit=$e2"
+ echo "RESULT $d $MULTI_SHAPE exit=$e2$(reason "$e2" "$d/site-$MULTI_SHAPE.out")"
[[ $e1 -eq 0 && $e2 -eq 0 ]] || fail=1
done
echo "SUMMARY: $([[ $fail -eq 0 ]] && echo ALL PASSED || echo FAILURES ABOVE) $(date)"