diff --git a/systems/Frontier/nrhs_fault_36.job b/systems/Frontier/nrhs_fault_36.job index 2a8a02e3c..fb32c1cf7 100644 --- a/systems/Frontier/nrhs_fault_36.job +++ b/systems/Frontier/nrhs_fault_36.job @@ -23,8 +23,6 @@ # (the runtime's log goes to stderr, so the faulting # rank's Grid.stderr. ends with the culprit) # --log ...,Memory MemoryManager transfers in program order -# --debug-heartbeat periodic backtrace per rank: a HANG (2026-08-28, single-RHS solve -# after 12 RHS with FineSloppyComms=0) shows where every rank sits # # Cell A: serialised (5-10x slower: setup ~10 min + solve until the fault). # Cell B: NOT serialised, same logging -- if serialisation makes the fault @@ -102,7 +100,7 @@ run_cell () { echo "----- $name : $* -----" export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \ - --debug-stdout --debug-heartbeat --log Error,Warning,Message,Memory > log.fault36.$name 2>&1 + --debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1 echo " exit $?"; sleep 30 f=$(grep -l "Memory access fault" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1) if [ -n "$f" ]; then @@ -117,5 +115,16 @@ run_cell () { fi } -run_cell A_serialised AMD_SERIALIZE_KERNEL=3 AMD_SERIALIZE_COPY=3 AMD_LOG_LEVEL=3 -run_cell B_plain AMD_LOG_LEVEL=1 +#run_cell A_serialised AMD_SERIALIZE_KERNEL=3 AMD_SERIALIZE_COPY=3 AMD_LOG_LEVEL=3 # 5371703-era: OOM found; done +#run_cell B_plain AMD_LOG_LEVEL=1 +# Ladder for the NO_TRANSLATION / hang fault (both at NRHS=12): +# 5371703 sloppy ON, no kdreg2 -> NO_TRANSLATION (Dhop halo) at outer 36 +# 5371826 sloppy OFF, no kdreg2 -> 12-RHS converged, HANG in single-RHS solve +# 5372414 sloppy OFF, kdreg2 (env also had FI_MR_CACHE_MAX_COUNT=786432) -> NO_TRANSLATION (Couter, 2 reqs) at outer 60 +# CORRECTION (fi_mr(3)): FI_MR_CACHE_MONITOR={memhooks,kdreg2,userfaultfd,disabled} governs +# SYSTEM memory only. Device (HMEM_ROCR) registrations have their own monitor, +# FI_MR_ROCR_CACHE_MONITOR_ENABLED=0|1, and Grid's comms window is hipMalloc'd device memory +# -- so 5372414 (kdreg2) never exercised the translations that fail. Unambiguous global +# switch: FI_MR_CACHE_MAX_COUNT=0 disables the registration cache for ALL memory types. +run_cell D_nocache FI_MR_CACHE_MAX_COUNT=0 FI_LOG_LEVEL=info FI_LOG_PROV=core # step 3: no cache, any memory +run_cell E_rocrmon0 FI_MR_ROCR_CACHE_MONITOR_ENABLED=0 FI_LOG_LEVEL=info FI_LOG_PROV=core # step 3b: device-memory monitor off, cache on diff --git a/systems/WorkArounds.txt b/systems/WorkArounds.txt index 7ffdce822..942e1ec3e 100644 --- a/systems/WorkArounds.txt +++ b/systems/WorkArounds.txt @@ -217,6 +217,19 @@ by analogy, and the fix is UNTESTED here. Test ladder, one knob per run, NRHS=1 (4) --disable-accelerator-aware-mpi build. REVISIT this entry with the result; if (2) does not cure it, remove kdreg2 from the job scripts' justification (it stays as OLCF's recommendation regardless). + 2026-08-28 22:18, job 5372414: step (2) DID NOT cure it -- kdreg2 set (and the + environment already carried FI_MR_CACHE_MAX_COUNT=786432), sloppy comms OFF, still + NO_TRANSLATION at outer step 60 (Couter, Waitall count=2). + CORRECTION 2026-08-28 23:xx (fi_mr(3) man page): FI_MR_CACHE_MONITOR governs SYSTEM + memory only; device (HMEM_ROCR) registrations are monitored by + FI_MR_ROCR_CACHE_MONITOR_ENABLED=0|1. Grid's comms window is hipMalloc'd device + memory, so the kdreg2 step tested nothing relevant -- the ALPS PLT bug (#11451) and + the Frontier NO_TRANSLATION are the same *library* but different monitors. + Attributed by gdb on a hung rank (5371826): main thread in MPI_Waitall <- Grid + 0x5d539c3 (StencilSendToRecvFromComplete/CommsComplete class), i.e. the HALO + EXCHANGE; the two aborts' request counts (16 = fine Dhop, 2 = coarse PaddedCell + direction) agree. Next: FI_MR_CACHE_MAX_COUNT=0 (all memory) and + FI_MR_ROCR_CACHE_MONITOR_ENABLED=0, one per cell. Symptom (Frontier, x86, Cray MPICH 8.1.x, ROCm 7.2): device-buffer MPI fails with