portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-29 13:59:36 +01:00
Code Releases Activity

More FI_MR related

Browse Source
This commit is contained in:
Peter Boyle
2026-08-28 22:01:36 -04:00
parent 43c6573ca4
commit e03797e882
15 changed files with 95 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
systems/Frontier/allreduce_bench.job
+1
View File
@@ -43,6 +43,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/batched_linalg.job
+1
View File
@@ -51,6 +51,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=1
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/nrhs_fault.job
+1
View File
@@ -60,6 +60,7 @@ export OMP_NUM_THREADS=7
# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
ulimit -c 0
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/nrhs_fault_36.job
+4 -1
View File
@@ -23,6 +23,8 @@
# (the runtime's log goes to stderr, so the faulting
# rank's Grid.stderr.<rank> ends with the culprit)
# --log ...,Memory MemoryManager transfers in program order
# --debug-heartbeat periodic backtrace per rank: a HANG (2026-08-28, single-RHS solve
# after 12 RHS with FineSloppyComms=0) shows where every rank sits
#
# Cell A: serialised (5-10x slower: setup ~10 min + solve until the fault).
# Cell B: NOT serialised, same logging -- if serialisation makes the fault
@@ -61,6 +63,7 @@ export OMP_NUM_THREADS=7
# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
ulimit -c 0
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
@@ -99,7 +102,7 @@ run_cell () {
echo "----- $name : $* -----"
export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \
--debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
--debug-stdout --debug-heartbeat --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
echo " exit $?"; sleep 30
f=$(grep -l "Memory access fault" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
if [ -n "$f" ]; then
systems/Frontier/schur2d_env.job
+1
View File
@@ -47,6 +47,7 @@ chmod +x ./select_gpu
root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/schur2d_ladder.job
+1
View File
@@ -47,6 +47,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/slate_debug.job
+1
View File
@@ -46,6 +46,7 @@ chmod +x ./select_gpu
root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/slate_vs_schur2d.job
+1
View File
@@ -58,6 +58,7 @@ source $root/sourceme-rocm7.2.sh
# hung, V2 with OMP_NUM_THREADS=1 completed, 2026-08-25). Each rank has a
# whole GCD; the parallelism is on the device for both.
export OMP_NUM_THREADS=1
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/smoother_modes.job
+1
View File
@@ -52,6 +52,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/sourceme-rocm6.4.sh
+1
View File
@@ -13,3 +13,4 @@ module load rocm/6.4.0
export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/rocm-6.4.0/lib/llvm/lib/:$LD_LIBRARY_PATH
module load emacs
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
systems/Frontier/sourceme-rocm7.0.sh
+1
View File
@@ -11,3 +11,4 @@ export LD_LIBRARY_PATH=/opt/rocm-7.0.2/lib/llvm/lib/:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
module load emacs
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
systems/Frontier/sourceme-rocm7.2.sh
+1
View File
@@ -29,3 +29,4 @@ export LD_LIBRARY_PATH=$BLASPP/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$SLATE/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
module load emacs
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
systems/WorkArounds.txt
+35
View File
@@ -204,3 +204,38 @@ Aurora/unitrace
--------------------------------------------------------------------
Tursa/nsight-sys
--------------------------------------------------------------------
============================================================================
2026-08-28 libfabric memory-registration-cache "memhooks" monitor -- DEFECTIVE
(libfabric issue #11451, filed by PB, HPE JIRA opened)
============================================================================
STATUS: the ALPS/aarch64 case below is DEMONSTRATED (issue + both fixes verified
there). The Frontier case is a HYPOTHESIS as of 2026-08-28: the NO_TRANSLATION
failure is observed, its attribution to the memhooks MR cache is by mechanism and
by analogy, and the fix is UNTESTED here. Test ladder, one knob per run, NRHS=12:
(1) FineSloppyComms=0 (2) FI_MR_CACHE_MONITOR=kdreg2 (3) FI_MR_CACHE_MAX_COUNT=0
(4) --disable-accelerator-aware-mpi build.
REVISIT this entry with the result; if (2) does not cure it, remove kdreg2 from the
job scripts' justification (it stays as OLCF's recommendation regardless).
Symptom (Frontier, x86, Cray MPICH 8.1.x, ROCm 7.2): device-buffer MPI fails
with
MPI_Waitall ... MPIDI_OFI_handle_cq_error: OFI poll failed
(ofi_events.c:MPIDI_OFI_handle_cq_error: Input/output error - NO_TRANSLATION)
on a LIVE, never-freed hipMalloc buffer (the sloppy-comms compressed halo buffer,
a static deviceVector) once the process does sustained hipMalloc/hipFree churn
(MemoryManager eviction at NRHS>=12, EvictAll/DropCache). Translations cached by
the provider go stale.
Symptom (ALPS/CSCS, aarch64 Grace/H200, cray-mpich 8.1.32, libfabric 1.22): the
memhooks monitor intercepts munmap by PATCHING THE PLT at MPI_Init
(ofi_memhooks_start -> ofi_write_patch with a garbage data_size); the write overruns
munmap@plt into the NEXT PLT entry (MPI_Comm_dup in Grid's binary), leaving
`br x15` where an adrp belongs -> segfault on first MPI_Comm_dup. Diagnosed with
a hardware watchpoint on the PLT entry during MPI_Init; see the issue for the
gdb transcript. Reproduce on one node with MPICH_SINGLE_HOST_ENABLED=0.
Fix (either): export FI_MR_CACHE_MONITOR=kdreg2 (kernel-driven invalidation; OLCF's
own recommendation, NOT the default)
export FI_MR_CACHE_MAX_COUNT=0 (no registration cache at all)
On ALPS this is a CORRECTNESS requirement for MPI on Slingshot, not a tuning.
Every systems/Frontier job and sourceme now sets kdreg2 on OLCF's recommendation;
whether it resolves the Frontier NO_TRANSLATION is the pending test above.