From e03797e88220b683c5296c1d880685e29040e80e Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Fri, 28 Aug 2026 22:01:36 -0400 Subject: [PATCH] More FI_MR related --- ...mple_pvdagm_v2_3level_DenseCoarseMatrix.cc | 21 +++++++++++ skills/gpu-runtime-correctness.md | 24 +++++++++++++ systems/Frontier/allreduce_bench.job | 1 + systems/Frontier/batched_linalg.job | 1 + systems/Frontier/nrhs_fault.job | 1 + systems/Frontier/nrhs_fault_36.job | 5 ++- systems/Frontier/schur2d_env.job | 1 + systems/Frontier/schur2d_ladder.job | 1 + systems/Frontier/slate_debug.job | 1 + systems/Frontier/slate_vs_schur2d.job | 1 + systems/Frontier/smoother_modes.job | 1 + systems/Frontier/sourceme-rocm6.4.sh | 1 + systems/Frontier/sourceme-rocm7.0.sh | 1 + systems/Frontier/sourceme-rocm7.2.sh | 1 + systems/WorkArounds.txt | 35 +++++++++++++++++++ 15 files changed, 95 insertions(+), 1 deletion(-) diff --git a/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc b/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc index a85457098..d40a8eff1 100644 --- a/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc +++ b/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc @@ -188,6 +188,27 @@ void ParseEnvironment(void) std::cout << GridLogMessage << "PARAM: NBASIS " << NBASIS << std::endl; std::cout << GridLogMessage << "PARAM: NRHS " << Nrhs << std::endl; std::cout << GridLogMessage << "PARAM: COARSEN_BATCH " << CoarsenBatch << std::endl; + // EVERY knob this programme parsed, so a log identifies its own run (2026-08-28: + // four jobs in the queue differing in FineSloppyComms/NRHS/mmax and none of it + // printed). Interim until the Serializable parameter struct replaces all of this. + auto P = [](const char *n, auto v){ std::cout << GridLogMessage << "PARAM: " << std::left << std::setw(20) << n << std::right << " " << v << std::endl; }; + P("FineSmootherShift",FineSmootherShift); P("FineSmootherOrder",FineSmootherOrder); P("FineSmootherMmax",FineSmootherMmax); + P("FineSmootherMode",FineSmootherMode); P("FineChebLo",FineChebLo); P("FineChebHi",FineChebHi); + P("FineSloppyComms",FineSloppyComms); + P("CoarseSmootherShift",CoarseSmootherShift); P("CoarseSmootherNstep",CoarseSmootherNstep); P("CoarseSmootherMmax",CoarseSmootherMmax); + P("CoarseSmootherMode",CoarseSmootherMode); P("CoarseChebLo",CoarseChebLo); P("CoarseChebHi",CoarseChebHi); + P("CoarseSolverTol",CoarseSolverTol); P("CoarseSolverOrder",CoarseSolverOrder); P("CoarseSolverMmax",CoarseSolverMmax); + P("OuterTol",OuterTol); P("OuterMmax",OuterMmax); P("OuterNstep",OuterNstep); + P("PowerIterations",PowerIterations); P("PolyRecordIters",PolyRecordIters); P("PolyRecordStart",PolyRecordStart); + P("PolyRecordSelect",PolyRecordSelect); P("PolyRefresh",PolyRefresh); P("PolyVerbose",PolyVerbose); + P("SmootherCoeffLog",SmootherCoeffLog); + // library-side and runtime knobs, read straight from the environment as the library will + const char *envs[] = {"BLOCK","BLOCK2","SUBSPACE_FILE","CONFIG","HOT_START","MRHS_COARSEN","V1_CHECK", + "DENSE_CC","DENSE_SCHUR","DENSE_SCHUR2D","DENSE_DEVICE_SUM","DENSE_SPLITK","DENSE_NB", + "SCHUR2D_LEAF_SPAN","SCHUR2D_LEAF_LU","SCHUR2D_PROBE","SUMMA_HANDSHAKE","DENSE_APPLY_PROFILE","SLAB_FILE", + "SOLVE_SRHS","GRID_ALLOC_NCACHE_LARGE","OMP_NUM_THREADS", + "MPICH_GPU_SUPPORT_ENABLED","FI_MR_CACHE_MONITOR","FI_MR_CACHE_MAX_COUNT","AMD_SERIALIZE_KERNEL","AMD_LOG_LEVEL"}; + for(auto e : envs) P(e, getenv(e) ? std::string(getenv(e)) : std::string("(unset)")); } template diff --git a/skills/gpu-runtime-correctness.md b/skills/gpu-runtime-correctness.md index 4f9b4bd70..6049c329b 100644 --- a/skills/gpu-runtime-correctness.md +++ b/skills/gpu-runtime-correctness.md @@ -129,3 +129,27 @@ script suppresses it (Slurm propagates rlimits to `srun` tasks by default) and Deleted-while-open dumps on NFS persist as hidden `.nfs*` files against quota until the holder exits. Run fault hunts from a Lustre directory. Source: ROCgdb documentation, "AMD GPU" chapter, core-dump section. + +## Attributing memory corruption in stripped vendor libraries: the watchpoint + +When code you cannot read (MPI, libfabric, driver: stripped, no debug info) corrupts +something, source-level breakpoints are useless. The tools that need no source +(libfabric #11451, PB, aarch64 PLT corruption at MPI_Init, is the worked example): + +1. Find the corrupted word: `p/x *(uint32_t *)0x408ed0` before and after the suspect + call; compare with the binary's own disassembly (`objdump -d`, or `layout asm`). +2. `watch *0x408ed0` (hardware watchpoint on the ADDRESS, not a breakpoint on a + function), then run the suspect call: the trap fires at the instruction that writes. +3. `bt` at the trap names the writer and its arguments (there: `ofi_write_patch( + data_size=2876502996, address=)` -- a garbage length overrunning into + the next PLT entry). +4. `layout asm` + `info registers` at the later crash show the mechanism (`br x15` with + x15=0x10 where an `adrp` belonged); `info symbol ` names stray addresses. +5. Trace the written opcode back to the library's source if you have it (0xd61f01e0 = + libfabric's `br(15)` in util_mem_hooks.c) -- irrefutable. +6. Make it reproducible where a debugger attaches: `MPICH_SINGLE_HOST_ENABLED=0` + forces the CXI/fabric path on ONE node. + +Report standard (PB, from EDA/QCDOC/BG-Q QA): symptom -> instrument -> the line -> +the fix. A report without the mechanism is noise; one with it gets an HPE JIRA the +same day. Fix for the above: `FI_MR_CACHE_MONITOR=kdreg2` (see systems/WorkArounds.txt). diff --git a/systems/Frontier/allreduce_bench.job b/systems/Frontier/allreduce_bench.job index 65ed0929b..9627ec795 100644 --- a/systems/Frontier/allreduce_bench.job +++ b/systems/Frontier/allreduce_bench.job @@ -43,6 +43,7 @@ root=$HOME/ParallelIO/systems/Frontier source $root/sourceme-rocm7.2.sh export OMP_NUM_THREADS=7 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/batched_linalg.job b/systems/Frontier/batched_linalg.job index def0f71e9..bdaf34e39 100644 --- a/systems/Frontier/batched_linalg.job +++ b/systems/Frontier/batched_linalg.job @@ -51,6 +51,7 @@ root=$HOME/ParallelIO/systems/Frontier source $root/sourceme-rocm7.2.sh export OMP_NUM_THREADS=1 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/nrhs_fault.job b/systems/Frontier/nrhs_fault.job index d6b6ba756..0e679ab7f 100644 --- a/systems/Frontier/nrhs_fault.job +++ b/systems/Frontier/nrhs_fault.job @@ -60,6 +60,7 @@ export OMP_NUM_THREADS=7 # .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU # core dumps"); Slurm propagates the batch shell's limits to srun tasks by default. ulimit -c 0 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/nrhs_fault_36.job b/systems/Frontier/nrhs_fault_36.job index ef827a25e..2a8a02e3c 100644 --- a/systems/Frontier/nrhs_fault_36.job +++ b/systems/Frontier/nrhs_fault_36.job @@ -23,6 +23,8 @@ # (the runtime's log goes to stderr, so the faulting # rank's Grid.stderr. ends with the culprit) # --log ...,Memory MemoryManager transfers in program order +# --debug-heartbeat periodic backtrace per rank: a HANG (2026-08-28, single-RHS solve +# after 12 RHS with FineSloppyComms=0) shows where every rank sits # # Cell A: serialised (5-10x slower: setup ~10 min + solve until the fault). # Cell B: NOT serialised, same logging -- if serialisation makes the fault @@ -61,6 +63,7 @@ export OMP_NUM_THREADS=7 # .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU # core dumps"); Slurm propagates the batch shell's limits to srun tasks by default. ulimit -c 0 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU @@ -99,7 +102,7 @@ run_cell () { echo "----- $name : $* -----" export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \ - --debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1 + --debug-stdout --debug-heartbeat --log Error,Warning,Message,Memory > log.fault36.$name 2>&1 echo " exit $?"; sleep 30 f=$(grep -l "Memory access fault" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1) if [ -n "$f" ]; then diff --git a/systems/Frontier/schur2d_env.job b/systems/Frontier/schur2d_env.job index c6748d565..9518386b9 100644 --- a/systems/Frontier/schur2d_env.job +++ b/systems/Frontier/schur2d_env.job @@ -47,6 +47,7 @@ chmod +x ./select_gpu root=$HOME/ParallelIO/systems/Frontier source $root/sourceme-rocm7.2.sh +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/schur2d_ladder.job b/systems/Frontier/schur2d_ladder.job index 71016a364..891c8184b 100644 --- a/systems/Frontier/schur2d_ladder.job +++ b/systems/Frontier/schur2d_ladder.job @@ -47,6 +47,7 @@ root=$HOME/ParallelIO/systems/Frontier source $root/sourceme-rocm7.2.sh export OMP_NUM_THREADS=7 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/slate_debug.job b/systems/Frontier/slate_debug.job index 255e1f9f3..4e8ae0f63 100644 --- a/systems/Frontier/slate_debug.job +++ b/systems/Frontier/slate_debug.job @@ -46,6 +46,7 @@ chmod +x ./select_gpu root=$HOME/ParallelIO/systems/Frontier source $root/sourceme-rocm7.2.sh +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/slate_vs_schur2d.job b/systems/Frontier/slate_vs_schur2d.job index 1aa92fa55..6dd672c17 100644 --- a/systems/Frontier/slate_vs_schur2d.job +++ b/systems/Frontier/slate_vs_schur2d.job @@ -58,6 +58,7 @@ source $root/sourceme-rocm7.2.sh # hung, V2 with OMP_NUM_THREADS=1 completed, 2026-08-25). Each rank has a # whole GCD; the parallelism is on the device for both. export OMP_NUM_THREADS=1 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/smoother_modes.job b/systems/Frontier/smoother_modes.job index 144b48e87..cb0b7a9d1 100644 --- a/systems/Frontier/smoother_modes.job +++ b/systems/Frontier/smoother_modes.job @@ -52,6 +52,7 @@ root=$HOME/ParallelIO/systems/Frontier source $root/sourceme-rocm7.2.sh export OMP_NUM_THREADS=7 +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) export MPICH_GPU_SUPPORT_ENABLED=1 export MPICH_SMP_SINGLE_COPY_MODE=CMA export MPICH_OFI_NIC_POLICY=GPU diff --git a/systems/Frontier/sourceme-rocm6.4.sh b/systems/Frontier/sourceme-rocm6.4.sh index 95752e02c..66fff35ae 100644 --- a/systems/Frontier/sourceme-rocm6.4.sh +++ b/systems/Frontier/sourceme-rocm6.4.sh @@ -13,3 +13,4 @@ module load rocm/6.4.0 export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/opt/rocm-6.4.0/lib/llvm/lib/:$LD_LIBRARY_PATH module load emacs +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) diff --git a/systems/Frontier/sourceme-rocm7.0.sh b/systems/Frontier/sourceme-rocm7.0.sh index a5a64c314..872ef1be5 100644 --- a/systems/Frontier/sourceme-rocm7.0.sh +++ b/systems/Frontier/sourceme-rocm7.0.sh @@ -11,3 +11,4 @@ export LD_LIBRARY_PATH=/opt/rocm-7.0.2/lib/llvm/lib/:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH module load emacs +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) diff --git a/systems/Frontier/sourceme-rocm7.2.sh b/systems/Frontier/sourceme-rocm7.2.sh index 2701c15e0..dcfc2f7d1 100644 --- a/systems/Frontier/sourceme-rocm7.2.sh +++ b/systems/Frontier/sourceme-rocm7.2.sh @@ -29,3 +29,4 @@ export LD_LIBRARY_PATH=$BLASPP/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$SLATE/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH module load emacs +export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451) diff --git a/systems/WorkArounds.txt b/systems/WorkArounds.txt index 7191b4ffb..7ffdce822 100644 --- a/systems/WorkArounds.txt +++ b/systems/WorkArounds.txt @@ -204,3 +204,38 @@ Aurora/unitrace -------------------------------------------------------------------- Tursa/nsight-sys -------------------------------------------------------------------- + +============================================================================ +2026-08-28 libfabric memory-registration-cache "memhooks" monitor -- DEFECTIVE + (libfabric issue #11451, filed by PB, HPE JIRA opened) +============================================================================ +STATUS: the ALPS/aarch64 case below is DEMONSTRATED (issue + both fixes verified +there). The Frontier case is a HYPOTHESIS as of 2026-08-28: the NO_TRANSLATION +failure is observed, its attribution to the memhooks MR cache is by mechanism and +by analogy, and the fix is UNTESTED here. Test ladder, one knob per run, NRHS=12: + (1) FineSloppyComms=0 (2) FI_MR_CACHE_MONITOR=kdreg2 (3) FI_MR_CACHE_MAX_COUNT=0 + (4) --disable-accelerator-aware-mpi build. +REVISIT this entry with the result; if (2) does not cure it, remove kdreg2 from the +job scripts' justification (it stays as OLCF's recommendation regardless). + +Symptom (Frontier, x86, Cray MPICH 8.1.x, ROCm 7.2): device-buffer MPI fails +with + MPI_Waitall ... MPIDI_OFI_handle_cq_error: OFI poll failed + (ofi_events.c:MPIDI_OFI_handle_cq_error: Input/output error - NO_TRANSLATION) +on a LIVE, never-freed hipMalloc buffer (the sloppy-comms compressed halo buffer, +a static deviceVector) once the process does sustained hipMalloc/hipFree churn +(MemoryManager eviction at NRHS>=12, EvictAll/DropCache). Translations cached by +the provider go stale. +Symptom (ALPS/CSCS, aarch64 Grace/H200, cray-mpich 8.1.32, libfabric 1.22): the +memhooks monitor intercepts munmap by PATCHING THE PLT at MPI_Init +(ofi_memhooks_start -> ofi_write_patch with a garbage data_size); the write overruns +munmap@plt into the NEXT PLT entry (MPI_Comm_dup in Grid's binary), leaving +`br x15` where an adrp belongs -> segfault on first MPI_Comm_dup. Diagnosed with +a hardware watchpoint on the PLT entry during MPI_Init; see the issue for the +gdb transcript. Reproduce on one node with MPICH_SINGLE_HOST_ENABLED=0. +Fix (either): export FI_MR_CACHE_MONITOR=kdreg2 (kernel-driven invalidation; OLCF's + own recommendation, NOT the default) + export FI_MR_CACHE_MAX_COUNT=0 (no registration cache at all) +On ALPS this is a CORRECTNESS requirement for MPI on Slingshot, not a tuning. +Every systems/Frontier job and sourceme now sets kdreg2 on OLCF's recommendation; +whether it resolves the Frontier NO_TRANSLATION is the pending test above.