FI investigatins

Grid/allocator/MemoryManager.cc

@@ -272,7 +272,6 @@ void MemoryManager::InitMessage(void) {
 #endif
 
 }
-
 void *MemoryManager::Insert(void *ptr,size_t bytes,int type) 
 {
 #ifdef ALLOCATION_CACHE
@@ -286,6 +285,46 @@ void *MemoryManager::Insert(void *ptr,size_t bytes,int type)
   return ptr;
 #endif
 }
+void MemoryManager::DropCache(void)
+{
+  // Release every block held in the allocation caches (the "recent
+  // allocations" pools: Small/Large/Huge x Cpu/Acc/Shared).  Blocks in these
+  // pools are freed from the caller's point of view but still occupy memory;
+  // after a large setup phase (SUMMA scratch, coarsening temporaries) they can
+  // hold gigabytes of device memory that hipMalloc then cannot get.  Each pool
+  // is freed with the call that pairs with its allocator.
+  for(int sz=0;sz<3;sz++) {
+    int t;
+    t = Acc+sz;
+    for(int e=0;e<Ncache[t];e++) {
+      if( Entries[t][e].valid ) {
+	acceleratorFreeDevice(Entries[t][e].address);
+	Entries[t][e].valid = 0; Entries[t][e].bytes = 0; Entries[t][e].address = NULL;
+      }
+    }
+    CacheBytes[t]=0; Victim[t]=0;
+    t = Shared+sz;
+    for(int e=0;e<Ncache[t];e++) {
+      if( Entries[t][e].valid ) {
+	acceleratorFreeShared(Entries[t][e].address);
+	Entries[t][e].valid = 0; Entries[t][e].bytes = 0; Entries[t][e].address = NULL;
+      }
+    }
+    CacheBytes[t]=0; Victim[t]=0;
+    t = Cpu+sz;
+    for(int e=0;e<Ncache[t];e++) {
+      if( Entries[t][e].valid ) {
+#ifdef GRID_UVM
+	acceleratorFreeShared(Entries[t][e].address);
+#else
+	acceleratorFreeCpu(Entries[t][e].address);
+#endif
+	Entries[t][e].valid = 0; Entries[t][e].bytes = 0; Entries[t][e].address = NULL;
+      }
+    }
+    CacheBytes[t]=0; Victim[t]=0;
+  }
+}
 
 void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim, uint64_t &cacheBytes) 
 {

Grid/allocator/MemoryManager.h

@@ -216,6 +216,7 @@ public:
   static void Print(void);
   static void PrintAll(void);
   static void EvictAll(void);
+  static void DropCache(void);
   static void PrintState( void* CpuPtr);
   static int   isOpen   (void* CpuPtr);
   static void  ViewClose(void* CpuPtr,ViewMode mode);

examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc

@@ -188,6 +188,27 @@ void ParseEnvironment(void)
   std::cout << GridLogMessage << "PARAM: NBASIS             " << NBASIS       << std::endl;
   std::cout << GridLogMessage << "PARAM: NRHS               " << Nrhs         << std::endl;
   std::cout << GridLogMessage << "PARAM: COARSEN_BATCH      " << CoarsenBatch << std::endl;
+  // EVERY knob this programme parsed, so a log identifies its own run (2026-08-28:
+  // four jobs in the queue differing in FineSloppyComms/NRHS/mmax and none of it
+  // printed).  Interim until the Serializable parameter struct replaces all of this.
+  auto P = [](const char *n, auto v){ std::cout << GridLogMessage << "PARAM: " << std::left << std::setw(20) << n << std::right << " " << v << std::endl; };
+  P("FineSmootherShift",FineSmootherShift);   P("FineSmootherOrder",FineSmootherOrder);   P("FineSmootherMmax",FineSmootherMmax);
+  P("FineSmootherMode",FineSmootherMode);     P("FineChebLo",FineChebLo);                 P("FineChebHi",FineChebHi);
+  P("FineSloppyComms",FineSloppyComms);
+  P("CoarseSmootherShift",CoarseSmootherShift); P("CoarseSmootherNstep",CoarseSmootherNstep); P("CoarseSmootherMmax",CoarseSmootherMmax);
+  P("CoarseSmootherMode",CoarseSmootherMode); P("CoarseChebLo",CoarseChebLo);             P("CoarseChebHi",CoarseChebHi);
+  P("CoarseSolverTol",CoarseSolverTol);       P("CoarseSolverOrder",CoarseSolverOrder);   P("CoarseSolverMmax",CoarseSolverMmax);
+  P("OuterTol",OuterTol);                     P("OuterMmax",OuterMmax);                   P("OuterNstep",OuterNstep);
+  P("PowerIterations",PowerIterations);       P("PolyRecordIters",PolyRecordIters);       P("PolyRecordStart",PolyRecordStart);
+  P("PolyRecordSelect",PolyRecordSelect);     P("PolyRefresh",PolyRefresh);               P("PolyVerbose",PolyVerbose);
+  P("SmootherCoeffLog",SmootherCoeffLog);
+  // library-side and runtime knobs, read straight from the environment as the library will
+  const char *envs[] = {"BLOCK","BLOCK2","SUBSPACE_FILE","CONFIG","HOT_START","MRHS_COARSEN","V1_CHECK",
+                        "DENSE_CC","DENSE_SCHUR","DENSE_SCHUR2D","DENSE_DEVICE_SUM","DENSE_SPLITK","DENSE_NB",
+                        "SCHUR2D_LEAF_SPAN","SCHUR2D_LEAF_LU","SCHUR2D_PROBE","SUMMA_HANDSHAKE","DENSE_APPLY_PROFILE","SLAB_FILE",
+                        "SOLVE_SRHS","GRID_ALLOC_NCACHE_LARGE","OMP_NUM_THREADS",
+                        "MPICH_GPU_SUPPORT_ENABLED","FI_MR_CACHE_MONITOR","FI_MR_CACHE_MAX_COUNT","AMD_SERIALIZE_KERNEL","AMD_LOG_LEVEL"};
+  for(auto e : envs) P(e, getenv(e) ? std::string(getenv(e)) : std::string("(unset)"));
 }
 
 template <class Field>
@@ -967,6 +988,9 @@ int main (int argc, char ** argv)
       // coarsening temporaries) goes back to host, so the solve's working set
       // starts from a clean device and the cap applies to it alone.
       MemoryManager::EvictAll();
+      // ...and release the allocation caches' held blocks (setup-era deviceVector
+      // scratch that is "free" to the caller but not to hipMalloc).
+      MemoryManager::DropCache();
       MemoryManager::PrintBytes();
       acceleratorMem();
     }

skills/gpu-runtime-correctness.md

@@ -129,3 +129,27 @@ script suppresses it (Slurm propagates rlimits to `srun` tasks by default) and
 Deleted-while-open dumps on NFS persist as hidden `.nfs*` files against quota
 until the holder exits.  Run fault hunts from a Lustre directory.
 Source: ROCgdb documentation, "AMD GPU" chapter, core-dump section.
+
+## Attributing memory corruption in stripped vendor libraries: the watchpoint
+
+When code you cannot read (MPI, libfabric, driver: stripped, no debug info) corrupts
+something, source-level breakpoints are useless.  The tools that need no source
+(libfabric #11451, PB, aarch64 PLT corruption at MPI_Init, is the worked example):
+
+1. Find the corrupted word: `p/x *(uint32_t *)0x408ed0` before and after the suspect
+   call; compare with the binary's own disassembly (`objdump -d`, or `layout asm`).
+2. `watch *0x408ed0` (hardware watchpoint on the ADDRESS, not a breakpoint on a
+   function), then run the suspect call: the trap fires at the instruction that writes.
+3. `bt` at the trap names the writer and its arguments (there: `ofi_write_patch(
+   data_size=2876502996, address=<munmap@plt>)` -- a garbage length overrunning into
+   the next PLT entry).
+4. `layout asm` + `info registers` at the later crash show the mechanism (`br x15` with
+   x15=0x10 where an `adrp` belonged); `info symbol <addr>` names stray addresses.
+5. Trace the written opcode back to the library's source if you have it (0xd61f01e0 =
+   libfabric's `br(15)` in util_mem_hooks.c) -- irrefutable.
+6. Make it reproducible where a debugger attaches: `MPICH_SINGLE_HOST_ENABLED=0`
+   forces the CXI/fabric path on ONE node.
+
+Report standard (PB, from EDA/QCDOC/BG-Q QA): symptom -> instrument -> the line ->
+the fix.  A report without the mechanism is noise; one with it gets an HPE JIRA the
+same day.  Fix for the above: `FI_MR_CACHE_MONITOR=kdreg2` (see systems/WorkArounds.txt).

systems/Frontier/allreduce_bench.job

@@ -43,6 +43,7 @@ root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
 export OMP_NUM_THREADS=7
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/batched_linalg.job

@@ -51,6 +51,7 @@ root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
 export OMP_NUM_THREADS=1
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/nrhs_fault.job

@@ -60,6 +60,7 @@ export OMP_NUM_THREADS=7
 # .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
 # core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
 ulimit -c 0
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/nrhs_fault_36.job

@@ -61,6 +61,7 @@ export OMP_NUM_THREADS=7
 # .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
 # core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
 ulimit -c 0
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU
@@ -114,5 +115,16 @@ run_cell () {
   fi
 }
 
-run_cell   A_serialised   AMD_SERIALIZE_KERNEL=3 AMD_SERIALIZE_COPY=3 AMD_LOG_LEVEL=3
-run_cell   B_plain        AMD_LOG_LEVEL=1
+#run_cell   A_serialised   AMD_SERIALIZE_KERNEL=3 AMD_SERIALIZE_COPY=3 AMD_LOG_LEVEL=3   # 5371703-era: OOM found; done
+#run_cell   B_plain        AMD_LOG_LEVEL=1
+# Ladder for the NO_TRANSLATION / hang fault (both at NRHS=12):
+#   5371703  sloppy ON,  no kdreg2            -> NO_TRANSLATION (Dhop halo) at outer 36
+#   5371826  sloppy OFF, no kdreg2            -> 12-RHS converged, HANG in single-RHS solve
+#   5372414  sloppy OFF, kdreg2 (env also had FI_MR_CACHE_MAX_COUNT=786432) -> NO_TRANSLATION (Couter, 2 reqs) at outer 60
+# CORRECTION (fi_mr(3)): FI_MR_CACHE_MONITOR={memhooks,kdreg2,userfaultfd,disabled} governs
+# SYSTEM memory only.  Device (HMEM_ROCR) registrations have their own monitor,
+# FI_MR_ROCR_CACHE_MONITOR_ENABLED=0|1, and Grid's comms window is hipMalloc'd device memory
+# -- so 5372414 (kdreg2) never exercised the translations that fail.  Unambiguous global
+# switch: FI_MR_CACHE_MAX_COUNT=0 disables the registration cache for ALL memory types.
+run_cell   D_nocache      FI_MR_CACHE_MAX_COUNT=0 FI_LOG_LEVEL=info FI_LOG_PROV=core           # step 3: no cache, any memory
+run_cell   E_rocrmon0     FI_MR_ROCR_CACHE_MONITOR_ENABLED=0 FI_LOG_LEVEL=info FI_LOG_PROV=core  # step 3b: device-memory monitor off, cache on

systems/Frontier/schur2d_env.job

@@ -47,6 +47,7 @@ chmod +x ./select_gpu
 root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/schur2d_ladder.job

@@ -47,6 +47,7 @@ root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
 export OMP_NUM_THREADS=7
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/slate_debug.job

@@ -46,6 +46,7 @@ chmod +x ./select_gpu
 root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/slate_vs_schur2d.job

@@ -58,6 +58,7 @@ source $root/sourceme-rocm7.2.sh
 # hung, V2 with OMP_NUM_THREADS=1 completed, 2026-08-25).  Each rank has a
 # whole GCD; the parallelism is on the device for both.
 export OMP_NUM_THREADS=1
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/smoother_modes.job

@@ -52,6 +52,7 @@ root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
 export OMP_NUM_THREADS=7
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU

systems/Frontier/sourceme-rocm6.4.sh

@@ -13,3 +13,4 @@ module load rocm/6.4.0
 export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=/opt/rocm-6.4.0/lib/llvm/lib/:$LD_LIBRARY_PATH
 module load emacs
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)

systems/Frontier/sourceme-rocm7.0.sh

@@ -11,3 +11,4 @@ export LD_LIBRARY_PATH=/opt/rocm-7.0.2/lib/llvm/lib/:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
 
 module load emacs
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)

systems/Frontier/sourceme-rocm7.2.sh

@@ -29,3 +29,4 @@ export LD_LIBRARY_PATH=$BLASPP/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$SLATE/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
 module load emacs
+export FI_MR_CACHE_MONITOR=kdreg2       # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)

systems/WorkArounds.txt

@@ -204,3 +204,51 @@ Aurora/unitrace
 --------------------------------------------------------------------
 Tursa/nsight-sys
 --------------------------------------------------------------------
+
+============================================================================
+2026-08-28  libfabric memory-registration-cache "memhooks" monitor -- DEFECTIVE
+           (libfabric issue #11451, filed by PB, HPE JIRA opened)
+============================================================================
+STATUS: the ALPS/aarch64 case below is DEMONSTRATED (issue + both fixes verified
+there).  The Frontier case is a HYPOTHESIS as of 2026-08-28: the NO_TRANSLATION
+failure is observed, its attribution to the memhooks MR cache is by mechanism and
+by analogy, and the fix is UNTESTED here.  Test ladder, one knob per run, NRHS=12:
+  (1) FineSloppyComms=0  (2) FI_MR_CACHE_MONITOR=kdreg2  (3) FI_MR_CACHE_MAX_COUNT=0
+  (4) --disable-accelerator-aware-mpi build.
+REVISIT this entry with the result; if (2) does not cure it, remove kdreg2 from the
+job scripts' justification (it stays as OLCF's recommendation regardless).
+  2026-08-28 22:18, job 5372414: step (2) DID NOT cure it -- kdreg2 set (and the
+  environment already carried FI_MR_CACHE_MAX_COUNT=786432), sloppy comms OFF, still
+  NO_TRANSLATION at outer step 60 (Couter, Waitall count=2).
+  CORRECTION 2026-08-28 23:xx (fi_mr(3) man page): FI_MR_CACHE_MONITOR governs SYSTEM
+  memory only; device (HMEM_ROCR) registrations are monitored by
+  FI_MR_ROCR_CACHE_MONITOR_ENABLED=0|1.  Grid's comms window is hipMalloc'd device
+  memory, so the kdreg2 step tested nothing relevant -- the ALPS PLT bug (#11451) and
+  the Frontier NO_TRANSLATION are the same *library* but different monitors.
+  Attributed by gdb on a hung rank (5371826): main thread in MPI_Waitall <- Grid
+  0x5d539c3 (StencilSendToRecvFromComplete/CommsComplete class), i.e. the HALO
+  EXCHANGE; the two aborts' request counts (16 = fine Dhop, 2 = coarse PaddedCell
+  direction) agree.  Next: FI_MR_CACHE_MAX_COUNT=0 (all memory) and
+  FI_MR_ROCR_CACHE_MONITOR_ENABLED=0, one per cell.
+
+Symptom (Frontier, x86, Cray MPICH 8.1.x, ROCm 7.2): device-buffer MPI fails
+with
+   MPI_Waitall ... MPIDI_OFI_handle_cq_error: OFI poll failed
+   (ofi_events.c:MPIDI_OFI_handle_cq_error: Input/output error - NO_TRANSLATION)
+on a LIVE, never-freed hipMalloc buffer (the sloppy-comms compressed halo buffer,
+a static deviceVector) once the process does sustained hipMalloc/hipFree churn
+(MemoryManager eviction at NRHS>=12, EvictAll/DropCache).  Translations cached by
+the provider go stale.
+Symptom (ALPS/CSCS, aarch64 Grace/H200, cray-mpich 8.1.32, libfabric 1.22): the
+memhooks monitor intercepts munmap by PATCHING THE PLT at MPI_Init
+(ofi_memhooks_start -> ofi_write_patch with a garbage data_size); the write overruns
+munmap@plt into the NEXT PLT entry (MPI_Comm_dup in Grid's binary), leaving
+`br x15` where an adrp belongs -> segfault on first MPI_Comm_dup.  Diagnosed with
+a hardware watchpoint on the PLT entry during MPI_Init; see the issue for the
+gdb transcript.  Reproduce on one node with MPICH_SINGLE_HOST_ENABLED=0.
+Fix (either):   export FI_MR_CACHE_MONITOR=kdreg2      (kernel-driven invalidation; OLCF's
+                                                        own recommendation, NOT the default)
+                export FI_MR_CACHE_MAX_COUNT=0         (no registration cache at all)
+On ALPS this is a CORRECTNESS requirement for MPI on Slingshot, not a tuning.
+Every systems/Frontier job and sourceme now sets kdreg2 on OLCF's recommendation;
+whether it resolves the Frontier NO_TRANSLATION is the pending test above.