portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-30 06:19:35 +01:00
Code Releases Activity

More FI_MR related

Browse Source
This commit is contained in:
Peter Boyle
2026-08-28 22:01:36 -04:00
parent 43c6573ca4
commit e03797e882
15 changed files with 95 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
systems/Frontier/allreduce_bench.job
+1
View File
@@ -43,6 +43,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/batched_linalg.job
+1
View File
@@ -51,6 +51,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=1
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/nrhs_fault.job
+1
View File
@@ -60,6 +60,7 @@ export OMP_NUM_THREADS=7
# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
ulimit -c 0
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/nrhs_fault_36.job
+4 -1
View File
@@ -23,6 +23,8 @@
# (the runtime's log goes to stderr, so the faulting
# rank's Grid.stderr.<rank> ends with the culprit)
# --log ...,Memory MemoryManager transfers in program order
# --debug-heartbeat periodic backtrace per rank: a HANG (2026-08-28, single-RHS solve
# after 12 RHS with FineSloppyComms=0) shows where every rank sits
#
# Cell A: serialised (5-10x slower: setup ~10 min + solve until the fault).
# Cell B: NOT serialised, same logging -- if serialisation makes the fault
@@ -61,6 +63,7 @@ export OMP_NUM_THREADS=7
# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
ulimit -c 0
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
@@ -99,7 +102,7 @@ run_cell () {
echo "----- $name : $* -----"
export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \
--debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
--debug-stdout --debug-heartbeat --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
echo " exit $?"; sleep 30
f=$(grep -l "Memory access fault" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
if [ -n "$f" ]; then
systems/Frontier/schur2d_env.job
+1
View File
@@ -47,6 +47,7 @@ chmod +x ./select_gpu
root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/schur2d_ladder.job
+1
View File
@@ -47,6 +47,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/slate_debug.job
+1
View File
@@ -46,6 +46,7 @@ chmod +x ./select_gpu
root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/slate_vs_schur2d.job
+1
View File
@@ -58,6 +58,7 @@ source $root/sourceme-rocm7.2.sh
# hung, V2 with OMP_NUM_THREADS=1 completed, 2026-08-25). Each rank has a
# whole GCD; the parallelism is on the device for both.
export OMP_NUM_THREADS=1
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/smoother_modes.job
+1
View File
@@ -52,6 +52,7 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
systems/Frontier/sourceme-rocm6.4.sh
+1
View File
@@ -13,3 +13,4 @@ module load rocm/6.4.0
export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/opt/rocm-6.4.0/lib/llvm/lib/:$LD_LIBRARY_PATH
module load emacs
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
systems/Frontier/sourceme-rocm7.0.sh
+1
View File
@@ -11,3 +11,4 @@ export LD_LIBRARY_PATH=/opt/rocm-7.0.2/lib/llvm/lib/:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
module load emacs
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
systems/Frontier/sourceme-rocm7.2.sh
+1
View File
@@ -29,3 +29,4 @@ export LD_LIBRARY_PATH=$BLASPP/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$SLATE/lib64:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
module load emacs
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)