Frontier jobs

systems/Frontier/slate_debug.job

@@ -0,0 +1,84 @@
+#!/bin/bash -l
+#SBATCH --job-name=slate-debug
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --cpus-per-task=7
+#SBATCH --gpus-per-node=8
+#SBATCH --time=0:30:00
+#SBATCH --account=phy157_dwf
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+#SBATCH -S 0
+#SBATCH -q debug
+
+##############################################################################
+# Localise the multi-rank SLATE hang (Test_schur2d_vs_slate, 8 ranks, 2x4).
+# Each variant is a separate srun with unbuffered output (-u) and a hard
+# timeout, and the binary prints flushed "stage:" markers, so the log shows
+# the LAST stage reached in each variant:
+#
+#   V1 : as the production job     (warm-up on, OMP_NUM_THREADS=7)
+#   V2 : OMP_NUM_THREADS=1         -> is it threads x GPU-aware MPI?
+#   V3 : S2D_NOWARM=1, OMP=7       -> does the timed leg hang at the same
+#                                     SLATE call as the warm-up did?
+#   V4 : OMP_NUM_THREADS=1 + NOWARM
+#   V5 : MPICH_GPU_SUPPORT_ENABLED=0 -> SLATE Devices target with host-staged
+#                                     MPI (blaspp copies tiles itself)
+#
+# Read: for each variant the final "stage:" line and whether "SLATE :" was
+# printed.  A variant that completes identifies the fix.
+##############################################################################
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 7 6 5 4)
+export NUMA_MAP=(3 3 1 1 2 2 0 0)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
+export HIP_VISIBLE_DEVICES=\$GPU
+unset ROCR_VISIBLE_DEVICES
+if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+chmod +x ./select_gpu
+
+root=$HOME/ParallelIO/systems/Frontier
+source $root/sourceme-rocm7.2.sh
+
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_OFI_NIC_POLICY=GPU
+export MPICH_MAX_THREAD_SAFETY=multiple
+module load libfabric
+
+BIN=$root/tests/debug/Test_schur2d_vs_slate
+OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
+export S2D_N=4096
+
+echo "binary: $(ls -l $BIN)"
+ldd $BIN | grep -E "slate|blaspp|lapackpp|omp|mpi_cray"
+module list 2>&1 | grep -i slate && echo "WARNING: a slate module is loaded"
+
+run_variant () {
+  name=$1; shift
+  echo "========================================================="
+  echo "$name :  $*"
+  echo "========================================================="
+  env "$@" timeout 240 srun -u -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 2>&1 | tee $name.out | grep -E "stage:|warm-up|GRID  :|SLATE :|rror|Assert|signal"
+  rc=${PIPESTATUS[0]}
+  echo "$name exit code $rc  (124 = timeout)   last stage: $(grep 'stage:' $name.out | tail -1 | sed 's/.*stage: //')"
+}
+
+run_variant V1_default        OMP_NUM_THREADS=7
+run_variant V2_omp1           OMP_NUM_THREADS=1
+run_variant V3_nowarm         OMP_NUM_THREADS=7 S2D_NOWARM=1
+run_variant V4_omp1_nowarm    OMP_NUM_THREADS=1 S2D_NOWARM=1
+run_variant V5_hostmpi        OMP_NUM_THREADS=7 MPICH_GPU_SUPPORT_ENABLED=0
+
+echo "========================================================="
+echo "summary"
+echo "========================================================="
+for v in V1_default V2_omp1 V3_nowarm V4_omp1_nowarm V5_hostmpi; do
+  printf "%-18s SLATE-line:%s  last-stage: %s\n" $v "$(grep -c 'SLATE :' $v.out)" "$(grep 'stage:' $v.out | tail -1 | sed 's/.*stage: //')"
+done

systems/Frontier/slate_vs_schur2d.job

@@ -0,0 +1,110 @@
+#!/bin/bash -l
+#SBATCH --job-name=slate-vs-schur2d
+#SBATCH --nodes=36
+#SBATCH --ntasks-per-node=8
+#SBATCH --cpus-per-task=7
+#SBATCH --gpus-per-node=8
+#SBATCH --time=1:00:00
+#SBATCH --account=phy157_dwf
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+#SBATCH -S 0
+##SBATCH -q debug
+
+##############################################################################
+# Like-for-like: Grid's 2D block-cyclic Schur inverse vs SLATE getrf+getri,
+# same matrix, same ranks, same job, every layout cost on the clock
+# (tests/debug/Test_schur2d_vs_slate.cc).  Both inverses are certified by the
+# same SUMMA instrument; both legs print a one-line timing breakdown.
+#
+#   S1 : 1 node,  8 ranks, N=4096    -- API/device/thread-level shakeout
+#   S2 : 1 node,  8 ranks, N=13824   -- one-node production-shape rehearsal
+#   S3 : 36 nodes, 288 ranks, N=138240 (nb=480, grid 16x18) -- THE comparison
+#
+# S3 is gated on S1 passing (rc=0 and both certificates printed): a broken
+# SLATE leg should not burn the 36-node allocation.
+#
+# Requires Grid configured with --with-slate/--with-blaspp/--with-lapackpp
+# (HAVE_SLATE in Config.h); sourceme puts the three lib64 dirs on
+# LD_LIBRARY_PATH.  The binary initialises MPI at MPI_THREAD_MULTIPLE itself.
+##############################################################################
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 7 6 5 4)
+export NUMA_MAP=(3 3 1 1 2 2 0 0)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
+export HIP_VISIBLE_DEVICES=\$GPU
+unset ROCR_VISIBLE_DEVICES
+
+if [ \$SLURM_PROCID = "0" ]
+then
+echo \$*
+fi
+
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+
+chmod +x ./select_gpu
+
+root=$HOME/ParallelIO/systems/Frontier
+source $root/sourceme-rocm7.2.sh
+
+export OMP_NUM_THREADS=7
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_OFI_NIC_POLICY=GPU
+export MPICH_MAX_THREAD_SAFETY=multiple   # SLATE calls MPI from OpenMP tasks
+
+module load libfabric
+
+BIN=$root/tests/debug/Test_schur2d_vs_slate
+OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
+
+echo "========================================================="
+echo "binary and runtime linkage"
+echo "========================================================="
+ls -l $BIN
+ldd $BIN | grep -E "slate|blaspp|lapackpp|omp|rocblas|mpi_cray"
+# the cpu-env slate module must NOT be loaded: it would shadow the ROCm build
+module list 2>&1 | grep -i slate && echo "WARNING: a slate module is loaded"
+
+##############################################################################
+echo "========================================================="
+echo "S1: shakeout, 1 node, 8 ranks, N=4096 (nb=512, grid 2x4)"
+echo "========================================================="
+##############################################################################
+S2D_N=4096 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 \
+     2>&1 | tee s1.out
+S1RC=${PIPESTATUS[0]}
+S1CERT=$(grep -c "certificate" s1.out)
+echo "S1 exit code $S1RC, certificates printed $S1CERT (expect 2)"
+
+##############################################################################
+echo "========================================================="
+echo "S2: one-node rehearsal, N=13824 (nb=1728, grid 2x4)"
+echo "========================================================="
+##############################################################################
+S2D_N=13824 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1
+
+##############################################################################
+echo "========================================================="
+echo "S3: THE comparison, 36 nodes, 288 ranks, N=138240 (nb=480, grid 16x18)"
+echo "========================================================="
+##############################################################################
+if [ "$S1RC" -eq 0 ] && [ "$S1CERT" -eq 2 ]
+then
+  S2D_N=138240 srun -N36 -n288 ./select_gpu $BIN --mpi 3.6.4.4 --grid 48.48.48.96 $OPTS1
+  echo "S3 exit code $?"
+else
+  echo "S3 SKIPPED: S1 failed (rc=$S1RC, certificates=$S1CERT)"
+fi
+
+echo "========================================================="
+echo "summary (both legs, all stages)"
+echo "========================================================="
+grep -h -E "Grid-vs-SLATE|GRID  :|SLATE :" slurm-$SLURM_JOB_ID.out 2>/dev/null
+grep -h "git commit hash" slurm-$SLURM_JOB_ID.out 2>/dev/null | sort -u
+echo "========================================================="