Frontier jobs

systems/Frontier/slate_vs_schur2d.job

@@ -0,0 +1,110 @@
+#!/bin/bash -l
+#SBATCH --job-name=slate-vs-schur2d
+#SBATCH --nodes=36
+#SBATCH --ntasks-per-node=8
+#SBATCH --cpus-per-task=7
+#SBATCH --gpus-per-node=8
+#SBATCH --time=1:00:00
+#SBATCH --account=phy157_dwf
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+#SBATCH -S 0
+##SBATCH -q debug
+
+##############################################################################
+# Like-for-like: Grid's 2D block-cyclic Schur inverse vs SLATE getrf+getri,
+# same matrix, same ranks, same job, every layout cost on the clock
+# (tests/debug/Test_schur2d_vs_slate.cc).  Both inverses are certified by the
+# same SUMMA instrument; both legs print a one-line timing breakdown.
+#
+#   S1 : 1 node,  8 ranks, N=4096    -- API/device/thread-level shakeout
+#   S2 : 1 node,  8 ranks, N=13824   -- one-node production-shape rehearsal
+#   S3 : 36 nodes, 288 ranks, N=138240 (nb=480, grid 16x18) -- THE comparison
+#
+# S3 is gated on S1 passing (rc=0 and both certificates printed): a broken
+# SLATE leg should not burn the 36-node allocation.
+#
+# Requires Grid configured with --with-slate/--with-blaspp/--with-lapackpp
+# (HAVE_SLATE in Config.h); sourceme puts the three lib64 dirs on
+# LD_LIBRARY_PATH.  The binary initialises MPI at MPI_THREAD_MULTIPLE itself.
+##############################################################################
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 7 6 5 4)
+export NUMA_MAP=(3 3 1 1 2 2 0 0)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
+export HIP_VISIBLE_DEVICES=\$GPU
+unset ROCR_VISIBLE_DEVICES
+
+if [ \$SLURM_PROCID = "0" ]
+then
+echo \$*
+fi
+
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+
+chmod +x ./select_gpu
+
+root=$HOME/ParallelIO/systems/Frontier
+source $root/sourceme-rocm7.2.sh
+
+export OMP_NUM_THREADS=7
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_OFI_NIC_POLICY=GPU
+export MPICH_MAX_THREAD_SAFETY=multiple   # SLATE calls MPI from OpenMP tasks
+
+module load libfabric
+
+BIN=$root/tests/debug/Test_schur2d_vs_slate
+OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
+
+echo "========================================================="
+echo "binary and runtime linkage"
+echo "========================================================="
+ls -l $BIN
+ldd $BIN | grep -E "slate|blaspp|lapackpp|omp|rocblas|mpi_cray"
+# the cpu-env slate module must NOT be loaded: it would shadow the ROCm build
+module list 2>&1 | grep -i slate && echo "WARNING: a slate module is loaded"
+
+##############################################################################
+echo "========================================================="
+echo "S1: shakeout, 1 node, 8 ranks, N=4096 (nb=512, grid 2x4)"
+echo "========================================================="
+##############################################################################
+S2D_N=4096 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 \
+     2>&1 | tee s1.out
+S1RC=${PIPESTATUS[0]}
+S1CERT=$(grep -c "certificate" s1.out)
+echo "S1 exit code $S1RC, certificates printed $S1CERT (expect 2)"
+
+##############################################################################
+echo "========================================================="
+echo "S2: one-node rehearsal, N=13824 (nb=1728, grid 2x4)"
+echo "========================================================="
+##############################################################################
+S2D_N=13824 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1
+
+##############################################################################
+echo "========================================================="
+echo "S3: THE comparison, 36 nodes, 288 ranks, N=138240 (nb=480, grid 16x18)"
+echo "========================================================="
+##############################################################################
+if [ "$S1RC" -eq 0 ] && [ "$S1CERT" -eq 2 ]
+then
+  S2D_N=138240 srun -N36 -n288 ./select_gpu $BIN --mpi 3.6.4.4 --grid 48.48.48.96 $OPTS1
+  echo "S3 exit code $?"
+else
+  echo "S3 SKIPPED: S1 failed (rc=$S1RC, certificates=$S1CERT)"
+fi
+
+echo "========================================================="
+echo "summary (both legs, all stages)"
+echo "========================================================="
+grep -h -E "Grid-vs-SLATE|GRID  :|SLATE :" slurm-$SLURM_JOB_ID.out 2>/dev/null
+grep -h "git commit hash" slurm-$SLURM_JOB_ID.out 2>/dev/null | sort -u
+echo "========================================================="