diff --git a/systems/Frontier/slate_debug.job b/systems/Frontier/slate_debug.job new file mode 100644 index 000000000..255e1f9f3 --- /dev/null +++ b/systems/Frontier/slate_debug.job @@ -0,0 +1,84 @@ +#!/bin/bash -l +#SBATCH --job-name=slate-debug +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=0:30:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 +#SBATCH -q debug + +############################################################################## +# Localise the multi-rank SLATE hang (Test_schur2d_vs_slate, 8 ranks, 2x4). +# Each variant is a separate srun with unbuffered output (-u) and a hard +# timeout, and the binary prints flushed "stage:" markers, so the log shows +# the LAST stage reached in each variant: +# +# V1 : as the production job (warm-up on, OMP_NUM_THREADS=7) +# V2 : OMP_NUM_THREADS=1 -> is it threads x GPU-aware MPI? +# V3 : S2D_NOWARM=1, OMP=7 -> does the timed leg hang at the same +# SLATE call as the warm-up did? +# V4 : OMP_NUM_THREADS=1 + NOWARM +# V5 : MPICH_GPU_SUPPORT_ENABLED=0 -> SLATE Devices target with host-staged +# MPI (blaspp copies tiles itself) +# +# Read: for each variant the final "stage:" line and whether "SLATE :" was +# printed. A variant that completes identifies the fix. +############################################################################## + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES +if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi +exec numactl -m \$NUMA -N \$NUMA \$* +EOF +chmod +x ./select_gpu + +root=$HOME/ParallelIO/systems/Frontier +source $root/sourceme-rocm7.2.sh + +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU +export MPICH_MAX_THREAD_SAFETY=multiple +module load libfabric + +BIN=$root/tests/debug/Test_schur2d_vs_slate +OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000" +export S2D_N=4096 + +echo "binary: $(ls -l $BIN)" +ldd $BIN | grep -E "slate|blaspp|lapackpp|omp|mpi_cray" +module list 2>&1 | grep -i slate && echo "WARNING: a slate module is loaded" + +run_variant () { + name=$1; shift + echo "=========================================================" + echo "$name : $*" + echo "=========================================================" + env "$@" timeout 240 srun -u -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 2>&1 | tee $name.out | grep -E "stage:|warm-up|GRID :|SLATE :|rror|Assert|signal" + rc=${PIPESTATUS[0]} + echo "$name exit code $rc (124 = timeout) last stage: $(grep 'stage:' $name.out | tail -1 | sed 's/.*stage: //')" +} + +run_variant V1_default OMP_NUM_THREADS=7 +run_variant V2_omp1 OMP_NUM_THREADS=1 +run_variant V3_nowarm OMP_NUM_THREADS=7 S2D_NOWARM=1 +run_variant V4_omp1_nowarm OMP_NUM_THREADS=1 S2D_NOWARM=1 +run_variant V5_hostmpi OMP_NUM_THREADS=7 MPICH_GPU_SUPPORT_ENABLED=0 + +echo "=========================================================" +echo "summary" +echo "=========================================================" +for v in V1_default V2_omp1 V3_nowarm V4_omp1_nowarm V5_hostmpi; do + printf "%-18s SLATE-line:%s last-stage: %s\n" $v "$(grep -c 'SLATE :' $v.out)" "$(grep 'stage:' $v.out | tail -1 | sed 's/.*stage: //')" +done diff --git a/systems/Frontier/slate_vs_schur2d.job b/systems/Frontier/slate_vs_schur2d.job new file mode 100644 index 000000000..4873f800f --- /dev/null +++ b/systems/Frontier/slate_vs_schur2d.job @@ -0,0 +1,110 @@ +#!/bin/bash -l +#SBATCH --job-name=slate-vs-schur2d +#SBATCH --nodes=36 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=1:00:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 +##SBATCH -q debug + +############################################################################## +# Like-for-like: Grid's 2D block-cyclic Schur inverse vs SLATE getrf+getri, +# same matrix, same ranks, same job, every layout cost on the clock +# (tests/debug/Test_schur2d_vs_slate.cc). Both inverses are certified by the +# same SUMMA instrument; both legs print a one-line timing breakdown. +# +# S1 : 1 node, 8 ranks, N=4096 -- API/device/thread-level shakeout +# S2 : 1 node, 8 ranks, N=13824 -- one-node production-shape rehearsal +# S3 : 36 nodes, 288 ranks, N=138240 (nb=480, grid 16x18) -- THE comparison +# +# S3 is gated on S1 passing (rc=0 and both certificates printed): a broken +# SLATE leg should not burn the 36-node allocation. +# +# Requires Grid configured with --with-slate/--with-blaspp/--with-lapackpp +# (HAVE_SLATE in Config.h); sourceme puts the three lib64 dirs on +# LD_LIBRARY_PATH. The binary initialises MPI at MPI_THREAD_MULTIPLE itself. +############################################################################## + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES + +if [ \$SLURM_PROCID = "0" ] +then +echo \$* +fi + +exec numactl -m \$NUMA -N \$NUMA \$* +EOF + +chmod +x ./select_gpu + +root=$HOME/ParallelIO/systems/Frontier +source $root/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=7 +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU +export MPICH_MAX_THREAD_SAFETY=multiple # SLATE calls MPI from OpenMP tasks + +module load libfabric + +BIN=$root/tests/debug/Test_schur2d_vs_slate +OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000" + +echo "=========================================================" +echo "binary and runtime linkage" +echo "=========================================================" +ls -l $BIN +ldd $BIN | grep -E "slate|blaspp|lapackpp|omp|rocblas|mpi_cray" +# the cpu-env slate module must NOT be loaded: it would shadow the ROCm build +module list 2>&1 | grep -i slate && echo "WARNING: a slate module is loaded" + +############################################################################## +echo "=========================================================" +echo "S1: shakeout, 1 node, 8 ranks, N=4096 (nb=512, grid 2x4)" +echo "=========================================================" +############################################################################## +S2D_N=4096 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 \ + 2>&1 | tee s1.out +S1RC=${PIPESTATUS[0]} +S1CERT=$(grep -c "certificate" s1.out) +echo "S1 exit code $S1RC, certificates printed $S1CERT (expect 2)" + +############################################################################## +echo "=========================================================" +echo "S2: one-node rehearsal, N=13824 (nb=1728, grid 2x4)" +echo "=========================================================" +############################################################################## +S2D_N=13824 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 + +############################################################################## +echo "=========================================================" +echo "S3: THE comparison, 36 nodes, 288 ranks, N=138240 (nb=480, grid 16x18)" +echo "=========================================================" +############################################################################## +if [ "$S1RC" -eq 0 ] && [ "$S1CERT" -eq 2 ] +then + S2D_N=138240 srun -N36 -n288 ./select_gpu $BIN --mpi 3.6.4.4 --grid 48.48.48.96 $OPTS1 + echo "S3 exit code $?" +else + echo "S3 SKIPPED: S1 failed (rc=$S1RC, certificates=$S1CERT)" +fi + +echo "=========================================================" +echo "summary (both legs, all stages)" +echo "=========================================================" +grep -h -E "Grid-vs-SLATE|GRID :|SLATE :" slurm-$SLURM_JOB_ID.out 2>/dev/null +grep -h "git commit hash" slurm-$SLURM_JOB_ID.out 2>/dev/null | sort -u +echo "========================================================="