#!/bin/bash -l
#SBATCH --job-name=slate-vs-schur2d
#SBATCH --nodes=36
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=7
#SBATCH --gpus-per-node=8
#SBATCH --time=1:00:00
#SBATCH --account=phy157_dwf
#SBATCH --gpu-bind=none
#SBATCH --exclusive
#SBATCH --mem=0
#SBATCH -S 0
##SBATCH -q debug

##############################################################################
# Like-for-like: Grid's 2D block-cyclic Schur inverse vs SLATE getrf+getri,
# same matrix, same ranks, same job, every layout cost on the clock
# (tests/debug/Test_schur2d_vs_slate.cc).  Both inverses are certified by the
# same SUMMA instrument; both legs print a one-line timing breakdown.
#
#   S1 : 1 node,  8 ranks, N=4096    -- API/device/thread-level shakeout
#   S2 : 1 node,  8 ranks, N=13824   -- one-node production-shape rehearsal
#   S3 : 36 nodes, 288 ranks, N=138240 (nb=480, grid 18x16) -- THE comparison
#
# S3 is gated on S1 passing (rc=0 and both certificates printed): a broken
# SLATE leg should not burn the 36-node allocation.
#
# Requires Grid configured with --with-slate/--with-blaspp/--with-lapackpp
# (HAVE_SLATE in Config.h); sourceme puts the three lib64 dirs on
# LD_LIBRARY_PATH.  The binary initialises MPI at MPI_THREAD_MULTIPLE itself.
##############################################################################

cat << EOF > select_gpu
#!/bin/bash
export GPU_MAP=(0 1 2 3 7 6 5 4)
export NUMA_MAP=(3 3 1 1 2 2 0 0)
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
export HIP_VISIBLE_DEVICES=\$GPU
unset ROCR_VISIBLE_DEVICES

if [ \$SLURM_PROCID = "0" ]
then
echo \$*
fi

exec numactl -m \$NUMA -N \$NUMA \$*
EOF

chmod +x ./select_gpu

root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh

# ONE host thread per rank, for both codes.  Grid uses no host OpenMP on the
# GPU build; SLATE with 7 threads issuing device-buffer MPI concurrently
# deadlocks in getrf's tile broadcast under Cray MPICH (slate_debug.job V1
# hung, V2 with OMP_NUM_THREADS=1 completed, 2026-08-25).  Each rank has a
# whole GCD; the parallelism is on the device for both.
export OMP_NUM_THREADS=1
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
export MPICH_MAX_THREAD_SAFETY=multiple   # SLATE still requests MPI_THREAD_MULTIPLE

module load libfabric

BIN=$root/tests/debug/Test_schur2d_vs_slate
# Three legs per stage: Grid 2D Schur, SLATE getrf+getri, SLATE getrf+getrs(I).
# getri is a host loop in SLATE (minutes at N=138240; measured once already):
# S2D_SKIP_GETRI=1 drops it so S3 is ~3 min.  Unset to measure it again.
export S2D_SKIP_GETRI=${S2D_SKIP_GETRI:-1}
OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"

echo "========================================================="
echo "binary and runtime linkage"
echo "========================================================="
ls -l $BIN
ldd $BIN | grep -E "slate|blaspp|lapackpp|omp|rocblas|mpi_cray"
# the cpu-env slate module must NOT be loaded: it would shadow the ROCm build
module list 2>&1 | grep -i slate && echo "WARNING: a slate module is loaded"

##############################################################################
echo "========================================================="
echo "S1: shakeout, 1 node, 8 ranks, N=4096 (nb=512, grid 2x4)"
echo "========================================================="
##############################################################################
S2D_N=4096 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 \
     2>&1 | tee s1.out
S1RC=${PIPESTATUS[0]}
S1CERT=$(grep -c "certificate" s1.out)
echo "S1 exit code $S1RC, certificates printed $S1CERT (expect 2, or 3 with getri)"

##############################################################################
echo "========================================================="
echo "S2: one-node rehearsal, N=13824 (nb=1728, grid 2x4)"
echo "========================================================="
##############################################################################
S2D_N=13824 srun -N1 -n8 ./select_gpu $BIN --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1

##############################################################################
echo "========================================================="
echo "S3: THE comparison, 36 nodes, 288 ranks, N=138240 (nb=480, grid 18x16)"
echo "========================================================="
##############################################################################
if [ "$S1RC" -eq 0 ] && [ "$S1CERT" -ge 2 ]
then
  S2D_N=138240 srun -N36 -n288 ./select_gpu $BIN --mpi 3.6.4.4 --grid 48.48.48.96 $OPTS1
  echo "S3 exit code $?"
else
  echo "S3 SKIPPED: S1 failed (rc=$S1RC, certificates=$S1CERT)"
fi

echo "========================================================="
echo "summary (both legs, all stages)"
echo "========================================================="
grep -h -E "Grid-vs-SLATE|GRID  :|SLATE :|SLATE-getrs" slurm-$SLURM_JOB_ID.out 2>/dev/null
grep -h "git commit hash" slurm-$SLURM_JOB_ID.out 2>/dev/null | sort -u
echo "========================================================="