mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-29 22:09:35 +01:00
98 lines
4.8 KiB
Bash
98 lines
4.8 KiB
Bash
#!/bin/bash -l
|
|
#SBATCH --job-name=schur2d-env
|
|
#SBATCH --nodes=36
|
|
#SBATCH --ntasks-per-node=8
|
|
#SBATCH --cpus-per-task=7
|
|
#SBATCH --gpus-per-node=8
|
|
#SBATCH --time=0:50:00
|
|
#SBATCH --account=phy157_dwf
|
|
#SBATCH --gpu-bind=none
|
|
#SBATCH --exclusive
|
|
#SBATCH --mem=0
|
|
#SBATCH -S 0
|
|
|
|
##############################################################################
|
|
# Environment walk for the 2D Schur inverse (2026-08-27).
|
|
#
|
|
# The SAME inverse (N=138240, 18x16, nb 480, SCHUR2D_LEAF_SPAN=9, LU leaf)
|
|
# runs 20.8 s in Test_schur2d_vs_slate and 26.5 s in the production example,
|
|
# and the LOCAL GPU work is what differs: GEMM 2.47 vs 3.11 s, leaf inverse
|
|
# 0.53 vs 0.76 s for identical calls; rings about equal. The environments
|
|
# differ in (a) OMP_NUM_THREADS 1 vs 7, (b) MPI_THREAD_MULTIPLE init (SLATE
|
|
# harness) vs SERIALIZED (example), (c) ~10.6 GB of fine-grid state resident
|
|
# on the device in the example. Test_schur2d_scale is the SLATE-free harness
|
|
# (same library code, plain Grid_init); walk it from the harness environment
|
|
# to the example's one knob at a time.
|
|
#
|
|
# Readout per cell (identical work in every cell):
|
|
# "gemm min/max" vs 2.45/2.61 (harness) / 3.01/3.24 (example)
|
|
# "BIG LEAVES ... inverse (max over ranks)" vs 0.53 / 0.76
|
|
# "SCHUR fp64 ... invert" / "GRID : ... invert" vs 20.8 / 26.5
|
|
# "Schur2D PROBE leaf W=" one-GCD getri / getrf+getrs / zgemm TF/s
|
|
##############################################################################
|
|
|
|
cat << EOF > select_gpu
|
|
#!/bin/bash
|
|
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
|
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
|
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
|
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
|
export HIP_VISIBLE_DEVICES=\$GPU
|
|
unset ROCR_VISIBLE_DEVICES
|
|
if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
|
|
exec numactl -m \$NUMA -N \$NUMA \$*
|
|
EOF
|
|
chmod +x ./select_gpu
|
|
|
|
root=$HOME/ParallelIO/systems/Frontier
|
|
source $root/sourceme-rocm7.2.sh
|
|
|
|
export FI_MR_CACHE_MONITOR=kdreg2 # REQUIRED for device-buffer MPI on Slingshot: libfabric memhooks monitor (default) is defective, see systems/WorkArounds.txt (libfabric #11451)
|
|
export MPICH_GPU_SUPPORT_ENABLED=1
|
|
export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
|
export MPICH_OFI_NIC_POLICY=GPU
|
|
module load libfabric
|
|
|
|
BIN=$root/tests/debug/Test_schur2d_scale
|
|
OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
|
|
vol=48.48.48.96
|
|
MPI_GEOM=3.6.4.4
|
|
|
|
export S2D_N=138240
|
|
export S2D_NB=480
|
|
export SCHUR2D_LEAF_SPAN=9
|
|
export SCHUR2D_LEAF_LU=1
|
|
export SUMMA_HANDSHAKE=1
|
|
# SCHUR2D_PROBE unset: probe on, including the one-GCD leaf microbenchmark
|
|
|
|
run_cell () {
|
|
name=$1; shift
|
|
echo "----- $name : $* -----"
|
|
fname=log.env.$name
|
|
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 > $fname 2>&1
|
|
echo " exit $?"; sleep 30
|
|
grep -h "PROBE banner\|device ballast\|pre-heat\|PROBE leaf\|BIG LEAVES\|gemm min/max\|invert\|certificate" $fname \
|
|
| sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-170
|
|
}
|
|
|
|
# name environment
|
|
run_cell E1_omp1 OMP_NUM_THREADS=1
|
|
#run_cell E2_omp1_multiple (MPI_THREAD_MULTIPLE init: 21.2 s, NIL; the GRID_MPI_THREAD_MULTIPLE hack was removed 2026-08-28)
|
|
run_cell E3_omp7 OMP_NUM_THREADS=7 # the example's thread count
|
|
run_cell E4_omp7_ballast OMP_NUM_THREADS=7 S2D_BALLAST_GB=10.6 # + the example's device residency
|
|
run_cell E5_omp1_ballast OMP_NUM_THREADS=1 S2D_BALLAST_GB=10.6 # ballast alone
|
|
run_cell E6_omp1_again OMP_NUM_THREADS=1 # repeat E1: the noise floor
|
|
# E1-E4 (2026-08-27): 24.5 (first cell, cold) / 21.2 / 21.3 / 21.35 s -- thread level, OMP, residency all cleared.
|
|
# Remaining difference from the example (26.5-27.6 s, local kernels 20-40% slower, wires equal): the example
|
|
# inverts after ~100 s of full load on every GCD. Pre-heat reproduces that; "last zgemm" TF/s vs 23.7 is the tell.
|
|
run_cell E7_omp7_preheat OMP_NUM_THREADS=7 S2D_PREHEAT_S=100 # sustained load before the invert
|
|
run_cell E8_omp7_preheat_ballast OMP_NUM_THREADS=7 S2D_PREHEAT_S=100 S2D_BALLAST_GB=10.6 # the example's full state
|
|
|
|
echo "========================================================="
|
|
echo "summary"
|
|
for f in log.env.E*; do
|
|
echo "$f: $(grep -h "invert" $f | grep -h "TOTAL\|took" | sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-80 | head -1) gemm $(grep -oh "gemm min/max [0-9./]*" $f) leaf $(grep -oh "inverse (max over ranks) [0-9.]*" $f)"
|
|
done
|
|
echo "harness (SLATE build, OMP=1, MULTIPLE): invert 20.8 gemm 2.45/2.61 leaf 0.53; example (OMP=7, SERIALIZED, 10.6 GB resident): 26.5 3.01/3.24 0.76"
|
|
echo "========================================================="
|