From 5f8f396e9b589eb27aa98f5bfe1246fffcbbdebc Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 26 Aug 2026 14:22:46 -0400 Subject: [PATCH] More jobs for testing Mgrid --- systems/Frontier/batched_linalg.job | 96 ++++++++++++++++++ systems/Frontier/schur2d_ladder.job | 149 ++++++++++++++++++++++++++++ systems/Frontier/smoother_modes.job | 125 +++++++++++++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 systems/Frontier/batched_linalg.job create mode 100644 systems/Frontier/schur2d_ladder.job create mode 100644 systems/Frontier/smoother_modes.job diff --git a/systems/Frontier/batched_linalg.job b/systems/Frontier/batched_linalg.job new file mode 100644 index 000000000..def0f71e9 --- /dev/null +++ b/systems/Frontier/batched_linalg.job @@ -0,0 +1,96 @@ +#!/bin/bash -l +#SBATCH --job-name=batched-linalg +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=0:20:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 +#SBATCH -q debug + +############################################################################## +# GPU gate for the batched Krylov linear algebra (Lattice_reduction.h: +# innerProductMulti / axpyMulti / axpyMultiNorm with the by-value ViewPack +# kernel argument) and the PGCR that sits on it. Nothing here has run on a +# GPU before this job. +# +# B1 : Test_multi_reduction 8 ranks, 2x4 -- every batch width {2,4,8,16}, +# >16 chunking, fine (SpinColour) and coarse (iVector on a +# 5D grid) fields, against the sequential reference, tol 1e-12. +# B2 : Test_multi_reduction 1 rank -- same, no comms: separates a +# kernel fault from a reduction/allreduce fault if B1 fails. +# B3 : Test_pgcr_history 8 ranks -- solver on top: bitwise- +# repeatable history reuse, true residual, second mmax. +# B4 : Test_ring_allreduce 8 ranks, 2x4 -- P2P ring allreduce vs +# MPI_Allreduce, all types/sizes, bitwise repeat, 16 MB timing. +# (Also not yet run on Frontier; cheap to include.) +# +# Each stage is a separate srun with unbuffered output and its own timeout; +# the summary at the end lists pass/fail per stage from the binaries' own +# "ALL PASS"/"FAILURES" lines, and any GRID_ASSERT or signal. +############################################################################## + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES +if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi +exec numactl -m \$NUMA -N \$NUMA \$* +EOF +chmod +x ./select_gpu + +root=$HOME/ParallelIO/systems/Frontier +source $root/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=1 +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU +module load libfabric + +OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000" +T=$root/tests/debug + +echo "=========================================================" +echo "binaries (git hash line is libGrid's build, not the test's)" +echo "=========================================================" +ls -l $T/Test_multi_reduction $T/Test_pgcr_history $T/Test_ring_allreduce + +run_stage () { + name=$1; nranks=$2; mpi=$3; grid=$4; bin=$5 + echo "=========================================================" + echo "$name : $bin -n$nranks --mpi $mpi --grid $grid" + echo "=========================================================" + timeout 300 srun -u -N1 -n$nranks ./select_gpu $T/$bin --mpi $mpi --grid $grid $OPTS1 2>&1 | tee $name.out \ + | grep -E "PASS|FAIL|T4 timing|GRID_ASSERT|Signal|signal|abort|Bus error|Segmentation" + rc=${PIPESTATUS[0]} + echo "$name exit code $rc (124 = timeout)" +} + +run_stage B1_multi_reduction_8 8 1.1.2.4 16.16.16.32 Test_multi_reduction +run_stage B2_multi_reduction_1 1 1.1.1.1 16.16.16.16 Test_multi_reduction +run_stage B3_pgcr_history_8 8 1.1.2.4 16.16.16.16 Test_pgcr_history +run_stage B4_ring_allreduce_8 8 1.1.2.4 16.16.16.32 Test_ring_allreduce + +echo "=========================================================" +echo "summary" +echo "=========================================================" +for s in B1_multi_reduction_8 B2_multi_reduction_1 B3_pgcr_history_8 B4_ring_allreduce_8; do + verdict=$(grep -oE "ALL PASS|FAILURES" $s.out | head -1) + fails=$(grep -c "\*\* FAIL \*\*" $s.out) + asserts=$(grep -cE "GRID_ASSERT|Signal|Bus error|Segmentation" $s.out) + printf "%-24s %-10s fails=%s asserts/signals=%s\n" $s "${verdict:-NO-VERDICT}" $fails $asserts +done +echo "--- any FAIL lines ---" +grep -h "\*\* FAIL \*\*" B*.out | head -20 +echo "--- ring allreduce timing (16 MB ComplexF) ---" +grep -h "T4 timing" B4_ring_allreduce_8.out +echo "=========================================================" diff --git a/systems/Frontier/schur2d_ladder.job b/systems/Frontier/schur2d_ladder.job new file mode 100644 index 000000000..fa28bf100 --- /dev/null +++ b/systems/Frontier/schur2d_ladder.job @@ -0,0 +1,149 @@ +#!/bin/bash -l +#SBATCH --job-name=schur2d-ladder +#SBATCH --nodes=36 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=2:00:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 +##SBATCH -q debug + +############################################################################## +# Staged shakeout of the 2D block-cyclic dense inverse (DENSE_SCHUR2D). +# +# F1 : unit tests, 1 node / 8 ranks (seconds) +# F2 : scale rehearsal, 1 node, N=13824 (seconds) +# F3 : scale rehearsal, 36 nodes, N=138240 (the timing number) +# F4 : production example, 1D baseline then 2D, same nodes +# +# F4's 2D leg is gated on F3 passing: if the synthetic production-shape +# inverse fails or is slow, do not spend the 400 s multigrid setup on it. +############################################################################## + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES + +if [ \$SLURM_PROCID = "0" ] +then +echo \$* +fi + +exec numactl -m \$NUMA -N \$NUMA \$* +EOF + +chmod +x ./select_gpu + +root=$HOME/ParallelIO/systems/Frontier +source $root/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=7 +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU + +module load libfabric + +# Stale-environment protection: none of the gather knobs apply to the 2D +# path, and DENSE_GATHER=1 without FORCE would (correctly) abort at start. +unset DENSE_GATHER DENSE_GATHER_FORCE DENSE_GATHER_DEBUG DENSE_GATHER_MIN_BYTES + +OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000" + +############################################################################## +echo "=========================================================" +echo "F1: unit tests, 1 node, 8 ranks" +echo "=========================================================" +############################################################################## +for t in Test_blockcyclic Test_summa Test_schur2d Test_schur2d_redist +do +echo "----- F1 $t -----" +srun -N1 -n8 ./select_gpu $root/tests/debug/$t --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 +done + +############################################################################## +echo "=========================================================" +echo "F2: scale rehearsal, 1 node, N=13824 (nb=1728, grid 2x4)" +echo "=========================================================" +############################################################################## +S2D_N=13824 srun -N1 -n8 ./select_gpu $root/tests/debug/Test_schur2d_scale \ + --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1 + +############################################################################## +echo "=========================================================" +echo "F3: scale rehearsal, 36 nodes, N=138240 (nb=480, grid 16x18)" +echo " THE number: invert phase vs the 1D baseline 328 s" +echo "=========================================================" +############################################################################## +S2D_N=138240 srun -N36 -n288 ./select_gpu $root/tests/debug/Test_schur2d_scale \ + --mpi 3.6.4.4 --grid 48.48.48.96 $OPTS1 +F3RC=$? +echo "F3 exit code $F3RC" + +############################################################################## +echo "=========================================================" +echo "F4: production example (1D baseline first, then 2D if F3 passed)" +echo "=========================================================" +############################################################################## + +vol=48.48.48.96 +MPI_GEOM=3.6.4.4 + +export CoarseSmootherNstep=2 +export FineSmootherOrder=6 + +export SUBSPACE_FILE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb64.scidac +unset SLAB_FILE # force a fresh import in both legs + +export DENSE_SCHUR=1 +export MASS=0.00078 + +export BLOCK=2.2.3.3 # L1->L2 blocking +export BLOCK2=4.4.2.4 # L2->L3 blocking + +export FineSmootherShift=0.1 +export CoarseSmootherShift=0.1 +export CoarseSolverTol=0.05 +export CoarseSolverOrder=200 +export L3_TOL=3.0e-1 +export L3_MAXIT=2 +export L3_NSTEP=50 +export OuterTol=1e-8 +export OuterMmax=4 +export OuterNstep=8 +export DENSE_CC=1 +export DENSE_APPLY_PROFILE=1 +unset DENSE_CC_CHECK +export DENSE_SPLITK=128 +export DENSE_DEVICE_SUM=1 +export GRID_ALLOC_NCACHE_LARGE=64 +export NRHS=4 + +echo "----- F4a: 1D baseline (DENSE_SCHUR2D unset) -----" +unset DENSE_SCHUR2D +srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \ + --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap + +if [ $F3RC -eq 0 ] +then +echo "----- F4b: 2D block-cyclic (DENSE_SCHUR2D=1) -----" +export DENSE_SCHUR2D=1 +srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \ + --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap +else +echo "----- F4b SKIPPED: F3 failed (rc=$F3RC), not spending the setup on it -----" +fi + +echo "=========================================================" +echo "ladder complete; binaries were:" +grep -h "git commit hash" slurm-$SLURM_JOB_ID.out 2>/dev/null | sort -u +echo "=========================================================" diff --git a/systems/Frontier/smoother_modes.job b/systems/Frontier/smoother_modes.job new file mode 100644 index 000000000..664219233 --- /dev/null +++ b/systems/Frontier/smoother_modes.job @@ -0,0 +1,125 @@ +#!/bin/bash -l +#SBATCH --job-name=smoother-modes +#SBATCH --nodes=36 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=1:00:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 + +############################################################################## +# The 1402.2585 p.13 comparison on this machine: adaptive GCR smoothers vs +# the same polynomials frozen (GCRReplaySmoother: recorded for the first +# PolyRecordIters outer steps, then replayed with no inner products), vs a +# Chebyshev 1/x fit on the measured interval. Same banked point otherwise. +# +# M1 gcr / gcr reference +# M2 replay / replay both levels frozen after PolyRecordIters steps +# M3 replay / gcr fine frozen only (the fine smoother is the reduction- +# heavy one at Nrhs=1) +# M4 cheb / gcr fine Chebyshev [FineChebLo,FineChebHi] order Fso +# M5 cheb / cheb +# +# Laptop 8^4 findings (hot config, Ls=4, NBASIS=8): replay/replay converges +# (28 vs 23 outer); cheb on the FINE level diverges there while cheb on the +# coarse level is fine -- the tiny operator has modes the fixed polynomial +# amplifies (left of / off the axis) that GCR handles adaptively. Production +# spectrum is near-normal with edge 130.5 (shift 0.1) so M4/M5 may behave +# differently; they are cheap to include and cheap to discard. +# +# Readouts per mode: Fouter count, s/RHS, and the FINAL exact-halo residual. +############################################################################## + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES +if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi +exec numactl -m \$NUMA -N \$NUMA \$* +EOF +chmod +x ./select_gpu + +root=$HOME/ParallelIO/systems/Frontier +source $root/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=7 +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU +module load libfabric +unset DENSE_GATHER DENSE_GATHER_FORCE DENSE_GATHER_DEBUG DENSE_GATHER_MIN_BYTES + +OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000" +vol=48.48.48.96 +MPI_GEOM=3.6.4.4 + +# banked point (2026-08-26 sweep): Css 2.0, Nstep 2, Fso 6, mmax 4, svm 8 +export FineSmootherShift=0.1 +export FineSmootherOrder=6 +export FineSmootherMmax=4 +export CoarseSmootherShift=2.0 +export CoarseSmootherNstep=2 +export CoarseSmootherMmax=2 +export CoarseSolverTol=0.05 +export CoarseSolverOrder=200 +export CoarseSolverMmax=8 +export OuterTol=1e-8 +export OuterMmax=6 +export OuterNstep=12 +export SUBSPACE_FILE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb64.scidac +unset SLAB_FILE +export DENSE_SCHUR=1 +export DENSE_SCHUR2D=1 +export MASS=0.00078 +export BLOCK=2.2.3.3 +export BLOCK2=4.4.2.4 +export L3_TOL=3.0e-1 +export L3_MAXIT=2 +export L3_NSTEP=50 +export DENSE_CC=1 +export DENSE_APPLY_PROFILE=1 +unset DENSE_CC_CHECK +export DENSE_SPLITK=128 +export DENSE_DEVICE_SUM=2 # cartesian P2P ring: no 8 MB device-allreduce cliff (NRHS=12 abort) +export GRID_ALLOC_NCACHE_LARGE=64 +export NRHS=4 +export PowerIterations=0 +export SmootherCoeffLog=0 + +# frozen-polynomial controls +export PolyRecordIters=4 # outer steps recorded before the switch +export FineChebLo=3.0 # harvested |R|<0.1 edge / PowerIteration edge x1.05 +export FineChebHi=137.0 +export CoarseChebLo=8.0 +export CoarseChebHi=47.0 # shift 2.0: edge 43.3 x1.08 + +run_mode () { + name=$1; export FineSmootherMode=$2; export CoarseSmootherMode=$3 + echo "----- $name : FineSmootherMode=$FineSmootherMode CoarseSmootherMode=$CoarseSmootherMode -----" + fname=log.modes.$name + srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \ + --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap > $fname 2>&1 + echo " $(grep -h 'V2 3-level solve Nrhs' $fname | tr '\n' ' ')" + echo " $(grep -h 'Fouter MrhsPGCR: Converged' $fname | sed 's/.*Converged/Converged/' | tr '\n' ' ')" + echo " $(grep -h 'FINAL Nrhs .: worst' $fname | tr '\n' ' ')" + grep -h "SwitchableSmoother\|GCRCoefficients .*calls" $fname | head -6 +} + +run_mode M1_gcr_gcr gcr gcr +run_mode M2_replay_replay replay replay +run_mode M3_replay_gcr replay gcr +run_mode M4_cheb_gcr cheb gcr +run_mode M5_cheb_cheb cheb cheb + +echo "=========================================================" +echo "summary" +grep -h "V2 3-level solve Nrhs 1" log.modes.* | sed 's/.*V2/V2/' +echo "========================================================="