mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-28 13:29:35 +01:00
More jobs for testing Mgrid
This commit is contained in:
Peter Boyle
@@ -0,0 +1,96 @@
|
||||
#!/bin/bash -l
|
||||
#SBATCH --job-name=batched-linalg
|
||||
#SBATCH --nodes=1
|
||||
#SBATCH --ntasks-per-node=8
|
||||
#SBATCH --cpus-per-task=7
|
||||
#SBATCH --gpus-per-node=8
|
||||
#SBATCH --time=0:20:00
|
||||
#SBATCH --account=phy157_dwf
|
||||
#SBATCH --gpu-bind=none
|
||||
#SBATCH --exclusive
|
||||
#SBATCH --mem=0
|
||||
#SBATCH -S 0
|
||||
#SBATCH -q debug
|
||||
|
||||
##############################################################################
|
||||
# GPU gate for the batched Krylov linear algebra (Lattice_reduction.h:
|
||||
# innerProductMulti / axpyMulti / axpyMultiNorm with the by-value ViewPack
|
||||
# kernel argument) and the PGCR that sits on it. Nothing here has run on a
|
||||
# GPU before this job.
|
||||
#
|
||||
# B1 : Test_multi_reduction 8 ranks, 2x4 -- every batch width {2,4,8,16},
|
||||
# >16 chunking, fine (SpinColour) and coarse (iVector<CComplex,8> on a
|
||||
# 5D grid) fields, against the sequential reference, tol 1e-12.
|
||||
# B2 : Test_multi_reduction 1 rank -- same, no comms: separates a
|
||||
# kernel fault from a reduction/allreduce fault if B1 fails.
|
||||
# B3 : Test_pgcr_history 8 ranks -- solver on top: bitwise-
|
||||
# repeatable history reuse, true residual, second mmax.
|
||||
# B4 : Test_ring_allreduce 8 ranks, 2x4 -- P2P ring allreduce vs
|
||||
# MPI_Allreduce, all types/sizes, bitwise repeat, 16 MB timing.
|
||||
# (Also not yet run on Frontier; cheap to include.)
|
||||
#
|
||||
# Each stage is a separate srun with unbuffered output and its own timeout;
|
||||
# the summary at the end lists pass/fail per stage from the binaries' own
|
||||
# "ALL PASS"/"FAILURES" lines, and any GRID_ASSERT or signal.
|
||||
##############################################################################
|
||||
|
||||
cat << EOF > select_gpu
|
||||
#!/bin/bash
|
||||
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
||||
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
||||
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
||||
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
||||
export HIP_VISIBLE_DEVICES=\$GPU
|
||||
unset ROCR_VISIBLE_DEVICES
|
||||
if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
|
||||
exec numactl -m \$NUMA -N \$NUMA \$*
|
||||
EOF
|
||||
chmod +x ./select_gpu
|
||||
|
||||
root=$HOME/ParallelIO/systems/Frontier
|
||||
source $root/sourceme-rocm7.2.sh
|
||||
|
||||
export OMP_NUM_THREADS=1
|
||||
export MPICH_GPU_SUPPORT_ENABLED=1
|
||||
export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
||||
export MPICH_OFI_NIC_POLICY=GPU
|
||||
module load libfabric
|
||||
|
||||
OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
|
||||
T=$root/tests/debug
|
||||
|
||||
echo "========================================================="
|
||||
echo "binaries (git hash line is libGrid's build, not the test's)"
|
||||
echo "========================================================="
|
||||
ls -l $T/Test_multi_reduction $T/Test_pgcr_history $T/Test_ring_allreduce
|
||||
|
||||
run_stage () {
|
||||
name=$1; nranks=$2; mpi=$3; grid=$4; bin=$5
|
||||
echo "========================================================="
|
||||
echo "$name : $bin -n$nranks --mpi $mpi --grid $grid"
|
||||
echo "========================================================="
|
||||
timeout 300 srun -u -N1 -n$nranks ./select_gpu $T/$bin --mpi $mpi --grid $grid $OPTS1 2>&1 | tee $name.out \
|
||||
| grep -E "PASS|FAIL|T4 timing|GRID_ASSERT|Signal|signal|abort|Bus error|Segmentation"
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "$name exit code $rc (124 = timeout)"
|
||||
}
|
||||
|
||||
run_stage B1_multi_reduction_8 8 1.1.2.4 16.16.16.32 Test_multi_reduction
|
||||
run_stage B2_multi_reduction_1 1 1.1.1.1 16.16.16.16 Test_multi_reduction
|
||||
run_stage B3_pgcr_history_8 8 1.1.2.4 16.16.16.16 Test_pgcr_history
|
||||
run_stage B4_ring_allreduce_8 8 1.1.2.4 16.16.16.32 Test_ring_allreduce
|
||||
|
||||
echo "========================================================="
|
||||
echo "summary"
|
||||
echo "========================================================="
|
||||
for s in B1_multi_reduction_8 B2_multi_reduction_1 B3_pgcr_history_8 B4_ring_allreduce_8; do
|
||||
verdict=$(grep -oE "ALL PASS|FAILURES" $s.out | head -1)
|
||||
fails=$(grep -c "\*\* FAIL \*\*" $s.out)
|
||||
asserts=$(grep -cE "GRID_ASSERT|Signal|Bus error|Segmentation" $s.out)
|
||||
printf "%-24s %-10s fails=%s asserts/signals=%s\n" $s "${verdict:-NO-VERDICT}" $fails $asserts
|
||||
done
|
||||
echo "--- any FAIL lines ---"
|
||||
grep -h "\*\* FAIL \*\*" B*.out | head -20
|
||||
echo "--- ring allreduce timing (16 MB ComplexF) ---"
|
||||
grep -h "T4 timing" B4_ring_allreduce_8.out
|
||||
echo "========================================================="
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/bin/bash -l
|
||||
#SBATCH --job-name=schur2d-ladder
|
||||
#SBATCH --nodes=36
|
||||
#SBATCH --ntasks-per-node=8
|
||||
#SBATCH --cpus-per-task=7
|
||||
#SBATCH --gpus-per-node=8
|
||||
#SBATCH --time=2:00:00
|
||||
#SBATCH --account=phy157_dwf
|
||||
#SBATCH --gpu-bind=none
|
||||
#SBATCH --exclusive
|
||||
#SBATCH --mem=0
|
||||
#SBATCH -S 0
|
||||
##SBATCH -q debug
|
||||
|
||||
##############################################################################
|
||||
# Staged shakeout of the 2D block-cyclic dense inverse (DENSE_SCHUR2D).
|
||||
#
|
||||
# F1 : unit tests, 1 node / 8 ranks (seconds)
|
||||
# F2 : scale rehearsal, 1 node, N=13824 (seconds)
|
||||
# F3 : scale rehearsal, 36 nodes, N=138240 (the timing number)
|
||||
# F4 : production example, 1D baseline then 2D, same nodes
|
||||
#
|
||||
# F4's 2D leg is gated on F3 passing: if the synthetic production-shape
|
||||
# inverse fails or is slow, do not spend the 400 s multigrid setup on it.
|
||||
##############################################################################
|
||||
|
||||
cat << EOF > select_gpu
|
||||
#!/bin/bash
|
||||
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
||||
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
||||
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
||||
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
||||
export HIP_VISIBLE_DEVICES=\$GPU
|
||||
unset ROCR_VISIBLE_DEVICES
|
||||
|
||||
if [ \$SLURM_PROCID = "0" ]
|
||||
then
|
||||
echo \$*
|
||||
fi
|
||||
|
||||
exec numactl -m \$NUMA -N \$NUMA \$*
|
||||
EOF
|
||||
|
||||
chmod +x ./select_gpu
|
||||
|
||||
root=$HOME/ParallelIO/systems/Frontier
|
||||
source $root/sourceme-rocm7.2.sh
|
||||
|
||||
export OMP_NUM_THREADS=7
|
||||
export MPICH_GPU_SUPPORT_ENABLED=1
|
||||
export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
||||
export MPICH_OFI_NIC_POLICY=GPU
|
||||
|
||||
module load libfabric
|
||||
|
||||
# Stale-environment protection: none of the gather knobs apply to the 2D
|
||||
# path, and DENSE_GATHER=1 without FORCE would (correctly) abort at start.
|
||||
unset DENSE_GATHER DENSE_GATHER_FORCE DENSE_GATHER_DEBUG DENSE_GATHER_MIN_BYTES
|
||||
|
||||
OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
|
||||
|
||||
##############################################################################
|
||||
echo "========================================================="
|
||||
echo "F1: unit tests, 1 node, 8 ranks"
|
||||
echo "========================================================="
|
||||
##############################################################################
|
||||
for t in Test_blockcyclic Test_summa Test_schur2d Test_schur2d_redist
|
||||
do
|
||||
echo "----- F1 $t -----"
|
||||
srun -N1 -n8 ./select_gpu $root/tests/debug/$t --mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1
|
||||
done
|
||||
|
||||
##############################################################################
|
||||
echo "========================================================="
|
||||
echo "F2: scale rehearsal, 1 node, N=13824 (nb=1728, grid 2x4)"
|
||||
echo "========================================================="
|
||||
##############################################################################
|
||||
S2D_N=13824 srun -N1 -n8 ./select_gpu $root/tests/debug/Test_schur2d_scale \
|
||||
--mpi 1.1.2.4 --grid 16.16.16.16 $OPTS1
|
||||
|
||||
##############################################################################
|
||||
echo "========================================================="
|
||||
echo "F3: scale rehearsal, 36 nodes, N=138240 (nb=480, grid 16x18)"
|
||||
echo " THE number: invert phase vs the 1D baseline 328 s"
|
||||
echo "========================================================="
|
||||
##############################################################################
|
||||
S2D_N=138240 srun -N36 -n288 ./select_gpu $root/tests/debug/Test_schur2d_scale \
|
||||
--mpi 3.6.4.4 --grid 48.48.48.96 $OPTS1
|
||||
F3RC=$?
|
||||
echo "F3 exit code $F3RC"
|
||||
|
||||
##############################################################################
|
||||
echo "========================================================="
|
||||
echo "F4: production example (1D baseline first, then 2D if F3 passed)"
|
||||
echo "========================================================="
|
||||
##############################################################################
|
||||
|
||||
vol=48.48.48.96
|
||||
MPI_GEOM=3.6.4.4
|
||||
|
||||
export CoarseSmootherNstep=2
|
||||
export FineSmootherOrder=6
|
||||
|
||||
export SUBSPACE_FILE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb64.scidac
|
||||
unset SLAB_FILE # force a fresh import in both legs
|
||||
|
||||
export DENSE_SCHUR=1
|
||||
export MASS=0.00078
|
||||
|
||||
export BLOCK=2.2.3.3 # L1->L2 blocking
|
||||
export BLOCK2=4.4.2.4 # L2->L3 blocking
|
||||
|
||||
export FineSmootherShift=0.1
|
||||
export CoarseSmootherShift=0.1
|
||||
export CoarseSolverTol=0.05
|
||||
export CoarseSolverOrder=200
|
||||
export L3_TOL=3.0e-1
|
||||
export L3_MAXIT=2
|
||||
export L3_NSTEP=50
|
||||
export OuterTol=1e-8
|
||||
export OuterMmax=4
|
||||
export OuterNstep=8
|
||||
export DENSE_CC=1
|
||||
export DENSE_APPLY_PROFILE=1
|
||||
unset DENSE_CC_CHECK
|
||||
export DENSE_SPLITK=128
|
||||
export DENSE_DEVICE_SUM=1
|
||||
export GRID_ALLOC_NCACHE_LARGE=64
|
||||
export NRHS=4
|
||||
|
||||
echo "----- F4a: 1D baseline (DENSE_SCHUR2D unset) -----"
|
||||
unset DENSE_SCHUR2D
|
||||
srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \
|
||||
--mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap
|
||||
|
||||
if [ $F3RC -eq 0 ]
|
||||
then
|
||||
echo "----- F4b: 2D block-cyclic (DENSE_SCHUR2D=1) -----"
|
||||
export DENSE_SCHUR2D=1
|
||||
srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \
|
||||
--mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap
|
||||
else
|
||||
echo "----- F4b SKIPPED: F3 failed (rc=$F3RC), not spending the setup on it -----"
|
||||
fi
|
||||
|
||||
echo "========================================================="
|
||||
echo "ladder complete; binaries were:"
|
||||
grep -h "git commit hash" slurm-$SLURM_JOB_ID.out 2>/dev/null | sort -u
|
||||
echo "========================================================="
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/bin/bash -l
|
||||
#SBATCH --job-name=smoother-modes
|
||||
#SBATCH --nodes=36
|
||||
#SBATCH --ntasks-per-node=8
|
||||
#SBATCH --cpus-per-task=7
|
||||
#SBATCH --gpus-per-node=8
|
||||
#SBATCH --time=1:00:00
|
||||
#SBATCH --account=phy157_dwf
|
||||
#SBATCH --gpu-bind=none
|
||||
#SBATCH --exclusive
|
||||
#SBATCH --mem=0
|
||||
#SBATCH -S 0
|
||||
|
||||
##############################################################################
|
||||
# The 1402.2585 p.13 comparison on this machine: adaptive GCR smoothers vs
|
||||
# the same polynomials frozen (GCRReplaySmoother: recorded for the first
|
||||
# PolyRecordIters outer steps, then replayed with no inner products), vs a
|
||||
# Chebyshev 1/x fit on the measured interval. Same banked point otherwise.
|
||||
#
|
||||
# M1 gcr / gcr reference
|
||||
# M2 replay / replay both levels frozen after PolyRecordIters steps
|
||||
# M3 replay / gcr fine frozen only (the fine smoother is the reduction-
|
||||
# heavy one at Nrhs=1)
|
||||
# M4 cheb / gcr fine Chebyshev [FineChebLo,FineChebHi] order Fso
|
||||
# M5 cheb / cheb
|
||||
#
|
||||
# Laptop 8^4 findings (hot config, Ls=4, NBASIS=8): replay/replay converges
|
||||
# (28 vs 23 outer); cheb on the FINE level diverges there while cheb on the
|
||||
# coarse level is fine -- the tiny operator has modes the fixed polynomial
|
||||
# amplifies (left of / off the axis) that GCR handles adaptively. Production
|
||||
# spectrum is near-normal with edge 130.5 (shift 0.1) so M4/M5 may behave
|
||||
# differently; they are cheap to include and cheap to discard.
|
||||
#
|
||||
# Readouts per mode: Fouter count, s/RHS, and the FINAL exact-halo residual.
|
||||
##############################################################################
|
||||
|
||||
cat << EOF > select_gpu
|
||||
#!/bin/bash
|
||||
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
||||
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
||||
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
||||
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
||||
export HIP_VISIBLE_DEVICES=\$GPU
|
||||
unset ROCR_VISIBLE_DEVICES
|
||||
if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
|
||||
exec numactl -m \$NUMA -N \$NUMA \$*
|
||||
EOF
|
||||
chmod +x ./select_gpu
|
||||
|
||||
root=$HOME/ParallelIO/systems/Frontier
|
||||
source $root/sourceme-rocm7.2.sh
|
||||
|
||||
export OMP_NUM_THREADS=7
|
||||
export MPICH_GPU_SUPPORT_ENABLED=1
|
||||
export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
||||
export MPICH_OFI_NIC_POLICY=GPU
|
||||
module load libfabric
|
||||
unset DENSE_GATHER DENSE_GATHER_FORCE DENSE_GATHER_DEBUG DENSE_GATHER_MIN_BYTES
|
||||
|
||||
OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
|
||||
vol=48.48.48.96
|
||||
MPI_GEOM=3.6.4.4
|
||||
|
||||
# banked point (2026-08-26 sweep): Css 2.0, Nstep 2, Fso 6, mmax 4, svm 8
|
||||
export FineSmootherShift=0.1
|
||||
export FineSmootherOrder=6
|
||||
export FineSmootherMmax=4
|
||||
export CoarseSmootherShift=2.0
|
||||
export CoarseSmootherNstep=2
|
||||
export CoarseSmootherMmax=2
|
||||
export CoarseSolverTol=0.05
|
||||
export CoarseSolverOrder=200
|
||||
export CoarseSolverMmax=8
|
||||
export OuterTol=1e-8
|
||||
export OuterMmax=6
|
||||
export OuterNstep=12
|
||||
export SUBSPACE_FILE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb64.scidac
|
||||
unset SLAB_FILE
|
||||
export DENSE_SCHUR=1
|
||||
export DENSE_SCHUR2D=1
|
||||
export MASS=0.00078
|
||||
export BLOCK=2.2.3.3
|
||||
export BLOCK2=4.4.2.4
|
||||
export L3_TOL=3.0e-1
|
||||
export L3_MAXIT=2
|
||||
export L3_NSTEP=50
|
||||
export DENSE_CC=1
|
||||
export DENSE_APPLY_PROFILE=1
|
||||
unset DENSE_CC_CHECK
|
||||
export DENSE_SPLITK=128
|
||||
export DENSE_DEVICE_SUM=2 # cartesian P2P ring: no 8 MB device-allreduce cliff (NRHS=12 abort)
|
||||
export GRID_ALLOC_NCACHE_LARGE=64
|
||||
export NRHS=4
|
||||
export PowerIterations=0
|
||||
export SmootherCoeffLog=0
|
||||
|
||||
# frozen-polynomial controls
|
||||
export PolyRecordIters=4 # outer steps recorded before the switch
|
||||
export FineChebLo=3.0 # harvested |R|<0.1 edge / PowerIteration edge x1.05
|
||||
export FineChebHi=137.0
|
||||
export CoarseChebLo=8.0
|
||||
export CoarseChebHi=47.0 # shift 2.0: edge 43.3 x1.08
|
||||
|
||||
run_mode () {
|
||||
name=$1; export FineSmootherMode=$2; export CoarseSmootherMode=$3
|
||||
echo "----- $name : FineSmootherMode=$FineSmootherMode CoarseSmootherMode=$CoarseSmootherMode -----"
|
||||
fname=log.modes.$name
|
||||
srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \
|
||||
--mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap > $fname 2>&1
|
||||
echo " $(grep -h 'V2 3-level solve Nrhs' $fname | tr '\n' ' ')"
|
||||
echo " $(grep -h 'Fouter MrhsPGCR: Converged' $fname | sed 's/.*Converged/Converged/' | tr '\n' ' ')"
|
||||
echo " $(grep -h 'FINAL Nrhs .: worst' $fname | tr '\n' ' ')"
|
||||
grep -h "SwitchableSmoother\|GCRCoefficients .*calls" $fname | head -6
|
||||
}
|
||||
|
||||
run_mode M1_gcr_gcr gcr gcr
|
||||
run_mode M2_replay_replay replay replay
|
||||
run_mode M3_replay_gcr replay gcr
|
||||
run_mode M4_cheb_gcr cheb gcr
|
||||
run_mode M5_cheb_cheb cheb cheb
|
||||
|
||||
echo "========================================================="
|
||||
echo "summary"
|
||||
grep -h "V2 3-level solve Nrhs 1" log.modes.* | sed 's/.*V2/V2/'
|
||||
echo "========================================================="
|
||||
Reference in New Issue
Block a user