diff --git a/systems/Frontier/pvdagm_multigrid.job b/systems/Frontier/pvdagm_multigrid.job new file mode 100644 index 000000000..14ef25a26 --- /dev/null +++ b/systems/Frontier/pvdagm_multigrid.job @@ -0,0 +1,133 @@ +#!/bin/bash -l +#SBATCH --job-name=pvdagm-multigrid +#SBATCH --nodes=36 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=1:30:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -S 0 + +############################################################################## +# Production run of the three-level mrhs PVdagM multigrid on the library +# objects (examples/Example_pvdagm_multigrid.cc), 48^3x96 Ls=24, 288 GCDs. +# +# All algorithm/physics parameters are in ONE serialisable struct read from +# XML (--pvdagm-params); there are NO Grid environment knobs any more. The +# library-internal choices the old nrhs_fault_36.job set by env are now the +# only code path (2D block-cyclic dense inverse, ring-allgather apply, +# inverseLU big leaves) and are gone from the environment. +# +# The binary must be built with -DNBASIS=64 to match the subspace file. +############################################################################## + +LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle +RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID} +mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR" + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +export HIP_VISIBLE_DEVICES=\$GPU +unset ROCR_VISIBLE_DEVICES +if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi +exec numactl -m \$NUMA -N \$NUMA \$* +EOF +chmod +x ./select_gpu + +root=$HOME/ParallelIO/systems/Frontier +source $root/sourceme-rocm7.2.sh +# sourceme sets FI_HMEM_ROCR_USE_DMABUF=0 -- the standing avoidance of the CXI +# NO_TRANSLATION fault at NRHS>=12 (libfabric #12775). + +export OMP_NUM_THREADS=7 +ulimit -c 0 # no 22 GB GPU core dumps +export FI_MR_CACHE_MONITOR=kdreg2 # site default; device-buffer MPI on Slingshot +export MPICH_GPU_SUPPORT_ENABLED=1 +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_OFI_NIC_POLICY=GPU +export GRID_ALLOC_NCACHE_LARGE=64 # allocator large-ring depth (Grid allocator, not a solver knob) +module load libfabric + +BIN=$root/examples/Example_pvdagm_multigrid +OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000 --comms-overlap" +vol=48.48.48.96 +MPI_GEOM=3.6.4.4 + +# Paths substituted into the XML below. +SUBSPACE=$LUSTRE/subspace_nb64.scidac +CONFIG=/ccs/home/poare/ckpoint_lat.1000 # gauge field (empty => hot start) + +############################################################################## +# The parameter file. Values are the banked adaptive operating point carried +# over from nrhs_fault_36.job. Tags mirror PVdagMDriverParams / +# PVdagMMultiGridParams exactly; a mistyped tag deserialises silently to the +# struct default, so keep this in step with the headers. +############################################################################## +cat << EOF > params.xml + + + + 24 + 0.00078 + 1.8 + 1.5 + 0.5 + $CONFIG + 12 + 1 + + + 2233 + 4424 + 9 + $SUBSPACE + 1 + + + 0.164 + + + 2.022 + + + 0.052008 + + + 1e-81000612 + + + 9 + + + + +EOF +echo "=== params.xml ==="; cat params.xml; echo "==================" + +run_cell () { + name=$1; shift + echo "----- $name : $* -----" + export GRID_STDOUT_ROOT=$RUNDIR/mg_$name + env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS \ + --pvdagm-params params.xml --debug-stdout --log Error,Warning,Message,Performance > log.mg.$name 2>&1 + echo " exit $?"; sleep 30 + f=$(grep -l "Memory access fault\|NO_TRANSLATION" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1) + if [ -n "$f" ]; then + echo " FAULT in $f (rank $(basename $f | sed 's/Grid.stderr.//'))"; tail -20 $f | cut -c1-160 + else + echo " no fault; convergence + timing:" + grep -h "3-level solve\|FINAL Nrhs\|MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -8 | cut -c1-140 + fi +} + +# Two identical cells: the NRHS>=12 fault was intermittent (~1/3) before the +# dmabuf avoidance, so run it twice to confirm a clean pair. +run_cell run1 +run_cell run2