mirror of
https://github.com/paboyle/Grid.git
synced 2026-09-12 04:23:18 +01:00
149 lines
5.2 KiB
Bash
149 lines
5.2 KiB
Bash
#!/bin/bash -l
|
|
#SBATCH --job-name=pvdagm-multigrid
|
|
#SBATCH --nodes=36
|
|
#SBATCH --ntasks-per-node=8
|
|
#SBATCH --cpus-per-task=7
|
|
#SBATCH --gpus-per-node=8
|
|
#SBATCH --time=1:30:00
|
|
#SBATCH --account=phy157_dwf
|
|
#SBATCH --gpu-bind=none
|
|
#SBATCH --exclusive
|
|
#SBATCH --mem=0
|
|
#SBATCH -S 0
|
|
|
|
##############################################################################
|
|
# Production run of the three-level mrhs PVdagM multigrid on the library
|
|
# objects (examples/Example_pvdagm_multigrid.cc), 48^3x96 Ls=24, 288 GCDs.
|
|
#
|
|
# All algorithm/physics parameters are in ONE serialisable struct read from
|
|
# XML (--pvdagm-params); there are NO Grid environment knobs any more. The
|
|
# library-internal choices the old nrhs_fault_36.job set by env are now the
|
|
# only code path (2D block-cyclic dense inverse, ring-allgather apply,
|
|
# inverseLU big leaves) and are gone from the environment.
|
|
#
|
|
# The binary must be built with -DNBASIS=64 to match the subspace file.
|
|
##############################################################################
|
|
|
|
root=$HOME/PVdagM/Grid/systems/Frontier
|
|
source $root/sourceme-rocm7.2.sh
|
|
|
|
# Paths substituted into the XML below.
|
|
LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
|
|
SUBSPACE=$LUSTRE/subspace_nb64.scidac
|
|
CONFIG=/ccs/home/paboyle/configs/ckpoint_lat.1000
|
|
RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
|
|
mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
|
|
|
|
|
|
cat << EOF > select_gpu
|
|
#!/bin/bash
|
|
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
|
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
|
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
|
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
|
export HIP_VISIBLE_DEVICES=\$GPU
|
|
unset ROCR_VISIBLE_DEVICES
|
|
if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
|
|
exec numactl -m \$NUMA -N \$NUMA \$*
|
|
EOF
|
|
chmod +x ./select_gpu
|
|
|
|
|
|
export OMP_NUM_THREADS=7
|
|
ulimit -c 0 # no 22 GB GPU core dumps
|
|
export FI_MR_CACHE_MONITOR=kdreg2 # site default; device-buffer MPI on Slingshot
|
|
export MPICH_GPU_SUPPORT_ENABLED=1
|
|
export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
|
export MPICH_OFI_NIC_POLICY=GPU
|
|
|
|
#### This raise the probability of failure ####
|
|
export GRID_ALLOC_NCACHE_LARGE=0 # allocator large-ring depth (Grid allocator, not a solver knob) -- default is 8
|
|
##############################################################################
|
|
|
|
|
|
module load libfabric
|
|
|
|
BIN=$root/examples/Example_pvdagm_multigrid
|
|
OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 0 --device-mem 32000 --comms-overlap"
|
|
vol=48.48.48.96
|
|
MPI_GEOM=3.6.4.4
|
|
|
|
|
|
##############################################################################
|
|
# The parameter file. Values are the banked adaptive operating point carried
|
|
# over from nrhs_fault_36.job. Tags mirror PVdagMDriverParams /
|
|
# PVdagMMultiGridParams exactly; a mistyped tag deserialises silently to the
|
|
# struct default, so keep this in step with the headers.
|
|
##############################################################################
|
|
cat << EOF > params.xml
|
|
<?xml version="1.0"?>
|
|
<grid>
|
|
<PVdagMDriver>
|
|
<Ls>24</Ls>
|
|
<Mass>0.00078</Mass>
|
|
<M5>1.8</M5>
|
|
<MobiusB>1.5</MobiusB>
|
|
<MobiusC>0.5</MobiusC>
|
|
<Config>$CONFIG</Config>
|
|
<Nrhs>12</Nrhs>
|
|
<SolveSingleRHS>1</SolveSingleRHS>
|
|
<MultiGrid>
|
|
<Setup>
|
|
<Block><elem>2</elem><elem>2</elem><elem>3</elem><elem>3</elem></Block>
|
|
<Block2><elem>4</elem><elem>4</elem><elem>2</elem><elem>4</elem></Block2>
|
|
<CoarsenBatch>9</CoarsenBatch>
|
|
<SubspaceFile>$SUBSPACE</SubspaceFile>
|
|
<FineSloppyComms>1</FineSloppyComms>
|
|
</Setup>
|
|
<FineSmoother>
|
|
<Shift>0.1</Shift><Nstep>6</Nstep><Mmax>4</Mmax>
|
|
</FineSmoother>
|
|
<CoarseSmoother>
|
|
<Shift>2.0</Shift><Nstep>2</Nstep><Mmax>2</Mmax>
|
|
</CoarseSmoother>
|
|
<CoarseSolver>
|
|
<Tol>0.05</Tol><Order>200</Order><Mmax>8</Mmax>
|
|
</CoarseSolver>
|
|
<Outer>
|
|
<Tol>1e-8</Tol><MaxIterations>1000</MaxIterations><Mmax>6</Mmax><Nstep>12</Nstep>
|
|
</Outer>
|
|
<Dense>
|
|
<LeafSpan>9</LeafSpan>
|
|
</Dense>
|
|
</MultiGrid>
|
|
</PVdagMDriver>
|
|
</grid>
|
|
EOF
|
|
echo "=== params.xml ==="; cat params.xml; echo "=================="
|
|
|
|
run_cell () {
|
|
name=$1; shift
|
|
echo "----- $name : $* -----"
|
|
export GRID_STDOUT_ROOT=$RUNDIR/mg_$name
|
|
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS \
|
|
--pvdagm-params params.xml --debug-stdout --log Error,Warning,Message,Performance > log.mg.$name 2>&1
|
|
echo " exit $?"; sleep 30
|
|
f=$(grep -l "Memory access fault\|NO_TRANSLATION" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
|
|
if [ -n "$f" ]; then
|
|
echo " FAULT in $f (rank $(basename $f | sed 's/Grid.stderr.//'))"; tail -20 $f | cut -c1-160
|
|
else
|
|
echo " no fault; convergence + timing:"
|
|
grep -h "3-level solve\|FINAL Nrhs\|MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -8 | cut -c1-140
|
|
fi
|
|
}
|
|
|
|
# the NRHS>=12 fault was intermittent (~1/3) before the
|
|
# dmabuf avoidance, so run it twice to confirm a clean pair.
|
|
|
|
# sourceme sets FI_HMEM_ROCR_USE_DMABUF=0 -- the standing avoidance of the CXI
|
|
# NO_TRANSLATION fault at NRHS>=12 (libfabric #12775).
|
|
export FI_HMEM_ROCR_USE_DMABUF=0
|
|
run_cell dmabuf0_run1
|
|
run_cell dmabuf0_run2
|
|
run_cell dmabuf0_run3
|
|
|
|
export FI_HMEM_ROCR_USE_DMABUF=1
|
|
run_cell dmabuf1_run1
|
|
run_cell dmabuf1_run2
|
|
run_cell dmabuf1_run3
|