mirror of
https://github.com/paboyle/Grid.git
synced 2026-09-10 03:33:16 +01:00
New script for ORNL to try
This commit is contained in:
Peter Boyle
@@ -0,0 +1,133 @@
|
||||
#!/bin/bash -l
|
||||
#SBATCH --job-name=pvdagm-multigrid
|
||||
#SBATCH --nodes=36
|
||||
#SBATCH --ntasks-per-node=8
|
||||
#SBATCH --cpus-per-task=7
|
||||
#SBATCH --gpus-per-node=8
|
||||
#SBATCH --time=1:30:00
|
||||
#SBATCH --account=phy157_dwf
|
||||
#SBATCH --gpu-bind=none
|
||||
#SBATCH --exclusive
|
||||
#SBATCH --mem=0
|
||||
#SBATCH -S 0
|
||||
|
||||
##############################################################################
|
||||
# Production run of the three-level mrhs PVdagM multigrid on the library
|
||||
# objects (examples/Example_pvdagm_multigrid.cc), 48^3x96 Ls=24, 288 GCDs.
|
||||
#
|
||||
# All algorithm/physics parameters are in ONE serialisable struct read from
|
||||
# XML (--pvdagm-params); there are NO Grid environment knobs any more. The
|
||||
# library-internal choices the old nrhs_fault_36.job set by env are now the
|
||||
# only code path (2D block-cyclic dense inverse, ring-allgather apply,
|
||||
# inverseLU big leaves) and are gone from the environment.
|
||||
#
|
||||
# The binary must be built with -DNBASIS=64 to match the subspace file.
|
||||
##############################################################################
|
||||
|
||||
LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
|
||||
RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
|
||||
mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
|
||||
|
||||
cat << EOF > select_gpu
|
||||
#!/bin/bash
|
||||
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
||||
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
||||
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
||||
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
||||
export HIP_VISIBLE_DEVICES=\$GPU
|
||||
unset ROCR_VISIBLE_DEVICES
|
||||
if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
|
||||
exec numactl -m \$NUMA -N \$NUMA \$*
|
||||
EOF
|
||||
chmod +x ./select_gpu
|
||||
|
||||
root=$HOME/ParallelIO/systems/Frontier
|
||||
source $root/sourceme-rocm7.2.sh
|
||||
# sourceme sets FI_HMEM_ROCR_USE_DMABUF=0 -- the standing avoidance of the CXI
|
||||
# NO_TRANSLATION fault at NRHS>=12 (libfabric #12775).
|
||||
|
||||
export OMP_NUM_THREADS=7
|
||||
ulimit -c 0 # no 22 GB GPU core dumps
|
||||
export FI_MR_CACHE_MONITOR=kdreg2 # site default; device-buffer MPI on Slingshot
|
||||
export MPICH_GPU_SUPPORT_ENABLED=1
|
||||
export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
||||
export MPICH_OFI_NIC_POLICY=GPU
|
||||
export GRID_ALLOC_NCACHE_LARGE=64 # allocator large-ring depth (Grid allocator, not a solver knob)
|
||||
module load libfabric
|
||||
|
||||
BIN=$root/examples/Example_pvdagm_multigrid
|
||||
OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000 --comms-overlap"
|
||||
vol=48.48.48.96
|
||||
MPI_GEOM=3.6.4.4
|
||||
|
||||
# Paths substituted into the XML below.
|
||||
SUBSPACE=$LUSTRE/subspace_nb64.scidac
|
||||
CONFIG=/ccs/home/poare/ckpoint_lat.1000 # gauge field (empty => hot start)
|
||||
|
||||
##############################################################################
|
||||
# The parameter file. Values are the banked adaptive operating point carried
|
||||
# over from nrhs_fault_36.job. Tags mirror PVdagMDriverParams /
|
||||
# PVdagMMultiGridParams exactly; a mistyped tag deserialises silently to the
|
||||
# struct default, so keep this in step with the headers.
|
||||
##############################################################################
|
||||
cat << EOF > params.xml
|
||||
<?xml version="1.0"?>
|
||||
<grid>
|
||||
<PVdagMDriver>
|
||||
<Ls>24</Ls>
|
||||
<Mass>0.00078</Mass>
|
||||
<M5>1.8</M5>
|
||||
<MobiusB>1.5</MobiusB>
|
||||
<MobiusC>0.5</MobiusC>
|
||||
<Config>$CONFIG</Config>
|
||||
<Nrhs>12</Nrhs>
|
||||
<SolveSingleRHS>1</SolveSingleRHS>
|
||||
<MultiGrid>
|
||||
<Setup>
|
||||
<Block><elem>2</elem><elem>2</elem><elem>3</elem><elem>3</elem></Block>
|
||||
<Block2><elem>4</elem><elem>4</elem><elem>2</elem><elem>4</elem></Block2>
|
||||
<CoarsenBatch>9</CoarsenBatch>
|
||||
<SubspaceFile>$SUBSPACE</SubspaceFile>
|
||||
<FineSloppyComms>1</FineSloppyComms>
|
||||
</Setup>
|
||||
<FineSmoother>
|
||||
<Shift>0.1</Shift><Nstep>6</Nstep><Mmax>4</Mmax>
|
||||
</FineSmoother>
|
||||
<CoarseSmoother>
|
||||
<Shift>2.0</Shift><Nstep>2</Nstep><Mmax>2</Mmax>
|
||||
</CoarseSmoother>
|
||||
<CoarseSolver>
|
||||
<Tol>0.05</Tol><Order>200</Order><Mmax>8</Mmax>
|
||||
</CoarseSolver>
|
||||
<Outer>
|
||||
<Tol>1e-8</Tol><MaxIterations>1000</MaxIterations><Mmax>6</Mmax><Nstep>12</Nstep>
|
||||
</Outer>
|
||||
<Dense>
|
||||
<LeafSpan>9</LeafSpan>
|
||||
</Dense>
|
||||
</MultiGrid>
|
||||
</PVdagMDriver>
|
||||
</grid>
|
||||
EOF
|
||||
echo "=== params.xml ==="; cat params.xml; echo "=================="
|
||||
|
||||
run_cell () {
|
||||
name=$1; shift
|
||||
echo "----- $name : $* -----"
|
||||
export GRID_STDOUT_ROOT=$RUNDIR/mg_$name
|
||||
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS \
|
||||
--pvdagm-params params.xml --debug-stdout --log Error,Warning,Message,Performance > log.mg.$name 2>&1
|
||||
echo " exit $?"; sleep 30
|
||||
f=$(grep -l "Memory access fault\|NO_TRANSLATION" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
|
||||
if [ -n "$f" ]; then
|
||||
echo " FAULT in $f (rank $(basename $f | sed 's/Grid.stderr.//'))"; tail -20 $f | cut -c1-160
|
||||
else
|
||||
echo " no fault; convergence + timing:"
|
||||
grep -h "3-level solve\|FINAL Nrhs\|MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -8 | cut -c1-140
|
||||
fi
|
||||
}
|
||||
|
||||
# Two identical cells: the NRHS>=12 fault was intermittent (~1/3) before the
|
||||
# dmabuf avoidance, so run it twice to confirm a clean pair.
|
||||
run_cell run1
|
||||
run_cell run2
|
||||
Reference in New Issue
Block a user