mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-16 07:29:35 +01:00
138 lines
6.4 KiB
Bash
138 lines
6.4 KiB
Bash
#!/bin/bash
|
|
|
|
# Standalone MPI-only I/O reproducer on Aurora. Host only -- no SYCL, no
|
|
# gpu_tile_compact.sh -- so unlike halo_mpi this needs nothing but MPI:
|
|
#
|
|
# mpicxx -O2 -std=c++11 io_mpi.cc -o io_mpi
|
|
#
|
|
# 12 ranks per node, one per tile, which is what the machine is. That is a
|
|
# deliberate difference from io_frontier.slurm (8 per node, one per GCD), so
|
|
# read the comparison carefully -- see WHAT IS AND IS NOT COMPARABLE below.
|
|
|
|
# Run io_aurora_debug.pbs first. If the cross validation fails there, this
|
|
# scan is 2 hours of 128 nodes producing numbers for a broken file.
|
|
|
|
#PBS -q prod
|
|
#PBS -l filesystems=flare
|
|
#PBS -l filesystems=home
|
|
#PBS -l select=128
|
|
#PBS -l walltime=02:00:00
|
|
#PBS -A 15479
|
|
|
|
cd $PBS_O_WORKDIR
|
|
cp $PBS_NODEFILE nodefile
|
|
|
|
# Only if mpiexec is not already in the environment. io_mpi is host only
|
|
# and needs no part of the Grid build environment.
|
|
#source ../../sourceme.sh
|
|
|
|
##########################################################################
|
|
# Environment. io_mpi has no OpenMP and never touches a GPU, so one thread
|
|
# per rank and a NUMA NIC policy rather than a GPU one.
|
|
#
|
|
# The MPICH_DBG_* variables are deliberately absent: at 1536 ranks they
|
|
# produce gigabytes of log and perturb the timings they would explain.
|
|
# MPICH_MPIIO_STATS/TIMERS are also off here -- they are per collective and
|
|
# 1536 ranks x 6 rungs x 3 reps is unreadable. Get them from the debug run.
|
|
##########################################################################
|
|
export OMP_NUM_THREADS=1
|
|
export MPICH_CH4_SHM=XPMEM
|
|
export MPICH_OFI_NIC_POLICY=NUMA
|
|
|
|
##########################################################################
|
|
# WHAT IS AND IS NOT COMPARABLE WITH THE FRONTIER SCAN
|
|
#
|
|
# Held identical at every rung of both scans:
|
|
# local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB/rank
|
|
# file view 32768 contiguous runs of 4608 B per rank
|
|
# aggregation k=2, row of 16, 8 extents of 18 MB
|
|
# (verified: 4.4.3.1 at 48 ranks and 4.4.3.2 at 96 ranks give exactly the
|
|
# same plan as Frontier's 4.4.2.1 at 32 ranks.)
|
|
#
|
|
# NOT identical, because 12 ranks/node is 1.5x the clients per node:
|
|
# record size at a given NODE count is 1.5x Frontier's
|
|
# client count at a given NODE count is 1.5x Frontier's
|
|
#
|
|
# So compare the two machines at equal RANK count (Aurora 4 nodes vs
|
|
# Frontier 6, and so on) if what you want is equal client count and equal
|
|
# record size; compare at equal NODE count if what you want is each machine
|
|
# used as it is meant to be used. Both are legitimate, they answer
|
|
# different questions, and a table that does not say which one it is
|
|
# reporting is worthless. The quantity that carries the MPI-IO pathology --
|
|
# per rank local volume and the resulting file view -- is invariant either
|
|
# way, which is the point.
|
|
##########################################################################
|
|
|
|
##########################################################################
|
|
# WHICH FILESYSTEM. Point this at Lustre for the like-for-like comparison
|
|
# with Frontier's Orion. DAOS is a different architecture -- its numbers
|
|
# are interesting but they are NOT a reproduction of the Frontier result,
|
|
# and mixing them into one table would misrepresent both. Label every set
|
|
# of numbers with the filesystem it came from.
|
|
##########################################################################
|
|
# PROJECT is the flare project DIRECTORY name, which is not the -A account
|
|
# number. Set it once; the mkdir below fails loudly rather than writing
|
|
# somewhere unintended.
|
|
PROJECT=LatticeQCD_aesp_CNDA
|
|
WORK=/lus/flare/projects/$PROJECT/$USER/iompi.$PBS_JOBID
|
|
mkdir -p $WORK || { echo "cannot create $WORK -- set PROJECT correctly"; exit 1; }
|
|
cd $WORK
|
|
|
|
# Match Frontier's default: no explicit striping. Record what was inherited.
|
|
lfs getstripe -d $WORK 2>/dev/null || echo "(no lfs getstripe -- not Lustre?)"
|
|
|
|
# The largest rung writes three files of 232 GB, so budget ~700 GB and check
|
|
# the quota before submitting. Each run unlinks the three files first, so
|
|
# that is peak usage, not cumulative.
|
|
|
|
NRANKS=12 # one per tile
|
|
|
|
BIN=$PBS_O_WORKDIR/io_mpi
|
|
ARGS="--target 4194304 --reps 3"
|
|
|
|
run () { # run <nodes> <grid> <mpi> <comment> [extra args...]
|
|
local nodes=$1 gr=$2 mp=$3 note=$4
|
|
local ntot=$(( nodes * NRANKS ))
|
|
shift 4
|
|
echo
|
|
echo "==================================================================="
|
|
echo "=== nodes=$nodes ranks=$ntot grid=$gr mpi=$mp $note"
|
|
echo "=== extra: $@"
|
|
echo "==================================================================="
|
|
mpiexec -np $ntot -ppn $NRANKS -envall $BIN --grid $gr --mpi $mp $ARGS "$@"
|
|
echo "=== exit $?"
|
|
}
|
|
|
|
#####################################################################
|
|
# Phase 0. Correctness. Both count branches of MPI_Alltoallv are
|
|
# covered; the labels below were checked, not assumed. The whole-file
|
|
# crc32 is serial, so keep these small.
|
|
#####################################################################
|
|
run 1 16.16.16.24 2.2.1.3 "correctness, UNIFORM counts, row of 4" --reps 0 --serial-crc
|
|
run 2 24.12.8.8 3.2.2.2 "correctness, NON-UNIFORM counts, row of 12" --reps 0 --serial-crc
|
|
run 4 16.16.32.24 2.2.4.3 "correctness, NON-UNIFORM, non-zero offset" --reps 0 --serial-crc --offset 1024
|
|
|
|
#####################################################################
|
|
# Phase 1. Weak scan at 151 MB/rank. Identical plan at every rung:
|
|
# k=2, row of 16, 8 extents of 18 MB, 32768 runs of 4608 B in the view.
|
|
#####################################################################
|
|
# nodes global lattice mpi record
|
|
run 4 32.32.96.128 4.4.3.1 "7.2 GB" --no-validate
|
|
run 8 32.32.96.256 4.4.3.2 "14.5 GB" --no-validate
|
|
run 16 32.32.96.512 4.4.3.4 "29.0 GB" --no-validate
|
|
run 32 32.32.192.512 4.4.6.4 "58.0 GB" --no-validate
|
|
run 64 32.32.192.1024 4.4.6.8 "116.0 GB" --no-validate
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB" --no-validate
|
|
|
|
#####################################################################
|
|
# Phase 2. The three questions a reviewer asks immediately.
|
|
#####################################################################
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB, durable" --no-validate --fsync --drop-cache
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB, cb hints" --no-validate \
|
|
--hints romio_cb_write=enable,romio_cb_read=enable,cb_nodes=128,cb_buffer_size=16777216
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB, mem subarray" --no-validate --mem-subarray
|
|
|
|
echo
|
|
echo "=== done. Files left in $WORK"
|
|
ls -l $WORK
|