mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-16 23:49:35 +01:00
117 lines
5.4 KiB
Bash
117 lines
5.4 KiB
Bash
#!/bin/bash
|
|
|
|
# Standalone MPI-only I/O reproducer on Aurora. Host only -- no SYCL, no
|
|
# gpu_tile_compact.sh -- so unlike halo_mpi this needs nothing but MPI:
|
|
#
|
|
# mpicxx -O2 -std=c++11 io_mpi.cc -o io_mpi
|
|
#
|
|
# 12 ranks per node, one per tile, which is what the machine is. That is a
|
|
# deliberate difference from io_frontier.slurm (8 per node, one per GCD), so
|
|
# read the comparison carefully -- see WHAT IS AND IS NOT COMPARABLE below.
|
|
|
|
#PBS -l select=128
|
|
#PBS -l walltime=02:00:00
|
|
#PBS -l filesystems=flare
|
|
#PBS -q prod
|
|
#PBS -A LatticeQCD_aesp_CNDA
|
|
|
|
cd $PBS_O_WORKDIR
|
|
|
|
##########################################################################
|
|
# WHAT IS AND IS NOT COMPARABLE WITH THE FRONTIER SCAN
|
|
#
|
|
# Held identical at every rung of both scans:
|
|
# local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB/rank
|
|
# file view 32768 contiguous runs of 4608 B per rank
|
|
# aggregation k=2, row of 16, 8 extents of 18 MB
|
|
# (verified: 4.4.3.1 at 48 ranks and 4.4.3.2 at 96 ranks give exactly the
|
|
# same plan as Frontier's 4.4.2.1 at 32 ranks.)
|
|
#
|
|
# NOT identical, because 12 ranks/node is 1.5x the clients per node:
|
|
# record size at a given NODE count is 1.5x Frontier's
|
|
# client count at a given NODE count is 1.5x Frontier's
|
|
#
|
|
# So compare the two machines at equal RANK count (Aurora 4 nodes vs
|
|
# Frontier 6, and so on) if what you want is equal client count and equal
|
|
# record size; compare at equal NODE count if what you want is each machine
|
|
# used as it is meant to be used. Both are legitimate, they answer
|
|
# different questions, and a table that does not say which one it is
|
|
# reporting is worthless. The quantity that carries the MPI-IO pathology --
|
|
# per rank local volume and the resulting file view -- is invariant either
|
|
# way, which is the point.
|
|
##########################################################################
|
|
|
|
##########################################################################
|
|
# WHICH FILESYSTEM. Point this at Lustre for the like-for-like comparison
|
|
# with Frontier's Orion. DAOS is a different architecture -- its numbers
|
|
# are interesting but they are NOT a reproduction of the Frontier result,
|
|
# and mixing them into one table would misrepresent both. Label every set
|
|
# of numbers with the filesystem it came from.
|
|
##########################################################################
|
|
WORK=/lus/flare/projects/LatticeQCD_aesp_CNDA/$USER/iompi.$PBS_JOBID
|
|
mkdir -p $WORK
|
|
cd $WORK
|
|
|
|
# Match Frontier's default: no explicit striping. Record what was inherited.
|
|
lfs getstripe -d $WORK 2>/dev/null || echo "(no lfs getstripe -- not Lustre?)"
|
|
|
|
# The largest rung writes three files of 232 GB, so budget ~700 GB and check
|
|
# the quota before submitting. Each run unlinks the three files first, so
|
|
# that is peak usage, not cumulative.
|
|
|
|
NRANKS=12 # one per tile
|
|
|
|
BIN=$PBS_O_WORKDIR/io_mpi
|
|
ARGS="--target 4194304 --reps 3"
|
|
|
|
# MPICH's own account of what it did; the first thing anyone reading the
|
|
# report will ask for.
|
|
# export MPICH_MPIIO_STATS=1
|
|
# export MPICH_MPIIO_TIMERS=1
|
|
|
|
run () { # run <nodes> <grid> <mpi> <comment> [extra args...]
|
|
local nodes=$1 gr=$2 mp=$3 note=$4
|
|
local ntot=$(( nodes * NRANKS ))
|
|
shift 4
|
|
echo
|
|
echo "==================================================================="
|
|
echo "=== nodes=$nodes ranks=$ntot grid=$gr mpi=$mp $note"
|
|
echo "=== extra: $@"
|
|
echo "==================================================================="
|
|
mpiexec -np $ntot -ppn $NRANKS $BIN --grid $gr --mpi $mp $ARGS "$@"
|
|
echo "=== exit $?"
|
|
}
|
|
|
|
#####################################################################
|
|
# Phase 0. Correctness. Both count branches of MPI_Alltoallv are
|
|
# covered; the labels below were checked, not assumed. The whole-file
|
|
# crc32 is serial, so keep these small.
|
|
#####################################################################
|
|
run 1 16.16.16.24 2.2.1.3 "correctness, UNIFORM counts, row of 4" --reps 0 --serial-crc
|
|
run 2 24.12.8.8 3.2.2.2 "correctness, NON-UNIFORM counts, row of 12" --reps 0 --serial-crc
|
|
run 4 16.16.32.24 2.2.4.3 "correctness, NON-UNIFORM, non-zero offset" --reps 0 --serial-crc --offset 1024
|
|
|
|
#####################################################################
|
|
# Phase 1. Weak scan at 151 MB/rank. Identical plan at every rung:
|
|
# k=2, row of 16, 8 extents of 18 MB, 32768 runs of 4608 B in the view.
|
|
#####################################################################
|
|
# nodes global lattice mpi record
|
|
run 4 32.32.96.128 4.4.3.1 "7.2 GB" --no-validate
|
|
run 8 32.32.96.256 4.4.3.2 "14.5 GB" --no-validate
|
|
run 16 32.32.96.512 4.4.3.4 "29.0 GB" --no-validate
|
|
run 32 32.32.192.512 4.4.6.4 "58.0 GB" --no-validate
|
|
run 64 32.32.192.1024 4.4.6.8 "116.0 GB" --no-validate
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB" --no-validate
|
|
|
|
#####################################################################
|
|
# Phase 2. The three questions a reviewer asks immediately.
|
|
#####################################################################
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB, durable" --no-validate --fsync --drop-cache
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB, cb hints" --no-validate \
|
|
--hints romio_cb_write=enable,romio_cb_read=enable,cb_nodes=128,cb_buffer_size=16777216
|
|
run 128 32.32.384.1024 4.4.12.8 "231.9 GB, mem subarray" --no-validate --mem-subarray
|
|
|
|
echo
|
|
echo "=== done. Files left in $WORK"
|
|
ls -l $WORK
|