mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-17 16:09:36 +01:00
110 lines
4.8 KiB
Bash
110 lines
4.8 KiB
Bash
#!/bin/bash -l
|
|
|
|
# Standalone MPI-only I/O reproducer on Frontier. No Grid, no accelerator,
|
|
# so no GCD/NUMA wrapper is needed -- the point of the exercise is that this
|
|
# depends on nothing but an MPI installation and a filesystem.
|
|
#
|
|
# mpicxx -O2 -std=c++11 io_mpi.cc -o io_mpi
|
|
#
|
|
# Weak scan: the local volume, and therefore the file view structure, is held
|
|
# identical at every rung and only the number of Lustre clients changes:
|
|
#
|
|
# local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB/rank
|
|
# file view 32768 contiguous runs of 4608 B per rank, at every rung
|
|
# aggregate k=2, row of 16, 8 extents of 18 MB, at every rung
|
|
#
|
|
# so any change in the relative bandwidth of the two lexicographic paths is a
|
|
# property of the client count alone.
|
|
#
|
|
# The PERF lines are MiB/s (bytes/1024/1024/s), which is what BinaryIO.h
|
|
# computes for lastPerf.mbytesPerSecond and prints as "MB/s", so the two
|
|
# tools can be compared directly. Grid's timed region is used here too:
|
|
# barrier, start, [plan build + exchange + I/O], barrier, stop, quoting the
|
|
# boss rank's stopwatch. --reuse-plan hoists the plan build out, which is
|
|
# how to show it is not where the time goes; do not use it when comparing
|
|
# against Grid's own numbers.
|
|
#
|
|
# io_aurora.pbs runs 12 ranks per node, one per tile, because that is what
|
|
# that machine is. The per rank local volume and the file view are the same
|
|
# there as here, but the record size and client count at a given NODE count
|
|
# are 1.5x. See the header of that script before tabulating the two
|
|
# together.
|
|
|
|
#SBATCH --job-name=ioMPI
|
|
#SBATCH --nodes=128
|
|
#SBATCH --ntasks-per-node=8
|
|
#SBATCH --cpus-per-task=7
|
|
#SBATCH --time=02:00:00
|
|
#SBATCH --account=phy157_dwf
|
|
#SBATCH --exclusive
|
|
#SBATCH --mem=0
|
|
|
|
module load cce/21.0.0
|
|
module load cpe/26.03
|
|
|
|
WORK=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/iompi.$SLURM_JOB_ID
|
|
mkdir -p $WORK
|
|
cd $WORK
|
|
|
|
# Do NOT stripe by default. Wide striping is what rescues the collective at
|
|
# scale and costs every other path a factor of 1.2-2.2; the default layout is
|
|
# what a user gets without knowing to ask. Uncomment to reproduce that
|
|
# interaction, and record which one you ran.
|
|
#lfs setstripe -c -1 -S 8M $WORK
|
|
lfs getstripe -d $WORK
|
|
|
|
BIN=$SLURM_SUBMIT_DIR/io_mpi
|
|
ARGS="--target 4194304 --reps 3"
|
|
|
|
# ROMIO's own view of what it did. Verbose, but the first thing anyone
|
|
# reading the report will ask for.
|
|
# export MPICH_MPIIO_STATS=1
|
|
# export MPICH_MPIIO_TIMERS=1
|
|
|
|
run () { # run <nodes> <grid> <mpi> <comment> [extra args...]
|
|
local nodes=$1 gr=$2 mp=$3 note=$4
|
|
local nranks=$(( nodes * 8 ))
|
|
shift 4
|
|
echo
|
|
echo "==================================================================="
|
|
echo "=== nodes=$nodes ranks=$nranks grid=$gr mpi=$mp $note"
|
|
echo "=== extra: $@"
|
|
echo "==================================================================="
|
|
srun -N$nodes -n$nranks --ntasks-per-node=8 $BIN --grid $gr --mpi $mp $ARGS "$@"
|
|
echo "=== exit $?"
|
|
}
|
|
|
|
#####################################################################
|
|
# Phase 0. Correctness, including the non-uniform Alltoallv branch
|
|
# (odd process factor in an un-split dimension). Small, and the
|
|
# whole-file crc32 is serial, so keep the volume down here.
|
|
#####################################################################
|
|
run 1 12.12.8.8 2.2.2.1 "correctness, uniform counts" --reps 0 --serial-crc
|
|
run 3 24.12.8.8 3.2.2.2 "correctness, NON-uniform counts" --reps 0 --serial-crc
|
|
run 4 16.16.16.32 2.2.2.4 "correctness, non-zero offset" --reps 0 --serial-crc --offset 1024
|
|
|
|
#####################################################################
|
|
# Phase 1. Weak scan, 151 MB/rank. Timing only.
|
|
#####################################################################
|
|
run 4 32.32.64.128 4.4.2.1 "4.8 GB" --no-validate
|
|
run 8 32.32.64.256 4.4.2.2 "9.7 GB" --no-validate
|
|
run 16 32.32.64.512 4.4.2.4 "19.3 GB" --no-validate
|
|
run 32 32.32.128.512 4.4.4.4 "38.6 GB" --no-validate
|
|
run 64 32.32.128.1024 4.4.4.8 "77.3 GB" --no-validate
|
|
run 128 32.32.256.1024 4.4.8.8 "154.6 GB" --no-validate
|
|
|
|
#####################################################################
|
|
# Phase 2. Answer the two questions a reviewer will ask immediately.
|
|
#####################################################################
|
|
# Is the gap an artefact of measuring cache rather than the filesystem?
|
|
run 128 32.32.256.1024 4.4.8.8 "154.6 GB, durable" --no-validate --fsync --drop-cache
|
|
# Does the collective recover if it is given the hints it wants?
|
|
run 128 32.32.256.1024 4.4.8.8 "154.6 GB, cb hints" --no-validate \
|
|
--hints romio_cb_write=enable,romio_cb_read=enable,cb_nodes=128,cb_buffer_size=16777216
|
|
# Does the degenerate memory subarray matter?
|
|
run 128 32.32.256.1024 4.4.8.8 "154.6 GB, mem subarray" --no-validate --mem-subarray
|
|
|
|
echo
|
|
echo "=== done. Files left in $WORK"
|
|
ls -l $WORK
|