From a6cdf20c18f44ffe4b35922cf7d0a02eef7e23c7 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 13 Aug 2026 20:08:56 -0400 Subject: [PATCH] IO tests --- systems/Aurora/aggregate_io.pbs | 144 ++++++++++++++++++++++++++ systems/Frontier/aggregate_io.slurm | 152 ++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+) create mode 100644 systems/Aurora/aggregate_io.pbs create mode 100644 systems/Frontier/aggregate_io.slurm diff --git a/systems/Aurora/aggregate_io.pbs b/systems/Aurora/aggregate_io.pbs new file mode 100644 index 000000000..4bc9ddac1 --- /dev/null +++ b/systems/Aurora/aggregate_io.pbs @@ -0,0 +1,144 @@ +#!/bin/bash + +# Grid's tests/IO/Test_aggregate_io on Aurora, 12 ranks per node (one per +# tile). This is the Grid-side counterpart of MPI_benchmark/io_aurora.pbs: +# same three paths, same comparison, but exercising the real BinaryIO.h +# code that production uses, with Grid's checksums and its munge/endian +# conversion in the loop. Run both -- if they disagree, the difference is +# in Grid, not in MPI-IO. +# +# Submit from the BUILD directory: BIN below is $PBS_O_WORKDIR relative. + +#PBS -q debug-scaling +#PBS -l filesystems=flare +#PBS -l filesystems=home +#PBS -l select=4 +#PBS -l walltime=01:00:00 +#PBS -A 15479 +##PBS -A LatticeQCD_aesp_CNDA + +cd $PBS_O_WORKDIR +cp $PBS_NODEFILE nodefile + +########################################################################## +# Grid source tree. Unlike io_mpi this IS a Grid binary -- SYCL, c-lime, +# the lot -- so sourceme.sh is required, and so is the tile wrapper. +########################################################################## +root=$HOME/Grid/systems/Aurora # <-- adjust to your source tree +source ${root}/sourceme.sh +TILE=${root}/benchmarks/gpu_tile.sh + +BIN=$PBS_O_WORKDIR/tests/IO/Test_aggregate_io +[ -x $BIN ] || { echo "no $BIN -- submit from the build directory"; exit 1; } +[ -x $TILE ] || { echo "no $TILE -- check \$root"; exit 1; } + +export OMP_NUM_THREADS=4 +export MPICH_OFI_NIC_POLICY=GPU +# The aggregate path's MPI_Alltoallv runs on HOST buffers, so NIC policy +# NUMA is arguably the better match here even though GPU is the standard +# for Grid on this machine. Worth one comparison run; do not change it +# silently, the rest of the Grid numbers on Aurora were taken with GPU. +#export MPICH_OFI_NIC_POLICY=NUMA + +########################################################################## +# Test_aggregate_io writes ref.bin / agg.bin / raw.bin / trunc.bin with +# RELATIVE paths, so they land in the cwd. Home is not the point of the +# exercise. +########################################################################## +# PROJECT is the flare project DIRECTORY name, not the -A account number. +PROJECT=LatticeQCD_aesp_CNDA +WORK=/lus/flare/projects/$PROJECT/$USER/aggio.$PBS_JOBID +mkdir -p $WORK || { echo "cannot create $WORK -- set PROJECT correctly"; exit 1; } +cd $WORK + +# Leave striping at the directory default, as the Frontier runs did. At +# 128 nodes there the aggregate path was 2.2x faster on the default layout +# than on -c -1, while MPI-IO was 2.4x slower on it; the two paths want +# opposite things. Set it only to reproduce that interaction, and say so. +#lfs setstripe -c -1 -S 8M $WORK +lfs getstripe -d $WORK 2>/dev/null || echo "(no lfs getstripe -- not Lustre?)" + +# ROMIO's own account of what the reference path did. Verbose; switch on +# deliberately. +#export MPICH_MPIIO_STATS=1 +#export MPICH_MPIIO_TIMERS=1 + +AT=8 # accelerator threads; immaterial to an I/O test, kept canonical + +run () { # run [extra args...] + local nodes=$1 gr=$2 mp=$3 note=$4 + local ntot=$(( nodes * 12 )) + shift 4 + echo + echo "===================================================================" + echo "=== nodes=$nodes ranks=$ntot grid=$gr mpi=$mp $note" + echo "=== extra: $@" + echo "===================================================================" + # Deliberately NOT removing ref.bin/agg.bin/raw.bin here. The test does + # it itself (Test_aggregate_io.cc:203 and :370) for exactly the stale-tail + # reason, and doing it in the script would destroy the files a later + # --io-read-only job needs. + mpiexec -np $ntot -ppn 12 -envall $TILE $BIN --mpi $mp --grid $gr \ + --shm-mpi 0 --shm 2048 --device-mem 32000 --accelerator-threads $AT "$@" + echo "=== exit $?" +} + +##################################################################### +# Phase 0. Logic corners, correctness only, no bandwidth claimed. +# +# These run the FULL default target sweep {1, 1024, 64K, 4M}, which walks +# k from 1 to Nd-1 and so exercises every shape of aggregation plan on a +# volume small enough that the serial byte-for-byte file comparison is +# affordable. 1.3 MB/rank means nothing for bandwidth and is not meant to. +# +# Both branches of MPI_Alltoallv are covered; the labels were checked with +# MPI_benchmark/io_mpi, not assumed. +##################################################################### +run 1 16.16.16.24 2.2.1.3 "UNIFORM counts, row of 4" --io-reps 0 +run 1 12.12.24.8 2.2.3.1 "NON-UNIFORM counts, row of 12" --io-reps 0 + +##################################################################### +# Phase 1. 151 MB/rank, 3.6 GB record, 24 ranks. Correctness AND +# performance at the production 4 MB target only -- the byte-for-byte +# comparison reads the whole file on one rank, so the four-target sweep +# is not affordable here. +##################################################################### +run 2 16.16.96.256 2.2.3.2 "3.6 GB, row of 4, 32 extents, correctness + perf" \ + --aggregate-target 4194304 + +# Same volume per rank, but an odd process factor in an un-split dimension +# puts the variable-count branch of Alltoallv where the bandwidth is real. +run 2 24.16.64.256 3.2.2.2 "3.6 GB, NON-UNIFORM counts, row of 6, 22 extents" \ + --aggregate-target 4194304 + +##################################################################### +# Phase 2. 604 MB/rank, 29.0 GB record, 48 ranks. Correctness off. +# +# Per-rank volume is held at 16.16.32.128 in ALL FOUR, so the only thing +# changing is how much work the aggregation has to do: +# +# 4.4.3.1 row 16 8 extents of 72 MB <- old path: 9 KB runs +# 4.2.3.2 row 8 16 extents of 36 MB +# 2.2.12.1 row 4 32 extents of 18 MB +# 1.1.3.16 row 1 128 extents of 4.5 MB <- no exchange at all +# +# The last is the interesting control: with the fast dimensions unsplit +# the aggregate path degenerates to plain large-extent POSIX and the +# Alltoallv is trivial, so it separates "the transposition is cheap" from +# "the writes are large". Note the test's own header warns that 1.1.X.Y +# makes the CORRECTNESS section vacuous -- rows of size one exchange +# nothing. That is fine here because correctness is off; it would not be +# fine as a correctness run. +##################################################################### +run 4 64.64.96.128 4.4.3.1 "29.0 GB, row of 16" --aggregate-target 4194304 --io-no-correctness +run 4 64.32.96.256 4.2.3.2 "29.0 GB, row of 8" --aggregate-target 4194304 --io-no-correctness +run 4 32.32.384.128 2.2.12.1 "29.0 GB, row of 4" --aggregate-target 4194304 --io-no-correctness +run 4 16.16.96.2048 1.1.3.16 "29.0 GB, row of 1, no exchange" \ + --aggregate-target 4194304 --io-no-correctness + +echo +echo "=== done. Output left in $WORK" +echo "=== For COLD read numbers, submit a second job on a fresh allocation" +echo "=== with WORK pointed at this directory and --io-read-only added." +echo "=== Reading back what this job just wrote only measures page cache." +ls -l $WORK diff --git a/systems/Frontier/aggregate_io.slurm b/systems/Frontier/aggregate_io.slurm new file mode 100644 index 000000000..02508d3cb --- /dev/null +++ b/systems/Frontier/aggregate_io.slurm @@ -0,0 +1,152 @@ +#!/bin/bash -l + +#SBATCH --job-name=runIO +#SBATCH --nodes=4 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=02:00:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 +#SBATCH -q debug + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +unset ROCR_VISIBLE_DEVICES +export HIP_VISIBLE_DEVICES=\$GPU + +rank=\$SLURM_PROCID +CMD="numactl -m \$NUMA -N \$NUMA \$*" +exec \$CMD +EOF + +chmod +x ./select_gpu + +# GPU_MAP/NUMA_MAP above describe a full node. At fewer than 8 ranks per node +# Slurm hands each task cpus-per-task cores block-wise from core 0, so the NUMA +# domain NUMA_MAP names is not in the task's allowed cpu mask and numactl fails +# with EINVAL before Grid starts. The GPU choice stays valid; only the memory +# binding does not. For the small correctness runs the binding is irrelevant, +# so use a wrapper that picks the GCD and leaves placement to Slurm. +cat << EOF > select_gcd +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +unset ROCR_VISIBLE_DEVICES +export HIP_VISIBLE_DEVICES=\$GPU +exec \$* +EOF + +chmod +x ./select_gcd +SELECT=$SLURM_SUBMIT_DIR/select_gpu +SELECT_NONUMA=$SLURM_SUBMIT_DIR/select_gcd + +root=$HOME/ParallelIO/systems/Frontier/ +source ${root}/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=7 +export MPICH_GPU_SUPPORT_ENABLED=1 + +# Test_aggregate_io writes ref.bin / agg.bin / trunc.bin with *relative* paths, +# so they land in the cwd of each rank, which is the submission directory unless +# we move. Home is NFS; the whole point of the exercise is Lustre. +WORK=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/aggio.$SLURM_JOB_ID +mkdir -p $WORK +cd $WORK + +# Striping has to be set on the directory before the files are created, and +# all three paths inherit it, so the A/B comparison stays fair either way. +# Left unset: at 128 nodes the aggregate path is 2.2x faster on the default +# layout than on -c -1, while MPI-IO is 2.4x slower on it. See the table in +# aggregate_io_scaling.slurm. Set it only to reproduce that interaction. +#lfs setstripe -c 32 -S 8M $WORK +lfs getstripe -d $WORK # record the inherited layout; needed to read the numbers + +BIN=$SLURM_SUBMIT_DIR/tests/IO/Test_aggregate_io + +# ROMIO / Cray MPICH collective buffering report for the reference path. +# Verbose; switch on when you want to see what the old path is doing. +# export MPICH_MPIIO_STATS=1 +# export MPICH_MPIIO_TIMERS=1 + +# Always state ntasks-per-node explicitly. Letting srun derive it from -n +# gives an uneven packing (3,2,2,2 for -n9 over 4 nodes) which no fixed +# localid -> GCD map can describe. Ranks per node must not exceed 8. +run () { # run [extra args...] + local nodes=$1 rpn=$2 gr=$3 mp=$4 note=$5 wrap=$SELECT + local nranks=$(( $1 * $2 )) + shift 5 + if [ $rpn -ne 8 ]; then wrap=$SELECT_NONUMA; fi # NUMA_MAP only valid at 8 + echo + echo "===================================================================" + echo "=== N=$nodes x $rpn = $nranks grid=$gr mpi=$mp $note" + echo "=== extra: $@" + echo "===================================================================" + srun -N$nodes -n$nranks --ntasks-per-node=$rpn $wrap $BIN --grid $gr --mpi $mp \ + --accelerator-threads 8 --shm 2048 --device-mem 32000 "$@" + echo "=== exit $?" +} + +##################################################################### +# Phase 0. Non-uniform AllToAllV, correctness only. +# +# The variable-count branch of CartesianCommunicator::AllToAllV only runs +# when the send counts differ between destinations, which needs an odd +# process factor in an un-split (fast) dimension. It does NOT need a toy +# volume -- see phase 1b, which hits it at 226 MB/rank. The one small run +# kept here is the only way to reach the non-uniform case at k=1 with many +# extents per rank; it is a logic corner, carries --io-reps 0, and reports +# no bandwidth. 2.2.4.4 per rank is 9 KB and means nothing for I/O. +##################################################################### +run 3 3 6.6.4.4 3.3.1.1 "non-uniform at all four targets, k=1 and k=3" --io-reps 0 + +##################################################################### +# Phase 1. Medium volume, 4.8 GB record, 151 MB/rank. +# Correctness plus performance at the production 4MB target only; the +# byte-for-byte comparison reads the whole file on one rank, so the +# full four target sweep is not affordable above a few GB. +##################################################################### +run 4 8 32.32.64.128 4.4.2.1 "4.8 GB, correctness + perf" --aggregate-target 4194304 + +##################################################################### +# Phase 1b. Non-uniform AllToAllV at a volume that means something. +# 24 ranks = 3 nodes x 8, so the NUMA map stays valid. +# 48.48.64.64 / 3.2.2.2 k=2 row=6 <=6 extents of 13.5 MB NON-UNIFORM +# 5.4 GB record, 226 MB/rank. Correctness and performance together, so +# the variable-count path is covered where the bandwidth is also real. +##################################################################### +run 3 8 48.48.64.64 3.2.2.2 "5.4 GB, NON-UNIFORM counts, correctness + perf" --aggregate-target 4194304 + +##################################################################### +# Phase 2. Production volume, 19.3 GB record, 604 MB/rank. +# Correctness off. Modelled extents at the 4MB target (fobj = 576 B): +# 1.1.4.8 k=2 row=1 16 extents of 36 MB <- old path gives 36 KB runs +# 4.4.2.1 k=2 row=16 8 extents of 72 MB <- old path gives 9 KB runs +# 2.2.2.4 k=2 row=4 8 extents +# 8.4.1.1 k=2 row=32 4 extents +##################################################################### +run 4 8 64.64.64.128 1.1.4.8 "19.3 GB, baseline decomposition" --aggregate-target 4194304 --io-no-correctness +run 4 8 64.64.64.128 4.4.2.1 "19.3 GB, row of 32" --aggregate-target 4194304 --io-no-correctness +run 4 8 64.64.64.128 2.2.2.4 "19.3 GB, row of 4" --aggregate-target 4194304 --io-no-correctness +run 4 8 64.64.64.128 8.4.1.1 "19.3 GB, row of 32, x heavy" --aggregate-target 4194304 --io-no-correctness + +# 10.9 GB with non-uniform counts, 453 MB/rank, 27 MB extents, 24 ranks. +run 3 8 48.96.64.64 3.4.2.1 "10.9 GB, NON-UNIFORM counts" --aggregate-target 4194304 --io-no-correctness + +##################################################################### +# Phase 3. Large volume, 43.5 GB record, 1.36 GB/rank. +# Host side buffering is 3 copies of the local slice, so ~4 GB/rank +# and ~33 GB/node. Two repetitions only. +##################################################################### +run 4 8 64.64.96.192 1.1.4.8 "43.5 GB, baseline" --aggregate-target 4194304 --io-no-correctness --io-reps 2 +run 4 8 64.64.96.192 4.4.2.1 "43.5 GB, row of 32" --aggregate-target 4194304 --io-no-correctness --io-reps 2 + +echo +echo "=== done. Output left in $WORK" +ls -l $WORK