From 1fddd2c29b230f967606a5b686fca8dea5244787 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 13 Aug 2026 18:54:00 -0400 Subject: [PATCH] Updates to IO regression example for MPICH issue --- MPI_benchmark/io_aurora.pbs | 116 ++++++++++++++++++++++++++++++++ MPI_benchmark/io_frontier.slurm | 14 ++++ 2 files changed, 130 insertions(+) create mode 100644 MPI_benchmark/io_aurora.pbs diff --git a/MPI_benchmark/io_aurora.pbs b/MPI_benchmark/io_aurora.pbs new file mode 100644 index 000000000..579cd0e40 --- /dev/null +++ b/MPI_benchmark/io_aurora.pbs @@ -0,0 +1,116 @@ +#!/bin/bash + +# Standalone MPI-only I/O reproducer on Aurora. Host only -- no SYCL, no +# gpu_tile_compact.sh -- so unlike halo_mpi this needs nothing but MPI: +# +# mpicxx -O2 -std=c++11 io_mpi.cc -o io_mpi +# +# 12 ranks per node, one per tile, which is what the machine is. That is a +# deliberate difference from io_frontier.slurm (8 per node, one per GCD), so +# read the comparison carefully -- see WHAT IS AND IS NOT COMPARABLE below. + +#PBS -l select=128 +#PBS -l walltime=02:00:00 +#PBS -l filesystems=flare +#PBS -q prod +#PBS -A LatticeQCD_aesp_CNDA + +cd $PBS_O_WORKDIR + +########################################################################## +# WHAT IS AND IS NOT COMPARABLE WITH THE FRONTIER SCAN +# +# Held identical at every rung of both scans: +# local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB/rank +# file view 32768 contiguous runs of 4608 B per rank +# aggregation k=2, row of 16, 8 extents of 18 MB +# (verified: 4.4.3.1 at 48 ranks and 4.4.3.2 at 96 ranks give exactly the +# same plan as Frontier's 4.4.2.1 at 32 ranks.) +# +# NOT identical, because 12 ranks/node is 1.5x the clients per node: +# record size at a given NODE count is 1.5x Frontier's +# client count at a given NODE count is 1.5x Frontier's +# +# So compare the two machines at equal RANK count (Aurora 4 nodes vs +# Frontier 6, and so on) if what you want is equal client count and equal +# record size; compare at equal NODE count if what you want is each machine +# used as it is meant to be used. Both are legitimate, they answer +# different questions, and a table that does not say which one it is +# reporting is worthless. The quantity that carries the MPI-IO pathology -- +# per rank local volume and the resulting file view -- is invariant either +# way, which is the point. +########################################################################## + +########################################################################## +# WHICH FILESYSTEM. Point this at Lustre for the like-for-like comparison +# with Frontier's Orion. DAOS is a different architecture -- its numbers +# are interesting but they are NOT a reproduction of the Frontier result, +# and mixing them into one table would misrepresent both. Label every set +# of numbers with the filesystem it came from. +########################################################################## +WORK=/lus/flare/projects/LatticeQCD_aesp_CNDA/$USER/iompi.$PBS_JOBID +mkdir -p $WORK +cd $WORK + +# Match Frontier's default: no explicit striping. Record what was inherited. +lfs getstripe -d $WORK 2>/dev/null || echo "(no lfs getstripe -- not Lustre?)" + +# The largest rung writes three files of 232 GB, so budget ~700 GB and check +# the quota before submitting. Each run unlinks the three files first, so +# that is peak usage, not cumulative. + +NRANKS=12 # one per tile + +BIN=$PBS_O_WORKDIR/io_mpi +ARGS="--target 4194304 --reps 3" + +# MPICH's own account of what it did; the first thing anyone reading the +# report will ask for. +# export MPICH_MPIIO_STATS=1 +# export MPICH_MPIIO_TIMERS=1 + +run () { # run [extra args...] + local nodes=$1 gr=$2 mp=$3 note=$4 + local ntot=$(( nodes * NRANKS )) + shift 4 + echo + echo "===================================================================" + echo "=== nodes=$nodes ranks=$ntot grid=$gr mpi=$mp $note" + echo "=== extra: $@" + echo "===================================================================" + mpiexec -np $ntot -ppn $NRANKS $BIN --grid $gr --mpi $mp $ARGS "$@" + echo "=== exit $?" +} + +##################################################################### +# Phase 0. Correctness. Both count branches of MPI_Alltoallv are +# covered; the labels below were checked, not assumed. The whole-file +# crc32 is serial, so keep these small. +##################################################################### +run 1 16.16.16.24 2.2.1.3 "correctness, UNIFORM counts, row of 4" --reps 0 --serial-crc +run 2 24.12.8.8 3.2.2.2 "correctness, NON-UNIFORM counts, row of 12" --reps 0 --serial-crc +run 4 16.16.32.24 2.2.4.3 "correctness, NON-UNIFORM, non-zero offset" --reps 0 --serial-crc --offset 1024 + +##################################################################### +# Phase 1. Weak scan at 151 MB/rank. Identical plan at every rung: +# k=2, row of 16, 8 extents of 18 MB, 32768 runs of 4608 B in the view. +##################################################################### +# nodes global lattice mpi record +run 4 32.32.96.128 4.4.3.1 "7.2 GB" --no-validate +run 8 32.32.96.256 4.4.3.2 "14.5 GB" --no-validate +run 16 32.32.96.512 4.4.3.4 "29.0 GB" --no-validate +run 32 32.32.192.512 4.4.6.4 "58.0 GB" --no-validate +run 64 32.32.192.1024 4.4.6.8 "116.0 GB" --no-validate +run 128 32.32.384.1024 4.4.12.8 "231.9 GB" --no-validate + +##################################################################### +# Phase 2. The three questions a reviewer asks immediately. +##################################################################### +run 128 32.32.384.1024 4.4.12.8 "231.9 GB, durable" --no-validate --fsync --drop-cache +run 128 32.32.384.1024 4.4.12.8 "231.9 GB, cb hints" --no-validate \ + --hints romio_cb_write=enable,romio_cb_read=enable,cb_nodes=128,cb_buffer_size=16777216 +run 128 32.32.384.1024 4.4.12.8 "231.9 GB, mem subarray" --no-validate --mem-subarray + +echo +echo "=== done. Files left in $WORK" +ls -l $WORK diff --git a/MPI_benchmark/io_frontier.slurm b/MPI_benchmark/io_frontier.slurm index 498a4d536..49bd0effe 100644 --- a/MPI_benchmark/io_frontier.slurm +++ b/MPI_benchmark/io_frontier.slurm @@ -15,6 +15,20 @@ # # so any change in the relative bandwidth of the two lexicographic paths is a # property of the client count alone. +# +# The PERF lines are MiB/s (bytes/1024/1024/s), which is what BinaryIO.h +# computes for lastPerf.mbytesPerSecond and prints as "MB/s", so the two +# tools can be compared directly. Grid's timed region is used here too: +# barrier, start, [plan build + exchange + I/O], barrier, stop, quoting the +# boss rank's stopwatch. --reuse-plan hoists the plan build out, which is +# how to show it is not where the time goes; do not use it when comparing +# against Grid's own numbers. +# +# io_aurora.pbs runs 12 ranks per node, one per tile, because that is what +# that machine is. The per rank local volume and the file view are the same +# there as here, but the record size and client count at a given NODE count +# are 1.5x. See the header of that script before tabulating the two +# together. #SBATCH --job-name=ioMPI #SBATCH --nodes=128