From 1f55c13bc4fc84a1959b09b196f884661980e2a4 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Tue, 11 Aug 2026 13:37:50 -0400 Subject: [PATCH] Test script for Frontier on I/O --- systems/Frontier/aggregate_io_scaling.slurm | 104 ++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 systems/Frontier/aggregate_io_scaling.slurm diff --git a/systems/Frontier/aggregate_io_scaling.slurm b/systems/Frontier/aggregate_io_scaling.slurm new file mode 100644 index 000000000..f0107e726 --- /dev/null +++ b/systems/Frontier/aggregate_io_scaling.slurm @@ -0,0 +1,104 @@ +#!/bin/bash -l + +# Node count scan. Everything that could confound the comparison is held +# fixed and only the number of Lustre clients changes: +# +# local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB / rank +# aggregation k=2, row of 16, 8 extents of 18 MB (identical at +# every node count) +# old path contiguous run = lLattice[0]*576 = 4.6 KB (identical too) +# +# So the ratio of contiguous extent between the two layout preserving paths is +# a constant 4000x across the whole scan, and any change in their relative +# bandwidth is a property of the client count alone. That is the variable the +# 4 node runs could not move. +# +# Trim the tail of the run list if you want a smaller allocation; the +# geometries are independent. + +#SBATCH --job-name=ioScale +#SBATCH --nodes=128 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-node=8 +#SBATCH --time=02:00:00 +#SBATCH --account=phy157_dwf +#SBATCH --gpu-bind=none +#SBATCH --exclusive +#SBATCH --mem=0 + +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3 7 6 5 4) +export NUMA_MAP=(3 3 1 1 2 2 0 0) +export GPU=\${GPU_MAP[\$SLURM_LOCALID]} +export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]} +unset ROCR_VISIBLE_DEVICES +export HIP_VISIBLE_DEVICES=\$GPU + +rank=\$SLURM_PROCID +CMD="numactl -m \$NUMA -N \$NUMA \$*" +exec \$CMD +EOF + +chmod +x ./select_gpu +SELECT=$SLURM_SUBMIT_DIR/select_gpu + +root=$HOME/ParallelIO/systems/Frontier/ +source ${root}/sourceme-rocm7.2.sh + +export OMP_NUM_THREADS=7 +export MPICH_GPU_SUPPORT_ENABLED=1 + +WORK=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/aggscale.$SLURM_JOB_ID +mkdir -p $WORK +cd $WORK + +# The largest rung writes three files of 154 GB, so budget ~700 GB for the +# whole scan and check the quota before submitting. +# +# Do NOT stripe by default. Measured on Frontier at 128 nodes / 1024 ranks, +# 154.6 GB record, best of 3 (MB/s): +# +# default layout lfs setstripe -c -1 -S 8M +# raw 97673 81728 +# MPI-IO lexicographic 31675 77534 +# aggregate 189535 86063 +# +# The two layout preserving paths want opposite things. Wide striping is what +# rescues MPI-IO at scale, 32 -> 78 GB/s, which is presumably what careful +# striping has been buying in production. It costs the aggregate path a factor +# of 2.2, and costs raw as well. With the default layout MPI-IO plateaus at +# ~30 GB/s from 64 nodes upward -- the 20-40 GB/s seen in production -- while +# the aggregate path keeps scaling. Uncomment only to reproduce that. +#lfs setstripe -c -1 -S 8M $WORK +lfs getstripe -d $WORK # record the inherited layout; needed to read the numbers + +BIN=$SLURM_SUBMIT_DIR/tests/IO/Test_aggregate_io +ARGS="--aggregate-target 4194304 --io-no-correctness --io-reps 3" + +run () { # run + local nodes=$1 nranks=$(( $1 * 8 )) + echo + echo "===================================================================" + echo "=== SCAN nodes=$nodes ranks=$nranks grid=$2 mpi=$3 $4" + echo "===================================================================" + srun -N$nodes -n$nranks --ntasks-per-node=8 $SELECT $BIN --grid $2 --mpi $3 \ + --accelerator-threads 8 --shm 2048 --device-mem 32000 $ARGS + echo "=== exit $?" +} + +# nodes global lattice decomposition record +run 4 32.32.64.128 4.4.2.1 "4.8 GB" +run 8 32.32.64.256 4.4.2.2 "9.7 GB" +run 16 32.32.64.512 4.4.2.4 "19.3 GB" +run 32 32.32.128.512 4.4.4.4 "38.6 GB" +run 64 32.32.128.1024 4.4.4.8 "77.3 GB" +run 128 32.32.256.1024 4.4.8.8 "154.6 GB" + +echo +echo "=== done. Files left in $WORK" +echo "=== For cold read numbers, submit a SECOND job on a fresh allocation" +echo "=== with WORK pointed at this directory and --io-read-only added to" +echo "=== ARGS. Reading back what this job wrote only measures page cache." +ls -l $WORK