#!/bin/bash # Standalone MPI-only I/O reproducer on Aurora. Host only -- no SYCL, no # gpu_tile_compact.sh -- so unlike halo_mpi this needs nothing but MPI: # # mpicxx -O2 -std=c++11 io_mpi.cc -o io_mpi # # 12 ranks per node, one per tile, which is what the machine is. That is a # deliberate difference from io_frontier.slurm (8 per node, one per GCD), so # read the comparison carefully -- see WHAT IS AND IS NOT COMPARABLE below. # Run io_aurora_debug.pbs first. If the cross validation fails there, this # scan is 2 hours of 128 nodes producing numbers for a broken file. #PBS -q prod #PBS -l filesystems=flare #PBS -l filesystems=home #PBS -l select=128 #PBS -l walltime=02:00:00 #PBS -A 15479 cd $PBS_O_WORKDIR cp $PBS_NODEFILE nodefile # Only if mpiexec is not already in the environment. io_mpi is host only # and needs no part of the Grid build environment. #source ../../sourceme.sh ########################################################################## # Environment. io_mpi has no OpenMP and never touches a GPU, so one thread # per rank and a NUMA NIC policy rather than a GPU one. # # The MPICH_DBG_* variables are deliberately absent: at 1536 ranks they # produce gigabytes of log and perturb the timings they would explain. # MPICH_MPIIO_STATS/TIMERS are also off here -- they are per collective and # 1536 ranks x 6 rungs x 3 reps is unreadable. Get them from the debug run. ########################################################################## export OMP_NUM_THREADS=1 export MPICH_CH4_SHM=XPMEM export MPICH_OFI_NIC_POLICY=NUMA ########################################################################## # WHAT IS AND IS NOT COMPARABLE WITH THE FRONTIER SCAN # # Held identical at every rung of both scans: # local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB/rank # file view 32768 contiguous runs of 4608 B per rank # aggregation k=2, row of 16, 8 extents of 18 MB # (verified: 4.4.3.1 at 48 ranks and 4.4.3.2 at 96 ranks give exactly the # same plan as Frontier's 4.4.2.1 at 32 ranks.) # # NOT identical, because 12 ranks/node is 1.5x the clients per node: # record size at a given NODE count is 1.5x Frontier's # client count at a given NODE count is 1.5x Frontier's # # So compare the two machines at equal RANK count (Aurora 4 nodes vs # Frontier 6, and so on) if what you want is equal client count and equal # record size; compare at equal NODE count if what you want is each machine # used as it is meant to be used. Both are legitimate, they answer # different questions, and a table that does not say which one it is # reporting is worthless. The quantity that carries the MPI-IO pathology -- # per rank local volume and the resulting file view -- is invariant either # way, which is the point. ########################################################################## ########################################################################## # WHICH FILESYSTEM. Point this at Lustre for the like-for-like comparison # with Frontier's Orion. DAOS is a different architecture -- its numbers # are interesting but they are NOT a reproduction of the Frontier result, # and mixing them into one table would misrepresent both. Label every set # of numbers with the filesystem it came from. ########################################################################## # PROJECT is the flare project DIRECTORY name, which is not the -A account # number. Set it once; the mkdir below fails loudly rather than writing # somewhere unintended. PROJECT=LatticeQCD_aesp_CNDA WORK=/lus/flare/projects/$PROJECT/$USER/iompi.$PBS_JOBID mkdir -p $WORK || { echo "cannot create $WORK -- set PROJECT correctly"; exit 1; } cd $WORK # Match Frontier's default: no explicit striping. Record what was inherited. lfs getstripe -d $WORK 2>/dev/null || echo "(no lfs getstripe -- not Lustre?)" # The largest rung writes three files of 232 GB, so budget ~700 GB and check # the quota before submitting. Each run unlinks the three files first, so # that is peak usage, not cumulative. NRANKS=12 # one per tile BIN=$PBS_O_WORKDIR/io_mpi ARGS="--target 4194304 --reps 3" run () { # run [extra args...] local nodes=$1 gr=$2 mp=$3 note=$4 local ntot=$(( nodes * NRANKS )) shift 4 echo echo "===================================================================" echo "=== nodes=$nodes ranks=$ntot grid=$gr mpi=$mp $note" echo "=== extra: $@" echo "===================================================================" mpiexec -np $ntot -ppn $NRANKS -envall $BIN --grid $gr --mpi $mp $ARGS "$@" echo "=== exit $?" } ##################################################################### # Phase 0. Correctness. Both count branches of MPI_Alltoallv are # covered; the labels below were checked, not assumed. The whole-file # crc32 is serial, so keep these small. ##################################################################### run 1 16.16.16.24 2.2.1.3 "correctness, UNIFORM counts, row of 4" --reps 0 --serial-crc run 2 24.12.8.8 3.2.2.2 "correctness, NON-UNIFORM counts, row of 12" --reps 0 --serial-crc run 4 16.16.32.24 2.2.4.3 "correctness, NON-UNIFORM, non-zero offset" --reps 0 --serial-crc --offset 1024 ##################################################################### # Phase 1. Weak scan at 151 MB/rank. Identical plan at every rung: # k=2, row of 16, 8 extents of 18 MB, 32768 runs of 4608 B in the view. ##################################################################### # nodes global lattice mpi record run 4 32.32.96.128 4.4.3.1 "7.2 GB" --no-validate run 8 32.32.96.256 4.4.3.2 "14.5 GB" --no-validate run 16 32.32.96.512 4.4.3.4 "29.0 GB" --no-validate run 32 32.32.192.512 4.4.6.4 "58.0 GB" --no-validate run 64 32.32.192.1024 4.4.6.8 "116.0 GB" --no-validate run 128 32.32.384.1024 4.4.12.8 "231.9 GB" --no-validate ##################################################################### # Phase 2. The three questions a reviewer asks immediately. ##################################################################### run 128 32.32.384.1024 4.4.12.8 "231.9 GB, durable" --no-validate --fsync --drop-cache run 128 32.32.384.1024 4.4.12.8 "231.9 GB, cb hints" --no-validate \ --hints romio_cb_write=enable,romio_cb_read=enable,cb_nodes=128,cb_buffer_size=16777216 run 128 32.32.384.1024 4.4.12.8 "231.9 GB, mem subarray" --no-validate --mem-subarray echo echo "=== done. Files left in $WORK" ls -l $WORK