mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-12 13:53:30 +01:00
105 lines
3.9 KiB
Bash
105 lines
3.9 KiB
Bash
#!/bin/bash -l
|
|
|
|
# Node count scan. Everything that could confound the comparison is held
|
|
# fixed and only the number of Lustre clients changes:
|
|
#
|
|
# local volume 8.8.32.128 = 262144 sites x 576 B = 151 MB / rank
|
|
# aggregation k=2, row of 16, 8 extents of 18 MB (identical at
|
|
# every node count)
|
|
# old path contiguous run = lLattice[0]*576 = 4.6 KB (identical too)
|
|
#
|
|
# So the ratio of contiguous extent between the two layout preserving paths is
|
|
# a constant 4000x across the whole scan, and any change in their relative
|
|
# bandwidth is a property of the client count alone. That is the variable the
|
|
# 4 node runs could not move.
|
|
#
|
|
# Trim the tail of the run list if you want a smaller allocation; the
|
|
# geometries are independent.
|
|
|
|
#SBATCH --job-name=ioScale
|
|
#SBATCH --nodes=128
|
|
#SBATCH --ntasks-per-node=8
|
|
#SBATCH --cpus-per-task=7
|
|
#SBATCH --gpus-per-node=8
|
|
#SBATCH --time=02:00:00
|
|
#SBATCH --account=phy157_dwf
|
|
#SBATCH --gpu-bind=none
|
|
#SBATCH --exclusive
|
|
#SBATCH --mem=0
|
|
|
|
cat << EOF > select_gpu
|
|
#!/bin/bash
|
|
export GPU_MAP=(0 1 2 3 7 6 5 4)
|
|
export NUMA_MAP=(3 3 1 1 2 2 0 0)
|
|
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
|
|
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
|
|
unset ROCR_VISIBLE_DEVICES
|
|
export HIP_VISIBLE_DEVICES=\$GPU
|
|
|
|
rank=\$SLURM_PROCID
|
|
CMD="numactl -m \$NUMA -N \$NUMA \$*"
|
|
exec \$CMD
|
|
EOF
|
|
|
|
chmod +x ./select_gpu
|
|
SELECT=$SLURM_SUBMIT_DIR/select_gpu
|
|
|
|
root=$HOME/ParallelIO/systems/Frontier/
|
|
source ${root}/sourceme-rocm7.2.sh
|
|
|
|
export OMP_NUM_THREADS=7
|
|
export MPICH_GPU_SUPPORT_ENABLED=1
|
|
|
|
WORK=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/aggscale.$SLURM_JOB_ID
|
|
mkdir -p $WORK
|
|
cd $WORK
|
|
|
|
# The largest rung writes three files of 154 GB, so budget ~700 GB for the
|
|
# whole scan and check the quota before submitting.
|
|
#
|
|
# Do NOT stripe by default. Measured on Frontier at 128 nodes / 1024 ranks,
|
|
# 154.6 GB record, best of 3 (MB/s):
|
|
#
|
|
# default layout lfs setstripe -c -1 -S 8M
|
|
# raw 97673 81728
|
|
# MPI-IO lexicographic 31675 77534
|
|
# aggregate 189535 86063
|
|
#
|
|
# The two layout preserving paths want opposite things. Wide striping is what
|
|
# rescues MPI-IO at scale, 32 -> 78 GB/s, which is presumably what careful
|
|
# striping has been buying in production. It costs the aggregate path a factor
|
|
# of 2.2, and costs raw as well. With the default layout MPI-IO plateaus at
|
|
# ~30 GB/s from 64 nodes upward -- the 20-40 GB/s seen in production -- while
|
|
# the aggregate path keeps scaling. Uncomment only to reproduce that.
|
|
#lfs setstripe -c -1 -S 8M $WORK
|
|
lfs getstripe -d $WORK # record the inherited layout; needed to read the numbers
|
|
|
|
BIN=$SLURM_SUBMIT_DIR/tests/IO/Test_aggregate_io
|
|
ARGS="--aggregate-target 4194304 --io-no-correctness --io-reps 3"
|
|
|
|
run () { # run <nodes> <grid> <mpi> <comment>
|
|
local nodes=$1 nranks=$(( $1 * 8 ))
|
|
echo
|
|
echo "==================================================================="
|
|
echo "=== SCAN nodes=$nodes ranks=$nranks grid=$2 mpi=$3 $4"
|
|
echo "==================================================================="
|
|
srun -N$nodes -n$nranks --ntasks-per-node=8 $SELECT $BIN --grid $2 --mpi $3 \
|
|
--accelerator-threads 8 --shm 2048 --device-mem 32000 $ARGS
|
|
echo "=== exit $?"
|
|
}
|
|
|
|
# nodes global lattice decomposition record
|
|
run 4 32.32.64.128 4.4.2.1 "4.8 GB"
|
|
run 8 32.32.64.256 4.4.2.2 "9.7 GB"
|
|
run 16 32.32.64.512 4.4.2.4 "19.3 GB"
|
|
run 32 32.32.128.512 4.4.4.4 "38.6 GB"
|
|
run 64 32.32.128.1024 4.4.4.8 "77.3 GB"
|
|
run 128 32.32.256.1024 4.4.8.8 "154.6 GB"
|
|
|
|
echo
|
|
echo "=== done. Files left in $WORK"
|
|
echo "=== For cold read numbers, submit a SECOND job on a fresh allocation"
|
|
echo "=== with WORK pointed at this directory and --io-read-only added to"
|
|
echo "=== ARGS. Reading back what this job wrote only measures page cache."
|
|
ls -l $WORK
|