From b49db84b089bc8d91056bd878da5affa42c62bb4 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 1 Jun 2022 19:27:42 -0400 Subject: [PATCH] Slurm updates --- systems/Crusher/dwf8.slurm | 24 +++++------------------- systems/Crusher/sourceme.sh | 2 +- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/systems/Crusher/dwf8.slurm b/systems/Crusher/dwf8.slurm index 2113dcd1..866ec775 100644 --- a/systems/Crusher/dwf8.slurm +++ b/systems/Crusher/dwf8.slurm @@ -6,9 +6,8 @@ #SBATCH -J DWF #SBATCH -o DWF.%J #SBATCH -e DWF.%J -#SBATCH -N 8 -#SBATCH -n 64 -#SBATCH --exclusive +#SBATCH -N 1 +#SBATCH -n 1 #SBATCH --gpu-bind=map_gpu:0,1,2,3,7,6,5,4 DIR=. @@ -17,25 +16,12 @@ source setup.sh export MPICH_OFI_NIC_POLICY=GPU export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 export MPICH_GPU_SUPPORT_ENABLED=1 -export MPICH_SMP_SINGLE_COPY_MODE=XPMEM +#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM #export MPICH_SMP_SINGLE_COPY_MODE=CMA #export MPICH_SMP_SINGLE_COPY_MODE=NONE -export OMP_NUM_THREADS=1 +export OMP_NUM_THREADS=16 echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE -for vol in 64.64.64.256 64.64.64.128 32.32.32.256 32.32.32.128 -do -PARAMS=" --accelerator-threads 8 --grid $vol --mpi 2.2.2.8 --comms-overlap --shm 2048 --shm-mpi 1" -echo $PARAMS -srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_dwf_fp32 $PARAMS > dwf.${vol}.8node.shm-mpi1 -done - -PARAMS=" --accelerator-threads 8 --grid 64.64.64.32 --mpi 2.2.2.8 --comms-overlap --shm 2048 --shm-mpi 1" -echo $PARAMS -srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_ITT $PARAMS > itt.8node - -PARAMS=" --accelerator-threads 8 --grid 64.64.64.32 --mpi 2.2.2.8 --comms-overlap --shm 2048 --shm-mpi 0" -echo $PARAMS -srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_ITT $PARAMS > itt.8node_shm0 +srun --gpus-per-task 1 -N1 -n1 ./tests/Test_dwf_mixedcg_prec diff --git a/systems/Crusher/sourceme.sh b/systems/Crusher/sourceme.sh index 051014dc..ad0d6582 100644 --- a/systems/Crusher/sourceme.sh +++ b/systems/Crusher/sourceme.sh @@ -1,6 +1,6 @@ module load PrgEnv-gnu module load rocm/5.1.0 -module load cray-mpich/8.1.15 +module load cray-mpich/8.1.16 module load gmp #module load cray-fftw module load craype-accel-amd-gfx90a