From 8079dc2a145226b455608e87646992e737638db3 Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 22 Nov 2021 20:45:44 -0500 Subject: [PATCH] Cray MPI not working right yet --- systems/Spock/comms.slurm | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 systems/Spock/comms.slurm diff --git a/systems/Spock/comms.slurm b/systems/Spock/comms.slurm new file mode 100644 index 00000000..0841fda0 --- /dev/null +++ b/systems/Spock/comms.slurm @@ -0,0 +1,26 @@ +#!/bin/bash +# Begin LSF Directives +#SBATCH -A LGT104 +#SBATCH -t 01:00:00 +##SBATCH -U openmpThu +#SBATCH -p ecp +#SBATCH -J comms +#SBATCH -o comms.%J +#SBATCH -e comms.%J +#SBATCH -N 1 +#SBATCH -n 2 + +DIR=. +module list +export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 +export MPICH_GPU_SUPPORT_ENABLED=1 +#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM +#export MPICH_SMP_SINGLE_COPY_MODE=CMA +export MPICH_SMP_SINGLE_COPY_MODE=NONE +export OMP_NUM_THREADS=8 + +AT=8 +echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE +PARAMS=" --accelerator-threads ${AT} --grid 64.64.32.32 --mpi 2.1.1.1 " +srun -n2 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_comms_host_device $PARAMS +