diff --git a/systems/Spock/dwf.slurm b/systems/Spock/dwf.slurm new file mode 100644 index 00000000..7144a270 --- /dev/null +++ b/systems/Spock/dwf.slurm @@ -0,0 +1,26 @@ +#!/bin/bash +# Begin LSF Directives +#SBATCH -A LGT104 +#SBATCH -t 01:00:00 +##SBATCH -U openmpThu +#SBATCH -p ecp +#SBATCH -J DWF +#SBATCH -o DWF.%J +#SBATCH -e DWF.%J +#SBATCH -N 1 +#SBATCH -n 1 + +DIR=. +module list +export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 +export MPICH_GPU_SUPPORT_ENABLED=1 +#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM +#export MPICH_SMP_SINGLE_COPY_MODE=NONE +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export OMP_NUM_THREADS=8 + +AT=8 +echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE +PARAMS=" --accelerator-threads ${AT} --grid 32.32.32.32 --mpi 1.1.1.1 --comms-overlap" +srun -n1 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS + diff --git a/systems/Spock/dwf4.slurm b/systems/Spock/dwf4.slurm new file mode 100644 index 00000000..02a7b888 --- /dev/null +++ b/systems/Spock/dwf4.slurm @@ -0,0 +1,26 @@ +#!/bin/bash +# Begin LSF Directives +#SBATCH -A LGT104 +#SBATCH -t 01:00:00 +##SBATCH -U openmpThu +#SBATCH -p ecp +#SBATCH -J DWF +#SBATCH -o DWF.%J +#SBATCH -e DWF.%J +#SBATCH -N 1 +#SBATCH -n 4 + +DIR=. +module list +export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0 +export MPICH_GPU_SUPPORT_ENABLED=1 +#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM +#export MPICH_SMP_SINGLE_COPY_MODE=NONE +export MPICH_SMP_SINGLE_COPY_MODE=CMA +export OMP_NUM_THREADS=8 + +AT=8 +echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE +PARAMS=" --accelerator-threads ${AT} --grid 32.32.64.64 --mpi 1.1.2.2 --comms-overlap --shm 2048 --shm-mpi 0" +srun -n4 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS + diff --git a/systems/Spock/mpiwrapper.sh b/systems/Spock/mpiwrapper.sh new file mode 100755 index 00000000..76c4e364 --- /dev/null +++ b/systems/Spock/mpiwrapper.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +lrank=$SLURM_LOCALID + +export ROCR_VISIBLE_DEVICES=$SLURM_LOCALID + +echo "`hostname` - $lrank device=$ROCR_VISIBLE_DEVICES binding=$BINDING" + +$* + + +