2022-02-14 20:09:08 +00:00
|
|
|
#!/bin/bash
|
|
|
|
|
# Begin LSF Directives
|
|
|
|
|
#SBATCH -A LGT104
|
2022-02-14 22:55:16 +00:00
|
|
|
#SBATCH -t 01:00:00
|
|
|
|
|
##SBATCH -U openmpThu
|
2022-02-14 20:09:08 +00:00
|
|
|
#SBATCH -J DWF
|
|
|
|
|
#SBATCH -o DWF.%J
|
|
|
|
|
#SBATCH -e DWF.%J
|
2022-03-03 22:00:37 +00:00
|
|
|
#SBATCH -N 8
|
|
|
|
|
#SBATCH -n 64
|
2022-02-14 22:55:16 +00:00
|
|
|
#SBATCH --exclusive
|
2022-03-03 22:00:37 +00:00
|
|
|
#SBATCH --gpu-bind=map_gpu:0,1,2,3,7,6,5,4
|
2022-02-14 20:09:08 +00:00
|
|
|
|
|
|
|
|
DIR=.
|
|
|
|
|
module list
|
2022-03-03 22:00:37 +00:00
|
|
|
export MPICH_OFI_NIC_POLICY=GPU
|
2022-02-14 20:09:08 +00:00
|
|
|
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
|
|
|
|
|
export MPICH_GPU_SUPPORT_ENABLED=1
|
2022-03-03 22:00:37 +00:00
|
|
|
#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
|
2022-02-14 20:09:08 +00:00
|
|
|
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
|
2022-03-03 22:00:37 +00:00
|
|
|
export MPICH_SMP_SINGLE_COPY_MODE=NONE
|
2022-02-14 20:09:08 +00:00
|
|
|
export OMP_NUM_THREADS=1
|
|
|
|
|
|
|
|
|
|
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
|
2022-02-14 22:55:16 +00:00
|
|
|
|
2022-03-03 22:00:37 +00:00
|
|
|
PARAMS=" --accelerator-threads 16 --grid 64.64.64.256 --mpi 2.2.2.8 --comms-overlap --shm 2048 --shm-mpi 0"
|
|
|
|
|
echo $PARAMS
|
|
|
|
|
#srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_dwf_fp32 $PARAMS > dwf.64.64.64.256.8node
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
PARAMS=" --accelerator-threads 16 --grid 64.64.64.32 --mpi 4.4.4.1 --comms-overlap --shm 2048 --shm-mpi 1"
|
|
|
|
|
echo $PARAMS
|
|
|
|
|
srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_dwf_fp32 $PARAMS > dwf.64.64.64.32.8node
|
|
|
|
|
|
|
|
|
|
PARAMS=" --accelerator-threads 16 --grid 64.64.64.32 --mpi 4.4.4.1 --comms-overlap --shm 2048 --shm-mpi 0"
|
|
|
|
|
echo $PARAMS
|
|
|
|
|
#srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_dwf_fp32 $PARAMS > dwf.64.64.64.32.8node.shm0
|
|
|
|
|
|
|
|
|
|
PARAMS=" --accelerator-threads 16 --grid 64.64.64.32 --mpi 2.2.2.8 --comms-overlap --shm 2048 --shm-mpi 1"
|
|
|
|
|
echo $PARAMS
|
|
|
|
|
#srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_ITT $PARAMS > itt.8node
|
|
|
|
|
|
|
|
|
|
PARAMS=" --accelerator-threads 16 --grid 64.64.64.32 --mpi 2.2.2.8 --comms-overlap --shm 2048 --shm-mpi 0"
|
|
|
|
|
echo $PARAMS
|
|
|
|
|
#srun --gpus-per-task 1 -N8 -n64 ./benchmarks/Benchmark_ITT $PARAMS > itt.8node_shm0
|
|
|
|
|
|
2022-02-14 20:09:08 +00:00
|
|
|
|
|
|
|
|
|
