#!/bin/bash
#SBATCH -J dslash
#SBATCH -A tc002
#SBATCH -t 2:20:00
#SBATCH --nodelist=tu-c0r0n[00,03,06,09]
#SBATCH --exclusive
#SBATCH --nodes=4
#SBATCH --ntasks=16
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=8
#SBATCH --time=12:00:00
#SBATCH --partition=gpu
#SBATCH --gres=gpu:4
#SBATCH --output=%x.%j.out
#SBATCH --error=%x.%j.err

export OMP_NUM_THREADS=4
export OMPI_MCA_btl=^uct,openib
export UCX_TLS=gdr_copy,rc,rc_x,sm,cuda_copy,cuda_ipc
export UCX_RNDV_SCHEME=put_zcopy
export UCX_RNDV_THRESH=16384
export UCX_IB_GPU_DIRECT_RDMA=yes
export UCX_MEMTYPE_CACHE=n
OPT="--comms-overlap --comms-concurrent"


mpirun -np $SLURM_NTASKS -x LD_LIBRARY_PATH  --bind-to none \
	./mpiwrapper.sh \
	./benchmarks/Benchmark_dwf_fp32 \
	$OPT \
	--mpi 2.2.2.2 \
	--accelerator-threads 8 \
	--grid 64.64.64.64 \
	--shm 2048 > dwf.4node.perf