From a2cefaa53afcd8a5f9e05f8e7639f17bb8bfc20c Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 23 Sep 2022 16:49:14 -0400
Subject: [PATCH] Faster

---
 systems/Crusher/config-command |  6 ++++--
 systems/Crusher/dwf.slurm      | 23 +++++++++++++----------
 systems/Crusher/sourceme.sh    |  8 ++++++--
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/systems/Crusher/config-command b/systems/Crusher/config-command
index dfd8d127..854c2c01 100644
--- a/systems/Crusher/config-command
+++ b/systems/Crusher/config-command
@@ -6,10 +6,12 @@
 --enable-simd=GPU \
 --disable-fermion-reps \
 --with-gmp=$OLCF_GMP_ROOT \
+--with-fftw=$FFTW_DIR/.. \
 --with-mpfr=/opt/cray/pe/gcc/mpfr/3.1.4/ \
 --enable-gparity \
 CXX=hipcc MPICXX=mpicxx \
-CXXFLAGS="-fPIC -I{$ROCM_PATH}/include/ -std=c++14 -I${MPICH_DIR}/include " \
- LDFLAGS=" -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -lamdhip64 "
+CXXFLAGS="-fPIC -I{$ROCM_PATH}/include/ -std=c++14 -I${MPICH_DIR}/include -L/lib64 " \
+ LDFLAGS="-L/lib64 -L/opt/rocm-5.2.0/lib/ -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -lamdhip64 "
+
 
 
diff --git a/systems/Crusher/dwf.slurm b/systems/Crusher/dwf.slurm
index 757ad4a2..20239e80 100644
--- a/systems/Crusher/dwf.slurm
+++ b/systems/Crusher/dwf.slurm
@@ -10,23 +10,26 @@
 #SBATCH -n 8
 #SBATCH --exclusive
 #SBATCH --gpu-bind=map_gpu:0,1,2,3,7,6,5,4
+#export BIND="--cpu-bind=verbose,map_ldom:3,3,1,1,2,2,0,0"
 
 DIR=.
 source sourceme.sh
-export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
+export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=16384
 export MPICH_GPU_SUPPORT_ENABLED=1
-export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
-export OMP_NUM_THREADS=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export OMP_NUM_THREADS=4
+export MPICH_OFI_NIC_POLICY=GPU
 
-echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
+#PARAMS=" --accelerator-threads 8 --grid 64.64.32.16 --mpi 2.2.2.1 --comms-sequential --shm 2048 --shm-mpi 1"
+#srun --gpus-per-task 1 -n8 $BIND ./wrap.sh  ./benchmarks/Benchmark_dwf_fp32 $PARAMS
 
-echo working directory
-pwd
+PARAMS=" --accelerator-threads 8 --grid 16.16.32.32 --mpi 1.1.1.1 --comms-sequential --shm 2048 --shm-mpi 1"
+srun --gpus-per-task 1 -n1 $BIND ./wrap.sh  ./benchmarks/Benchmark_dwf_fp32 $PARAMS
 
-PARAMS=" --accelerator-threads 8 --grid 32.32.32.32 --mpi 1.1.1.1 --comms-sequential --shm 2048 --shm-mpi 0"
-srun --gpus-per-task 1 -n1  ./benchmarks/Benchmark_dwf_fp32 $PARAMS
+PARAMS=" --accelerator-threads 8 --grid 32.16.32.32 --mpi 1.1.1.2 --comms-sequential --shm 2048 --shm-mpi 1"
+srun --gpus-per-task 1 -n2 $BIND ./wrap.sh  ./benchmarks/Benchmark_dwf_fp32 $PARAMS
 
-PARAMS=" --accelerator-threads 8 --grid 64.64.64.32 --mpi 2.2.2.1 --comms-sequential --shm 2048 --shm-mpi 0"
-srun --gpus-per-task 1 -n8  ./benchmarks/Benchmark_dwf_fp32 $PARAMS
+PARAMS=" --accelerator-threads 8 --grid 32.32.32.64 --mpi 1.2.2.2 --comms-sequential --shm 2048 --shm-mpi 1"
+srun --gpus-per-task 1 -n8 $BIND ./wrap.sh  ./benchmarks/Benchmark_dwf_fp32 $PARAMS
 
 
diff --git a/systems/Crusher/sourceme.sh b/systems/Crusher/sourceme.sh
index ad0d6582..3cccb10a 100644
--- a/systems/Crusher/sourceme.sh
+++ b/systems/Crusher/sourceme.sh
@@ -1,8 +1,12 @@
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/crusher/spack-envs/base/opt/cray-sles15-zen3/gcc-11.2.0/gperftools-2.9.1-72ubwtuc5wcz2meqltbfdb76epufgzo2/lib
+module load emacs 
+#module load gperftools
 module load PrgEnv-gnu
-module load rocm/5.1.0
+module load rocm/5.2.0
 module load cray-mpich/8.1.16
+#module load cray-mpich/8.1.17
 module load gmp
-#module load cray-fftw
+module load cray-fftw
 module load craype-accel-amd-gfx90a
 export LD_LIBRARY_PATH=/opt/gcc/mpfr/3.1.4/lib:$LD_LIBRARY_PATH
 #Hack for lib