HMC running with new formulation

systems/Spock/config-command

@@ -6,6 +6,8 @@
 --enable-simd=GPU \
 --disable-fermion-reps \
 --disable-gparity \
+--with-gmp=$OLCF_GMP_ROOT \
+--with-mpfr=/opt/cray/pe/gcc/mpfr/3.1.4/ \
 CXX=hipcc MPICXX=mpicxx \
 CXXFLAGS="-fPIC -I/opt/rocm-4.3.0/include/ -std=c++14 -I${MPICH_DIR}/include " \
 --prefix=/ccs/home/chulwoo/Grid \

systems/Spock/dwf8.slurm

@@ -1,8 +1,7 @@
 #!/bin/bash
 # Begin LSF Directives
 #SBATCH -A LGT104
-#SBATCH -t 01:00:00
-##SBATCH -U openmpThu
+#SBATCH -t 3:00:00
 #SBATCH -p ecp
 #SBATCH -J DWF
 #SBATCH -o DWF.%J
@@ -14,13 +13,12 @@ DIR=.
 module list
 export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
 export MPICH_GPU_SUPPORT_ENABLED=1
-#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
-export MPICH_SMP_SINGLE_COPY_MODE=NONE
-#export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+
 export OMP_NUM_THREADS=8
 
 AT=8
 echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
-PARAMS=" --accelerator-threads ${AT} --grid 32.64.64.64 --mpi 1.2.2.2 --comms-overlap --shm 2048 --shm-mpi 0"
-srun -n8 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS
+PARAMS=" --accelerator-threads ${AT} --grid 16.16.16.48 --mpi 1.2.2.2 --comms-overlap --shm 2048 --shm-mpi 0"
+srun -N2 -n8 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./HMC/Mobius2p1f_DD_RHMC $PARAMS
 

systems/Spock/sourceme.sh

@@ -1,5 +1,9 @@
+module load emacs
 module load PrgEnv-gnu
 module load rocm/4.5.0
 module load gmp
 module load cray-fftw
 module load craype-accel-amd-gfx908
+export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
+export MPICH_GPU_SUPPORT_ENABLED=1
+export LD_LIBRARY_PATH=/opt/cray/pe/gcc/mpfr/3.1.4/lib/:$LD_LIBRARY_PATH