Merge branch 'develop' of https://github.com/paboyle/Grid into develop

Grid/perfmon/PerfCount.h

@@ -51,8 +51,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #endif
 #ifdef __x86_64__
 #ifdef GRID_CUDA
-accelerator_inline uint64_t __rdtsc(void) {  return 0; }
-accelerator_inline uint64_t __rdpmc(int ) {  return 0; }
+//accelerator_inline uint64_t __rdtsc(void) {  return 0; }
+//accelerator_inline uint64_t __rdpmc(int ) {  return 0; }
 #else
 #include <x86intrin.h>
 #endif

systems/Perlmutter/config-command

@@ -1,17 +1,16 @@
 DIR=`pwd`
-PREFIX=$DIR/../Prequisites/install/
+PREFIX=$HOME/DDHMC/Grid/systems/Prerequisites/install/
 ../../configure \
     --enable-comms=mpi \
     --enable-simd=GPU \
     --enable-shm=nvlink \
     --enable-gen-simd-width=64 \
-    --enable-accelerator=cuda \
-    --enable-setdevice \
-    --disable-accelerator-cshift \
     --with-gmp=$PREFIX \
+    --with-mpfr=$PREFIX \
+    --enable-accelerator=cuda \
     --disable-fermion-reps \
     --disable-unified \
     --disable-gparity \
     CXX=nvcc \
     LDFLAGS="-cudart shared " \
-    CXXFLAGS="-ccbin CC -gencode arch=compute_80,code=sm_80 -std=c++14 -cudart shared"
+    CXXFLAGS="-ccbin CC -gencode arch=compute_80,code=sm_80 -std=c++17 -cudart shared"

systems/Perlmutter/dwf384.slurm

@@ -0,0 +1,39 @@
+#!/bin/bash
+##SBATCH -A m5294_g
+#SBATCH -A mp13_g
+#m3886_g
+#SBATCH -C gpu
+#SBATCH -q premium
+#SBATCH -t 00:20
+#SBATCH -c 32
+#SBATCH -N 384
+#SBATCH -n 1536
+#SBATCH --ntasks-per-node=4
+#SBATCH --gpus-per-task=1
+#SBATCH --exclusive
+#SBATCH --gpu-bind=none
+
+export SLURM_CPU_BIND="cores"
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_RDMA_ENABLED_CUDA=1
+export MPICH_GPU_IPC_ENABLED=1
+export MPICH_GPU_EAGER_REGISTER_HOST_MEM=0
+export MPICH_GPU_NO_ASYNC_MEMCPY=0
+#export MPICH_SMP_SINGLE_COPY_MODE=CMA
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3)
+export NUMA_MAP=( 0 1 2 3 )
+export GPU=\$SLURM_LOCALID
+export NUMA=\$SLURM_LOCALID
+export CUDA_VISIBLE_DEVICES=\$GPU
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+
+chmod +x ./select_gpu
+
+export VOL=128.128.128.288
+OPT="--comms-overlap --shm-mpi 0"
+srun ./select_gpu ./benchmarks/Benchmark_dwf_fp32 --mpi 4.8.4.12 --grid $VOL --device-mem 16000 --accelerator-threads 8 --shm 2048 $OPT
+

systems/Perlmutter/dwf4.slurm

@@ -1,5 +1,6 @@
 #!/bin/bash
-#SBATCH -A m3886_g
+#SBATCH -A m5294_g
+#m3886_g
 #SBATCH -C gpu
 #SBATCH -q debug
 #SBATCH -t 0:20:00
@@ -19,9 +20,21 @@ export MPICH_GPU_EAGER_REGISTER_HOST_MEM=0
 export MPICH_GPU_NO_ASYNC_MEMCPY=0
 #export MPICH_SMP_SINGLE_COPY_MODE=CMA
 
-OPT="--comms-sequential --shm-mpi 1"
-VOL=64.64.64.64
-srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT
-#srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.1.1.4 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT
-#srun ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.8 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3)
+export NUMA_MAP=( 0 1 2 3 )
+export GPU=\$SLURM_LOCALID
+export NUMA=\$SLURM_LOCALID
+export CUDA_VISIBLE_DEVICES=\$GPU
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+
+chmod +x ./select_gpu
+
+OPT="--comms-sequential --shm-mpi 0"
+VOL=64.64.32.32
+srun ./select_gpu ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --device-mem 16000 --accelerator-threads 8 --shm 2048 $OPT
+OPT="--comms-overlap --shm-mpi 0"
+srun ./select_gpu ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --device-mem 16000 --accelerator-threads 8 --shm 2048 $OPT