From e4404efe5a9f98f2db0b671232638b0f5553202f Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Mon, 27 Apr 2026 13:53:28 -0700 Subject: [PATCH] Perlmutter compile update --- systems/Perlmutter/config-command | 9 ++++----- systems/Perlmutter/dwf4.slurm | 25 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/systems/Perlmutter/config-command b/systems/Perlmutter/config-command index 4f7ecee3..30b17b4e 100644 --- a/systems/Perlmutter/config-command +++ b/systems/Perlmutter/config-command @@ -1,17 +1,16 @@ DIR=`pwd` -PREFIX=$DIR/../Prequisites/install/ +PREFIX=$HOME/DDHMC/Grid/systems/Prerequisites/install/ ../../configure \ --enable-comms=mpi \ --enable-simd=GPU \ --enable-shm=nvlink \ --enable-gen-simd-width=64 \ - --enable-accelerator=cuda \ - --enable-setdevice \ - --disable-accelerator-cshift \ --with-gmp=$PREFIX \ + --with-mpfr=$PREFIX \ + --enable-accelerator=cuda \ --disable-fermion-reps \ --disable-unified \ --disable-gparity \ CXX=nvcc \ LDFLAGS="-cudart shared " \ - CXXFLAGS="-ccbin CC -gencode arch=compute_80,code=sm_80 -std=c++14 -cudart shared" + CXXFLAGS="-ccbin CC -gencode arch=compute_80,code=sm_80 -std=c++17 -cudart shared" diff --git a/systems/Perlmutter/dwf4.slurm b/systems/Perlmutter/dwf4.slurm index 426573d9..a891acf1 100644 --- a/systems/Perlmutter/dwf4.slurm +++ b/systems/Perlmutter/dwf4.slurm @@ -1,5 +1,6 @@ #!/bin/bash -#SBATCH -A m3886_g +#SBATCH -A m5294_g +#m3886_g #SBATCH -C gpu #SBATCH -q debug #SBATCH -t 0:20:00 @@ -19,9 +20,21 @@ export MPICH_GPU_EAGER_REGISTER_HOST_MEM=0 export MPICH_GPU_NO_ASYNC_MEMCPY=0 #export MPICH_SMP_SINGLE_COPY_MODE=CMA -OPT="--comms-sequential --shm-mpi 1" -VOL=64.64.64.64 -srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT -#srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.1.1.4 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT -#srun ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.8 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT +cat << EOF > select_gpu +#!/bin/bash +export GPU_MAP=(0 1 2 3) +export NUMA_MAP=( 0 1 2 3 ) +export GPU=\$SLURM_LOCALID +export NUMA=\$SLURM_LOCALID +export CUDA_VISIBLE_DEVICES=\$GPU +exec numactl -m \$NUMA -N \$NUMA \$* +EOF + +chmod +x ./select_gpu + +OPT="--comms-sequential --shm-mpi 0" +VOL=64.64.32.32 +srun ./select_gpu ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --device-mem 16000 --accelerator-threads 8 --shm 2048 $OPT +OPT="--comms-overlap --shm-mpi 0" +srun ./select_gpu ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --device-mem 16000 --accelerator-threads 8 --shm 2048 $OPT