portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-06-12 20:27:06 +01:00
Code Releases Activity

Merge branch 'develop' of https://github.com/paboyle/Grid into develop

Browse Source
This commit is contained in:
Peter Boyle
2023-05-11 12:29:09 -04:00
parent 91efd08179 3aa43e6065
commit 9c8750f261
368 changed files with 25710 additions and 4770 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
systems/Spock/comms.slurm → systems/Crusher/comms.slurm
View File

18
systems/Crusher/config-command
View File

@ -1,12 +1,22 @@
CLIME=`spack find --paths c-lime@2-3-9 | grep c-lime| cut -c 15-`
../../configure --enable-comms=mpi-auto \
--with-lime=$CLIME \
--enable-unified=no \
--enable-shm=nvlink \
--enable-tracing=timer \
--enable-accelerator=hip \
--enable-gen-simd-width=64 \
--enable-simd=GPU \
--disable-accelerator-cshift \
--with-gmp=$OLCF_GMP_ROOT \
--with-fftw=$FFTW_DIR/.. \
--with-mpfr=/opt/cray/pe/gcc/mpfr/3.1.4/ \
--disable-fermion-reps \
--disable-gparity \
CXX=hipcc MPICXX=mpicxx \
CXXFLAGS="-fPIC -I/opt/rocm-4.5.0/include/ -std=c++14 -I${MPICH_DIR}/include " \
LDFLAGS=" -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa "
HIPFLAGS = --amdgpu-target=gfx90a
CXXFLAGS="-fPIC -I{$ROCM_PATH}/include/ -std=c++14 -I${MPICH_DIR}/include -L/lib64 " \
LDFLAGS="-L/lib64 -L/opt/rocm-5.2.0/lib/ -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -lamdhip64 "
#--enable-simd=GPU-RRII \

31
systems/Crusher/dwf.slurm
View File

@ -3,28 +3,33 @@
#SBATCH -A LGT104
#SBATCH -t 01:00:00
##SBATCH -U openmpThu
##SBATCH -p ecp
#SBATCH -J DWF
#SBATCH -o DWF.%J
#SBATCH -e DWF.%J
#SBATCH -N 1
#SBATCH -n 1
#SBATCH --exclusive
#SBATCH -n 8
#SBATCH --exclusive
#SBATCH --gpu-bind=map_gpu:0,1,2,3,7,6,5,4
#export BIND="--cpu-bind=verbose,map_ldom:3,3,1,1,2,2,0,0"
DIR=.
module list
#export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
source sourceme.sh
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=16384
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
#export MPICH_SMP_SINGLE_COPY_MODE=NONE
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=4
export MPICH_OFI_NIC_POLICY=GPU
AT=8
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
#PARAMS=" --accelerator-threads 8 --grid 64.64.32.16 --mpi 2.2.2.1 --comms-sequential --shm 2048 --shm-mpi 1"
#srun --gpus-per-task 1 -n8 $BIND ./wrap.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS
PARAMS=" --accelerator-threads ${AT} --grid 24.24.24.24 --shm-mpi 0 --mpi 1.1.1.1"
PARAMS=" --accelerator-threads 8 --grid 16.16.32.32 --mpi 1.1.1.1 --comms-sequential --shm 2048 --shm-mpi 1"
srun --gpus-per-task 1 -n1 $BIND ./wrap.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS
srun --gpus-per-task 1 -n1 ./benchmarks/Benchmark_dwf_fp32 $PARAMS
PARAMS=" --accelerator-threads 8 --grid 32.16.32.32 --mpi 1.1.1.2 --comms-sequential --shm 2048 --shm-mpi 1"
srun --gpus-per-task 1 -n2 $BIND ./wrap.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS
PARAMS=" --accelerator-threads 8 --grid 32.32.32.64 --mpi 1.2.2.2 --comms-sequential --shm 2048 --shm-mpi 1"
srun --gpus-per-task 1 -n8 $BIND ./wrap.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS

16
systems/Crusher/dwf4.slurm
View File

@ -7,21 +7,19 @@
#SBATCH -o DWF.%J
#SBATCH -e DWF.%J
#SBATCH -N 1
#SBATCH -n 4
#SBATCH --exclusive
#SBATCH -n 2
#SBATCH --gpu-bind=map_gpu:0,1
DIR=.
module list
source setup.sh
export MPICH_OFI_NIC_POLICY=GPU
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
export MPICH_GPU_SUPPORT_ENABLED=1
#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
export MPICH_SMP_SINGLE_COPY_MODE=NONE
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=4
export OMP_NUM_THREADS=16
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
PARAMS=" --accelerator-threads 8 --grid 32.32.64.64 --mpi 1.1.2.2 --comms-overlap --shm 2048 --shm-mpi 0"
srun --gpus-per-task 1 -n4 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS
srun --gpus-per-task 1 -N1 -n2 ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.2 --grid 16.16.32.64 --shm-mpi 1 --shm 2048 --comms-sequential --accelerator-threads 8

17
systems/Crusher/dwf8.slurm
View File

@ -8,20 +8,21 @@
#SBATCH -e DWF.%J
#SBATCH -N 1
#SBATCH -n 8
#SBATCH --exclusive
##SBATCH --gpu-bind=map_gpu:0,1,2,3,7,6,5,4
#SBATCH --gpu-bind=map_gpu:0,1,2,3,6,7,4,5
DIR=.
module list
source setup.sh
export MPICH_OFI_NIC_POLICY=GPU
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
#export MPICH_SMP_SINGLE_COPY_MODE=NONE
#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=1
#export MPICH_SMP_SINGLE_COPY_MODE=NONE
export OMP_NUM_THREADS=16
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
PARAMS=" --accelerator-threads 8 --grid 32.64.64.64 --mpi 1.2.2.2 --comms-overlap --shm 2048 --shm-mpi 0"
srun --gpus-per-task 1 -n8 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS
srun --gpus-per-task 1 -N1 -n8 ./benchmarks/Benchmark_comms_host_device --mpi 2.2.2.1 --shm-mpi 1 --shm 2048 --comms-sequential --accelerator-threads 8

5
systems/Crusher/mpiwrapper.sh
View File

@ -1,10 +1,11 @@
#!/bin/bash
lrank=$SLURM_LOCALID
lgpu=(0 1 2 3 7 6 5 4)
export ROCR_VISIBLE_DEVICES=$SLURM_LOCALID
export ROCR_VISIBLE_DEVICES=${lgpu[$lrank]}
echo "`hostname` - $lrank device=$ROCR_VISIBLE_DEVICES binding=$BINDING"
echo "`hostname` - $lrank device=$ROCR_VISIBLE_DEVICES "
$*

12
systems/Crusher/sourceme.sh
View File

@ -1,5 +1,15 @@
. /autofs/nccs-svm1_home1/paboyle/Crusher/Grid/spack/share/spack/setup-env.sh
spack load c-lime
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/crusher/spack-envs/base/opt/cray-sles15-zen3/gcc-11.2.0/gperftools-2.9.1-72ubwtuc5wcz2meqltbfdb76epufgzo2/lib
module load emacs
#module load gperftools
module load PrgEnv-gnu
module load rocm/4.5.0
module load rocm/5.3.0
#module load cray-mpich/8.1.16
module load cray-mpich/8.1.17
module load gmp
module load cray-fftw
module load craype-accel-amd-gfx90a
export LD_LIBRARY_PATH=/opt/gcc/mpfr/3.1.4/lib:$LD_LIBRARY_PATH
#Hack for lib
export LD_LIBRARY_PATH=`pwd`:$LD_LIBRARY_PATH

9
systems/Crusher/wrap.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
export HIP_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES
unset ROCR_VISIBLE_DEVICES
#rank=$SLURM_PROCID
#rocprof -d rocprof.$rank -o rocprof.$rank/results.rank$SLURM_PROCID.csv --sys-trace $@
$@

2
systems/PVC/benchmarks/run-1tile.sh
View File

@ -4,7 +4,7 @@
#SBATCH -p QZ1J-ICX-PVC
##SBATCH -p QZ1J-SPR-PVC-2C
source /nfs/site/home/paboylex/ATS/GridNew/Grid/systems/PVC-nightly/setup.sh
#source /nfs/site/home/paboylex/ATS/GridNew/Grid/systems/PVC-nightly/setup.sh
export NT=8

15
systems/PVC/benchmarks/run-2tile-mpi.sh
View File

@ -4,7 +4,7 @@
#SBATCH -p QZ1J-ICX-PVC
source /nfs/site/home/paboylex/ATS/GridNew/Grid/systems/PVC-nightly/setup.sh
#source /nfs/site/home/paboylex/ATS/GridNew/Grid/systems/PVC-nightly/setup.sh
export NT=16
@ -19,16 +19,15 @@ export SYCL_DEVICE_FILTER=gpu,level_zero
export I_MPI_OFFLOAD_CELL=tile
export EnableImplicitScaling=0
export EnableWalkerPartition=0
export SYCL_PI_LEVEL_ZERO_DEVICE_SCOPE_EVENTS=1
export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
#export SYCL_PI_LEVEL_ZERO_DEVICE_SCOPE_EVENTS=1
#export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE=0
for i in 0
for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
do
mpiexec -launcher ssh -n 2 -host localhost ./wrap4gpu.sh ./Benchmark_dwf_fp32 --mpi 1.1.1.2 --grid 32.32.32.64 --accelerator-threads $NT --shm-mpi 1 --device-mem 32768
mpiexec -launcher ssh -n 2 -host localhost ./wrap4gpu.sh ./Benchmark_dwf_fp32 --mpi 2.1.1.1 --grid 64.32.32.32 --accelerator-threads $NT --shm-mpi 1 --device-mem 32768
mpiexec -launcher ssh -n 2 -host localhost ./wrap.sh ./Benchmark_dwf_fp32 --mpi 1.1.1.2 --grid 32.32.32.64 --accelerator-threads $NT --shm-mpi 0 --device-mem 32768 > 1.1.1.2.log$i
mpiexec -launcher ssh -n 2 -host localhost ./wrap.sh ./Benchmark_dwf_fp32 --mpi 2.1.1.1 --grid 64.32.32.32 --accelerator-threads $NT --shm-mpi 0 --device-mem 32768 > 2.1.1.1.log$i
done
#mpiexec -launcher ssh -n 2 -host localhost ./wrap4gpu.sh ./Benchmark_halo --mpi 1.1.1.2 --grid 32.32.32.64 --accelerator-threads $NT --shm-mpi 1 > halo.2tile.1x2.log
#mpiexec -launcher ssh -n 2 -host localhost ./wrap4gpu.sh ./Benchmark_halo --mpi 2.1.1.1 --grid 64.32.32.32 --accelerator-threads $NT --shm-mpi 1 > halo.2tile.2x1.log
mpiexec -launcher ssh -n 2 -host localhost ./wrap.sh ./Benchmark_dwf_fp32 --mpi 2.1.1.1 --grid 64.32.32.32 --accelerator-threads $NT --comms-sequential --shm-mpi 0

7
systems/PVC/benchmarks/wrap.sh
View File

@ -5,10 +5,5 @@ export ZE_AFFINITY_MASK=0.$MPI_LOCALRANKID
echo Ranke $MPI_LOCALRANKID ZE_AFFINITY_MASK is $ZE_AFFINITY_MASK
if [ $MPI_LOCALRANKID = "0" ]
then
# ~psteinbr/build_pti/ze_tracer -h $@
onetrace --chrome-device-timeline $@
else
$@
fi

2
systems/PVC/config-command
View File

@ -1,4 +1,4 @@
INSTALL=/nfs/site/home/azusayax/install
INSTALL=/nfs/site/home/paboylx/prereqs/
../../configure \
--enable-simd=GPU \
--enable-gen-simd-width=64 \

11
systems/PVC/setup.sh
View File

@ -1,9 +1,16 @@
export https_proxy=http://proxy-chain.intel.com:911
export LD_LIBRARY_PATH=/nfs/site/home/azusayax/install/lib:$LD_LIBRARY_PATH
#export LD_LIBRARY_PATH=/nfs/site/home/azusayax/install/lib:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$HOME/prereqs/lib/:$LD_LIBRARY_PATH
module load intel-release
source /opt/intel/oneapi/PVC_setup.sh
module load intel-comp-rt/embargo-ci-neo
#source /opt/intel/oneapi/PVC_setup.sh
#source /opt/intel/oneapi/ATS_setup.sh
#module load intel-nightly/20230331
#module load intel-comp-rt/ci-neo-master/026093
#module load intel/mpich
module load intel/mpich/pvc45.3
export PATH=~/ATS/pti-gpu/tools/onetrace/:$PATH

5
systems/Perlmutter/config-command
View File

@ -1,9 +1,14 @@
DIR=`pwd`
PREFIX=$DIR/../Prequisites/install/
../../configure \
--enable-comms=mpi \
--enable-simd=GPU \
--enable-shm=nvlink \
--enable-gen-simd-width=64 \
--enable-accelerator=cuda \
--enable-setdevice \
--disable-accelerator-cshift \
--with-gmp=$PREFIX \
--disable-fermion-reps \
--disable-unified \
--disable-gparity \

31
systems/Perlmutter/dwf4.slurm
View File

@ -1,24 +1,27 @@
#!/bin/bash
#SBATCH -A mp13
#SBATCH -A m3886_g
#SBATCH -C gpu
#SBATCH -q regular
#SBATCH -q debug
#SBATCH -t 0:20:00
#SBATCH -n 16
#SBATCH --ntasks-per-node=4
#SBATCH -c 32
#SBATCH --exclusive
#SBATCH -N 1
#SBATCH -n 4
#SBATCH --ntasks-per-node=4
#SBATCH --gpus-per-task=1
#SBATCH --gpu-bind=map_gpu:0,1,2,3
#SBATCH --exclusive
#SBATCH --gpu-bind=none
export SLURM_CPU_BIND="cores"
export MPICH_RDMA_ENABLED_CUDA=1
export MPICH_GPU_SUPPORT_ENABLED=1
srun ./benchmarks/Benchmark_comms_host_device --mpi 2.2.2.2 --accelerator-threads 8 > comms.4node
export MPICH_RDMA_ENABLED_CUDA=1
export MPICH_GPU_IPC_ENABLED=1
export MPICH_GPU_EAGER_REGISTER_HOST_MEM=0
export MPICH_GPU_NO_ASYNC_MEMCPY=0
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
OPT="--comms-overlap --comms-concurrent --shm-mpi 0"
srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.2.2 --grid 64.64.64.64 --accelerator-threads 8 --shm 2048 $OPT > dwf.64.64.64.64.4node.opt0
srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.2.2 --grid 48.48.48.48 --accelerator-threads 8 --shm 2048 $OPT > dwf.48.48.48.48.4node.opt0
OPT="--comms-sequential --shm-mpi 1"
VOL=64.64.64.64
srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.1.1 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT
#srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.1.1.4 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT
#srun ./benchmarks/Benchmark_dwf_fp32 --mpi 1.1.1.8 --grid $VOL --accelerator-threads 8 --shm 2048 $OPT
OPT="--comms-overlap --comms-concurrent --shm-mpi 1"
srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.2.2 --grid 64.64.64.64 --accelerator-threads 8 --shm 2048 $OPT > dwf.64.64.64.64.4node.opt1
srun ./benchmarks/Benchmark_dwf_fp32 --mpi 2.2.2.2 --grid 48.48.48.48 --accelerator-threads 8 --shm 2048 $OPT > dwf.48.48.48.48.4node.opt1

2
systems/Perlmutter/sourceme.sh
View File

@ -1,4 +1,4 @@
export CRAY_ACCEL_TARGET=nvidia80
module load PrgEnv-gnu cpe-cuda cuda
module load PrgEnv-gnu cpe-cuda cudatoolkit/11.4

12
systems/Spock/config-command
View File

@ -1,12 +0,0 @@
../../configure --enable-comms=mpi-auto \
--enable-unified=no \
--enable-shm=nvlink \
--enable-accelerator=hip \
--enable-gen-simd-width=64 \
--enable-simd=GPU \
--disable-fermion-reps \
--disable-gparity \
CXX=hipcc MPICXX=mpicxx \
CXXFLAGS="-fPIC -I/opt/rocm-4.3.0/include/ -std=c++14 -I${MPICH_DIR}/include " \
--prefix=/ccs/home/chulwoo/Grid \
LDFLAGS=" -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa "

26
systems/Spock/dwf.slurm
View File

@ -1,26 +0,0 @@
#!/bin/bash
# Begin LSF Directives
#SBATCH -A LGT104
#SBATCH -t 01:00:00
##SBATCH -U openmpThu
#SBATCH -p ecp
#SBATCH -J DWF
#SBATCH -o DWF.%J
#SBATCH -e DWF.%J
#SBATCH -N 1
#SBATCH -n 1
DIR=.
module list
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
export MPICH_GPU_SUPPORT_ENABLED=1
#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
#export MPICH_SMP_SINGLE_COPY_MODE=NONE
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=8
AT=8
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
PARAMS=" --accelerator-threads ${AT} --grid 32.32.32.32 --mpi 1.1.1.1 --comms-overlap"
srun -n1 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS

26
systems/Spock/dwf4.slurm
View File

@ -1,26 +0,0 @@
#!/bin/bash
# Begin LSF Directives
#SBATCH -A LGT104
#SBATCH -t 01:00:00
##SBATCH -U openmpThu
#SBATCH -p ecp
#SBATCH -J DWF
#SBATCH -o DWF.%J
#SBATCH -e DWF.%J
#SBATCH -N 1
#SBATCH -n 4
DIR=.
module list
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
export MPICH_GPU_SUPPORT_ENABLED=1
#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
export MPICH_SMP_SINGLE_COPY_MODE=NONE
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=8
AT=8
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
PARAMS=" --accelerator-threads ${AT} --grid 32.32.64.64 --mpi 1.1.2.2 --comms-overlap --shm 2048 --shm-mpi 0"
srun -n4 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS

26
systems/Spock/dwf8.slurm
View File

@ -1,26 +0,0 @@
#!/bin/bash
# Begin LSF Directives
#SBATCH -A LGT104
#SBATCH -t 01:00:00
##SBATCH -U openmpThu
#SBATCH -p ecp
#SBATCH -J DWF
#SBATCH -o DWF.%J
#SBATCH -e DWF.%J
#SBATCH -N 2
#SBATCH -n 8
DIR=.
module list
export MPIR_CVAR_GPU_EAGER_DEVICE_MEM=0
export MPICH_GPU_SUPPORT_ENABLED=1
#export MPICH_SMP_SINGLE_COPY_MODE=XPMEM
export MPICH_SMP_SINGLE_COPY_MODE=NONE
#export MPICH_SMP_SINGLE_COPY_MODE=CMA
export OMP_NUM_THREADS=8
AT=8
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
PARAMS=" --accelerator-threads ${AT} --grid 32.64.64.64 --mpi 1.2.2.2 --comms-overlap --shm 2048 --shm-mpi 0"
srun -n8 --label -c$OMP_NUM_THREADS --gpus-per-task=1 ./mpiwrapper.sh ./benchmarks/Benchmark_dwf_fp32 $PARAMS

12
systems/Spock/mpiwrapper.sh
View File

@ -1,12 +0,0 @@
#!/bin/bash
lrank=$SLURM_LOCALID
export ROCR_VISIBLE_DEVICES=$SLURM_LOCALID
echo "`hostname` - $lrank device=$ROCR_VISIBLE_DEVICES binding=$BINDING"
$*

5
systems/Spock/sourceme.sh
View File

@ -1,5 +0,0 @@
module load PrgEnv-gnu
module load rocm/4.3.0
module load gmp
module load cray-fftw
module load craype-accel-amd-gfx908

179
systems/Summit/comms.4node
View File

@ -1,179 +0,0 @@
OPENMPI detected
AcceleratorCudaInit[0]: ========================
AcceleratorCudaInit[0]: Device Number : 0
AcceleratorCudaInit[0]: ========================
AcceleratorCudaInit[0]: Device identifier: Tesla V100-SXM2-16GB
AcceleratorCudaInit[0]: totalGlobalMem: 16911433728
AcceleratorCudaInit[0]: managedMemory: 1
AcceleratorCudaInit[0]: isMultiGpuBoard: 0
AcceleratorCudaInit[0]: warpSize: 32
AcceleratorCudaInit[0]: pciBusID: 4
AcceleratorCudaInit[0]: pciDeviceID: 0
AcceleratorCudaInit[0]: maxGridSize (2147483647,65535,65535)
AcceleratorCudaInit: rank 0 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 0 device 0 bus id: 0004:04:00.0
AcceleratorCudaInit: ================================================
SharedMemoryMpi: World communicator of size 24
SharedMemoryMpi: Node communicator of size 6
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 1073741824bytes at 0x200060000000 for comms buffers
Setting up IPC
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
__|_ | | | | | | | | | | | | _|__
__|_ _|__
__|_ GGGG RRRR III DDDD _|__
__|_ G R R I D D _|__
__|_ G R R I D D _|__
__|_ G GG RRRR I D D _|__
__|_ G G R R I D D _|__
__|_ GGGG R R III DDDD _|__
__|_ _|__
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
| | | | | | | | | | | | | |
Copyright (C) 2015 Peter Boyle, Azusa Yamaguchi, Guido Cossu, Antonin Portelli and other authors
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
Current Grid git commit hash=7cb1ff7395a5833ded6526c43891bd07a0436290: (HEAD -> develop, origin/develop, origin/HEAD) clean
Grid : Message : ================================================
Grid : Message : MPI is initialised and logging filters activated
Grid : Message : ================================================
Grid : Message : Requested 1073741824 byte stencil comms buffers
AcceleratorCudaInit: rank 1 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 1 device 1 bus id: 0004:05:00.0
AcceleratorCudaInit: rank 2 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 2 device 2 bus id: 0004:06:00.0
AcceleratorCudaInit: rank 5 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 5 device 5 bus id: 0035:05:00.0
AcceleratorCudaInit: rank 4 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 4 device 4 bus id: 0035:04:00.0
AcceleratorCudaInit: rank 3 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 3 device 3 bus id: 0035:03:00.0
Grid : Message : MemoryManager Cache 13529146982 bytes
Grid : Message : MemoryManager::Init() setting up
Grid : Message : MemoryManager::Init() cache pool for recent allocations: SMALL 8 LARGE 2
Grid : Message : MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory
Grid : Message : MemoryManager::Init() Using cudaMalloc
Grid : Message : 2.137929 s : Grid is setup to use 6 threads
Grid : Message : 2.137941 s : Number of iterations to average: 250
Grid : Message : 2.137950 s : ====================================================================================================
Grid : Message : 2.137958 s : = Benchmarking sequential halo exchange from host memory
Grid : Message : 2.137966 s : ====================================================================================================
Grid : Message : 2.137974 s : L Ls bytes MB/s uni MB/s bidi
AcceleratorCudaInit: rank 22 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 10 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 15 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 21 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 20 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 7 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 9 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 11 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 8 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 6 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 19 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 23 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 18 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 12 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 16 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 13 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 14 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 17 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
Grid : Message : 2.604949 s : 8 8 393216 89973.9 179947.8
Grid : Message : 2.668249 s : 8 8 393216 18650.3 37300.5
Grid : Message : 2.732288 s : 8 8 393216 18428.5 36857.1
Grid : Message : 2.753565 s : 8 8 393216 55497.2 110994.4
Grid : Message : 2.808960 s : 12 8 1327104 100181.5 200363.0
Grid : Message : 3.226900 s : 12 8 1327104 20600.5 41201.0
Grid : Message : 3.167459 s : 12 8 1327104 24104.6 48209.2
Grid : Message : 3.227660 s : 12 8 1327104 66156.7 132313.5
Grid : Message : 3.413570 s : 16 8 3145728 56174.4 112348.8
Grid : Message : 3.802697 s : 16 8 3145728 24255.9 48511.7
Grid : Message : 4.190498 s : 16 8 3145728 24336.7 48673.4
Grid : Message : 4.385171 s : 16 8 3145728 48484.1 96968.2
Grid : Message : 4.805284 s : 20 8 6144000 46380.5 92761.1
Grid : Message : 5.562975 s : 20 8 6144000 24328.5 48656.9
Grid : Message : 6.322562 s : 20 8 6144000 24266.7 48533.4
Grid : Message : 6.773598 s : 20 8 6144000 40868.5 81736.9
Grid : Message : 7.600999 s : 24 8 10616832 40198.3 80396.6
Grid : Message : 8.912917 s : 24 8 10616832 24279.5 48559.1
Grid : Message : 10.220961 s : 24 8 10616832 24350.2 48700.4
Grid : Message : 11.728250 s : 24 8 10616832 37390.9 74781.8
Grid : Message : 12.497258 s : 28 8 16859136 36792.2 73584.5
Grid : Message : 14.585387 s : 28 8 16859136 24222.2 48444.3
Grid : Message : 16.664783 s : 28 8 16859136 24323.4 48646.8
Grid : Message : 17.955238 s : 28 8 16859136 39194.7 78389.4
Grid : Message : 20.136479 s : 32 8 25165824 35718.3 71436.5
Grid : Message : 23.241958 s : 32 8 25165824 24311.4 48622.9
Grid : Message : 26.344810 s : 32 8 25165824 24331.9 48663.7
Grid : Message : 28.384420 s : 32 8 25165824 37016.3 74032.7
Grid : Message : 28.388879 s : ====================================================================================================
Grid : Message : 28.388894 s : = Benchmarking sequential halo exchange from GPU memory
Grid : Message : 28.388909 s : ====================================================================================================
Grid : Message : 28.388924 s : L Ls bytes MB/s uni MB/s bidi
Grid : Message : 28.553993 s : 8 8 393216 8272.4 16544.7
Grid : Message : 28.679592 s : 8 8 393216 9395.4 18790.8
Grid : Message : 28.811112 s : 8 8 393216 8971.0 17942.0
Grid : Message : 28.843770 s : 8 8 393216 36145.6 72291.2
Grid : Message : 28.981754 s : 12 8 1327104 49591.6 99183.2
Grid : Message : 29.299764 s : 12 8 1327104 12520.8 25041.7
Grid : Message : 29.620288 s : 12 8 1327104 12422.2 24844.4
Grid : Message : 29.657645 s : 12 8 1327104 106637.5 213275.1
Grid : Message : 29.952933 s : 16 8 3145728 43939.2 87878.5
Grid : Message : 30.585411 s : 16 8 3145728 14922.1 29844.2
Grid : Message : 31.219781 s : 16 8 3145728 14877.2 29754.4
Grid : Message : 31.285017 s : 16 8 3145728 144724.3 289448.7
Grid : Message : 31.706443 s : 20 8 6144000 54676.2 109352.4
Grid : Message : 32.739205 s : 20 8 6144000 17848.0 35696.1
Grid : Message : 33.771852 s : 20 8 6144000 17849.9 35699.7
Grid : Message : 33.871981 s : 20 8 6144000 184141.4 368282.8
Grid : Message : 34.536808 s : 24 8 10616832 55784.3 111568.6
Grid : Message : 36.275648 s : 24 8 10616832 18317.6 36635.3
Grid : Message : 37.997181 s : 24 8 10616832 18501.7 37003.4
Grid : Message : 38.140442 s : 24 8 10616832 222383.9 444767.9
Grid : Message : 39.177222 s : 28 8 16859136 56609.7 113219.4
Grid : Message : 41.874755 s : 28 8 16859136 18749.9 37499.8
Grid : Message : 44.529381 s : 28 8 16859136 19052.9 38105.8
Grid : Message : 44.742192 s : 28 8 16859136 237717.1 475434.2
Grid : Message : 46.184000 s : 32 8 25165824 57091.2 114182.4
Grid : Message : 50.734740 s : 32 8 25165824 19411.0 38821.9
Grid : Message : 53.931228 s : 32 8 25165824 19570.6 39141.2
Grid : Message : 54.238467 s : 32 8 25165824 245765.6 491531.2
Grid : Message : 54.268664 s : ====================================================================================================
Grid : Message : 54.268680 s : = All done; Bye Bye
Grid : Message : 54.268691 s : ====================================================================================================

7
systems/Summit/config-command
View File

@ -2,11 +2,12 @@
--enable-simd=GPU \
--enable-gen-simd-width=32 \
--enable-unified=no \
--enable-shm=nvlink \
--disable-gparity \
--enable-setdevice \
--enable-shm=no \
--enable-gparity \
--disable-setdevice \
--disable-fermion-reps \
--enable-accelerator=cuda \
--enable-accelerator-cshift \
--prefix /ccs/home/paboyle/prefix \
CXX=nvcc \
LDFLAGS=-L/ccs/home/paboyle/prefix/lib/ \

218
systems/Summit/dwf.24.4node
View File

@ -10,19 +10,16 @@ AcceleratorCudaInit[0]: warpSize: 32
AcceleratorCudaInit[0]: pciBusID: 4
AcceleratorCudaInit[0]: pciDeviceID: 0
AcceleratorCudaInit[0]: maxGridSize (2147483647,65535,65535)
AcceleratorCudaInit: rank 0 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
local rank 0 device 0 bus id: 0004:04:00.0
AcceleratorCudaInit: ================================================
SharedMemoryMpi: World communicator of size 24
SharedMemoryMpi: Node communicator of size 6
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 2147483648bytes at 0x200080000000 for comms buffers
AcceleratorCudaInit: rank 3 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 3 device 3 bus id: 0035:03:00.0
AcceleratorCudaInit: rank 5 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 5 device 5 bus id: 0035:05:00.0
SharedMemoryMpi: Node communicator of size 1
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 1073741824bytes at 0x200080000000 - 2000bfffffff for comms buffers
Setting up IPC
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
@ -36,6 +33,11 @@ __|_ G GG RRRR I D D _|__
__|_ G G R R I D D _|__
__|_ GGGG R R III DDDD _|__
__|_ _|__
local rank 5 device 0 bus id: 0035:05:00.0
local rank 1 device 0 bus id: 0004:05:00.0
local rank 2 device 0 bus id: 0004:06:00.0
local rank 3 device 0 bus id: 0035:03:00.0
local rank 4 device 0 bus id: 0035:04:00.0
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
| | | | | | | | | | | | | |
@ -45,15 +47,6 @@ Copyright (C) 2015 Peter Boyle, Azusa Yamaguchi, Guido Cossu, Antonin Portelli a
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
AcceleratorCudaInit: rank 4 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 4 device 4 bus id: 0035:04:00.0
AcceleratorCudaInit: rank 1 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 1 device 1 bus id: 0004:05:00.0
AcceleratorCudaInit: rank 2 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 2 device 2 bus id: 0004:06:00.0
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
@ -61,146 +54,63 @@ This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
Current Grid git commit hash=7cb1ff7395a5833ded6526c43891bd07a0436290: (HEAD -> develop, origin/develop, origin/HEAD) clean
Current Grid git commit hash=1713de35c0dc339564661dd7df8a72583f889e91: (HEAD -> feature/dirichlet) uncommited changes
Grid : Message : ================================================
Grid : Message : MPI is initialised and logging filters activated
Grid : Message : ================================================
Grid : Message : Requested 2147483648 byte stencil comms buffers
Grid : Message : MemoryManager Cache 8388608000 bytes
Grid : Message : Requested 1073741824 byte stencil comms buffers
Grid : Message : MemoryManager Cache 4194304000 bytes
Grid : Message : MemoryManager::Init() setting up
Grid : Message : MemoryManager::Init() cache pool for recent allocations: SMALL 8 LARGE 2
Grid : Message : MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory
Grid : Message : MemoryManager::Init() Using cudaMalloc
Grid : Message : 1.731905 s : Grid Layout
Grid : Message : 1.731915 s : Global lattice size : 48 48 48 72
Grid : Message : 1.731928 s : OpenMP threads : 6
Grid : Message : 1.731938 s : MPI tasks : 2 2 2 3
AcceleratorCudaInit: rank 9 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 23 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 22 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 21 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 18 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 6 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 7 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 10 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 8 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 11 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 20 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 19 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 13 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 12 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 14 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 16 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 15 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 17 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
Grid : Message : 2.683494 s : Making s innermost grids
Grid : Message : 2.780034 s : Initialising 4d RNG
Grid : Message : 2.833099 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 2.833121 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 2.916841 s : Initialising 5d RNG
Grid : Message : 3.762880 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 3.762902 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 5.264345 s : Initialised RNGs
Grid : Message : 6.489904 s : Drawing gauge field
Grid : Message : 6.729262 s : Random gauge initialised
Grid : Message : 7.781273 s : Setting up Cshift based reference
Grid : Message : 8.725313 s : *****************************************************************
Grid : Message : 8.725332 s : * Kernel options --dslash-generic, --dslash-unroll, --dslash-asm
Grid : Message : 8.725342 s : *****************************************************************
Grid : Message : 8.725352 s : *****************************************************************
Grid : Message : 8.725362 s : * Benchmarking DomainWallFermionR::Dhop
Grid : Message : 8.725372 s : * Vectorising space-time by 4
Grid : Message : 8.725383 s : * VComplexF size is 32 B
Grid : Message : 8.725395 s : * SINGLE precision
Grid : Message : 8.725405 s : * Using Overlapped Comms/Compute
Grid : Message : 8.725415 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 8.725425 s : *****************************************************************
Grid : Message : 9.465229 s : Called warmup
Grid : Message : 58.646066 s : Called Dw 3000 times in 4.91764e+07 us
Grid : Message : 58.646121 s : mflop/s = 1.02592e+07
Grid : Message : 58.646134 s : mflop/s per rank = 427468
Grid : Message : 58.646145 s : mflop/s per node = 2.56481e+06
Grid : Message : 58.646156 s : RF GiB/s (base 2) = 20846.5
Grid : Message : 58.646166 s : mem GiB/s (base 2) = 13029.1
Grid : Message : 58.648008 s : norm diff 1.04778e-13
Grid : Message : 58.734885 s : #### Dhop calls report
Grid : Message : 58.734897 s : WilsonFermion5D Number of DhopEO Calls : 6002
Grid : Message : 58.734909 s : WilsonFermion5D TotalTime /Calls : 8217.71 us
Grid : Message : 58.734922 s : WilsonFermion5D CommTime /Calls : 7109.5 us
Grid : Message : 58.734933 s : WilsonFermion5D FaceTime /Calls : 446.623 us
Grid : Message : 58.734943 s : WilsonFermion5D ComputeTime1/Calls : 18.0558 us
Grid : Message : 58.734953 s : WilsonFermion5D ComputeTime2/Calls : 731.097 us
Grid : Message : 58.734979 s : Average mflops/s per call : 4.8157e+09
Grid : Message : 58.734989 s : Average mflops/s per call per rank : 2.00654e+08
Grid : Message : 58.734999 s : Average mflops/s per call per node : 1.20393e+09
Grid : Message : 58.735008 s : Average mflops/s per call (full) : 1.04183e+07
Grid : Message : 58.735017 s : Average mflops/s per call per rank (full): 434094
Grid : Message : 58.735026 s : Average mflops/s per call per node (full): 2.60456e+06
Grid : Message : 58.735035 s : WilsonFermion5D Stencil
Grid : Message : 58.735043 s : WilsonFermion5D StencilEven
Grid : Message : 58.735051 s : WilsonFermion5D StencilOdd
Grid : Message : 58.735059 s : WilsonFermion5D Stencil Reporti()
Grid : Message : 58.735067 s : WilsonFermion5D StencilEven Reporti()
Grid : Message : 58.735075 s : WilsonFermion5D StencilOdd Reporti()
Grid : Message : 64.934380 s : Compare to naive wilson implementation Dag to verify correctness
Grid : Message : 64.934740 s : Called DwDag
Grid : Message : 64.934870 s : norm dag result 12.0422
Grid : Message : 64.120756 s : norm dag ref 12.0422
Grid : Message : 64.149389 s : norm dag diff 7.6644e-14
Grid : Message : 64.317786 s : Calling Deo and Doe and //assert Deo+Doe == Dunprec
Grid : Message : 64.465331 s : src_e0.499995
Grid : Message : 64.524653 s : src_o0.500005
Grid : Message : 64.558706 s : *********************************************************
Grid : Message : 64.558717 s : * Benchmarking DomainWallFermionF::DhopEO
Grid : Message : 64.558727 s : * Vectorising space-time by 4
Grid : Message : 64.558737 s : * SINGLE precision
Grid : Message : 64.558745 s : * Using Overlapped Comms/Compute
Grid : Message : 64.558753 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 64.558761 s : *********************************************************
Grid : Message : 92.702145 s : Deo mflop/s = 8.97692e+06
Grid : Message : 92.702185 s : Deo mflop/s per rank 374038
Grid : Message : 92.702198 s : Deo mflop/s per node 2.24423e+06
Grid : Message : 92.702209 s : #### Dhop calls report
Grid : Message : 92.702223 s : WilsonFermion5D Number of DhopEO Calls : 3001
Grid : Message : 92.702240 s : WilsonFermion5D TotalTime /Calls : 9377.88 us
Grid : Message : 92.702257 s : WilsonFermion5D CommTime /Calls : 8221.84 us
Grid : Message : 92.702277 s : WilsonFermion5D FaceTime /Calls : 543.548 us
Grid : Message : 92.702301 s : WilsonFermion5D ComputeTime1/Calls : 20.936 us
Grid : Message : 92.702322 s : WilsonFermion5D ComputeTime2/Calls : 732.33 us
Grid : Message : 92.702376 s : Average mflops/s per call : 4.13001e+09
Grid : Message : 92.702387 s : Average mflops/s per call per rank : 1.72084e+08
Grid : Message : 92.702397 s : Average mflops/s per call per node : 1.0325e+09
Grid : Message : 92.702407 s : Average mflops/s per call (full) : 9.12937e+06
Grid : Message : 92.702416 s : Average mflops/s per call per rank (full): 380391
Grid : Message : 92.702426 s : Average mflops/s per call per node (full): 2.28234e+06
Grid : Message : 92.702435 s : WilsonFermion5D Stencil
Grid : Message : 92.702443 s : WilsonFermion5D StencilEven
Grid : Message : 92.702451 s : WilsonFermion5D StencilOdd
Grid : Message : 92.702459 s : WilsonFermion5D Stencil Reporti()
Grid : Message : 92.702467 s : WilsonFermion5D StencilEven Reporti()
Grid : Message : 92.702475 s : WilsonFermion5D StencilOdd Reporti()
Grid : Message : 92.772983 s : r_e6.02121
Grid : Message : 92.786384 s : r_o6.02102
Grid : Message : 92.799622 s : res12.0422
Grid : Message : 93.860500 s : norm diff 0
Grid : Message : 93.162026 s : norm diff even 0
Grid : Message : 93.197529 s : norm diff odd 0
Grid : Message : 0.179000 s : ++++++++++++++++++++++++++++++++++++++++++++++++
Grid : Message : 0.196000 s : Testing with full communication
Grid : Message : 0.211000 s : ++++++++++++++++++++++++++++++++++++++++++++++++
Grid : Message : 0.225000 s : Grid Layout
Grid : Message : 0.233000 s : Global lattice size : 48 48 48 72
Grid : Message : 0.246000 s : OpenMP threads : 6
Grid : Message : 0.255000 s : MPI tasks : 2 2 2 3
Grid : Message : 0.182200 s : Initialising 4d RNG
Grid : Message : 0.233863 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 0.233886 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 0.245805 s : Initialising 5d RNG
Grid : Message : 1.710720 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 1.710950 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 2.220272 s : Drawing gauge field
Grid : Message : 2.418119 s : Random gauge initialised
Grid : Message : 2.418142 s : Applying BCs for Dirichlet Block5 [0 0 0 0 0]
Grid : Message : 2.418156 s : Applying BCs for Dirichlet Block4 [0 0 0 0]
Grid : Message : 2.489588 s : Setting up Cshift based reference
Grid : Message : 13.921239 s : *****************************************************************
Grid : Message : 13.921261 s : * Kernel options --dslash-generic, --dslash-unroll, --dslash-asm
Grid : Message : 13.921270 s : *****************************************************************
Grid : Message : 13.921279 s : *****************************************************************
Grid : Message : 13.921288 s : * Benchmarking DomainWallFermionR::Dhop
Grid : Message : 13.921296 s : * Vectorising space-time by 4
Grid : Message : 13.921305 s : * VComplexF size is 32 B
Grid : Message : 13.921314 s : * SINGLE precision
Grid : Message : 13.921321 s : * Using Overlapped Comms/Compute
Grid : Message : 13.921328 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 13.921335 s : *****************************************************************
Grid : Message : 14.821339 s : Called warmup
Grid : Message : 23.975467 s : Called Dw 300 times in 9.15155e+06 us
Grid : Message : 23.975528 s : mflop/s = 5.51286e+06
Grid : Message : 23.975543 s : mflop/s per rank = 229702
Grid : Message : 23.975557 s : mflop/s per node = 229702
Grid : Message : 23.989684 s : norm diff 5.09279e-313 Line 291
Grid : Message : 39.450493 s : ----------------------------------------------------------------
Grid : Message : 39.450517 s : Compare to naive wilson implementation Dag to verify correctness
Grid : Message : 39.450526 s : ----------------------------------------------------------------
Grid : Message : 39.450534 s : Called DwDag
Grid : Message : 39.450542 s : norm dag result nan
Grid : Message : 39.451564 s : norm dag ref nan
Grid : Message : 39.455714 s : norm dag diff nan Line 354

197
systems/Summit/dwf.32.4node
View File

@ -10,14 +10,21 @@ AcceleratorCudaInit[0]: warpSize: 32
AcceleratorCudaInit[0]: pciBusID: 4
AcceleratorCudaInit[0]: pciDeviceID: 0
AcceleratorCudaInit[0]: maxGridSize (2147483647,65535,65535)
AcceleratorCudaInit: rank 0 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
local rank 0 device 0 bus id: 0004:04:00.0
AcceleratorCudaInit: ================================================
SharedMemoryMpi: World communicator of size 24
SharedMemoryMpi: Node communicator of size 6
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 2147483648bytes at 0x200080000000 for comms buffers
SharedMemoryMpi: Node communicator of size 1
local rank 3 device 0 bus id: 0004:04:00.0
local rank 2 device 0 bus id: 0004:04:00.0
local rank 1 device 0 bus id: 0004:04:00.0
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 1073741824bytes at 0x200080000000 - 2000bfffffff for comms buffers
Setting up IPC
local rank 5 device 0 bus id: 0004:04:00.0
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
@ -39,168 +46,46 @@ Copyright (C) 2015 Peter Boyle, Azusa Yamaguchi, Guido Cossu, Antonin Portelli a
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
local rank 4 device 0 bus id: 0004:04:00.0
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
AcceleratorCudaInit: rank 2 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 2 device 2 bus id: 0004:06:00.0
AcceleratorCudaInit: rank 1 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 1 device 1 bus id: 0004:05:00.0
AcceleratorCudaInit: rank 4 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 4 device 4 bus id: 0035:04:00.0
AcceleratorCudaInit: rank 3 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 3 device 3 bus id: 0035:03:00.0
AcceleratorCudaInit: rank 5 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
local rank 5 device 5 bus id: 0035:05:00.0
GNU General Public License for more details.
Current Grid git commit hash=7cb1ff7395a5833ded6526c43891bd07a0436290: (HEAD -> develop, origin/develop, origin/HEAD) clean
Current Grid git commit hash=1713de35c0dc339564661dd7df8a72583f889e91: (HEAD -> feature/dirichlet) uncommited changes
Grid : Message : ================================================
Grid : Message : MPI is initialised and logging filters activated
Grid : Message : ================================================
Grid : Message : Requested 2147483648 byte stencil comms buffers
Grid : Message : MemoryManager Cache 8388608000 bytes
Grid : Message : Requested 1073741824 byte stencil comms buffers
Grid : Message : MemoryManager::Init() setting up
Grid : Message : MemoryManager::Init() cache pool for recent allocations: SMALL 8 LARGE 2
Grid : Message : MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory
Grid : Message : MemoryManager::Init() Using cudaMalloc
Grid : Message : 1.544984 s : Grid Layout
Grid : Message : 1.544992 s : Global lattice size : 64 64 64 96
Grid : Message : 1.545003 s : OpenMP threads : 6
Grid : Message : 1.545011 s : MPI tasks : 2 2 2 3
AcceleratorCudaInit: rank 8 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 6 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 11 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 16 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 17 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 13 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 12 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 21 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 23 setting device to node rank 5
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 22 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 19 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 18 setting device to node rank 0
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 7 setting device to node rank 1
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 10 setting device to node rank 4
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 9 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 14 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 15 setting device to node rank 3
AcceleratorCudaInit: Configure options --enable-setdevice=yes
AcceleratorCudaInit: rank 20 setting device to node rank 2
AcceleratorCudaInit: Configure options --enable-setdevice=yes
Grid : Message : 2.994920 s : Making s innermost grids
Grid : Message : 2.232502 s : Initialising 4d RNG
Grid : Message : 2.397047 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 2.397069 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 2.653140 s : Initialising 5d RNG
Grid : Message : 5.285347 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 5.285369 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 9.994738 s : Initialised RNGs
Grid : Message : 13.153426 s : Drawing gauge field
Grid : Message : 13.825697 s : Random gauge initialised
Grid : Message : 18.537657 s : Setting up Cshift based reference
Grid : Message : 22.296755 s : *****************************************************************
Grid : Message : 22.296781 s : * Kernel options --dslash-generic, --dslash-unroll, --dslash-asm
Grid : Message : 22.296791 s : *****************************************************************
Grid : Message : 22.296800 s : *****************************************************************
Grid : Message : 22.296809 s : * Benchmarking DomainWallFermionR::Dhop
Grid : Message : 22.296818 s : * Vectorising space-time by 4
Grid : Message : 22.296828 s : * VComplexF size is 32 B
Grid : Message : 22.296838 s : * SINGLE precision
Grid : Message : 22.296847 s : * Using Overlapped Comms/Compute
Grid : Message : 22.296855 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 22.296863 s : *****************************************************************
Grid : Message : 24.746452 s : Called warmup
Grid : Message : 137.525756 s : Called Dw 3000 times in 1.12779e+08 us
Grid : Message : 137.525818 s : mflop/s = 1.41383e+07
Grid : Message : 137.525831 s : mflop/s per rank = 589097
Grid : Message : 137.525843 s : mflop/s per node = 3.53458e+06
Grid : Message : 137.525854 s : RF GiB/s (base 2) = 28728.7
Grid : Message : 137.525864 s : mem GiB/s (base 2) = 17955.5
Grid : Message : 137.693645 s : norm diff 1.04885e-13
Grid : Message : 137.965585 s : #### Dhop calls report
Grid : Message : 137.965598 s : WilsonFermion5D Number of DhopEO Calls : 6002
Grid : Message : 137.965612 s : WilsonFermion5D TotalTime /Calls : 18899.7 us
Grid : Message : 137.965624 s : WilsonFermion5D CommTime /Calls : 16041.4 us
Grid : Message : 137.965634 s : WilsonFermion5D FaceTime /Calls : 859.705 us
Grid : Message : 137.965644 s : WilsonFermion5D ComputeTime1/Calls : 70.5881 us
Grid : Message : 137.965654 s : WilsonFermion5D ComputeTime2/Calls : 2094.8 us
Grid : Message : 137.965682 s : Average mflops/s per call : 3.87638e+09
Grid : Message : 137.965692 s : Average mflops/s per call per rank : 1.61516e+08
Grid : Message : 137.965702 s : Average mflops/s per call per node : 9.69095e+08
Grid : Message : 137.965712 s : Average mflops/s per call (full) : 1.43168e+07
Grid : Message : 137.965721 s : Average mflops/s per call per rank (full): 596533
Grid : Message : 137.965730 s : Average mflops/s per call per node (full): 3.5792e+06
Grid : Message : 137.965740 s : WilsonFermion5D Stencil
Grid : Message : 137.965748 s : WilsonFermion5D StencilEven
Grid : Message : 137.965756 s : WilsonFermion5D StencilOdd
Grid : Message : 137.965764 s : WilsonFermion5D Stencil Reporti()
Grid : Message : 137.965772 s : WilsonFermion5D StencilEven Reporti()
Grid : Message : 137.965780 s : WilsonFermion5D StencilOdd Reporti()
Grid : Message : 156.554605 s : Compare to naive wilson implementation Dag to verify correctness
Grid : Message : 156.554632 s : Called DwDag
Grid : Message : 156.554642 s : norm dag result 12.0421
Grid : Message : 156.639265 s : norm dag ref 12.0421
Grid : Message : 156.888281 s : norm dag diff 7.62057e-14
Grid : Message : 157.609797 s : Calling Deo and Doe and //assert Deo+Doe == Dunprec
Grid : Message : 158.208630 s : src_e0.499996
Grid : Message : 158.162447 s : src_o0.500004
Grid : Message : 158.267780 s : *********************************************************
Grid : Message : 158.267791 s : * Benchmarking DomainWallFermionF::DhopEO
Grid : Message : 158.267801 s : * Vectorising space-time by 4
Grid : Message : 158.267811 s : * SINGLE precision
Grid : Message : 158.267820 s : * Using Overlapped Comms/Compute
Grid : Message : 158.267828 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 158.267836 s : *********************************************************
Grid : Message : 216.487829 s : Deo mflop/s = 1.37283e+07
Grid : Message : 216.487869 s : Deo mflop/s per rank 572011
Grid : Message : 216.487881 s : Deo mflop/s per node 3.43206e+06
Grid : Message : 216.487893 s : #### Dhop calls report
Grid : Message : 216.487903 s : WilsonFermion5D Number of DhopEO Calls : 3001
Grid : Message : 216.487913 s : WilsonFermion5D TotalTime /Calls : 19399.6 us
Grid : Message : 216.487923 s : WilsonFermion5D CommTime /Calls : 16475.4 us
Grid : Message : 216.487933 s : WilsonFermion5D FaceTime /Calls : 972.393 us
Grid : Message : 216.487943 s : WilsonFermion5D ComputeTime1/Calls : 49.8474 us
Grid : Message : 216.487953 s : WilsonFermion5D ComputeTime2/Calls : 2089.93 us
Grid : Message : 216.488001 s : Average mflops/s per call : 5.39682e+09
Grid : Message : 216.488011 s : Average mflops/s per call per rank : 2.24867e+08
Grid : Message : 216.488020 s : Average mflops/s per call per node : 1.3492e+09
Grid : Message : 216.488030 s : Average mflops/s per call (full) : 1.39479e+07
Grid : Message : 216.488039 s : Average mflops/s per call per rank (full): 581162
Grid : Message : 216.488048 s : Average mflops/s per call per node (full): 3.48697e+06
Grid : Message : 216.488057 s : WilsonFermion5D Stencil
Grid : Message : 216.488065 s : WilsonFermion5D StencilEven
Grid : Message : 216.488073 s : WilsonFermion5D StencilOdd
Grid : Message : 216.488081 s : WilsonFermion5D Stencil Reporti()
Grid : Message : 216.488089 s : WilsonFermion5D StencilEven Reporti()
Grid : Message : 216.488097 s : WilsonFermion5D StencilOdd Reporti()
Grid : Message : 217.384495 s : r_e6.02113
Grid : Message : 217.426121 s : r_o6.02096
Grid : Message : 217.472636 s : res12.0421
Grid : Message : 218.200068 s : norm diff 0
Grid : Message : 218.645673 s : norm diff even 0
Grid : Message : 218.816561 s : norm diff odd 0
Grid : Message : MemoryManager::Init() Unified memory space
Grid : Message : MemoryManager::Init() Using cudaMallocManaged
Grid : Message : 0.139000 s : ++++++++++++++++++++++++++++++++++++++++++++++++
Grid : Message : 0.151000 s : Testing with full communication
Grid : Message : 0.158000 s : ++++++++++++++++++++++++++++++++++++++++++++++++
Grid : Message : 0.165000 s : Grid Layout
Grid : Message : 0.171000 s : Global lattice size : 64 64 64 96
Grid : Message : 0.181000 s : OpenMP threads : 6
Grid : Message : 0.189000 s : MPI tasks : 2 2 2 3
Grid : Message : 0.177717 s : Initialising 4d RNG
Grid : Message : 0.342461 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 0.342483 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 0.370454 s : Initialising 5d RNG
Grid : Message : 3.174160 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 3.174420 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 22.119339 s : Drawing gauge field
Grid : Message : 38.113060 s : Random gauge initialised
Grid : Message : 38.113320 s : Applying BCs for Dirichlet Block5 [0 0 0 0 0]
Grid : Message : 38.113470 s : Applying BCs for Dirichlet Block4 [0 0 0 0]
Grid : Message : 43.906786 s : Setting up Cshift based reference

30
systems/Summit/dwf16.lsf
View File

@ -1,25 +1,39 @@
#!/bin/bash
#BSUB -P LGT104
#BSUB -W 2:00
#BSUB -W 0:20
#BSUB -nnodes 16
#BSUB -J DWF
export OMP_NUM_THREADS=6
export PAMI_IBV_ADAPTER_AFFINITY=1
export PAMI_ENABLE_STRIPING=1
export OPT="--comms-concurrent --comms-overlap "
APP="./benchmarks/Benchmark_comms_host_device --mpi 4.4.4.3 "
jsrun --nrs 16 -a6 -g6 -c42 -dpacked -b packed:7 --latency_priority gpu-cpu --smpiargs=-gpu $APP > comms.16node.log
DIR=.
source sourceme.sh
APP="./benchmarks/Benchmark_dwf_fp32 --grid 96.96.96.72 --mpi 4.4.4.3 --shm 2048 --shm-force-mpi 1 --device-mem 8000 --shm-force-mpi 1 $OPT "
jsrun --nrs 16 -a6 -g6 -c42 -dpacked -b packed:7 --latency_priority gpu-cpu --smpiargs=-gpu $APP > dwf.16node.24.log
echo MPICH_SMP_SINGLE_COPY_MODE $MPICH_SMP_SINGLE_COPY_MODE
APP="./benchmarks/Benchmark_dwf_fp32 --grid 128.128.128.96 --mpi 4.4.4.3 --shm 2048 --shm-force-mpi 1 --device-mem 8000 --shm-force-mpi 1 $OPT "
jsrun --nrs 16 -a6 -g6 -c42 -dpacked -b packed:7 --latency_priority gpu-cpu --smpiargs=-gpu $APP > dwf.16node.32.log
VOLS=( 32.32.32.16 32.32.32.64 64.32.32.64 64.32.64.64 64.64.64.64 64.64.64.128 64.64.64.256 64.64.64.512 128.64.64.64.512)
MPI=( 1.1.1.1 1.1.1.4 2.1.1.4 2.1.2.4 2.2.2.4 2.2.2.8 2.2.2.16 2.2.2.32 4.4.2.32 )
RANKS=( 1 4 8 16 32 64 128 256 1024)
NODES=( 1 1 2 4 8 16 32 64 128)
INTS=( 0 1 2 3 4 5 6 7 8)
for i in 5
do
vol=${VOLS[$i]}
nodes=${NODES[$i]}
mpi=${MPI[$i]}
ranks=${RANKS[$i]}
JSRUN="jsrun --nrs $nodes -a4 -g4 -c42 -dpacked -b packed:10 --latency_priority gpu-cpu --smpiargs=-gpu"
PARAMS=" --accelerator-threads 8 --grid $vol --mpi $mpi --comms-sequential --shm 2048 --shm-mpi 0"
$JSRUN ./benchmarks/Benchmark_dwf_fp32 $PARAMS > run.v${vol}.n${nodes}.m${mpi}.seq.ker
PARAMS=" --accelerator-threads 8 --grid $vol --mpi $mpi --comms-overlap --shm 2048 --shm-mpi 0"
$JSRUN ./benchmarks/Benchmark_dwf_fp32 $PARAMS > run.v${vol}.n${nodes}.m${mpi}.over.ker
done

13
systems/Summit/dwf4.lsf
View File

@ -7,16 +7,15 @@
export OMP_NUM_THREADS=6
export PAMI_IBV_ADAPTER_AFFINITY=1
export PAMI_ENABLE_STRIPING=1
export PAMI_DISABLE_IPC=1
export OPT="--comms-concurrent --comms-overlap "
#export GRID_ALLOC_NCACHE_LARGE=1
export APP="./benchmarks/Benchmark_comms_host_device --mpi 2.2.2.3 "
jsrun --nrs 4 -a6 -g6 -c42 -dpacked -b packed:7 --latency_priority gpu-cpu --smpiargs=-gpu $APP > comms.4node
APP="./benchmarks/Benchmark_dwf_fp32 --grid 48.48.48.72 --mpi 2.2.2.3 --shm 2048 --shm-force-mpi 1 --device-mem 8000 --shm-force-mpi 1 $OPT "
jsrun --nrs 4 -a6 -g6 -c42 -dpacked -b packed:7 --latency_priority gpu-cpu --smpiargs=-gpu $APP > dwf.24.4node
APP="./benchmarks/Benchmark_dwf_fp32 --grid 64.64.64.96 --mpi 2.2.2.3 --shm 2048 --shm-force-mpi 1 --device-mem 8000 --shm-force-mpi 1 $OPT "
jsrun --nrs 4 -a6 -g6 -c42 -dpacked -b packed:7 --latency_priority gpu-cpu --smpiargs=-gpu $APP > dwf.32.4node
APP="./wrap.sh ./benchmarks/Benchmark_dwf_fp32 --grid 48.48.48.72 --mpi 2.2.2.3 --shm 1024 --device-mem 4000 --shm-force-mpi 1 $OPT "
jsrun --nrs 24 -a1 -g1 -c6 -dpacked -b packed:6 --latency_priority gpu-cpu --smpiargs="-gpu" $APP > dwf.24.4node
APP="./wrap.sh ./benchmarks/Benchmark_comms_host_device --grid 48.48.48.72 --mpi 2.2.2.3 --shm 1024 --device-mem 4000 --shm-force-mpi 1 $OPT "
jsrun --smpiargs="-gpu" --nrs 4 -a6 -g6 -c42 -dpacked -b packed:6 $APP > comms.24.4node

328
systems/Tursa/dwf.4node.perf
View File

@ -1,25 +1,25 @@
tu-c0r0n00 - 0 device=0 binding=--interleave=0,1
tu-c0r0n00 - 1 device=1 binding=--interleave=2,3
tu-c0r0n09 - 1 device=1 binding=--interleave=2,3
tu-c0r0n00 - 2 device=2 binding=--interleave=4,5
tu-c0r0n06 - 0 device=0 binding=--interleave=0,1
tu-c0r0n06 - 1 device=1 binding=--interleave=2,3
tu-c0r0n09 - 0 device=0 binding=--interleave=0,1
tu-c0r0n09 - 2 device=2 binding=--interleave=4,5
tu-c0r0n03 - 1 device=1 binding=--interleave=2,3
tu-c0r0n06 - 2 device=2 binding=--interleave=4,5
tu-c0r0n09 - 3 device=3 binding=--interleave=6,7
tu-c0r0n00 - 3 device=3 binding=--interleave=6,7
tu-c0r0n03 - 0 device=0 binding=--interleave=0,1
tu-c0r0n03 - 2 device=2 binding=--interleave=4,5
tu-c0r0n06 - 3 device=3 binding=--interleave=6,7
tu-c0r0n03 - 3 device=3 binding=--interleave=6,7
tu-c0r3n00 - 0 device=0 binding=--interleave=0,1
tu-c0r3n00 - 1 device=1 binding=--interleave=2,3
tu-c0r3n00 - 2 device=2 binding=--interleave=4,5
tu-c0r3n00 - 3 device=3 binding=--interleave=6,7
tu-c0r3n06 - 1 device=1 binding=--interleave=2,3
tu-c0r3n06 - 3 device=3 binding=--interleave=6,7
tu-c0r3n06 - 0 device=0 binding=--interleave=0,1
tu-c0r3n06 - 2 device=2 binding=--interleave=4,5
tu-c0r3n03 - 1 device=1 binding=--interleave=2,3
tu-c0r3n03 - 2 device=2 binding=--interleave=4,5
tu-c0r3n03 - 0 device=0 binding=--interleave=0,1
tu-c0r3n03 - 3 device=3 binding=--interleave=6,7
tu-c0r3n09 - 0 device=0 binding=--interleave=0,1
tu-c0r3n09 - 1 device=1 binding=--interleave=2,3
tu-c0r3n09 - 2 device=2 binding=--interleave=4,5
tu-c0r3n09 - 3 device=3 binding=--interleave=6,7
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: ================================================
AcceleratorCudaInit: Configure options --enable-setdevice=no
OPENMPI detected
AcceleratorCudaInit[0]: ========================
AcceleratorCudaInit[0]: Device Number : 0
@ -33,11 +33,41 @@ AcceleratorCudaInit[0]: pciBusID: 3
AcceleratorCudaInit[0]: pciDeviceID: 0
AcceleratorCudaInit[0]: maxGridSize (2147483647,65535,65535)
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: ================================================
AcceleratorCudaInit: Configure options --enable-setdevice=no
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
OPENMPI detected
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-setdevice=no
AcceleratorCudaInit[0]: ========================
AcceleratorCudaInit[0]: Device Number : 0
AcceleratorCudaInit[0]: ========================
@ -50,43 +80,25 @@ AcceleratorCudaInit[0]: pciBusID: 3
AcceleratorCudaInit[0]: pciDeviceID: 0
AcceleratorCudaInit[0]: maxGridSize (2147483647,65535,65535)
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: assume user either uses
AcceleratorCudaInit: a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: Configure options --enable-setdevice=no
local rank 1 device 0 bus id: 0000:44:00.0
AcceleratorCudaInit: ================================================
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
local rank 0 device 0 bus id: 0000:03:00.0
AcceleratorCudaInit: ================================================
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: ================================================
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: ================================================
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: ================================================
OPENMPI detected
AcceleratorCudaInit: using default device
AcceleratorCudaInit: assume user either uses a) IBM jsrun, or
AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding
AcceleratorCudaInit: Configure options --enable-summit, --enable-select-gpu=no
AcceleratorCudaInit: ================================================
local rank 0 device 0 bus id: 0000:03:00.0
AcceleratorCudaInit: ================================================
AcceleratorCudaInit: ================================================
local rank 2 device 0 bus id: 0000:84:00.0
SharedMemoryMpi: World communicator of size 16
SharedMemoryMpi: Node communicator of size 4
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 2147483648bytes at 0x7fcd80000000 for comms buffers
0SharedMemoryMpi: SharedMemoryMPI.cc acceleratorAllocDevice 2147483648bytes at 0x153960000000 for comms buffers
Setting up IPC
__|__|__|__|__|__|__|__|__|__|__|__|__|__|__
@ -116,7 +128,7 @@ This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
Current Grid git commit hash=9d2238148c56e3fbadfa95dcabf2b83d4bde14cd: (HEAD -> develop) uncommited changes
Current Grid git commit hash=da06d15f73184ceb15d66d4e7e702b02fed7b940: (HEAD -> feature/dirichlet, develop) uncommited changes
Grid : Message : ================================================
Grid : Message : MPI is initialised and logging filters activated
@ -124,122 +136,102 @@ Grid : Message : ================================================
Grid : Message : Requested 2147483648 byte stencil comms buffers
Grid : Message : MemoryManager Cache 34004218675 bytes
Grid : Message : MemoryManager::Init() setting up
Grid : Message : MemoryManager::Init() cache pool for recent allocations: SMALL 32 LARGE 8
Grid : Message : MemoryManager::Init() cache pool for recent allocations: SMALL 8 LARGE 2
Grid : Message : MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory
Grid : Message : MemoryManager::Init() Using cudaMalloc
Grid : Message : 1.198523 s : Grid Layout
Grid : Message : 1.198530 s : Global lattice size : 64 64 64 64
Grid : Message : 1.198534 s : OpenMP threads : 4
Grid : Message : 1.198535 s : MPI tasks : 2 2 2 2
Grid : Message : 1.397615 s : Making s innermost grids
Grid : Message : 1.441828 s : Initialising 4d RNG
Grid : Message : 1.547973 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 1.547998 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 1.954777 s : Initialising 5d RNG
Grid : Message : 3.633825 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 3.633869 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 12.162710 s : Initialised RNGs
Grid : Message : 15.882520 s : Drawing gauge field
Grid : Message : 15.816362 s : Random gauge initialised
Grid : Message : 17.279671 s : Setting up Cshift based reference
Grid : Message : 26.331426 s : *****************************************************************
Grid : Message : 26.331452 s : * Kernel options --dslash-generic, --dslash-unroll, --dslash-asm
Grid : Message : 26.331454 s : *****************************************************************
Grid : Message : 26.331456 s : *****************************************************************
Grid : Message : 26.331458 s : * Benchmarking DomainWallFermionR::Dhop
Grid : Message : 26.331459 s : * Vectorising space-time by 8
Grid : Message : 26.331463 s : * VComplexF size is 64 B
Grid : Message : 26.331465 s : * SINGLE precision
Grid : Message : 26.331467 s : * Using Overlapped Comms/Compute
Grid : Message : 26.331468 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 26.331469 s : *****************************************************************
Grid : Message : 28.413717 s : Called warmup
Grid : Message : 56.418423 s : Called Dw 3000 times in 2.80047e+07 us
Grid : Message : 56.418476 s : mflop/s = 3.79581e+07
Grid : Message : 56.418479 s : mflop/s per rank = 2.37238e+06
Grid : Message : 56.418481 s : mflop/s per node = 9.48953e+06
Grid : Message : 56.418483 s : RF GiB/s (base 2) = 77130
Grid : Message : 56.418485 s : mem GiB/s (base 2) = 48206.3
Grid : Message : 56.422076 s : norm diff 1.03481e-13
Grid : Message : 56.456894 s : #### Dhop calls report
Grid : Message : 56.456899 s : WilsonFermion5D Number of DhopEO Calls : 6002
Grid : Message : 56.456903 s : WilsonFermion5D TotalTime /Calls : 4710.93 us
Grid : Message : 56.456905 s : WilsonFermion5D CommTime /Calls : 3196.15 us
Grid : Message : 56.456908 s : WilsonFermion5D FaceTime /Calls : 494.392 us
Grid : Message : 56.456910 s : WilsonFermion5D ComputeTime1/Calls : 44.4107 us
Grid : Message : 56.456912 s : WilsonFermion5D ComputeTime2/Calls : 1037.75 us
Grid : Message : 56.456921 s : Average mflops/s per call : 3.55691e+09
Grid : Message : 56.456925 s : Average mflops/s per call per rank : 2.22307e+08
Grid : Message : 56.456928 s : Average mflops/s per call per node : 8.89228e+08
Grid : Message : 56.456930 s : Average mflops/s per call (full) : 3.82915e+07
Grid : Message : 56.456933 s : Average mflops/s per call per rank (full): 2.39322e+06
Grid : Message : 56.456952 s : Average mflops/s per call per node (full): 9.57287e+06
Grid : Message : 56.456954 s : WilsonFermion5D Stencil
Grid : Message : 56.457016 s : Stencil calls 3001
Grid : Message : 56.457022 s : Stencil halogtime 0
Grid : Message : 56.457024 s : Stencil gathertime 55.9154
Grid : Message : 56.457026 s : Stencil gathermtime 20.1073
Grid : Message : 56.457028 s : Stencil mergetime 18.5585
Grid : Message : 56.457030 s : Stencil decompresstime 0.0639787
Grid : Message : 56.457032 s : Stencil comms_bytes 4.02653e+08
Grid : Message : 56.457034 s : Stencil commtime 6379.93
Grid : Message : 56.457036 s : Stencil 63.1124 GB/s per rank
Grid : Message : 56.457038 s : Stencil 252.45 GB/s per node
Grid : Message : 56.457040 s : WilsonFermion5D StencilEven
Grid : Message : 56.457048 s : WilsonFermion5D StencilOdd
Grid : Message : 56.457062 s : WilsonFermion5D Stencil Reporti()
Grid : Message : 56.457065 s : WilsonFermion5D StencilEven Reporti()
Grid : Message : 56.457066 s : WilsonFermion5D StencilOdd Reporti()
Grid : Message : 79.259261 s : Compare to naive wilson implementation Dag to verify correctness
Grid : Message : 79.259287 s : Called DwDag
Grid : Message : 79.259288 s : norm dag result 12.0421
Grid : Message : 79.271740 s : norm dag ref 12.0421
Grid : Message : 79.287759 s : norm dag diff 7.63236e-14
Grid : Message : 79.328100 s : Calling Deo and Doe and //assert Deo+Doe == Dunprec
Grid : Message : 79.955951 s : src_e0.499997
Grid : Message : 80.633620 s : src_o0.500003
Grid : Message : 80.164163 s : *********************************************************
Grid : Message : 80.164168 s : * Benchmarking DomainWallFermionF::DhopEO
Grid : Message : 80.164170 s : * Vectorising space-time by 8
Grid : Message : 80.164172 s : * SINGLE precision
Grid : Message : 80.164174 s : * Using Overlapped Comms/Compute
Grid : Message : 80.164177 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 80.164178 s : *********************************************************
Grid : Message : 93.797635 s : Deo mflop/s = 3.93231e+07
Grid : Message : 93.797670 s : Deo mflop/s per rank 2.45769e+06
Grid : Message : 93.797672 s : Deo mflop/s per node 9.83077e+06
Grid : Message : 93.797674 s : #### Dhop calls report
Grid : Message : 93.797675 s : WilsonFermion5D Number of DhopEO Calls : 3001
Grid : Message : 93.797677 s : WilsonFermion5D TotalTime /Calls : 4542.83 us
Grid : Message : 93.797679 s : WilsonFermion5D CommTime /Calls : 2978.97 us
Grid : Message : 93.797681 s : WilsonFermion5D FaceTime /Calls : 602.287 us
Grid : Message : 93.797683 s : WilsonFermion5D ComputeTime1/Calls : 67.1416 us
Grid : Message : 93.797685 s : WilsonFermion5D ComputeTime2/Calls : 1004.07 us
Grid : Message : 93.797713 s : Average mflops/s per call : 3.30731e+09
Grid : Message : 93.797717 s : Average mflops/s per call per rank : 2.06707e+08
Grid : Message : 93.797719 s : Average mflops/s per call per node : 8.26827e+08
Grid : Message : 93.797721 s : Average mflops/s per call (full) : 3.97084e+07
Grid : Message : 93.797727 s : Average mflops/s per call per rank (full): 2.48178e+06
Grid : Message : 93.797732 s : Average mflops/s per call per node (full): 9.92711e+06
Grid : Message : 93.797735 s : WilsonFermion5D Stencil
Grid : Message : 93.797746 s : WilsonFermion5D StencilEven
Grid : Message : 93.797758 s : WilsonFermion5D StencilOdd
Grid : Message : 93.797769 s : Stencil calls 3001
Grid : Message : 93.797773 s : Stencil halogtime 0
Grid : Message : 93.797776 s : Stencil gathertime 56.7458
Grid : Message : 93.797780 s : Stencil gathermtime 22.6504
Grid : Message : 93.797782 s : Stencil mergetime 21.1913
Grid : Message : 93.797786 s : Stencil decompresstime 0.0556481
Grid : Message : 93.797788 s : Stencil comms_bytes 2.01327e+08
Grid : Message : 93.797791 s : Stencil commtime 2989.33
Grid : Message : 93.797795 s : Stencil 67.3484 GB/s per rank
Grid : Message : 93.797798 s : Stencil 269.394 GB/s per node
Grid : Message : 93.797801 s : WilsonFermion5D Stencil Reporti()
Grid : Message : 93.797803 s : WilsonFermion5D StencilEven Reporti()
Grid : Message : 93.797805 s : WilsonFermion5D StencilOdd Reporti()
Grid : Message : 93.873429 s : r_e6.02111
Grid : Message : 93.879931 s : r_o6.02102
Grid : Message : 93.885912 s : res12.0421
Grid : Message : 94.876555 s : norm diff 0
Grid : Message : 95.485643 s : norm diff even 0
Grid : Message : 95.581236 s : norm diff odd 0
Grid : Message : 1.875883 s : Grid Layout
Grid : Message : 1.875893 s : Global lattice size : 64 64 64 64
Grid : Message : 1.875897 s : OpenMP threads : 4
Grid : Message : 1.875898 s : MPI tasks : 2 2 2 2
Grid : Message : 1.993571 s : Initialising 4d RNG
Grid : Message : 2.881990 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 2.882370 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 2.495044 s : Initialising 5d RNG
Grid : Message : 4.120900 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 4.121350 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 15.268010 s : Drawing gauge field
Grid : Message : 16.234025 s : Random gauge initialised
Grid : Message : 16.234057 s : Applying BCs
Grid : Message : 16.365565 s : Setting up Cshift based reference
Grid : Message : 44.512418 s : *****************************************************************
Grid : Message : 44.512448 s : * Kernel options --dslash-generic, --dslash-unroll, --dslash-asm
Grid : Message : 44.512450 s : *****************************************************************
Grid : Message : 44.512451 s : *****************************************************************
Grid : Message : 44.512452 s : * Benchmarking DomainWallFermionR::Dhop
Grid : Message : 44.512453 s : * Vectorising space-time by 8
Grid : Message : 44.512454 s : * VComplexF size is 64 B
Grid : Message : 44.512456 s : * SINGLE precision
Grid : Message : 44.512459 s : * Using Overlapped Comms/Compute
Grid : Message : 44.512460 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 44.512461 s : *****************************************************************
Grid : Message : 46.389070 s : Called warmup
Grid : Message : 49.211265 s : Called Dw 300 times in 2.82203e+06 us
Grid : Message : 49.211295 s : mflop/s = 3.76681e+07
Grid : Message : 49.211297 s : mflop/s per rank = 2.35425e+06
Grid : Message : 49.211299 s : mflop/s per node = 9.41702e+06
Grid : Message : 49.211301 s : RF GiB/s (base 2) = 76540.6
Grid : Message : 49.211308 s : mem GiB/s (base 2) = 47837.9
Grid : Message : 49.214868 s : norm diff 1.06409e-13
Grid : Message : 92.647781 s : Compare to naive wilson implementation Dag to verify correctness
Grid : Message : 92.647816 s : Called DwDag
Grid : Message : 92.647817 s : norm dag result 12.0421
Grid : Message : 92.801806 s : norm dag ref 12.0421
Grid : Message : 92.817724 s : norm dag diff 7.21921e-14
Grid : Message : 92.858973 s : Calling Deo and Doe and //assert Deo+Doe == Dunprec
Grid : Message : 93.210378 s : src_e0.499997
Grid : Message : 93.583286 s : src_o0.500003
Grid : Message : 93.682468 s : *********************************************************
Grid : Message : 93.682471 s : * Benchmarking DomainWallFermionF::DhopEO
Grid : Message : 93.682472 s : * Vectorising space-time by 8
Grid : Message : 93.682473 s : * SINGLE precision
Grid : Message : 93.682475 s : * Using Overlapped Comms/Compute
Grid : Message : 93.682476 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 93.682477 s : *********************************************************
Grid : Message : 95.162342 s : Deo mflop/s = 3.92487e+07
Grid : Message : 95.162387 s : Deo mflop/s per rank 2.45305e+06
Grid : Message : 95.162389 s : Deo mflop/s per node 9.81219e+06
Grid : Message : 95.232801 s : r_e6.02111
Grid : Message : 95.240061 s : r_o6.02102
Grid : Message : 95.245975 s : res12.0421
Grid : Message : 95.833402 s : norm diff 0
Grid : Message : 96.573829 s : norm diff even 0
Grid : Message : 96.868272 s : norm diff odd 0
Dirichlet block [0 64 64 32 32]
Grid : Message : 97.756909 s : Grid Layout
Grid : Message : 97.756911 s : Global lattice size : 64 64 64 64
Grid : Message : 97.756921 s : OpenMP threads : 4
Grid : Message : 97.756922 s : MPI tasks : 2 2 2 2
Grid : Message : 97.897085 s : Initialising 4d RNG
Grid : Message : 97.965061 s : Intialising parallel RNG with unique string 'The 4D RNG'
Grid : Message : 97.965097 s : Seed SHA256: 49db4542db694e3b1a74bf2592a8c1b83bfebbe18401693c2609a4c3af1
Grid : Message : 98.367431 s : Initialising 5d RNG
Grid : Message : 99.752745 s : Intialising parallel RNG with unique string 'The 5D RNG'
Grid : Message : 99.752790 s : Seed SHA256: b6316f2fac44ce14111f93e0296389330b077bfd0a7b359f781c58589f8a
Grid : Message : 111.290148 s : Drawing gauge field
Grid : Message : 112.349289 s : Random gauge initialised
Grid : Message : 112.349320 s : Applying BCs
Grid : Message : 113.948740 s : Setting up Cshift based reference
Grid : Message : 140.320415 s : *****************************************************************
Grid : Message : 140.320443 s : * Kernel options --dslash-generic, --dslash-unroll, --dslash-asm
Grid : Message : 140.320444 s : *****************************************************************
Grid : Message : 140.320445 s : *****************************************************************
Grid : Message : 140.320446 s : * Benchmarking DomainWallFermionR::Dhop
Grid : Message : 140.320447 s : * Vectorising space-time by 8
Grid : Message : 140.320448 s : * VComplexF size is 64 B
Grid : Message : 140.320450 s : * SINGLE precision
Grid : Message : 140.320451 s : * Using Overlapped Comms/Compute
Grid : Message : 140.320452 s : * Using GENERIC Nc WilsonKernels
Grid : Message : 140.320453 s : *****************************************************************
Grid : Message : 142.296150 s : Called warmup
Grid : Message : 144.397678 s : Called Dw 300 times in 2.36719e+06 us
Grid : Message : 144.397700 s : mflop/s = 4.49058e+07
Grid : Message : 144.397702 s : mflop/s per rank = 2.80661e+06
Grid : Message : 144.397704 s : mflop/s per node = 1.12265e+07
Grid : Message : 144.397706 s : RF GiB/s (base 2) = 91247.6
Grid : Message : 144.397708 s : mem GiB/s (base 2) = 57029.7
Grid : Message : 144.401269 s : norm diff 9.78944e-14
Grid : Message : 186.885460 s : Compare to naive wilson implementation Dag to verify correctness
Grid : Message : 186.885492 s : Called DwDag
Grid : Message : 186.885493 s : norm dag result 10.4157
Grid : Message : 186.897154 s : norm dag ref 11.2266
Grid : Message : 186.912538 s : norm dag diff 0.484633

7
systems/Tursa/dwf4.slurm
View File

@ -1,14 +1,13 @@
#!/bin/bash
#SBATCH -J dslash
#SBATCH -A tc002
#SBATCH -t 2:20:00
#SBATCH --nodelist=tu-c0r0n[00,03,06,09]
#SBATCH -A dp207
#SBATCH --exclusive
#SBATCH --nodes=4
#SBATCH --ntasks=16
#SBATCH --qos=standard
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=8
#SBATCH --time=12:00:00
#SBATCH --time=0:05:00
#SBATCH --partition=gpu
#SBATCH --gres=gpu:4
#SBATCH --output=%x.%j.out