add Readme.md to Quda benchmark

Quda/Benchmark_Quda.cpp

@@ -226,7 +226,7 @@ void benchmark_dwf()
 #endif
   printfQuda("%5s %15s %15s\n", "L", "time (usec)", "Gflop/s/rank");
   int Ls = 12;
-  for (int L : {8, 12, 16, 24, 32, 48})
+  for (int L : {8, 12, 16, 24})
   {
     auto U = make_gauge_field(L);
     auto src = make_source(L, Ls);
@@ -305,7 +305,7 @@ void benchmark_axpy()
 
   printfQuda("%5s %15s %15s %15s %15s\n", "L", "size (MiB/rank)", "time (usec)",
              "GiB/s/rank", "Gflop/s/rank");
-  std::vector L_list = {8, 12, 16, 24, 32};
+  std::vector L_list = {8, 12, 16, 24, 32, 48};
   for (int L : L_list)
   {
     // IMPORTANT: all of `param.x`, `field_elements`, `field.Bytes()`

Quda/Readme.md

@@ -0,0 +1,25 @@
+# QUDA benchmarks
+
+This folder contains benchmarks for the [QUDA](https://github.com/lattice/quda) library.
+
+- `Benchmark_Quda`: This benchmark measure floating point performances of fermion
+matrices (Wilson and DWF), as well as memory bandwidth (using a simple `axpy` operation). Measurements are
+performed for a fixed range of problem sizes.
+
+## Building
+After setting up your compilation environment (Tursa: `source /home/dp207/dp207/shared/env/production/env-{base,gpu}.sh`):
+```bash
+./build-quda.sh <env_dir>          # build Quda
+./build-benchmark.sh <env_dir>     # build benchmark
+```
+where `<env_dir>` is an arbitrary directory where every product will be stored.
+
+## Running the Benchmark
+
+The benchmark should be run as
+```bash
+mpirun -np <ranks> <env_dir>/prefix/qudabench/Benchmark_Quda
+```
+where `<ranks>` is the total number of GPU's to use. On Tursa this is 4 times the number of nodes.
+
+Note: on Tursa, the `wrapper.sh` script that is typically used with Grid is not necessary.

Quda/build-benchmark.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090,SC1091
+
+set -euo pipefail
+
+if (( $# != 1 )); then
+    echo "usage: $(basename "$0") <environment directory>" 1>&2
+    exit 1
+fi
+env_dir=$1
+
+# TODO: this is Tursa specific. have not figured out the correct way to do this.
+EXTRA_LIBS="/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so"
+
+# NOTE: these flags need to be in sync with Qudas compilation options (see build-quda.sh)
+BUILD_FLAGS="-O3 -std=c++17 -DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=12 -DQUDA_RECONSTRUCT=4"
+
+call_dir=$(pwd -P)
+script_dir="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
+cd "${env_dir}"
+env_dir=$(pwd -P)
+cd "${call_dir}"
+BUILD_DIR="${env_dir}/build/Quda-benchmarks"
+PREFIX_DIR="${env_dir}/prefix/qudabench"
+QUDA_DIR=${env_dir}/prefix/quda
+mkdir -p "${BUILD_DIR}"
+mkdir -p "${PREFIX_DIR}"
+
+LINK_FLAGS="-Wl,-rpath,$QUDA_DIR/lib: $QUDA_DIR/lib/libquda.so $EXTRA_LIBS -lpthread -lmpi"
+
+g++ $BUILD_FLAGS  -I$QUDA_DIR/include   -c -o $BUILD_DIR/Benchmark_Quda.o  $script_dir/Benchmark_Quda.cpp
+g++ -g -O3 $BUILD_DIR/Benchmark_Quda.o -o $PREFIX_DIR/Benchmark_Quda $LINK_FLAGS -lmpi

Quda/build-quda.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090,SC1091
+
+BUILD_FLAGS="-O3 -std=c++17"
+QUDA_FLAGS="-DQUDA_MPI=ON -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=4 -DQUDA_GPU_ARCH=sm_80"
+
+set -euo pipefail
+
+if (( $# != 1 )); then
+    echo "usage: $(basename "$0") <environment directory>" 1>&2
+    exit 1
+fi
+env_dir=$1
+
+call_dir=$(pwd -P)
+mkdir -p ${env_dir}
+cd "${env_dir}"
+env_dir=$(pwd -P)
+cd "${call_dir}"
+
+build_dir="${env_dir}/build/quda"
+if [ -d "${build_dir}" ]; then
+    echo "error: directory '${build_dir}' exists"
+    exit 1
+fi
+mkdir -p "${build_dir}"
+
+git clone https://github.com/lattice/quda.git "${build_dir}"
+cd "${build_dir}"
+
+mkdir build; cd build
+cmake .. $QUDA_FLAGS -DCMAKE_INSTALL_PREFIX=${env_dir}/prefix/quda
+make -j128
+make install
+
+cd "${call_dir}"

Quda/build.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-#CXX=/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen/gcc-8.4.1/gcc-9.4.0-g3vyv3te4ah634euh7phyokb3fiurprp/bin/g++
-QUDA_BUILD=/home/dp207/dp207/dc-burg2/quda_build
-QUDA_SRC=/home/dp207/dp207/dc-burg2/quda
-#QUDA_BUILD=
-
-FLAGS="-DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=7 -g -O3 -Wall -Wextra -std=c++17 "
-$CXX $FLAGS -I$QUDA_BUILD/include/targets/cuda -I$QUDA_SRC/include -I$QUDA_BUILD/include -isystem $QUDA_SRC/include/externals -isystem $QUDA_BUILD/_deps/eigen-src -c -o Benchmark_Quda.o  Benchmark_Quda.cpp
-LINK_FLAGS="-Wl,-rpath,$QUDA_BUILD/tests:$QUDA_BUILD/lib:/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs: $QUDA_BUILD/lib/libquda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcudart_static.a -ldl /usr/lib64/librt.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcublas.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcufft.so -lpthread"
-$CXX -g -O3 Benchmark_Quda.o -o Benchmark_Quda $LINK_FLAGS -lmpi