Compare commits

...

3 Commits

Author SHA1 Message Date
7193ef4c4a add Readme.md to Quda benchmark 2023-06-09 18:09:49 +01:00
bd68360c2c clean up build script a bit 2023-06-09 18:09:31 +01:00
6a11511000 better range of lattice sizes 2023-06-09 18:07:36 +01:00
5 changed files with 95 additions and 12 deletions

View File

@ -226,7 +226,7 @@ void benchmark_dwf()
#endif
printfQuda("%5s %15s %15s\n", "L", "time (usec)", "Gflop/s/rank");
int Ls = 12;
for (int L : {8, 12, 16, 24, 32, 48})
for (int L : {8, 12, 16, 24})
{
auto U = make_gauge_field(L);
auto src = make_source(L, Ls);
@ -305,7 +305,7 @@ void benchmark_axpy()
printfQuda("%5s %15s %15s %15s %15s\n", "L", "size (MiB/rank)", "time (usec)",
"GiB/s/rank", "Gflop/s/rank");
std::vector L_list = {8, 12, 16, 24, 32};
std::vector L_list = {8, 12, 16, 24, 32, 48};
for (int L : L_list)
{
// IMPORTANT: all of `param.x`, `field_elements`, `field.Bytes()`

25
Quda/Readme.md Normal file
View File

@ -0,0 +1,25 @@
# QUDA benchmarks
This folder contains benchmarks for the [QUDA](https://github.com/lattice/quda) library.
- `Benchmark_Quda`: This benchmark measure floating point performances of fermion
matrices (Wilson and DWF), as well as memory bandwidth (using a simple `axpy` operation). Measurements are
performed for a fixed range of problem sizes.
## Building
After setting up your compilation environment (Tursa: `source /home/dp207/dp207/shared/env/production/env-{base,gpu}.sh`):
```bash
./build-quda.sh <env_dir> # build Quda
./build-benchmark.sh <env_dir> # build benchmark
```
where `<env_dir>` is an arbitrary directory where every product will be stored.
## Running the Benchmark
The benchmark should be run as
```bash
mpirun -np <ranks> <env_dir>/prefix/qudabench/Benchmark_Quda
```
where `<ranks>` is the total number of GPU's to use. On Tursa this is 4 times the number of nodes.
Note: on Tursa, the `wrapper.sh` script that is typically used with Grid is not necessary.

32
Quda/build-benchmark.sh Executable file
View File

@ -0,0 +1,32 @@
#!/usr/bin/env bash
# shellcheck disable=SC1090,SC1091
set -euo pipefail
if (( $# != 1 )); then
echo "usage: $(basename "$0") <environment directory>" 1>&2
exit 1
fi
env_dir=$1
# TODO: this is Tursa specific. have not figured out the correct way to do this.
EXTRA_LIBS="/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so"
# NOTE: these flags need to be in sync with Qudas compilation options (see build-quda.sh)
BUILD_FLAGS="-O3 -std=c++17 -DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=12 -DQUDA_RECONSTRUCT=4"
call_dir=$(pwd -P)
script_dir="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
cd "${env_dir}"
env_dir=$(pwd -P)
cd "${call_dir}"
BUILD_DIR="${env_dir}/build/Quda-benchmarks"
PREFIX_DIR="${env_dir}/prefix/qudabench"
QUDA_DIR=${env_dir}/prefix/quda
mkdir -p "${BUILD_DIR}"
mkdir -p "${PREFIX_DIR}"
LINK_FLAGS="-Wl,-rpath,$QUDA_DIR/lib: $QUDA_DIR/lib/libquda.so $EXTRA_LIBS -lpthread -lmpi"
g++ $BUILD_FLAGS -I$QUDA_DIR/include -c -o $BUILD_DIR/Benchmark_Quda.o $script_dir/Benchmark_Quda.cpp
g++ -g -O3 $BUILD_DIR/Benchmark_Quda.o -o $PREFIX_DIR/Benchmark_Quda $LINK_FLAGS -lmpi

36
Quda/build-quda.sh Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/env bash
# shellcheck disable=SC1090,SC1091
BUILD_FLAGS="-O3 -std=c++17"
QUDA_FLAGS="-DQUDA_MPI=ON -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=4 -DQUDA_GPU_ARCH=sm_80"
set -euo pipefail
if (( $# != 1 )); then
echo "usage: $(basename "$0") <environment directory>" 1>&2
exit 1
fi
env_dir=$1
call_dir=$(pwd -P)
mkdir -p ${env_dir}
cd "${env_dir}"
env_dir=$(pwd -P)
cd "${call_dir}"
build_dir="${env_dir}/build/quda"
if [ -d "${build_dir}" ]; then
echo "error: directory '${build_dir}' exists"
exit 1
fi
mkdir -p "${build_dir}"
git clone https://github.com/lattice/quda.git "${build_dir}"
cd "${build_dir}"
mkdir build; cd build
cmake .. $QUDA_FLAGS -DCMAKE_INSTALL_PREFIX=${env_dir}/prefix/quda
make -j128
make install
cd "${call_dir}"

View File

@ -1,10 +0,0 @@
#!/bin/bash
#CXX=/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen/gcc-8.4.1/gcc-9.4.0-g3vyv3te4ah634euh7phyokb3fiurprp/bin/g++
QUDA_BUILD=/home/dp207/dp207/dc-burg2/quda_build
QUDA_SRC=/home/dp207/dp207/dc-burg2/quda
#QUDA_BUILD=
FLAGS="-DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=7 -g -O3 -Wall -Wextra -std=c++17 "
$CXX $FLAGS -I$QUDA_BUILD/include/targets/cuda -I$QUDA_SRC/include -I$QUDA_BUILD/include -isystem $QUDA_SRC/include/externals -isystem $QUDA_BUILD/_deps/eigen-src -c -o Benchmark_Quda.o Benchmark_Quda.cpp
LINK_FLAGS="-Wl,-rpath,$QUDA_BUILD/tests:$QUDA_BUILD/lib:/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs: $QUDA_BUILD/lib/libquda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcudart_static.a -ldl /usr/lib64/librt.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcublas.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcufft.so -lpthread"
$CXX -g -O3 Benchmark_Quda.o -o Benchmark_Quda $LINK_FLAGS -lmpi