portelli/lattice-benchmarks
3
1
Fork 2
Code Issues Pull Requests 2 Packages Projects Releases Wiki Activity

benchmark-quda #3

Open
simon.buerger wants to merge 16 commits from simon.buerger/lattice-benchmarks:benchmark-quda into main
pull from: simon.buerger/lattice-benchmarks:benchmark-quda
merge into: portelli:main
Conversation 1 Commits 16 Files Changed 7 +127 -31
2 changed files with 127 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b95984c230 - Show all commits

141
Quda/Benchmark_Quda.cpp
View File

@ -16,6 +16,7 @@ using namespace quda;
QudaPrecision smoother_halo_prec = QUDA_INVALID_PRECISION; QudaPrecision smoother_halo_prec = QUDA_INVALID_PRECISION;
// This is the MPI grid, i.e. the layout of ranks, not the lattice volume
std::array<int, 4> gridsize = {1, 1, 1, 4}; std::array<int, 4> gridsize = {1, 1, 1, 4};
void initComms(int argc, char **argv, std::array<int, 4> const &commDims) void initComms(int argc, char **argv, std::array<int, 4> const &commDims)
@ -31,6 +32,7 @@ void initComms(int argc, char **argv, std::array<int, 4> const &commDims)
rank = gridsize[i] * rank + coords[i]; rank = gridsize[i] * rank + coords[i];
return rank; return rank;
}; };
initCommsGridQuda(4, commDims.data(), lex_rank_from_coords, nullptr); initCommsGridQuda(4, commDims.data(), lex_rank_from_coords, nullptr);
for (int d = 0; d < 4; d++) for (int d = 0; d < 4; d++)
@ -45,32 +47,57 @@ cudaGaugeField make_gauge_field(std::array<int, 4> const &geom)
// dimension and type of the lattice object // dimension and type of the lattice object
param.nDim = 4; param.nDim = 4;
param.nColor = 3;
param.x[0] = geom[0]; param.x[0] = geom[0];
param.x[1] = geom[1]; param.x[1] = geom[1];
param.x[2] = geom[2]; param.x[2] = geom[2];
param.x[3] = geom[3]; param.x[3] = geom[3];
// number of colors. potentially confusingly, QUDA sometimes uses the word "color" to
// things unrelated with physical color. things like "nColor=32" do pop up in deflation
// solvers where it (to my understanding) refers to the number of (parallely processed)
// deflation vectors.
param.nColor = 3;
// boundary conditions (dont really care for benchmark)
param.t_boundary = QUDA_PERIODIC_T; param.t_boundary = QUDA_PERIODIC_T;
param.siteSubset = QUDA_FULL_SITE_SUBSET; // no even/odd, just a full lattice
param.link_type = QUDA_SU3_LINKS; // for this benchmark we only need "SINGLE" and/or "DOUBLE" precision. But smaller
// precisions are available in QUDA too
param.setPrecision(QUDA_DOUBLE_PRECISION); param.setPrecision(QUDA_DOUBLE_PRECISION);
param.create = QUDA_NULL_FIELD_CREATE; // do not (zero-) initilize the fields // no even/odd subset, we want a full lattice
param.location = QUDA_CUDA_FIELD_LOCATION; // field should live on the accelerator param.siteSubset = QUDA_FULL_SITE_SUBSET;
// turn off advanced features we dont care about for this benchmark // what kind of 3x3 matrices the field contains. A proper gauge field has SU(3)
// matrices, but (for example) smeared/thick links could have non-unitary links.
param.link_type = QUDA_SU3_LINKS;
// "NULL" does not initialize the field upon creation, "ZERO" would set everything to 0
param.create = QUDA_NULL_FIELD_CREATE;
// field should be allocated directly on the accelerator/GPU
param.location = QUDA_CUDA_FIELD_LOCATION;
// "reconstruct" here means reconstructing a SU(3) matrix from fewer than 18 real
// numbers (=3x3 complex numbers). Great feature in production (saving
// memory/cache/network bandwidth), not used for this benchmark.
param.reconstruct = QUDA_RECONSTRUCT_NO; param.reconstruct = QUDA_RECONSTRUCT_NO;
// "ghostExchange" would often be called "halo exchange" outside of Quda. This has
// nothing to do with ghost fields from continuum/perturbative qcd.
param.ghostExchange = QUDA_GHOST_EXCHANGE_NO; param.ghostExchange = QUDA_GHOST_EXCHANGE_NO;
// these control the physical data layout. Might be interesting to try out different // This controls the physical order of elements. "float2" is the the default
// settings
param.order = QUDA_FLOAT2_GAUGE_ORDER; param.order = QUDA_FLOAT2_GAUGE_ORDER;
param.geometry = QUDA_SCALAR_GEOMETRY;
// this means the field is a LORENTZ vector (which a gauge field must be). Has nothing
// to do with spin.
param.geometry = QUDA_VECTOR_GEOMETRY;
// create the field and fill with random SU(3) matrices // create the field and fill with random SU(3) matrices
// std::cout << param << std::endl; // double-check parameters
auto U = cudaGaugeField(param); auto U = cudaGaugeField(param);
quda::RNG rng(U, /*seed=*/1234); gaugeGauss(U, /*seed=*/1234, 1.0);
gaugeGauss(U, rng, 1.0);
return U; return U;
} }
@ -100,11 +127,11 @@ ColorSpinorField make_source(std::array<int, 4> const &geom)
auto src = ColorSpinorField(param); auto src = ColorSpinorField(param);
quda::RNG rng(src, 1234); quda::RNG rng(src, 1234);
spinorNoise(src, rng, QUDA_NOISE_GAUSS); spinorNoise(src, rng, QUDA_NOISE_GAUSS);
printfQuda( /*printfQuda(
"created src with norm = %f (sanity check: should be close to %f) and %f bytes\n", "created src with norm = %f (sanity check: should be close to %f) and %f bytes\n",
blas::norm2(src), 2.0 * 12 * geom[0] * geom[1] * geom[2] * geom[3], blas::norm2(src), 2.0 * 12 * geom[0] * geom[1] * geom[2] * geom[3],
src.Bytes() * 1.0); src.Bytes() * 1.0);*/
src.PrintDims(); // src.PrintDims();
return src; return src;
} }
@ -113,8 +140,16 @@ void benchmark(int L, int niter)
{ {
std::array<int, 4> geom = {L, L, L, L}; std::array<int, 4> geom = {L, L, L, L};
printfQuda("======================= benchmarking L=%d =======================\n", L);
auto U = make_gauge_field(geom); auto U = make_gauge_field(geom);
printfQuda("created random gauge field, %.3f GiB (sanity check: should be %.3f)\n",
U.Bytes() / 1024. / 1024. / 1024.,
1.0 * L * L * L * L * 4 * 18 * 8 / 1024. / 1024. / 1024.);
auto src = make_source(geom); auto src = make_source(geom);
printfQuda("created random source, %.3f GiB (sanity check: should be %.3f)\n",
src.Bytes() / 1024. / 1024. / 1024.,
1.0 * L * L * L * L * 12 * 2 * 8 / 1024. / 1024. / 1024.);
// create (Wilson) dirac operator // create (Wilson) dirac operator
DiracParam param; DiracParam param;
@ -134,9 +169,11 @@ void benchmark(int L, int niter)
geom[1], geom[2], geom[3], niter); geom[1], geom[2], geom[3], niter);
// couple iterations without timing to warm up // couple iterations without timing to warm up
printfQuda("warmup...\n");
for (int iter = 0; iter < 20; ++iter) for (int iter = 0; iter < 20; ++iter)
dirac.M(tmp, src); dirac.M(tmp, src);
printfQuda("running...\n");
dirac.Flops(); // reset flops counter dirac.Flops(); // reset flops counter
device_timer_t device_timer; device_timer_t device_timer;
device_timer.start(); device_timer.start();
@ -146,22 +183,82 @@ void benchmark(int L, int niter)
double secs = device_timer.last(); double secs = device_timer.last();
double gflops = (dirac.Flops() * 1e-9) / secs; double gflops = (dirac.Flops() * 1e-9) / secs;
printfQuda("Gflops = %6.1f\n", gflops); printfQuda("Gflops = %6.2f\n", gflops);
}
void benchmark_axpy(int L)
{
printfQuda("================ axpy L=%d ==============\n", L);
ColorSpinorParam param;
param.nColor = 3;
param.nSpin = 4;
param.nVec = 1;
param.pad = 0;
param.siteSubset = QUDA_FULL_SITE_SUBSET;
param.nDim = 4;
param.x[0] = L;
param.x[1] = L;
param.x[2] = L;
param.x[3] = L;
param.x[4] = 1; // no fifth dimension
param.pc_type = QUDA_4D_PC;
param.siteOrder = QUDA_EVEN_ODD_SITE_ORDER;
param.gammaBasis = QUDA_DEGRAND_ROSSI_GAMMA_BASIS;
param.create = QUDA_NULL_FIELD_CREATE; // do not (zero-) initilize the field
param.setPrecision(QUDA_DOUBLE_PRECISION);
param.location = QUDA_CUDA_FIELD_LOCATION;
// create the field and fill it with random values
auto fieldA = ColorSpinorField(param);
auto fieldB = ColorSpinorField(param);
quda::RNG rng(fieldA, 1234);
auto size_bytes = size_t(8) * 2 * param.x[0] * param.x[1] * param.x[2] * param.x[3] *
param.nColor * param.nSpin;
assert(fieldA.Bytes() == size_bytes); // sanity check
assert(fieldB.Bytes() == size_bytes); // sanity check
spinorNoise(fieldA, rng, QUDA_NOISE_GAUSS);
spinorNoise(fieldB, rng, QUDA_NOISE_GAUSS);
// number of (real) elements in the field = number of fma instructions to do
double flops_per_iter =
2 * param.x[0] * param.x[1] * param.x[2] * param.x[3] * param.nColor * param.nSpin;
int niter = 20;
printfQuda("warmup...\n");
for (int iter = 0; iter < 10; ++iter)
blas::axpy(1.234, fieldA, fieldB);
printfQuda("running...\n");
device_timer_t device_timer;
device_timer.start();
for (int iter = 0; iter < niter; ++iter)
blas::axpy(1.234, fieldA, fieldB); // fieldB += 1.234*fieldA
device_timer.stop();
double secs = device_timer.last();
double gflops = (flops_per_iter * niter) * 1e-9 / secs;
printfQuda("Gflops = %6.2f\n", gflops);
printfQuda("bytes = %6.2f GiB\n", 3. * fieldA.Bytes() / 1024. / 1024. / 1024.);
printfQuda("bandwidth = %6.2f GiB/s\n",
fieldA.Bytes() * 3 / 1024. / 1024. / 1024. * niter / secs);
} }
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
initComms(argc, argv, gridsize); initComms(argc, argv, gridsize);
// -1 for multi-gpu. otherwise this selects the device to be used initQuda(-1); // -1 for multi-gpu. otherwise this selects the device to be used
initQuda(-1);
// verbosity options are: // verbosity options are:
// SILENT, SUMMARIZE, VERBOSE, DEBUG_VERBOSE // SILENT, SUMMARIZE, VERBOSE, DEBUG_VERBOSE
setVerbosity(QUDA_SUMMARIZE); setVerbosity(QUDA_VERBOSE);
for (int L : {8, 16, 24, 32}) for (int L : {8, 12, 16, 24, 32})
benchmark(L, 1000); benchmark_axpy(L);
for (int L : {16, 24, 32, 48, 64})
benchmark(L, 100);
endQuda(); endQuda();
quda::comm_finalize(); quda::comm_finalize();

17
Quda/build.sh
View File

@ -1,11 +1,10 @@
#!/bin/bash #!/bin/bash
#CXX=/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen/gcc-8.4.1/gcc-9.4.0-g3vyv3te4ah634euh7phyokb3fiurprp/bin/g++
QUDA_BUILD=/home/dp207/dp207/dc-burg2/quda_build
QUDA_SRC=/home/dp207/dp207/dc-burg2/quda
#QUDA_BUILD=
set -e FLAGS="-DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=7 -g -O3 -Wall -Wextra -std=c++17 "
$CXX $FLAGS -I$QUDA_BUILD/include/targets/cuda -I$QUDA_SRC/include -I$QUDA_BUILD/include -isystem $QUDA_SRC/include/externals -isystem $QUDA_BUILD/_deps/eigen-src -c -o Benchmark_Quda.o Benchmark_Quda.cpp
LINK_FLAGS="-Wl,-rpath,$QUDA_BUILD/tests:$QUDA_BUILD/lib:/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs: $QUDA_BUILD/lib/libquda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcudart_static.a -ldl /usr/lib64/librt.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcublas.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcufft.so -lpthread"
COMPILE_FLAGS="-DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=7 -g -O3 -Wall -Wextra -pthread -std=c++17" $CXX -g -O3 Benchmark_Quda.o -o Benchmark_Quda $LINK_FLAGS -lmpi
LINK_FLAGS="-g -O3 -Wl,-rpath -Wl,/mnt/lustre/tursafs1/home/dp207/dp207/shared/env/versions/220428/prefix/ompi_gpu/lib -Wl,--enable-new-dtags -L/mnt/lustre/tursafs1/home/dp207/dp207/shared/env/versions/220428/prefix/ompi_gpu/lib -pthread -Wl,-rpath,/home/dp207/dp207/dc-burg2/quda_build/tests:/home/dp207/dp207/dc-burg2/quda_build/lib:/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs:/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64: ../../quda_install/lib/libquda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcudart_static.a -ldl /usr/lib64/librt.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcublas.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcufft.so -lpthread /home/dp207/dp207/shared/env/versions/220428/prefix/ompi_gpu/lib/libmpi.so"
$CXX $COMPILE_FLAGS -I/home/dp207/dp207/dc-burg2/quda_install/include/ -o Benchmark_Quda.o -c Benchmark_Quda.cpp
$CXX $LINK_FLAGS Benchmark_Quda.o -o Benchmark_Quda