add DWF benchmark

Grid/Benchmark_Grid.cpp

@@ -1,7 +1,7 @@
 /*
 Copyright © 2015 Peter Boyle <paboyle@ph.ed.ac.uk>
 Copyright © 2022 Antonin Portelli <antonin.portelli@me.com>
-Copyright © 2024 Simon Buerger <simon.buerger@rwth-aachen.de>
+Copyright © 2022 Simon Buerger <simon.buerger@rwth-aachen.de>
 
 This is a fork of Benchmark_ITT.cpp from Grid
 
@@ -29,43 +29,6 @@ int NN_global;
 
 nlohmann::json json_results;
 
-// NOTE: Grid::GridClock is just a typedef to
-// `std::chrono::high_resolution_clock`, but `Grid::usecond` rounds to
-// microseconds (no idea why, probably wasnt ever relevant before), so we need
-// our own wrapper here.
-double usecond_precise()
-{
-  using namespace std::chrono;
-  auto nsecs = duration_cast<nanoseconds>(GridClock::now() - Grid::theProgramStart);
-  return nsecs.count() * 1e-3;
-}
-
-std::vector<std::string> get_mpi_hostnames()
-{
-  int world_size;
-  MPI_Comm_size(MPI_COMM_WORLD, &world_size);
-
-  char hostname[MPI_MAX_PROCESSOR_NAME];
-  int name_len = 0;
-  MPI_Get_processor_name(hostname, &name_len);
-
-  // Allocate buffer to gather all hostnames
-  std::vector<char> all_hostnames(world_size * MPI_MAX_PROCESSOR_NAME);
-
-  // Use MPI_Allgather to gather all hostnames on all ranks
-  MPI_Allgather(hostname, MPI_MAX_PROCESSOR_NAME, MPI_CHAR, all_hostnames.data(),
-                MPI_MAX_PROCESSOR_NAME, MPI_CHAR, MPI_COMM_WORLD);
-
-  // Convert the gathered hostnames back into a vector of std::string
-  std::vector<std::string> hostname_list(world_size);
-  for (int i = 0; i < world_size; ++i)
-  {
-    hostname_list[i] = std::string(&all_hostnames[i * MPI_MAX_PROCESSOR_NAME]);
-  }
-
-  return hostname_list;
-}
-
 struct time_statistics
 {
   double mean;
@@ -110,8 +73,6 @@ class Benchmark
         {local[0] * mpi[0], local[1] * mpi[1], local[2] * mpi[2], local[3] * mpi[3]});
     GridCartesian *TmpGrid = SpaceTimeGrid::makeFourDimGrid(
         latt4, GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
-    Grid::Coordinate shm(4, 1);
-    GlobalSharedMemory::GetShmDims(mpi, shm);
 
     uint64_t NP = TmpGrid->RankCount();
     uint64_t NN = TmpGrid->NodeCount();
@@ -124,9 +85,7 @@ class Benchmark
     std::cout << GridLogMessage << "* OpenMP threads : " << GridThread::GetThreads()
               << std::endl;
 
-    std::cout << GridLogMessage << "* MPI layout     : " << GridCmdVectorIntToString(mpi)
+    std::cout << GridLogMessage << "* MPI tasks      : " << GridCmdVectorIntToString(mpi)
-              << std::endl;
-    std::cout << GridLogMessage << "* Shm layout     : " << GridCmdVectorIntToString(shm)
               << std::endl;
 
     std::cout << GridLogMessage << "* vReal          : " << sizeof(vReal) * 8 << "bits ; "
@@ -159,7 +118,6 @@ class Benchmark
     for (unsigned int i = 0; i < mpi.size(); ++i)
     {
       tmp["mpi"].push_back(mpi[i]);
-      tmp["shm"].push_back(shm[i]);
     }
     tmp["ranks"] = NP;
     tmp["nodes"] = NN;
@@ -174,8 +132,6 @@ class Benchmark
 
     Coordinate simd_layout = GridDefaultSimd(Nd, vComplexD::Nsimd());
     Coordinate mpi_layout = GridDefaultMpi();
-    Coordinate shm_layout(Nd, 1);
-    GlobalSharedMemory::GetShmDims(mpi_layout, shm_layout);
 
     for (int mu = 0; mu < Nd; mu++)
       if (mpi_layout[mu] > 1)
@@ -187,8 +143,8 @@ class Benchmark
     std::cout << GridLogMessage << "Benchmarking threaded STENCIL halo exchange in "
               << nmu << " dimensions" << std::endl;
     grid_small_sep();
-    grid_printf("%5s %5s %7s %15s %15s %15s %15s %15s\n", "L", "dir", "shm",
+    grid_printf("%5s %5s %15s %15s %15s %15s %15s\n", "L", "dir", "payload (B)",
-                "payload (B)", "time (usec)", "rate (GB/s/node)", "std dev", "max");
+                "time (usec)", "rate (GB/s/node)", "std dev", "max");
 
     for (int lat = 16; lat <= maxlat; lat += 8)
     {
@@ -217,10 +173,8 @@ class Benchmark
       for (int dir = 0; dir < 8; dir++)
       {
         int mu = dir % 4;
-        if (mpi_layout[mu] == 1) // skip directions that are not distributed
+        if (mpi_layout[mu] > 1)
-          continue;
+        {
-        bool is_shm = mpi_layout[mu] == shm_layout[mu];
-        bool is_partial_shm = !is_shm && shm_layout[mu] != 1;
 
           std::vector<double> times(Nloop);
           for (int i = 0; i < NWARMUP; i++)
@@ -238,8 +192,8 @@ class Benchmark
               int comm_proc = mpi_layout[mu] - 1;
               Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
             }
-          Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, (void *)&rbuf[dir][0],
+            Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
-                              recv_from_rank, bytes);
+                                (void *)&rbuf[dir][0], recv_from_rank, bytes);
           }
           for (int i = 0; i < Nloop; i++)
           {
@@ -259,8 +213,8 @@ class Benchmark
               int comm_proc = mpi_layout[mu] - 1;
               Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
             }
-          Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, (void *)&rbuf[dir][0],
+            Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
-                              recv_from_rank, bytes);
+                                (void *)&rbuf[dir][0], recv_from_rank, bytes);
             dbytes += bytes;
 
             double stop = usecond();
@@ -273,17 +227,12 @@ class Benchmark
           double rate = bidibytes / (timestat.mean / 1.e6) / 1024. / 1024. / 1024.;
           double rate_err = rate * timestat.err / timestat.mean;
           double rate_max = rate * timestat.mean / timestat.min;
-        grid_printf("%5d %5d %7s %15d %15.2f %15.2f %15.1f %15.2f\n", lat, dir,
+          grid_printf("%5d %5d %15d %15.2f %15.2f %15.1f %15.2f\n", lat, dir, bytes,
-                    is_shm           ? "yes"
+                      timestat.mean, rate, rate_err, rate_max);
-                    : is_partial_shm ? "partial"
-                                     : "no",
-                    bytes, timestat.mean, rate, rate_err, rate_max);
           nlohmann::json tmp;
           nlohmann::json tmp_rate;
           tmp["L"] = lat;
           tmp["dir"] = dir;
-        tmp["shared_mem"] = is_shm;
-        tmp["partial_shared_mem"] = is_partial_shm;
           tmp["bytes"] = bytes;
           tmp["time_usec"] = timestat.mean;
           tmp_rate["mean"] = rate;
@@ -292,6 +241,7 @@ class Benchmark
           tmp["rate_GBps"] = tmp_rate;
           json_results["comms"].push_back(tmp);
         }
+      }
       for (int d = 0; d < 8; d++)
       {
         acceleratorFreeDevice(xbuf[d]);
@@ -301,170 +251,6 @@ class Benchmark
     return;
   }
 
-  static void Latency(void)
-  {
-    int Nwarmup = 100;
-    int Nloop = 300;
-
-    std::cout << GridLogMessage << "Benchmarking point-to-point latency" << std::endl;
-    grid_small_sep();
-    grid_printf("from to      mean(usec)           err           max\n");
-
-    int ranks;
-    int me;
-    MPI_Comm_size(MPI_COMM_WORLD, &ranks);
-    MPI_Comm_rank(MPI_COMM_WORLD, &me);
-
-    int bytes = 8;
-    void *buf_from = acceleratorAllocDevice(bytes);
-    void *buf_to = acceleratorAllocDevice(bytes);
-    nlohmann::json json_latency;
-    for (int from = 0; from < ranks; ++from)
-      for (int to = 0; to < ranks; ++to)
-      {
-        if (from == to)
-          continue;
-
-        std::vector<double> t_time(Nloop);
-        time_statistics timestat;
-        MPI_Status status;
-
-        for (int i = -Nwarmup; i < Nloop; ++i)
-        {
-          double start = usecond_precise();
-          if (from == me)
-          {
-            auto err = MPI_Send(buf_from, bytes, MPI_CHAR, to, 0, MPI_COMM_WORLD);
-            assert(err == MPI_SUCCESS);
-          }
-          if (to == me)
-          {
-            auto err =
-                MPI_Recv(buf_to, bytes, MPI_CHAR, from, 0, MPI_COMM_WORLD, &status);
-            assert(err == MPI_SUCCESS);
-          }
-          double stop = usecond_precise();
-          if (i >= 0)
-            t_time[i] = stop - start;
-        }
-        // important: only 'from' and 'to' have meaningful timings. we use
-        // 'from's.
-        MPI_Bcast(t_time.data(), Nloop, MPI_DOUBLE, from, MPI_COMM_WORLD);
-
-        timestat.statistics(t_time);
-        grid_printf("%2d %2d %15.4f %15.3f %15.4f\n", from, to, timestat.mean,
-                    timestat.err, timestat.max);
-        nlohmann::json tmp;
-        tmp["from"] = from;
-        tmp["to"] = to;
-        tmp["time_usec"] = timestat.mean;
-        tmp["time_usec_error"] = timestat.err;
-        tmp["time_usec_min"] = timestat.min;
-        tmp["time_usec_max"] = timestat.max;
-        tmp["time_usec_full"] = t_time;
-        json_latency.push_back(tmp);
-      }
-    json_results["latency"] = json_latency;
-
-    acceleratorFreeDevice(buf_from);
-    acceleratorFreeDevice(buf_to);
-  }
-
-  static void P2P(void)
-  {
-    // IMPORTANT: The P2P benchmark uses "MPI_COMM_WORLD" communicator, which is
-    // not the quite the same as Grid.communicator. Practically speaking, the
-    // latter one contains the same MPI-ranks but in a different order. Grid
-    // does this make sure it can exploit ranks with shared memory (i.e.
-    // multiple ranks on the same node) as best as possible.
-
-    // buffer-size to benchmark. This number is the same as the largest one used
-    // in the "Comms()" benchmark. ( L=48, Ls=12, double-prec-complex,
-    // half-color-spin-vector. ). Mostly an arbitrary choice, but nice to match
-    // it here
-    size_t bytes = 127401984;
-
-    int Nwarmup = 20;
-    int Nloop = 100;
-
-    std::cout << GridLogMessage << "Benchmarking point-to-point bandwidth" << std::endl;
-    grid_small_sep();
-    grid_printf("from to      mean(usec)           err           min           "
-                "bytes    rate (GiB/s)\n");
-
-    int ranks;
-    int me;
-    MPI_Comm_size(MPI_COMM_WORLD, &ranks);
-    MPI_Comm_rank(MPI_COMM_WORLD, &me);
-
-    void *buf_from = acceleratorAllocDevice(bytes);
-    void *buf_to = acceleratorAllocDevice(bytes);
-    nlohmann::json json_p2p;
-    for (int from = 0; from < ranks; ++from)
-      for (int to = 0; to < ranks; ++to)
-      {
-        if (from == to)
-          continue;
-
-        std::vector<double> t_time(Nloop);
-        time_statistics timestat;
-        MPI_Status status;
-
-        for (int i = -Nwarmup; i < Nloop; ++i)
-        {
-          double start = usecond_precise();
-          if (from == me)
-          {
-            auto err = MPI_Send(buf_from, bytes, MPI_CHAR, to, 0, MPI_COMM_WORLD);
-            assert(err == MPI_SUCCESS);
-          }
-          if (to == me)
-          {
-            auto err =
-                MPI_Recv(buf_to, bytes, MPI_CHAR, from, 0, MPI_COMM_WORLD, &status);
-            assert(err == MPI_SUCCESS);
-          }
-          double stop = usecond_precise();
-          if (i >= 0)
-            t_time[i] = stop - start;
-        }
-        // important: only 'from' and 'to' have meaningful timings. we use
-        // 'from's.
-        MPI_Bcast(t_time.data(), Nloop, MPI_DOUBLE, from, MPI_COMM_WORLD);
-
-        timestat.statistics(t_time);
-        double rate = bytes / (timestat.mean / 1.e6) / 1024. / 1024. / 1024.;
-        double rate_err = rate * timestat.err / timestat.mean;
-        double rate_max = rate * timestat.mean / timestat.min;
-        double rate_min = rate * timestat.mean / timestat.max;
-
-        grid_printf("%2d %2d %15.4f %15.3f %15.4f %15d %15.2f\n", from, to, timestat.mean,
-                    timestat.err, timestat.min, bytes, rate);
-
-        nlohmann::json tmp;
-        tmp["from"] = from;
-        tmp["to"] = to;
-        tmp["bytes"] = bytes;
-        tmp["time_usec"] = timestat.mean;
-        tmp["time_usec_error"] = timestat.err;
-        tmp["time_usec_min"] = timestat.min;
-        tmp["time_usec_max"] = timestat.max;
-        tmp["time_usec_full"] = t_time;
-        nlohmann::json tmp_rate;
-        tmp_rate["mean"] = rate;
-        tmp_rate["error"] = rate_err;
-        tmp_rate["max"] = rate_max;
-        tmp_rate["min"] = rate_min;
-        tmp["rate_GBps"] = tmp_rate;
-
-        json_p2p.push_back(tmp);
-      }
-    json_results["p2p"] = json_p2p;
-
-    acceleratorFreeDevice(buf_from);
-    acceleratorFreeDevice(buf_to);
-  }
-
   static void Memory(void)
   {
     const int Nvec = 8;
@@ -726,6 +512,8 @@ class Benchmark
 
         FGrid->Broadcast(0, &ncall, sizeof(ncall));
 
+        Dw.ZeroCounters();
+
         time_statistics timestat;
         std::vector<double> t_time(ncall);
         for (uint64_t i = 0; i < ncall; i++)
@@ -920,6 +708,7 @@ class Benchmark
         uint64_t ncall = 500;
 
         FGrid->Broadcast(0, &ncall, sizeof(ncall));
+        Ds.ZeroCounters();
 
         time_statistics timestat;
         std::vector<double> t_time(ncall);
@@ -987,47 +776,11 @@ int main(int argc, char **argv)
 {
   Grid_init(&argc, &argv);
 
-  int Ls = 1;
-  bool do_su4 = true;
-  bool do_memory = true;
-  bool do_comms = true;
-  bool do_flops = true;
-
-  // NOTE: these two take O((number of ranks)^2) time, which might be a lot, so they are
-  // off by default
-  bool do_latency = false;
-  bool do_p2p = false;
-
   std::string json_filename = ""; // empty indicates no json output
   for (int i = 0; i < argc; i++)
   {
-    auto arg = std::string(argv[i]);
+    if (std::string(argv[i]) == "--json-out")
-    if (arg == "--json-out")
       json_filename = argv[i + 1];
-    if (arg == "--benchmark-su4")
-      do_su4 = true;
-    if (arg == "--benchmark-memory")
-      do_memory = true;
-    if (arg == "--benchmark-comms")
-      do_comms = true;
-    if (arg == "--benchmark-flops")
-      do_flops = true;
-    if (arg == "--benchmark-latency")
-      do_latency = true;
-    if (arg == "--benchmark-p2p")
-      do_p2p = true;
-    if (arg == "--no-benchmark-su4")
-      do_su4 = false;
-    if (arg == "--no-benchmark-memory")
-      do_memory = false;
-    if (arg == "--no-benchmark-comms")
-      do_comms = false;
-    if (arg == "--no-benchmark-flops")
-      do_flops = false;
-    if (arg == "--no-benchmark-latency")
-      do_latency = false;
-    if (arg == "--no-benchmark-p2p")
-      do_p2p = false;
   }
 
   CartesianCommunicator::SetCommunicatorPolicy(
@@ -1039,6 +792,12 @@ int main(int argc, char **argv)
 #endif
   Benchmark::Decomposition();
 
+  int do_su4 = 1;
+  int do_memory = 1;
+  int do_comms = 1;
+  int do_flops = 1;
+  int Ls = 1;
+
   int sel = 4;
   std::vector<int> L_list({8, 12, 16, 24, 32});
   int selm1 = sel - 1;
@@ -1071,22 +830,6 @@ int main(int argc, char **argv)
     Benchmark::Comms();
   }
 
-  if (do_latency)
-  {
-    grid_big_sep();
-    std::cout << GridLogMessage << " Latency benchmark " << std::endl;
-    grid_big_sep();
-    Benchmark::Latency();
-  }
-
-  if (do_p2p)
-  {
-    grid_big_sep();
-    std::cout << GridLogMessage << " Point-To-Point benchmark " << std::endl;
-    grid_big_sep();
-    Benchmark::P2P();
-  }
-
   if (do_flops)
   {
     Ls = 1;
@@ -1146,8 +889,6 @@ int main(int argc, char **argv)
     json_results["flops"] = tmp_flops;
   }
 
-  json_results["hostnames"] = get_mpi_hostnames();
-
   if (!json_filename.empty())
   {
     std::cout << GridLogMessage << "writing benchmark results to " << json_filename

Grid/systems/tursa/files/gpu-mpi-wrapper.sh

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 
 lrank=$OMPI_COMM_WORLD_LOCAL_RANK
-numa1=$((lrank))
+numa1=$(( 2 * lrank))
+numa2=$(( 2 * lrank + 1 ))
 netdev=mlx5_${lrank}:1
 
 export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
 export UCX_NET_DEVICES=${netdev}
-BINDING="--interleave=$numa1"
+BINDING="--interleave=$numa1,$numa2"
 
 echo "$(hostname) - $lrank device=$CUDA_VISIBLE_DEVICES binding=$BINDING"
 

Grid/systems/tursa/spack-bootstrap.sh

@@ -4,13 +4,7 @@ set -euo pipefail
 
 gcc_spec='gcc@9.4.0'
 cuda_spec='cuda@11.4.0'
-
+hdf5_spec='hdf5@1.10.7'
-# hdf5 and fftw depend on OpenMPI, which we install manually. To make sure this
-# dependency is picked by spack, we specify the compiler here explicitly. For
-# most other packages we dont really care about the compiler (i.e. system
-# compiler versus ${gcc_spec})
-hdf5_spec="hdf5@1.10.7+cxx+threadsafe%${gcc_spec}"
-fftw_spec="fftw%${gcc_spec}"
 
 if (( $# != 1 )); then
     echo "usage: $(basename "$0") <env dir>" 1>&2
@@ -44,23 +38,26 @@ rm external.yaml
 
 # Base compilers ###############################################################
 # configure system base
-
-spack env create base
-spack env activate base
 spack compiler find --scope site
 
-# install GCC, CUDA
+# install GCC, CUDA & LLVM
-spack add ${gcc_spec} ${cuda_spec}
+spack install ${gcc_spec} ${cuda_spec} llvm
-spack concretize
+
-spack env depfile -o Makefile.tmp
+spack load llvm
-make -j128 -f Makefile.tmp
 spack compiler find --scope site
+spack unload llvm
+
+spack load ${gcc_spec}
+spack compiler find --scope site
+spack unload ${gcc_spec}
 
 # Manual compilation of OpenMPI & UCX ##########################################
 # set build directories
 mkdir -p "${dir}"/build
 cd "${dir}"/build
 
+spack load ${gcc_spec} ${cuda_spec}
+
 cuda_path=$(spack find --format "{prefix}" cuda)
 gdrcopy_path=/mnt/lustre/tursafs1/apps/gdrcopy/2.3.1
 
@@ -80,7 +77,7 @@ mkdir -p build_gpu; cd build_gpu
              --enable-devel-headers --enable-examples --enable-optimizations   \
              --with-gdrcopy=${gdrcopy_path} --with-verbs --disable-logging     \
              --disable-debug --disable-assertions --enable-cma                 \
-             --with-knem=/opt/knem-1.1.4.90mlnx2/ --with-rdmacm                \
+             --with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm                \
              --without-rocm --without-ugni --without-java                      \
              --enable-compiler-opt=3 --with-cuda="${cuda_path}" --without-cm   \
              --with-rc --with-ud --with-dc --with-mlx5-dv --with-dm            \
@@ -96,7 +93,7 @@ mkdir -p build_cpu; cd build_cpu
              --enable-devel-headers --enable-examples --enable-optimizations   \
              --with-verbs --disable-logging --disable-debug                    \
              --disable-assertions --enable-mt --enable-cma                     \
-              --with-knem=/opt/knem-1.1.4.90mlnx2/--with-rdmacm                \
+             --with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm                \
              --without-rocm --without-ugni --without-java                      \
              --enable-compiler-opt=3 --without-cm --without-ugni --with-rc     \
              --with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --without-go
@@ -122,7 +119,7 @@ mkdir build_gpu; cd build_gpu
 ../configure --prefix="${dir}"/prefix/ompi_gpu --without-xpmem    \
              --with-ucx="${dir}"/prefix/ucx_gpu                   \
              --with-ucx-libdir="${dir}"/prefix/ucx_gpu/lib        \
-             --with-knem=/opt/knem-1.1.4.90mlnx2/                 \
+             --with-knem=/opt/knem-1.1.4.90mlnx1/                 \
              --enable-mca-no-build=btl-uct                        \
              --with-cuda="${cuda_path}" --disable-getpwuid        \
              --with-verbs --with-slurm --enable-mpi-fortran=all   \
@@ -136,7 +133,7 @@ mkdir build_cpu; cd build_cpu
 ../configure --prefix="${dir}"/prefix/ompi_cpu --without-xpmem    \
              --with-ucx="${dir}"/prefix/ucx_cpu                   \
              --with-ucx-libdir="${dir}"/prefix/ucx_cpu/lib        \
-             --with-knem=/opt/knem-1.1.4.90mlnx2/                 \
+             --with-knem=/opt/knem-1.1.4.90mlnx1/                 \
              --enable-mca-no-build=btl-uct --disable-getpwuid     \
              --with-verbs --with-slurm --enable-mpi-fortran=all   \
              --with-pmix=internal --with-libevent=internal
@@ -144,62 +141,60 @@ make -j 128
 make install
 cd "${dir}"
 
-ucx_spec_gpu="ucx@1.12.0.GPU%${gcc_spec}"
-ucx_spec_cpu="ucx@1.12.0.CPU%${gcc_spec}"
-openmpi_spec_gpu="openmpi@4.1.1.GPU%${gcc_spec}"
-openmpi_spec_cpu="openmpi@4.1.1.CPU%${gcc_spec}"
-
 # Add externals to spack
 echo "packages:
   ucx:
     externals:
-    - spec: \"${ucx_spec_gpu}\"
+    - spec: \"ucx@1.12.0.GPU%gcc@9.4.0\"
       prefix: ${dir}/prefix/ucx_gpu
-    - spec: \"${ucx_spec_cpu}\"
+    - spec: \"ucx@1.12.0.CPU%gcc@9.4.0\"
       prefix: ${dir}/prefix/ucx_cpu
     buildable: False
   openmpi:
     externals:
-    - spec: \"${openmpi_spec_gpu}\"
+    - spec: \"openmpi@4.1.1.GPU%gcc@9.4.0\"
       prefix: ${dir}/prefix/ompi_gpu
-    - spec: \"${openmpi_spec_cpu}\"
+    - spec: \"openmpi@4.1.1.CPU%gcc@9.4.0\"
       prefix: ${dir}/prefix/ompi_cpu
     buildable: False" > spack.yaml
 
 spack config --scope site add -f spack.yaml
 rm spack.yaml
-spack env deactivate
+spack install ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
+spack install ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
 
 cd "${cwd}"
 
 # environments #################################################################
 dev_tools=("autoconf" "automake" "libtool" "jq" "git")
+ompi_gpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.GPU)
+ompi_cpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.CPU)
 
 spack env create grid-gpu
 spack env activate grid-gpu
-spack compiler find --scope site
+spack add ${gcc_spec} ${cuda_spec} "${dev_tools[@]}" 
-spack add ${gcc_spec} ${cuda_spec} ${ucx_spec_gpu} ${openmpi_spec_gpu}
+spack add ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
-spack add ${hdf5_spec} ${fftw_spec}
+spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_gpu_hash}"
-spack add openssl gmp mpfr c-lime "${dev_tools[@]}"
+spack add fftw ^/"${ompi_gpu_hash}"
-spack concretize
+spack add openssl gmp mpfr c-lime
-spack env depfile -o Makefile.tmp
+spack install
-make -j128 -f Makefile.tmp
 spack env deactivate
 
 spack env create grid-cpu
 spack env activate grid-cpu
-spack compiler find --scope site
+spack add llvm "${dev_tools[@]}" 
-spack add ${gcc_spec} ${ucx_spec_cpu} ${openmpi_spec_cpu}
+spack add ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
-spack add ${hdf5_spec} ${fftw_spec}
+spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_cpu_hash}"
-spack add openssl gmp mpfr c-lime "${dev_tools[@]}"
+spack add fftw ^/"${ompi_cpu_hash}"
-spack concretize
+spack add openssl gmp mpfr c-lime
-spack env depfile -o Makefile.tmp
+spack install
-make -j128 -f Makefile.tmp
 spack env deactivate
 
+spack install jq git
+
 # Final setup ##################################################################
 spack clean
-#spack gc -y  # "spack gc" tends to get hung up for unknown reasons
+spack gc -y
 
 # add more environment variables in module loading
 spack config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH,LIBRARY_PATH]'

Quda/Benchmark_Quda.cpp

@@ -2,73 +2,23 @@
 #include <array>
 #include <blas_quda.h>
 #include <cassert>
-#include <chrono>
 #include <color_spinor_field.h>
-#include <communicator_quda.h>
 #include <dirac_quda.h>
-#include <fstream>
 #include <gauge_tools.h>
 #include <memory>
 #include <mpi.h>
 #include <stdio.h>
 #include <stdlib.h>
 
-// remove to use QUDA's own flop counting instead of Grid's convention
-#define FLOP_COUNTING_GRID
-
-#include "json.hpp"
-using nlohmann::json;
-json json_results;
-
 using namespace quda;
 
-// thanks chatGPT :)
+// remove to use QUDA's own flop counting instead of Grid's convention
-std::string get_timestamp()
+#define FLOP_COUNTING_GRID
-{
-  // Get the current time
-  auto now = std::chrono::system_clock::now();
-
-  // Convert the current time to a time_t object
-  std::time_t currentTime = std::chrono::system_clock::to_time_t(now);
-
-  // Format the time using std::put_time
-  std::stringstream ss;
-  ss << std::put_time(std::localtime(&currentTime), "%Y%m%d %H:%M:%S");
-
-  return ss.str();
-}
 
 // This is the MPI grid, i.e. the layout of ranks
 int nranks = -1;
 std::array<int, 4> mpi_grid = {1, 1, 1, 1};
 
-// run f() in a loop for roughly target_time seconds
-// returns seconds per iteration it took
-template <class F> double bench(F const &f, double target_time, int niter_warmup = 5)
-{
-  device_timer_t timer;
-  timer.start();
-  for (int iter = 0; iter < niter_warmup; ++iter)
-    f();
-  timer.stop();
-
-  double secs = timer.last() / niter_warmup;
-  int niter = std::max(1, int(target_time / secs));
-  // niter = std::min(1000, niter);
-  // printfQuda("during warmup took %f s/iter, deciding on %d iters\n", secs, niter);
-
-  // important: each rank has its own timer, so their measurements can slightly vary. But
-  // 'niter' needs to be consistent (bug took me a couple hours to track down)
-  comm_broadcast_global(&niter, sizeof(niter), 0);
-
-  timer.reset(__FUNCTION__, __FILE__, __LINE__);
-  timer.start();
-  for (int iter = 0; iter < niter; ++iter)
-    f();
-  timer.stop();
-  return timer.last() / niter;
-}
-
 void initComms(int argc, char **argv)
 {
   // init MPI communication
@@ -93,9 +43,6 @@ void initComms(int argc, char **argv)
   for (int d = 0; d < 4; d++)
     if (mpi_grid[d] > 1)
       commDimPartitionedSet(d);
-
-  json_results["geometry"]["ranks"] = nranks;
-  json_results["geometry"]["mpi"] = mpi_grid;
 }
 
 // creates a random gauge field. L = local(!) size
@@ -202,8 +149,11 @@ ColorSpinorField make_source(int L, int Ls = 1)
   return src;
 }
 
-void benchmark_wilson(std::vector<int> const &L_list, double target_time)
+void benchmark_wilson()
 {
+  int niter = 20;
+  int niter_warmup = 10;
+
   printfQuda("==================== wilson dirac operator ====================\n");
 #ifdef FLOP_COUNTING_GRID
   printfQuda("IMPORTANT: flop counting as in Benchmark_Grid\n");
@@ -213,10 +163,8 @@ void benchmark_wilson(std::vector<int> const &L_list, double target_time)
 #endif
   printfQuda("%5s %15s %15s\n", "L", "time (usec)", "Gflop/s/rank");
 
-  for (int L : L_list)
+  for (int L : {8, 12, 16, 24, 32, 48})
   {
-    // printfQuda("starting wilson L=%d\n", L);
-
     auto U = make_gauge_field(L);
     auto src = make_source(L);
 
@@ -231,41 +179,44 @@ void benchmark_wilson(std::vector<int> const &L_list, double target_time)
     // (the additional nullptr's are for smeared links and fancy preconditioners and such.
     // Not used for simple Wilson fermions)
     dirac.updateFields(&U, nullptr, nullptr, nullptr);
-    auto res = ColorSpinorField(ColorSpinorParam(src));
-    auto f = [&]() { dirac.Dslash(res, src, QUDA_EVEN_PARITY); };
 
-    // first run to get the quda tuning out of the way
+    auto tmp = ColorSpinorField(ColorSpinorParam(src));
+
+    // couple iterations without timing to warm up
+    for (int iter = 0; iter < niter_warmup; ++iter)
+      dirac.Dslash(tmp, src, QUDA_EVEN_PARITY);
+
+    // actual benchmark with timings
     dirac.Flops(); // reset flops counter
-    f();
+    device_timer_t device_timer;
-    double flops = 1.0 * dirac.Flops();
+    device_timer.start();
+    for (int iter = 0; iter < niter; ++iter)
+      dirac.Dslash(tmp, src, QUDA_EVEN_PARITY);
+    device_timer.stop();
 
-    // actual benchmarking
+    double secs = device_timer.last() / niter;
-    auto start_time = get_timestamp();
-    double secs = bench(f, target_time);
-    auto end_time = get_timestamp();
 
 #ifdef FLOP_COUNTING_GRID
     // this is the flop counting from Benchmark_Grid
     double Nc = 3;
     double Nd = 4;
     double Ns = 4;
-    flops = (Nc * (6 + (Nc - 1) * 8) * Ns * Nd + 2 * Nd * Nc * Ns + 2 * Nd * Nc * Ns * 2);
+    double flops =
+        (Nc * (6 + (Nc - 1) * 8) * Ns * Nd + 2 * Nd * Nc * Ns + 2 * Nd * Nc * Ns * 2);
     flops *= L * L * L * L / 2.0;
+#else
+    double flops = 1.0 * dirac.Flops() / niter;
 #endif
 
     printfQuda("%5d %15.2f %15.2f\n", L, secs * 1e6, flops / secs * 1e-9);
-
-    json tmp;
-    tmp["L"] = L;
-    tmp["Gflops_wilson"] = flops / secs * 1e-9;
-    tmp["start_time"] = start_time;
-    tmp["end_time"] = end_time;
-    json_results["flops"]["results"].push_back(tmp);
   }
 }
 
-void benchmark_dwf(std::vector<int> const &L_list, double target_time)
+void benchmark_dwf()
 {
+  int niter = 20;
+  int niter_warmup = 10;
+
   printfQuda("==================== domain wall dirac operator ====================\n");
 #ifdef FLOP_COUNTING_GRID
   printfQuda("IMPORTANT: flop counting as in Benchmark_Grid\n");
@@ -275,9 +226,8 @@ void benchmark_dwf(std::vector<int> const &L_list, double target_time)
 #endif
   printfQuda("%5s %15s %15s\n", "L", "time (usec)", "Gflop/s/rank");
   int Ls = 12;
-  for (int L : L_list)
+  for (int L : {8, 12, 16, 24, 32, 48})
   {
-    // printfQuda("starting dwf L=%d\n", L);
     auto U = make_gauge_field(L);
     auto src = make_source(L, Ls);
 
@@ -293,43 +243,45 @@ void benchmark_dwf(std::vector<int> const &L_list, double target_time)
     // insert gauge field into the dirac operator
     // (the additional nullptr's are for smeared links and fancy preconditioners and such)
     dirac.updateFields(&U, nullptr, nullptr, nullptr);
-    auto res = ColorSpinorField(ColorSpinorParam(src));
-    auto f = [&]() { dirac.Dslash(res, src, QUDA_EVEN_PARITY); };
 
-    // first run to get the quda tuning out of the way
+    auto tmp = ColorSpinorField(ColorSpinorParam(src));
+
+    // couple iterations without timing to warm up
+    for (int iter = 0; iter < niter_warmup; ++iter)
+      dirac.Dslash(tmp, src, QUDA_EVEN_PARITY);
+
+    // actual benchmark with timings
     dirac.Flops(); // reset flops counter
-    f();
+    device_timer_t device_timer;
-    double flops = 1.0 * dirac.Flops();
+    device_timer.start();
+    for (int iter = 0; iter < niter; ++iter)
+      dirac.Dslash(tmp, src, QUDA_EVEN_PARITY);
+    device_timer.stop();
 
-    // actual benchmarking
+    double secs = device_timer.last() / niter;
-    auto start_time = get_timestamp();
-    double secs = bench(f, target_time);
-    auto end_time = get_timestamp();
 
 #ifdef FLOP_COUNTING_GRID
     // this is the flop counting from Benchmark_Grid
     double Nc = 3;
     double Nd = 4;
     double Ns = 4;
-    flops = (Nc * (6 + (Nc - 1) * 8) * Ns * Nd + 2 * Nd * Nc * Ns + 2 * Nd * Nc * Ns * 2);
+    double flops =
+        (Nc * (6 + (Nc - 1) * 8) * Ns * Nd + 2 * Nd * Nc * Ns + 2 * Nd * Nc * Ns * 2);
     flops *= L * L * L * L * Ls / 2.0;
+#else
+    double flops = 1.0 * dirac.Flops() / niter;
 #endif
 
     printfQuda("%5d %15.2f %15.2f\n", L, secs * 1e6, flops / secs * 1e-9);
-    json tmp;
-    tmp["L"] = L;
-    tmp["Gflops_dwf4"] = flops / secs * 1e-9;
-    tmp["start_time"] = start_time;
-    tmp["end_time"] = end_time;
-    json_results["flops"]["results"].push_back(tmp);
   }
 }
 
-void benchmark_axpy(std::vector<int> const &L_list, double target_time)
+void benchmark_axpy()
 {
   // number of iterations for warmup / measurement
   // (feel free to change for noise/time tradeoff)
-  constexpr int niter_warmup = 5;
+  constexpr int niter_warmup = 10;
+  constexpr int niter = 20;
 
   printfQuda("==================== axpy / memory ====================\n");
 
@@ -353,9 +305,9 @@ void benchmark_axpy(std::vector<int> const &L_list, double target_time)
 
   printfQuda("%5s %15s %15s %15s %15s\n", "L", "size (MiB/rank)", "time (usec)",
              "GiB/s/rank", "Gflop/s/rank");
+  std::vector L_list = {8, 12, 16, 24, 32};
   for (int L : L_list)
   {
-    // printfQuda("starting axpy L=%d\n", L);
     // IMPORTANT: all of `param.x`, `field_elements`, `field.Bytes()`
     //            are LOCAL, i.e. per rank / per GPU
 
@@ -384,41 +336,26 @@ void benchmark_axpy(std::vector<int> const &L_list, double target_time)
     double flops = 2 * field_elements;
     double memory = 3 * sizeof(float) * field_elements;
 
-    auto f = [&]() { blas::axpy(1.234, fieldA, fieldB); };
+    // do some iterations to to let QUDA do its internal tuning and also stabilize cache
+    // behaviour and such
+    for (int iter = 0; iter < niter_warmup; ++iter)
+      blas::axpy(1.234, fieldA, fieldB);
 
-    // first run to get the quda tuning out of the way
+    // running the actual benchmark
-    f();
+    device_timer_t device_timer;
+    device_timer.start();
+    for (int iter = 0; iter < niter; ++iter)
+      blas::axpy(1.234, fieldA, fieldB);
+    device_timer.stop();
+    double secs = device_timer.last() / niter; // seconds per iteration
 
-    // actual benchmarking
+    printfQuda("%5d %15.2f %15.2f %15.2f %15.2f\n", L, memory / 1024. / 1024., secs * 1e6,
-    auto start_time = get_timestamp();
+               memory / secs / 1024. / 1024. / 1024., flops / secs * 1e-9);
-    double secs = bench(f, target_time);
-    auto end_time = get_timestamp();
-
-    double mem_MiB = memory / 1024. / 1024.;
-    double GBps = mem_MiB / 1024 / secs;
-    printfQuda("%5d %15.2f %15.2f %15.2f %15.2f\n", L, mem_MiB, secs * 1e6, GBps,
-               flops / secs * 1e-9);
-
-    json tmp;
-    tmp["L"] = L;
-    tmp["size_MB"] = mem_MiB;
-    tmp["GBps"] = GBps;
-    tmp["GFlops"] = flops / secs * 1e-9;
-    tmp["start_time"] = start_time;
-    tmp["end_time"] = end_time;
-    json_results["axpy"].push_back(tmp);
   }
 }
 
 int main(int argc, char **argv)
 {
-  std::string json_filename = ""; // empty indicates no json output
-  for (int i = 0; i < argc; i++)
-  {
-    if (std::string(argv[i]) == "--json-out")
-      json_filename = argv[i + 1];
-  }
-
   initComms(argc, argv);
 
   initQuda(-1); // -1 for multi-gpu. otherwise this selects the device to be used
@@ -430,28 +367,14 @@ int main(int argc, char **argv)
   printfQuda("MPI layout = %d %d %d %d\n", mpi_grid[0], mpi_grid[1], mpi_grid[2],
              mpi_grid[3]);
 
-  benchmark_axpy({8, 12, 16, 24, 32, 48}, 1.0);
+  benchmark_axpy();
 
   setVerbosity(QUDA_SILENT);
-  benchmark_wilson({8, 12, 16, 24, 32, 48}, 1.0);
+  benchmark_wilson();
-  benchmark_dwf({8, 12, 16, 24, 32}, 1.0);
+  benchmark_dwf();
   setVerbosity(QUDA_SUMMARIZE);
 
   printfQuda("==================== done with all benchmarks ====================\n");
-
-  if (!json_filename.empty())
-  {
-    printfQuda("writing benchmark results to %s\n", json_filename.c_str());
-
-    int me = 0;
-    MPI_Comm_rank(MPI_COMM_WORLD, &me);
-    if (me == 0)
-    {
-      std::ofstream json_file(json_filename);
-      json_file << std::setw(2) << json_results;
-    }
-  }
-
   endQuda();
   quda::comm_finalize();
   MPI_Finalize();

Quda/Readme.md

@@ -1,30 +0,0 @@
-# QUDA benchmarks
-
-This folder contains benchmarks for the [QUDA](https://github.com/lattice/quda) library.
-
-- `Benchmark_Quda`: This benchmark measure floating point performances of fermion
-matrices (Wilson and DWF), as well as memory bandwidth (using a simple `axpy` operation). Measurements are
-performed for a fixed range of problem sizes.
-
-## Building
-After setting up your compilation environment (Tursa: `source /home/dp207/dp207/shared/env/production/env-{base,gpu}.sh`):
-```bash
-./build-quda.sh <env_dir>          # build Quda
-./build-benchmark.sh <env_dir>     # build benchmark
-```
-where `<env_dir>` is an arbitrary directory where every product will be stored.
-
-## Running the Benchmark
-
-The benchmark should be run as
-```bash
-mpirun -np <ranks> <env_dir>/prefix/qudabench/Benchmark_Quda
-```
-where `<ranks>` is the total number of GPU's to use. On Tursa this is 4 times the number of nodes.
-
-Note:
-- on Tursa, the `wrapper.sh` script that is typically used with Grid is not necessary.
-- due to Qudas automatic tuning, the benchmark might take significantly longer to run than `Benchmark_Grid` (even though it does fewer things).
-  - setting `QUDA_ENABLE_TUNING=0` disables all tuning (degrades performance severely). By default, it is turned on.
-  - setting `QUDA_RESOURCE_PATH=<some folder>` enables Quda to save and reuse optimal tuning parameters, making repeated runs much faster
-  

Quda/build-benchmark.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC1090,SC1091
-
-set -euo pipefail
-
-if (( $# != 1 )); then
-    echo "usage: $(basename "$0") <environment directory>" 1>&2
-    exit 1
-fi
-env_dir=$1
-
-# TODO: this is Tursa specific. have not figured out the correct way to do this.
-EXTRA_LIBS="/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so"
-
-# NOTE: these flags need to be in sync with Qudas compilation options (see build-quda.sh)
-BUILD_FLAGS="-O3 -std=c++17 -DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=12 -DQUDA_RECONSTRUCT=4"
-
-call_dir=$(pwd -P)
-script_dir="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
-cd "${env_dir}"
-env_dir=$(pwd -P)
-cd "${call_dir}"
-BUILD_DIR="${env_dir}/build/Quda-benchmarks"
-PREFIX_DIR="${env_dir}/prefix/qudabench"
-QUDA_DIR=${env_dir}/prefix/quda
-mkdir -p "${BUILD_DIR}"
-mkdir -p "${PREFIX_DIR}"
-
-LINK_FLAGS="-Wl,-rpath,$QUDA_DIR/lib: $QUDA_DIR/lib/libquda.so $EXTRA_LIBS -lpthread -lmpi"
-
-g++ $BUILD_FLAGS -I$QUDA_DIR/include/targets/cuda  -I$QUDA_DIR/include   -c -o $BUILD_DIR/Benchmark_Quda.o  $script_dir/Benchmark_Quda.cpp
-g++ -g -O3 $BUILD_DIR/Benchmark_Quda.o -o $PREFIX_DIR/Benchmark_Quda $LINK_FLAGS -lmpi

Quda/build-quda.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC1090,SC1091
-
-BUILD_FLAGS="-O3 -std=c++17"
-QUDA_FLAGS="-DQUDA_MPI=ON -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=4 -DQUDA_GPU_ARCH=sm_80"
-
-set -euo pipefail
-
-if (( $# != 1 )); then
-    echo "usage: $(basename "$0") <environment directory>" 1>&2
-    exit 1
-fi
-env_dir=$1
-
-call_dir=$(pwd -P)
-mkdir -p ${env_dir}
-cd "${env_dir}"
-env_dir=$(pwd -P)
-cd "${call_dir}"
-
-build_dir="${env_dir}/build/quda"
-if [ -d "${build_dir}" ]; then
-    echo "error: directory '${build_dir}' exists"
-    exit 1
-fi
-mkdir -p "${build_dir}"
-
-git clone https://github.com/lattice/quda.git "${build_dir}"
-cd "${build_dir}"
-
-mkdir build; cd build
-cmake .. $QUDA_FLAGS -DCMAKE_INSTALL_PREFIX=${env_dir}/prefix/quda
-make -j128
-make install
-
-cd "${call_dir}"

Quda/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+#CXX=/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen/gcc-8.4.1/gcc-9.4.0-g3vyv3te4ah634euh7phyokb3fiurprp/bin/g++
+QUDA_BUILD=/home/dp207/dp207/dc-burg2/quda_build
+QUDA_SRC=/home/dp207/dp207/dc-burg2/quda
+#QUDA_BUILD=
+
+FLAGS="-DMPI_COMMS -DMULTI_GPU -DQUDA_PRECISION=14 -DQUDA_RECONSTRUCT=7 -g -O3 -Wall -Wextra -std=c++17 "
+$CXX $FLAGS -I$QUDA_BUILD/include/targets/cuda -I$QUDA_SRC/include -I$QUDA_BUILD/include -isystem $QUDA_SRC/include/externals -isystem $QUDA_BUILD/_deps/eigen-src -c -o Benchmark_Quda.o  Benchmark_Quda.cpp
+LINK_FLAGS="-Wl,-rpath,$QUDA_BUILD/tests:$QUDA_BUILD/lib:/home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs: $QUDA_BUILD/lib/libquda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libcuda.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/stubs/libnvidia-ml.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcudart_static.a -ldl /usr/lib64/librt.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcublas.so /home/dp207/dp207/shared/env/versions/220428/spack/opt/spack/linux-rhel8-zen2/gcc-9.4.0/cuda-11.4.0-etxow4jb23qdbs7j6txczy44cdatpj22/lib64/libcufft.so -lpthread"
+$CXX -g -O3 Benchmark_Quda.o -o Benchmark_Quda $LINK_FLAGS -lmpi