From c6349bb93f0f77d3459fd5030321f7aa56d22573 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Tue, 24 Jan 2023 18:44:47 +0000
Subject: [PATCH] renaming and formatting

---
 Grid/.clang-format                            |  14 +
 .../{Benchmark_ITT.cpp => Benchmark_Grid.cpp} |   5 +-
 Grid/Benchmark_comms_host_device.cpp          | 334 +++++++++---------
 3 files changed, 190 insertions(+), 163 deletions(-)
 create mode 100644 Grid/.clang-format
 rename Grid/{Benchmark_ITT.cpp => Benchmark_Grid.cpp} (99%)

diff --git a/Grid/.clang-format b/Grid/.clang-format
new file mode 100644
index 0000000..9156d50
--- /dev/null
+++ b/Grid/.clang-format
@@ -0,0 +1,14 @@
+{
+  BasedOnStyle: LLVM,
+  UseTab: Never,
+  IndentWidth: 2,
+  TabWidth: 2,
+  BreakBeforeBraces: Allman,
+  AllowShortIfStatementsOnASingleLine: false,
+  IndentCaseLabels: false,
+  ColumnLimit: 0,
+  AccessModifierOffset: -4,
+  NamespaceIndentation: All,
+  FixNamespaceComments: false,
+  SortIncludes: true,
+}
diff --git a/Grid/Benchmark_ITT.cpp b/Grid/Benchmark_Grid.cpp
similarity index 99%
rename from Grid/Benchmark_ITT.cpp
rename to Grid/Benchmark_Grid.cpp
index e618aff..fd2056e 100644
--- a/Grid/Benchmark_ITT.cpp
+++ b/Grid/Benchmark_Grid.cpp
@@ -2,6 +2,8 @@
 Copyright © 2015 Peter Boyle <paboyle@ph.ed.ac.uk>
 Copyright © 2022 Antonin Portelli <antonin.portelli@me.com>
 
+This is a refactoring of Benchmark_ITT.cpp from Grid
+
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
 as published by the Free Software Foundation; either version 2
@@ -67,6 +69,7 @@ Gamma::Algebra Gmu[] = {
     Gamma::Algebra::GammaY,
     Gamma::Algebra::GammaZ,
     Gamma::Algebra::GammaT};
+
 struct controls
 {
   int Opt;
@@ -76,7 +79,7 @@ struct controls
 
 class Benchmark
 {
-public:
+  public:
   static void Decomposition(void)
   {
 
diff --git a/Grid/Benchmark_comms_host_device.cpp b/Grid/Benchmark_comms_host_device.cpp
index 56227c4..5a4aae4 100644
--- a/Grid/Benchmark_comms_host_device.cpp
+++ b/Grid/Benchmark_comms_host_device.cpp
@@ -21,231 +21,241 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 using namespace std;
 using namespace Grid;
 
-struct time_statistics{
+struct time_statistics
+{
   double mean;
   double err;
   double min;
   double max;
 
-  void statistics(std::vector<double> v){
-      double sum = std::accumulate(v.begin(), v.end(), 0.0);
-      mean = sum / v.size();
+  void statistics(std::vector<double> v)
+  {
+    double sum = std::accumulate(v.begin(), v.end(), 0.0);
+    mean = sum / v.size();
 
-      std::vector<double> diff(v.size());
-      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
-      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
-      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+    std::vector<double> diff(v.size());
+    std::transform(v.begin(), v.end(), diff.begin(), [=](double x)
+                   { return x - mean; });
+    double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+    err = std::sqrt(sq_sum / (v.size() * (v.size() - 1)));
 
-      auto result = std::minmax_element(v.begin(), v.end());
-      min = *result.first;
-      max = *result.second;
-}
+    auto result = std::minmax_element(v.begin(), v.end());
+    min = *result.first;
+    max = *result.second;
+  }
 };
 
-void header(){
-  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
-            <<std::setw(11)<<"bytes\t\t"<<"MB/s uni"<<"\t"<<"MB/s bidi"<<std::endl;
-};
-
-int main (int argc, char ** argv)
+void header()
 {
-  Grid_init(&argc,&argv);
+  std::cout << GridLogMessage << " L  "
+            << "\t"
+            << " Ls  "
+            << "\t"
+            << std::setw(11) << "bytes\t\t"
+            << "MB/s uni"
+            << "\t"
+            << "MB/s bidi" << std::endl;
+};
 
-  Coordinate simd_layout = GridDefaultSimd(Nd,vComplexD::Nsimd());
-  Coordinate mpi_layout  = GridDefaultMpi();
+int main(int argc, char **argv)
+{
+  Grid_init(&argc, &argv);
+
+  Coordinate simd_layout = GridDefaultSimd(Nd, vComplexD::Nsimd());
+  Coordinate mpi_layout = GridDefaultMpi();
   int threads = GridThread::GetThreads();
-  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
 
-  int Nloop=250;
-  int nmu=0;
-  int maxlat=32;
-  for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
+  int Nloop = 250;
+  int nmu = 0;
+  int maxlat = 32;
+  for (int mu = 0; mu < Nd; mu++)
+    if (mpi_layout[mu] > 1)
+      nmu++;
 
-  std::cout << GridLogMessage << "Number of iterations to average: "<< Nloop << std::endl;
+  std::cout << GridLogMessage << "Number of iterations to average: " << Nloop << std::endl;
   std::vector<double> t_time(Nloop);
   //  time_statistics timestat;
 
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking sequential halo exchange from host memory "<<std::endl;
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout << GridLogMessage << "====================================================================================================" << std::endl;
+  std::cout << GridLogMessage << "= Benchmarking sequential halo exchange from host memory " << std::endl;
+  std::cout << GridLogMessage << "====================================================================================================" << std::endl;
   header();
 
-  for(int lat=8;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=8;Ls*=2){
+  for (int lat = 8; lat <= maxlat; lat += 4)
+  {
+    for (int Ls = 8; Ls <= 8; Ls *= 2)
+    {
 
-      Coordinate latt_size  ({lat*mpi_layout[0],
-	                      lat*mpi_layout[1],
-      			      lat*mpi_layout[2],
-      			      lat*mpi_layout[3]});
+      Coordinate latt_size({lat * mpi_layout[0],
+                            lat * mpi_layout[1],
+                            lat * mpi_layout[2],
+                            lat * mpi_layout[3]});
 
-      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      GridCartesian Grid(latt_size, simd_layout, mpi_layout);
       RealD Nrank = Grid._Nprocessors;
       RealD Nnode = Grid.NodeCount();
-      RealD ppn = Nrank/Nnode;
+      RealD ppn = Nrank / Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8);
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8);
+      std::vector<std::vector<HalfSpinColourVectorD>> xbuf(8);
+      std::vector<std::vector<HalfSpinColourVectorD>> rbuf(8);
 
-      for(int mu=0;mu<8;mu++){
-	xbuf[mu].resize(lat*lat*lat*Ls);
-	rbuf[mu].resize(lat*lat*lat*Ls);
+      for (int mu = 0; mu < 8; mu++)
+      {
+        xbuf[mu].resize(lat * lat * lat * Ls);
+        rbuf[mu].resize(lat * lat * lat * Ls);
       }
-      uint64_t bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      uint64_t bytes = lat * lat * lat * Ls * sizeof(HalfSpinColourVectorD);
 
       int ncomm;
 
-      for(int mu=0;mu<4;mu++){
-	if (mpi_layout[mu]>1 ) {
-	double start=usecond();
-	for(int i=0;i<Nloop;i++){
+      for (int mu = 0; mu < 4; mu++)
+      {
+        if (mpi_layout[mu] > 1)
+        {
+          double start = usecond();
+          for (int i = 0; i < Nloop; i++)
+          {
 
-	  ncomm=0;
-	
-	  
-	    ncomm++;
-	    int comm_proc=1;
-	    int xmit_to_rank;
-	    int recv_from_rank;
-	    
-	    {
-	      std::vector<CommsRequest_t> requests;
-	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	      Grid.SendToRecvFrom((void *)&xbuf[mu][0],
-				  xmit_to_rank,
-				  (void *)&rbuf[mu][0],
-				  recv_from_rank,
-				  bytes);
-	    }
+            ncomm = 0;
 
-	    comm_proc = mpi_layout[mu]-1;
-	    {
-	      std::vector<CommsRequest_t> requests;
-	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	      Grid.SendToRecvFrom((void *)&xbuf[mu+4][0],
-				  xmit_to_rank,
-				  (void *)&rbuf[mu+4][0],
-				  recv_from_rank,
-				  bytes);
-	    }
-	}
-	Grid.Barrier();
-	double stop=usecond();
-        double mean=(stop-start)/Nloop;      
-      double dbytes    = bytes*ppn;
-      double xbytes    = dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
+            ncomm++;
+            int comm_proc = 1;
+            int xmit_to_rank;
+            int recv_from_rank;
 
-      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
-               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)<<" "
-               <<std::right<< xbytes/mean<<"  "
-               << "\t\t"<<std::setw(7)<< bidibytes/mean<< std::endl;
+            {
+              std::vector<CommsRequest_t> requests;
+              Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
+              Grid.SendToRecvFrom((void *)&xbuf[mu][0],
+                                  xmit_to_rank,
+                                  (void *)&rbuf[mu][0],
+                                  recv_from_rank,
+                                  bytes);
+            }
 
+            comm_proc = mpi_layout[mu] - 1;
+            {
+              std::vector<CommsRequest_t> requests;
+              Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
+              Grid.SendToRecvFrom((void *)&xbuf[mu + 4][0],
+                                  xmit_to_rank,
+                                  (void *)&rbuf[mu + 4][0],
+                                  recv_from_rank,
+                                  bytes);
+            }
+          }
+          Grid.Barrier();
+          double stop = usecond();
+          double mean = (stop - start) / Nloop;
+          double dbytes = bytes * ppn;
+          double xbytes = dbytes * 2.0 * ncomm;
+          double rbytes = xbytes;
+          double bidibytes = xbytes + rbytes;
 
-	
-	}
+          std::cout << GridLogMessage << std::setw(4) << lat << "\t" << Ls << "\t"
+                    << std::setw(11) << bytes << std::fixed << std::setprecision(1) << std::setw(7) << " "
+                    << std::right << xbytes / mean << "  "
+                    << "\t\t" << std::setw(7) << bidibytes / mean << std::endl;
+        }
       }
-
-
-      
     }
   }
 
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= Benchmarking sequential halo exchange from GPU memory "<<std::endl;
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout << GridLogMessage << "====================================================================================================" << std::endl;
+  std::cout << GridLogMessage << "= Benchmarking sequential halo exchange from GPU memory " << std::endl;
+  std::cout << GridLogMessage << "====================================================================================================" << std::endl;
   header();
 
-  for(int lat=8;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=8;Ls*=2){
+  for (int lat = 8; lat <= maxlat; lat += 4)
+  {
+    for (int Ls = 8; Ls <= 8; Ls *= 2)
+    {
 
-      Coordinate latt_size  ({lat*mpi_layout[0],
-	                      lat*mpi_layout[1],
-      			      lat*mpi_layout[2],
-      			      lat*mpi_layout[3]});
+      Coordinate latt_size({lat * mpi_layout[0],
+                            lat * mpi_layout[1],
+                            lat * mpi_layout[2],
+                            lat * mpi_layout[3]});
 
-      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      GridCartesian Grid(latt_size, simd_layout, mpi_layout);
       RealD Nrank = Grid._Nprocessors;
       RealD Nnode = Grid.NodeCount();
-      RealD ppn = Nrank/Nnode;
-
+      RealD ppn = Nrank / Nnode;
 
       std::vector<HalfSpinColourVectorD *> xbuf(8);
       std::vector<HalfSpinColourVectorD *> rbuf(8);
 
-      uint64_t bytes = lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
-      for(int d=0;d<8;d++){
-	xbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes);
-	rbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes);
+      uint64_t bytes = lat * lat * lat * Ls * sizeof(HalfSpinColourVectorD);
+      for (int d = 0; d < 8; d++)
+      {
+        xbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes);
+        rbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes);
       }
 
       int ncomm;
 
-      for(int mu=0;mu<4;mu++){
-	if (mpi_layout[mu]>1 ) {
-	double start=usecond();
-	for(int i=0;i<Nloop;i++){
+      for (int mu = 0; mu < 4; mu++)
+      {
+        if (mpi_layout[mu] > 1)
+        {
+          double start = usecond();
+          for (int i = 0; i < Nloop; i++)
+          {
 
-	  ncomm=0;
-	
-	  
-	    ncomm++;
-	    int comm_proc=1;
-	    int xmit_to_rank;
-	    int recv_from_rank;
-	    
-	    {
-	      std::vector<CommsRequest_t> requests;
-	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	      Grid.SendToRecvFrom((void *)&xbuf[mu][0],
-				  xmit_to_rank,
-				  (void *)&rbuf[mu][0],
-				  recv_from_rank,
-				  bytes);
-	    }
+            ncomm = 0;
 
-	    comm_proc = mpi_layout[mu]-1;
-	    {
-	      std::vector<CommsRequest_t> requests;
-	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
-	      Grid.SendToRecvFrom((void *)&xbuf[mu+4][0],
-				  xmit_to_rank,
-				  (void *)&rbuf[mu+4][0],
-				  recv_from_rank,
-				  bytes);
-	    }
-	}
-	Grid.Barrier();
-	double stop=usecond();
-        double mean=(stop-start)/Nloop;      
-      double dbytes    = bytes*ppn;
-      double xbytes    = dbytes*2.0*ncomm;
-      double rbytes    = xbytes;
-      double bidibytes = xbytes+rbytes;
+            ncomm++;
+            int comm_proc = 1;
+            int xmit_to_rank;
+            int recv_from_rank;
 
-      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
-               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)<<" "
-               <<std::right<< xbytes/mean<<"  "
-               << "\t\t"<<std::setw(7)<< bidibytes/mean<< std::endl;
+            {
+              std::vector<CommsRequest_t> requests;
+              Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
+              Grid.SendToRecvFrom((void *)&xbuf[mu][0],
+                                  xmit_to_rank,
+                                  (void *)&rbuf[mu][0],
+                                  recv_from_rank,
+                                  bytes);
+            }
 
+            comm_proc = mpi_layout[mu] - 1;
+            {
+              std::vector<CommsRequest_t> requests;
+              Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
+              Grid.SendToRecvFrom((void *)&xbuf[mu + 4][0],
+                                  xmit_to_rank,
+                                  (void *)&rbuf[mu + 4][0],
+                                  recv_from_rank,
+                                  bytes);
+            }
+          }
+          Grid.Barrier();
+          double stop = usecond();
+          double mean = (stop - start) / Nloop;
+          double dbytes = bytes * ppn;
+          double xbytes = dbytes * 2.0 * ncomm;
+          double rbytes = xbytes;
+          double bidibytes = xbytes + rbytes;
 
-	
-	}
+          std::cout << GridLogMessage << std::setw(4) << lat << "\t" << Ls << "\t"
+                    << std::setw(11) << bytes << std::fixed << std::setprecision(1) << std::setw(7) << " "
+                    << std::right << xbytes / mean << "  "
+                    << "\t\t" << std::setw(7) << bidibytes / mean << std::endl;
+        }
       }
 
-      for(int d=0;d<8;d++){
-	acceleratorFreeDevice(xbuf[d]);
-	acceleratorFreeDevice(rbuf[d]);
+      for (int d = 0; d < 8; d++)
+      {
+        acceleratorFreeDevice(xbuf[d]);
+        acceleratorFreeDevice(rbuf[d]);
       }
-
-      
     }
   }
 
-
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
-  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout << GridLogMessage << "====================================================================================================" << std::endl;
+  std::cout << GridLogMessage << "= All done; Bye Bye" << std::endl;
+  std::cout << GridLogMessage << "====================================================================================================" << std::endl;
 
   Grid_finalize();
 }