Benchmark comms with the exact sequence in Stencil as it is now.

benchmarks/Benchmark_comms_host_device.cc

@@ -252,6 +252,131 @@ int main (int argc, char ** argv)
   }
 
 
+  /////////////////////////////////////////////////////////////////////////////
+  // Halo exchange through the PRODUCTION stencil path.
+  //
+  //   Prepare -> PollDtoH -> CopySynchronise -> Begin -> PollIRecv -> Complete
+  //
+  // verbatim as CartesianStencil::CommunicateBegin/Complete drive it, so this
+  // measures the code that actually runs inside the Dslash rather than a
+  // simplified stand-in built on SendToRecvFrom.
+  //
+  // THREE byte counts are reported, because they differ and conflating them
+  // has already produced an impossible number (272 GB/s/node against a
+  // 200 GB/s wire):
+  //
+  //   offered   : the whole halo this rank presents, all 8 directions.
+  //   off-node  : the subset whose partner is on another node, classified
+  //               GEOMETRICALLY via IsOffNode() -- independent of --shm-mpi.
+  //   to-MPI    : what StencilSendToRecvFrom{Prepare,Begin} actually handed to
+  //               MPI, i.e. their return value, exactly as the Stencil counts
+  //               it.  Under --shm-mpi 1 Grid delegates intranode transfers to
+  //               MPI as well, so this approaches "offered"; under --shm-mpi 0
+  //               Grid moves them itself and this approaches "off-node".
+  //
+  // Only the off-node column may be compared against wire speed.
+  /////////////////////////////////////////////////////////////////////////////
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking STENCIL-PATH halo exchange from GPU memory "<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "  L   Ls   offered   off-node    to-MPI      time   off-node   to-MPI"<<std::endl;
+  std::cout<<GridLogMessage << "            MB/nd      MB/nd      MB/nd    us/call   GB/s/nd  GB/s/nd"<<std::endl;
+
+  for(int lat=8;lat<=maxlat;lat+=4){
+    for(int Ls=8;Ls<=8;Ls*=2){
+
+      Coordinate latt_size  ({lat*mpi_layout[0],
+	                      lat*mpi_layout[1],
+	                      lat*mpi_layout[2],
+	                      lat*mpi_layout[3]});
+
+      GridCartesian Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+
+      uint64_t bytes = (uint64_t)lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+
+      std::vector<HalfSpinColourVectorD *> xbuf(8), rbuf(8);
+      for(int d=0;d<8;d++){
+	xbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes);
+	rbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes);
+      }
+
+      // Packet table in the same shape the Stencil builds: 8 directions,
+      // to/from ranks from ShiftedRanks, participation gated on mpi_layout.
+      // COMPACT packet list, exactly as the Stencil builds it: only the
+      // directions that actually communicate get a packet.  Passing a
+      // self-send with do_send=0 is not equivalent -- the assert
+      // "dest != _processor" in StencilSendToRecvFromBegin is unconditional.
+      std::vector<int> to_rank, from_rank, buf_id;
+      double offered = 0.0, offnode = 0.0;
+      for(int mu=0;mu<4;mu++){
+	if ( mpi_layout[mu] > 1 ) {
+	  int s,d;
+	  Grid.ShiftedRanks(mu,+1,s,d); to_rank.push_back(d); from_rank.push_back(s); buf_id.push_back(mu);
+	  Grid.ShiftedRanks(mu,-1,s,d); to_rank.push_back(d); from_rank.push_back(s); buf_id.push_back(mu+4);
+	}
+      }
+      int npkt = to_rank.size();
+      for(int i=0;i<npkt;i++){
+	offered += 2.0*bytes;                                  // send + receive
+	if ( Grid.IsOffNode(to_rank[i]) ) offnode += 2.0*bytes;
+      }
+
+      double mpibytes = 0.0, t = 0.0;
+      std::vector<CommsRequest_t> reqs;
+
+      for(int n=0;n<Nloop;n++){
+	reqs.resize(0);
+	double mb = 0.0;
+	Grid.StencilBarrier();
+	double t0=usecond();
+	for(int i=0;i<npkt;i++){
+	  mb += Grid.StencilSendToRecvFromPrepare(reqs,
+			 (void *)xbuf[buf_id[i]], to_rank[i],   1,
+			 (void *)rbuf[buf_id[i]], from_rank[i], 1,
+			 bytes,bytes,i);
+	}
+	Grid.StencilSendToRecvFromPollDtoH(reqs); /* Starts MPI */
+	acceleratorCopySynchronise();
+	for(int i=0;i<npkt;i++){
+	  // xmit and xmit_comp identical: no compression in this benchmark
+	  mb += Grid.StencilSendToRecvFromBegin(reqs,
+			 (void *)xbuf[buf_id[i]],(void *)xbuf[buf_id[i]], to_rank[i],   1,
+			 (void *)rbuf[buf_id[i]],(void *)rbuf[buf_id[i]], from_rank[i], 1,
+			 bytes,bytes,i);
+	}
+	Grid.StencilSendToRecvFromPollIRecv(reqs);
+	Grid.StencilSendToRecvFromComplete(reqs,0);
+	t += usecond()-t0;
+	mpibytes += mb;
+      }
+
+      // Sum bytes over the machine, average the time over ranks, report per
+      // node.  Reduced once, outside the timed region.  bytes/us == MB/s.
+      double sumMpi = mpibytes;        Grid.GlobalSum(sumMpi);
+      double sumOff = offnode*Nloop;   Grid.GlobalSum(sumOff);
+      double sumAll = offered*Nloop;   Grid.GlobalSum(sumAll);
+      double sumT   = t;               Grid.GlobalSum(sumT);
+      double avgT   = sumT/Nrank;
+
+      std::cout<<GridLogMessage
+	       << std::setw(4) << lat <<" "<< std::setw(4) << Ls
+	       <<"  "<< std::setw(9) << sumAll/Nloop/Nnode/1.0e6
+	       <<"  "<< std::setw(9) << sumOff/Nloop/Nnode/1.0e6
+	       <<"  "<< std::setw(9) << sumMpi/Nloop/Nnode/1.0e6
+	       <<"  "<< std::setw(9) << avgT/Nloop
+	       <<"  "<< std::setw(8) << sumOff/avgT/Nnode/1000.0
+	       <<"  "<< std::setw(8) << sumMpi/avgT/Nnode/1000.0
+	       <<std::endl;
+
+      for(int d=0;d<8;d++){
+	acceleratorFreeDevice(xbuf[d]);
+	acceleratorFreeDevice(rbuf[d]);
+      }
+    }
+  }
+
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;