diff --git a/benchmarks/Benchmark_comms_host_device.cc b/benchmarks/Benchmark_comms_host_device.cc index 53173d9f3..6c880ccfd 100644 --- a/benchmarks/Benchmark_comms_host_device.cc +++ b/benchmarks/Benchmark_comms_host_device.cc @@ -252,6 +252,131 @@ int main (int argc, char ** argv) } + ///////////////////////////////////////////////////////////////////////////// + // Halo exchange through the PRODUCTION stencil path. + // + // Prepare -> PollDtoH -> CopySynchronise -> Begin -> PollIRecv -> Complete + // + // verbatim as CartesianStencil::CommunicateBegin/Complete drive it, so this + // measures the code that actually runs inside the Dslash rather than a + // simplified stand-in built on SendToRecvFrom. + // + // THREE byte counts are reported, because they differ and conflating them + // has already produced an impossible number (272 GB/s/node against a + // 200 GB/s wire): + // + // offered : the whole halo this rank presents, all 8 directions. + // off-node : the subset whose partner is on another node, classified + // GEOMETRICALLY via IsOffNode() -- independent of --shm-mpi. + // to-MPI : what StencilSendToRecvFrom{Prepare,Begin} actually handed to + // MPI, i.e. their return value, exactly as the Stencil counts + // it. Under --shm-mpi 1 Grid delegates intranode transfers to + // MPI as well, so this approaches "offered"; under --shm-mpi 0 + // Grid moves them itself and this approaches "off-node". + // + // Only the off-node column may be compared against wire speed. + ///////////////////////////////////////////////////////////////////////////// + std::cout< xbuf(8), rbuf(8); + for(int d=0;d<8;d++){ + xbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes); + rbuf[d] = (HalfSpinColourVectorD *)acceleratorAllocDevice(bytes); + } + + // Packet table in the same shape the Stencil builds: 8 directions, + // to/from ranks from ShiftedRanks, participation gated on mpi_layout. + // COMPACT packet list, exactly as the Stencil builds it: only the + // directions that actually communicate get a packet. Passing a + // self-send with do_send=0 is not equivalent -- the assert + // "dest != _processor" in StencilSendToRecvFromBegin is unconditional. + std::vector to_rank, from_rank, buf_id; + double offered = 0.0, offnode = 0.0; + for(int mu=0;mu<4;mu++){ + if ( mpi_layout[mu] > 1 ) { + int s,d; + Grid.ShiftedRanks(mu,+1,s,d); to_rank.push_back(d); from_rank.push_back(s); buf_id.push_back(mu); + Grid.ShiftedRanks(mu,-1,s,d); to_rank.push_back(d); from_rank.push_back(s); buf_id.push_back(mu+4); + } + } + int npkt = to_rank.size(); + for(int i=0;i reqs; + + for(int n=0;n