diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h index 766862fec..687b96219 100644 --- a/Grid/stencil/Stencil.h +++ b/Grid/stencil/Stencil.h @@ -315,6 +315,10 @@ public: } int traceID; + double OffNodeBytes; + double CommTimer; + double InterNodeBandwidthMBps; + int face_table_computed; // int partialDirichlet; int fullDirichlet; @@ -539,11 +543,12 @@ public: #endif } traceID = traceStart("Stencil::CommunicateBegin"); - + OffNodeBytes=0; + CommTimer=-usecond(); for(int i=0;iBarrier(); - _grid->StencilSendToRecvFromPrepare(MpiReqs, + OffNodeBytes+=_grid->StencilSendToRecvFromPrepare(MpiReqs, Packets[i].compressed_send_buf, Packets[i].to_rank,Packets[i].do_send, Packets[i].compressed_recv_buf, @@ -560,7 +565,7 @@ public: for(int i=0;iBarrier(); - _grid->StencilSendToRecvFromBegin(MpiReqs, + OffNodeBytes+=_grid->StencilSendToRecvFromBegin(MpiReqs, Packets[i].send_buf,Packets[i].compressed_send_buf, Packets[i].to_rank,Packets[i].do_send, Packets[i].recv_buf,Packets[i].compressed_recv_buf, @@ -591,6 +596,8 @@ public: _grid->StencilSendToRecvFromComplete(MpiReqs,0); // MPI is done // if ( this->partialDirichlet ) DslashLogPartial(); traceStop(traceID); + CommTimer+=usecond(); + InterNodeBandwidthMBps = OffNodeBytes/CommTimer; if ( this->fullDirichlet ) DslashLogDirichlet(); else DslashLogFull(); diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc index c28b2686a..2d21976f8 100644 --- a/benchmarks/Benchmark_dwf.cc +++ b/benchmarks/Benchmark_dwf.cc @@ -276,12 +276,14 @@ void Benchmark(int Ls, Coordinate Dirichlet,bool sloppy) Dw.Dhop(src,result,0); std::cout<Barrier(); - + double volume=Ls; for(int mu=0;mu