portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-27 04:49:36 +01:00
Code Releases Activity

Bandwidth measures inside stencil

Browse Source
This commit is contained in:
Peter Boyle
2026-08-22 10:51:44 -04:00
parent 2e0c19c915
commit 152e9ecf3b
2 changed files with 14 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Grid/stencil/Stencil.h
+10 -3
View File
@@ -315,6 +315,10 @@ public:
} }
int traceID; int traceID;
double OffNodeBytes;
double CommTimer;
double InterNodeBandwidthMBps;
int face_table_computed; int face_table_computed;
// int partialDirichlet; // int partialDirichlet;
int fullDirichlet; int fullDirichlet;
@@ -539,11 +543,12 @@ public:
#endif #endif
} }
traceID = traceStart("Stencil::CommunicateBegin"); traceID = traceStart("Stencil::CommunicateBegin");
OffNodeBytes=0;
CommTimer=-usecond();
for(int i=0;i<Packets.size();i++){ for(int i=0;i<Packets.size();i++){
// std::cout << "Communicate prepare "<<i<<std::endl; // std::cout << "Communicate prepare "<<i<<std::endl;
// _grid->Barrier(); // _grid->Barrier();
_grid->StencilSendToRecvFromPrepare(MpiReqs, OffNodeBytes+=_grid->StencilSendToRecvFromPrepare(MpiReqs,
Packets[i].compressed_send_buf, Packets[i].compressed_send_buf,
Packets[i].to_rank,Packets[i].do_send, Packets[i].to_rank,Packets[i].do_send,
Packets[i].compressed_recv_buf, Packets[i].compressed_recv_buf,
@@ -560,7 +565,7 @@ public:
for(int i=0;i<Packets.size();i++){ for(int i=0;i<Packets.size();i++){
// std::cout << "Communicate Begin "<<i<<std::endl; // std::cout << "Communicate Begin "<<i<<std::endl;
// _grid->Barrier(); // _grid->Barrier();
_grid->StencilSendToRecvFromBegin(MpiReqs, OffNodeBytes+=_grid->StencilSendToRecvFromBegin(MpiReqs,
Packets[i].send_buf,Packets[i].compressed_send_buf, Packets[i].send_buf,Packets[i].compressed_send_buf,
Packets[i].to_rank,Packets[i].do_send, Packets[i].to_rank,Packets[i].do_send,
Packets[i].recv_buf,Packets[i].compressed_recv_buf, Packets[i].recv_buf,Packets[i].compressed_recv_buf,
@@ -591,6 +596,8 @@ public:
_grid->StencilSendToRecvFromComplete(MpiReqs,0); // MPI is done _grid->StencilSendToRecvFromComplete(MpiReqs,0); // MPI is done
// if ( this->partialDirichlet ) DslashLogPartial(); // if ( this->partialDirichlet ) DslashLogPartial();
traceStop(traceID); traceStop(traceID);
CommTimer+=usecond();
InterNodeBandwidthMBps = OffNodeBytes/CommTimer;
if ( this->fullDirichlet ) DslashLogDirichlet(); if ( this->fullDirichlet ) DslashLogDirichlet();
else DslashLogFull(); else DslashLogFull();
benchmarks/Benchmark_dwf.cc
+4 -1
View File
@@ -276,12 +276,14 @@ void Benchmark(int Ls, Coordinate Dirichlet,bool sloppy)
Dw.Dhop(src,result,0); Dw.Dhop(src,result,0);
std::cout<<GridLogMessage<<"Called warmup"<<std::endl; std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
double t0=usecond(); double t0=usecond();
double aveBW = 0;
for(int i=0;i<ncall;i++){ for(int i=0;i<ncall;i++){
Dw.Dhop(src,result,0); Dw.Dhop(src,result,0);
aveBW +=Dw.Stencil.InterNodeBandwidthMBps/ncall;
} }
double t1=usecond(); double t1=usecond();
FGrid->Barrier(); FGrid->Barrier();
double volume=Ls; for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu]; double volume=Ls; for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
double flops=single_site_flops*volume*ncall; double flops=single_site_flops*volume*ncall;
@@ -298,6 +300,7 @@ void Benchmark(int Ls, Coordinate Dirichlet,bool sloppy)
std::cout<<GridLogMessage << "mflop/s = "<< flops/(t1-t0)<<std::endl; std::cout<<GridLogMessage << "mflop/s = "<< flops/(t1-t0)<<std::endl;
std::cout<<GridLogMessage << "mflop/s per rank = "<< flops/(t1-t0)/NP<<std::endl; std::cout<<GridLogMessage << "mflop/s per rank = "<< flops/(t1-t0)/NP<<std::endl;
std::cout<<GridLogMessage << "mflop/s per node = "<< flops/(t1-t0)/NN<<std::endl; std::cout<<GridLogMessage << "mflop/s per node = "<< flops/(t1-t0)/NN<<std::endl;
std::cout<<GridLogMessage << "average bandwidth = "<< aveBW <<std::endl;
err = ref-result; err = ref-result;
n2e = norm2(err); n2e = norm2(err);
std::cout<<GridLogMessage << "norm diff "<< n2e<< " Line "<<__LINE__ <<std::endl; std::cout<<GridLogMessage << "norm diff "<< n2e<< " Line "<<__LINE__ <<std::endl;