portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-09-03 16:29:37 +01:00
Code Releases Activity

Mor ecomms probing

Browse Source
This commit is contained in:
Peter Boyle
2026-08-27 09:26:24 -04:00
parent 0979371c45
commit 0857f3aa82
2 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Grid/algorithms/multigrid/BlockCyclicSchurInverse.h
+7 -4
View File
@@ -290,14 +290,17 @@ public:
struct Ring { const char *name; int dest, src; }; struct Ring { const char *name; int dest, src; };
Ring rings[2] = { {"ringA(row, q+-1)", prow*Pc + (pcol+1)%Pc, prow*Pc + (pcol-1+Pc)%Pc}, Ring rings[2] = { {"ringA(row, q+-1)", prow*Pc + (pcol+1)%Pc, prow*Pc + (pcol-1+Pc)%Pc},
{"ringB(col, p+-1)", ((prow+1)%Pr)*Pc + pcol, ((prow-1+Pr)%Pr)*Pc + pcol} }; {"ringB(col, p+-1)", ((prow+1)%Pr)*Pc + pcol, ((prow-1+Pr)%Pr)*Pc + pcol} };
uint64_t sizes[3] = { 64ull*1024, 1024ull*1024, 8ull*1024*1024 }; // 2/3/4 MB added 2026-08-27: the SUMMA histogram put 93% of ring time in
uint64_t maxb = sizes[2]; // [2,4) MB messages at 0.3 GB/s while >=4 MB ran at 11-13 GB/s.
const int NSZ = 6;
uint64_t sizes[NSZ] = { 64ull*1024, 1024ull*1024, 2048ull*1024, 3072ull*1024, 4096ull*1024, 8ull*1024*1024 };
uint64_t maxb = sizes[NSZ-1];
deviceVector<char> dsend(maxb), drecv(maxb); deviceVector<char> dsend(maxb), drecv(maxb);
std::vector<char> hsend(maxb), hrecv(maxb); std::vector<char> hsend(maxb), hrecv(maxb);
for(int r=0;r<2;r++){ for(int r=0;r<2;r++){
if ( (r==0 && Pc==1) || (r==1 && Pr==1) ) continue; if ( (r==0 && Pc==1) || (r==1 && Pr==1) ) continue;
int off = grid->IsOffNode(rings[r].dest); int off = grid->IsOffNode(rings[r].dest);
for(int si=0;si<3;si++){ for(int si=0;si<NSZ;si++){
uint64_t bytes = sizes[si]; uint64_t bytes = sizes[si];
// warm one, time five, both memory spaces // warm one, time five, both memory spaces
grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes); grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes);
@@ -326,7 +329,7 @@ public:
/////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////
{ {
int r = (Pr>1) ? 1 : 0; int r = (Pr>1) ? 1 : 0;
uint64_t bytes = sizes[2]; uint64_t bytes = sizes[NSZ-1];
uint64_t big = 512ull*1024*1024; uint64_t big = 512ull*1024*1024;
deviceVector<char> bsend(big), brecv(big); deviceVector<char> bsend(big), brecv(big);
for(int variant=0; variant<3; variant++){ for(int variant=0; variant<3; variant++){
Grid/algorithms/multigrid/DenseCoarseMatrix.h
+1 -1
View File
@@ -1211,7 +1211,7 @@ public:
if ( getenv("DENSE_APPLY_PROFILE") ) { if ( getenv("DENSE_APPLY_PROFILE") ) {
std::cout << GridLogMessage << "DenseCoarseMatrix: apply6D profile:" std::cout << GridLogMessage << "DenseCoarseMatrix: apply6D profile:"
<< " pack " << (t1-t0)/1000.0 << " pack " << (t1-t0)/1000.0
<< " allreduce " << tprof[0]/1000.0 << (devSum==4 ? " allgather " : " allreduce ") << tprof[0]/1000.0
<< " H2D " << tprof[1]/1000.0 << " H2D " << tprof[1]/1000.0
<< " gemm+reduce "<< tprof[2]/1000.0 << " gemm+reduce "<< tprof[2]/1000.0
<< " D2H " << tprof[3]/1000.0 << " D2H " << tprof[3]/1000.0