portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-28 21:39:35 +01:00
Code Releases Activity

SUMMA telemetry: ring time histogram by message size

Browse Source 
Per log2-size bucket: messages, GB, seconds inside SendToRecvFrom, GB/s,
% of ring time.  Decomposes the 2 GB/s/rank average (probe: 11-20 GB/s
at 8 MB) into latency-bound small messages vs slow large ones vs wait.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RAhdQHrkzKmzfxpxW1whLn
This commit is contained in:
Peter Boyle
2026-08-27 02:02:30 -04:00
co-authored by Claude Fable 5
parent 96071aa492
commit 0979371c45
2 changed files with 31 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Grid/algorithms/multigrid/BlockCyclicSchurInverse.h
+19
View File
@@ -397,6 +397,25 @@ public:
<< " gemm min/max " << gmin << "/" << gmax
<< " | ring bytes/rank " << gb << " GB -> " << (ring>0 ? gb/ring : 0.0) << " GB/s/rank (boss)"
<< std::endl;
// Ring time decomposed by message size (boss rank; every rank sends the
// same sequence of sizes). Time is wall time inside SendToRecvFrom, so it
// includes waiting for the partner -- a bucket whose GB/s is far below the
// probe's for the same size is wait, not wire.
std::cout << GridLogMessage << "BlockCyclicSumma ring histogram (boss): size-bucket msgs GB secs GB/s %time" << std::endl;
std::streamsize oldprec = std::cout.precision();
for(int b=0;b<SUMMA.NHIST;b++){
if ( !SUMMA.histN[b] ) continue;
double sec = SUMMA.histUs[b]/1.0e6, g = SUMMA.histBytes[b]/1.0e9;
double lo = (double)(1ull<<b);
char sz[32]; if (lo>=1048576) snprintf(sz,32,"%6.1f MB",lo/1048576.); else if (lo>=1024) snprintf(sz,32,"%6.1f KB",lo/1024.); else snprintf(sz,32,"%6.0f B ",lo);
std::cout << GridLogMessage << " >=" << sz
<< std::setw(8) << SUMMA.histN[b]
<< std::setw(10) << std::setprecision(3) << g
<< std::setw(9) << std::setprecision(3) << sec
<< std::setw(9) << std::setprecision(3) << (sec>0 ? g/sec : 0.0)
<< std::setw(8) << std::setprecision(3) << (ring>0 ? 100.0*sec/ring : 0.0) << std::endl;
}
std::cout.precision(oldprec);
}
};
Grid/algorithms/multigrid/BlockCyclicSumma.h
+12 -1
View File
@@ -143,7 +143,14 @@ public:
deviceVector<ComplexD> Bbuf;
double tAlloc=0, tPack=0, tRingA=0, tRingB=0, tGemm=0;
uint64_t bytesRing=0, nRingMsg=0, nMultiply=0, nGemm=0;
void ResetTelemetry(void){ tAlloc=tPack=tRingA=tRingB=tGemm=0; bytesRing=nRingMsg=nMultiply=nGemm=0; }
// Per-message-size histogram (bucket = floor(log2 bytes)): count, bytes,
// microseconds -- decomposes the ring time by packet size so a low average
// GB/s can be attributed (many small latency-bound messages vs slow large
// ones vs partner-wait). The 8 MB probe runs at 11-20 GB/s; SUMMA averaged 2.
static const int NHIST=48;
uint64_t histN[NHIST]={0}, histBytes[NHIST]={0}; double histUs[NHIST]={0};
void HistAdd(uint64_t bytes, double us){ int b=0; while((bytes>>b)>1) b++; histN[b]++; histBytes[b]+=bytes; histUs[b]+=us; }
void ResetTelemetry(void){ tAlloc=tPack=tRingA=tRingB=tGemm=0; bytesRing=nRingMsg=nMultiply=nGemm=0; for(int b=0;b<NHIST;b++){histN[b]=histBytes[b]=0; histUs[b]=0;} }
static int Overlap(int64_t a0,int64_t a1,int64_t b0,int64_t b1)
{ return (a0 < b1) && (b0 < a1); }
@@ -273,9 +280,11 @@ public:
for(int t=1;t<Pc;t++){
int cs = (pcol - t + 1 + Pc*Pc) % Pc;
int cr = (pcol - t + Pc*Pc) % Pc;
double tm = usecond();
grid->SendToRecvFrom((void *)(&Abuf[0]+slotA*cs), dest,
(void *)(&Abuf[0]+slotA*cr), src,
slotA*sizeof(ComplexD));
HistAdd(slotA*sizeof(ComplexD), usecond()-tm);
bytesRing += slotA*sizeof(ComplexD); nRingMsg++;
}
tRingA += usecond();
@@ -291,9 +300,11 @@ public:
for(int t=1;t<Pr;t++){
int rs = (prow - t + 1 + Pr*Pr) % Pr;
int rr = (prow - t + Pr*Pr) % Pr;
double tm = usecond();
grid->SendToRecvFrom((void *)(&Bbuf[0]+slotB*rs), dest,
(void *)(&Bbuf[0]+slotB*rr), src,
slotB*sizeof(ComplexD));
HistAdd(slotB*sizeof(ComplexD), usecond()-tm);
bytesRing += slotB*sizeof(ComplexD); nRingMsg++;
}
tRingB += usecond();