diff --git a/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h b/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h index 0c9247eba..61f0eac50 100644 --- a/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h +++ b/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h @@ -404,7 +404,8 @@ public: // same sequence of sizes). Time is wall time inside SendToRecvFrom, so it // includes waiting for the partner -- a bucket whose GB/s is far below the // probe's for the same size is wait, not wire. - std::cout << GridLogMessage << "BlockCyclicSumma ring histogram (boss): size-bucket msgs GB secs GB/s %time" << std::endl; + std::cout << GridLogMessage << "BlockCyclicSumma ring histogram (boss): size-bucket msgs GB xfer-secs GB/s %time" + << (SUMMA.handshake>0 ? " handshake-secs (partner wait, excluded from xfer)" : "") << std::endl; std::streamsize oldprec = std::cout.precision(); for(int b=0;b0 ? g/sec : 0.0) - << std::setw(8) << std::setprecision(3) << (ring>0 ? 100.0*sec/ring : 0.0) << std::endl; + << std::setw(8) << std::setprecision(3) << (ring>0 ? 100.0*sec/ring : 0.0); + if ( SUMMA.handshake>0 ) std::cout << std::setw(12) << std::setprecision(3) << SUMMA.histHsUs[b]/1.0e6; + std::cout << std::endl; } std::cout.precision(oldprec); } diff --git a/Grid/algorithms/multigrid/BlockCyclicSumma.h b/Grid/algorithms/multigrid/BlockCyclicSumma.h index ab5e7a876..da3dc7bb5 100644 --- a/Grid/algorithms/multigrid/BlockCyclicSumma.h +++ b/Grid/algorithms/multigrid/BlockCyclicSumma.h @@ -148,9 +148,14 @@ public: // GB/s can be attributed (many small latency-bound messages vs slow large // ones vs partner-wait). The 8 MB probe runs at 11-20 GB/s; SUMMA averaged 2. static const int NHIST=48; - uint64_t histN[NHIST]={0}, histBytes[NHIST]={0}; double histUs[NHIST]={0}; - void HistAdd(uint64_t bytes, double us){ int b=0; while((bytes>>b)>1) b++; histN[b]++; histBytes[b]+=bytes; histUs[b]+=us; } - void ResetTelemetry(void){ tAlloc=tPack=tRingA=tRingB=tGemm=0; bytesRing=nRingMsg=nMultiply=nGemm=0; for(int b=0;b>b)>1) b++; histN[b]++; histBytes[b]+=bytes; histUs[b]+=us; histHsUs[b]+=hs; } + void ResetTelemetry(void){ tAlloc=tPack=tRingA=tRingB=tGemm=0; bytesRing=nRingMsg=nMultiply=nGemm=0; for(int b=0;b(slotA*Pc,1) ) Abuf.resize( std::max(slotA*Pc,1) ); if ( Bbuf.size() < std::max(slotB*Pr,1) ) Bbuf.resize( std::max(slotB*Pr,1) ); @@ -280,11 +286,12 @@ public: for(int t=1;tSendToRecvFrom((void *)&hsTx, dest, (void *)&hsRx, src, sizeof(int)); ths = usecond()-tm; tm = usecond(); } grid->SendToRecvFrom((void *)(&Abuf[0]+slotA*cs), dest, (void *)(&Abuf[0]+slotA*cr), src, slotA*sizeof(ComplexD)); - HistAdd(slotA*sizeof(ComplexD), usecond()-tm); + HistAdd(slotA*sizeof(ComplexD), usecond()-tm, ths); bytesRing += slotA*sizeof(ComplexD); nRingMsg++; } tRingA += usecond(); @@ -300,11 +307,12 @@ public: for(int t=1;tSendToRecvFrom((void *)&hsTx, dest, (void *)&hsRx, src, sizeof(int)); ths = usecond()-tm; tm = usecond(); } grid->SendToRecvFrom((void *)(&Bbuf[0]+slotB*rs), dest, (void *)(&Bbuf[0]+slotB*rr), src, slotB*sizeof(ComplexD)); - HistAdd(slotB*sizeof(ComplexD), usecond()-tm); + HistAdd(slotB*sizeof(ComplexD), usecond()-tm, ths); bytesRing += slotB*sizeof(ComplexD); nRingMsg++; } tRingB += usecond();