From 0857f3aa82f1e98be501cef450f201559abe262d Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 27 Aug 2026 09:26:24 -0400
Subject: [PATCH] Mor ecomms probing

---
 Grid/algorithms/multigrid/BlockCyclicSchurInverse.h | 11 +++++++----
 Grid/algorithms/multigrid/DenseCoarseMatrix.h       |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h b/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h
index ce00004d1..0c9247eba 100644
--- a/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h
+++ b/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h
@@ -290,14 +290,17 @@ public:
     struct Ring { const char *name; int dest, src; };
     Ring rings[2] = { {"ringA(row, q+-1)", prow*Pc + (pcol+1)%Pc,       prow*Pc + (pcol-1+Pc)%Pc},
                       {"ringB(col, p+-1)", ((prow+1)%Pr)*Pc + pcol,     ((prow-1+Pr)%Pr)*Pc + pcol} };
-    uint64_t sizes[3] = { 64ull*1024, 1024ull*1024, 8ull*1024*1024 };
-    uint64_t maxb = sizes[2];
+    // 2/3/4 MB added 2026-08-27: the SUMMA histogram put 93% of ring time in
+    // [2,4) MB messages at 0.3 GB/s while >=4 MB ran at 11-13 GB/s.
+    const int NSZ = 6;
+    uint64_t sizes[NSZ] = { 64ull*1024, 1024ull*1024, 2048ull*1024, 3072ull*1024, 4096ull*1024, 8ull*1024*1024 };
+    uint64_t maxb = sizes[NSZ-1];
     deviceVector<char> dsend(maxb), drecv(maxb);
     std::vector<char>  hsend(maxb), hrecv(maxb);
     for(int r=0;r<2;r++){
       if ( (r==0 && Pc==1) || (r==1 && Pr==1) ) continue;
       int off = grid->IsOffNode(rings[r].dest);
-      for(int si=0;si<3;si++){
+      for(int si=0;si<NSZ;si++){
         uint64_t bytes = sizes[si];
         // warm one, time five, both memory spaces
         grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes);
@@ -326,7 +329,7 @@ public:
     ///////////////////////////////////////////////////////////////////////
     {
       int r = (Pr>1) ? 1 : 0;
-      uint64_t bytes = sizes[2];
+      uint64_t bytes = sizes[NSZ-1];
       uint64_t big = 512ull*1024*1024;
       deviceVector<char> bsend(big), brecv(big);
       for(int variant=0; variant<3; variant++){
diff --git a/Grid/algorithms/multigrid/DenseCoarseMatrix.h b/Grid/algorithms/multigrid/DenseCoarseMatrix.h
index 1aa95f836..5b0873ec0 100644
--- a/Grid/algorithms/multigrid/DenseCoarseMatrix.h
+++ b/Grid/algorithms/multigrid/DenseCoarseMatrix.h
@@ -1211,7 +1211,7 @@ public:
     if ( getenv("DENSE_APPLY_PROFILE") ) {
       std::cout << GridLogMessage << "DenseCoarseMatrix: apply6D profile:"
                 << " pack "        << (t1-t0)/1000.0
-                << "  allreduce "  << tprof[0]/1000.0
+                << (devSum==4 ? "  allgather " : "  allreduce ")  << tprof[0]/1000.0
                 << "  H2D "        << tprof[1]/1000.0
                 << "  gemm+reduce "<< tprof[2]/1000.0
                 << "  D2H "        << tprof[3]/1000.0