From c4b089cf155875ad077c10a8aa9046206d4924ba Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 15 Aug 2026 09:51:56 -0400
Subject: [PATCH] More timing updates

---
 .../multigrid/RecursiveSchurInverse.h         | 36 +++++++++++++++----
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/Grid/algorithms/multigrid/RecursiveSchurInverse.h b/Grid/algorithms/multigrid/RecursiveSchurInverse.h
index 3c62ad14b..271cac8c1 100644
--- a/Grid/algorithms/multigrid/RecursiveSchurInverse.h
+++ b/Grid/algorithms/multigrid/RecursiveSchurInverse.h
@@ -128,10 +128,22 @@ public:
   double                tH2D;          // panel host->device
   double                tGemm;         // strided gemm + synchronise
   double                tLeaf;         // leaf inversions
+  double                tARmin;        // fastest single panel collective
+  double                tARmax;        // slowest single panel collective
   uint64_t              bytesAllreduce;
   uint64_t              nAllreduce;    // panel collectives
   uint64_t              nGatherGemm;   // GatherGemm calls
 
+  // PERSISTENT comms/staging buffers, grow-only across the whole walk.
+  // Fresh per-call host allocations defeat the MPI registration cache
+  // (every page unpinned every call => re-registration or bounce-buffer
+  // copies inside MPI on ~GB panels) -- the same reason the halo
+  // exchange uses allocate-once buffers.  Measured motivation: 0.48 GB/s
+  // effective allreduce payload at N=138k with per-call vectors.
+  std::vector<ComplexD>  panelBuf;
+  deviceVector<ComplexD> dPanelBuf;
+  std::vector<ComplexD>  stageBuf;
+
   ///////////////////////////////////////////////////////////////////////////
   // Ownership-table validation: a proper partition of [0,N).
   // Static and communicator-free so synthetic tables unit-test directly.
@@ -247,12 +259,12 @@ public:
     nGatherGemm++;
     GRID_TRACE("GatherGemm");
 
-    std::vector<ComplexD> stage;
+    std::vector<ComplexD> &stage = stageBuf;
     if ( owner )
     {
       GRID_ASSERT( colB + widthB <= B.cols );
       tStage -= usecond();
-      stage.resize((uint64_t)B.rows*n);
+      if ( stage.size() < (uint64_t)B.rows*n ) stage.resize((uint64_t)B.rows*n);
       acceleratorCopyFromDevice(B.ColumnWindow(colB), &stage[0],
                                 (uint64_t)B.rows*n*sizeof(ComplexD));
       tStage += usecond();
@@ -263,8 +275,10 @@ public:
     if ( kc > k ) kc = k;
     GRID_ASSERT( kc*n < 2147483647L );   // GlobalSumVector count is int
 
-    std::vector<ComplexD>   panel((uint64_t)kc*n);
-    deviceVector<ComplexD>  dPanel((uint64_t)kc*n);
+    std::vector<ComplexD>  &panel  = panelBuf;
+    deviceVector<ComplexD> &dPanel = dPanelBuf;
+    if ( panel.size()  < (uint64_t)kc*n ) panel.resize((uint64_t)kc*n);
+    if ( dPanel.size() < (uint64_t)kc*n ) dPanel.resize((uint64_t)kc*n);
     deviceVector<ComplexD*> ap(1);
     deviceVector<ComplexD*> bp(1);
     deviceVector<ComplexD*> cp(1);
@@ -298,9 +312,12 @@ public:
           tDeposit += usecond();
         }
       }
-      tAllreduce -= usecond();
+      double tar = -usecond();
       grid->GlobalSumVector(&panel[0], (int)(kchunk*n));
-      tAllreduce += usecond();
+      tar += usecond();
+      tAllreduce += tar;
+      tARmin = std::min(tARmin, tar);
+      tARmax = std::max(tARmax, tar);
       bytesAllreduce += (uint64_t)kchunk*n*sizeof(ComplexD);
       nAllreduce++;
 
@@ -580,6 +597,8 @@ public:
     tH2D           = 0.0;
     tGemm          = 0.0;
     tLeaf          = 0.0;
+    tARmin         = 1.0e30;
+    tARmax         = 0.0;
     bytesAllreduce = 0;
     nAllreduce     = 0;
     nGatherGemm    = 0;
@@ -632,6 +651,11 @@ public:
               << "  allreduce s min/max over ranks " << armin/1.0e6
               << " / " << armax/1.0e6
               << std::endl;
+    std::cout << GridLogMessage << "Schur comms per-call (boss):"
+              << "  min " << tARmin/1.0e3 << " ms"
+              << "  avg " << (nAllreduce ? tAllreduce/nAllreduce/1.0e3 : 0.0) << " ms"
+              << "  max " << tARmax/1.0e3 << " ms"
+              << std::endl;
   }
 };