Merge branch 'feature/splitGridCGBatch' into develop

broad unicode elimination effort

BLAS_benchmark/RocblasSolutionSweep.cc

@@ -0,0 +1,190 @@
+///////////////////////////////////////////////////////////////////////////////
+// Which rocBLAS kernel does the coarse multigrid GEMM get, and is a better
+// one available?
+//
+// The coarse operator issues a batched complex GEMM per stencil point with
+// M = K = nbasis, N = Nrhs and batch = the local coarse volume.  Traces of a
+// production run show the complex-single kernel taking as long as the
+// complex-double one while reading half the bytes, and the kernel names say
+// why: the library picks a 64x64 macro tile for single and 16x16 for double,
+// while the output block is only nbasis x Nrhs.  At nbasis 16 single
+// precision comes out absolutely slower than double.
+//
+// hipBLAS offers no way to override that choice, but rocBLAS's extended
+// entry point takes a solution index.  This walks the indices, keeps the ones
+// the library accepts for this problem, times each, and prints them against
+// the default heuristic.  No Grid, no MPI: one GPU, a few seconds.
+//
+//   hipcc -O3 -std=c++17 RocblasSolutionSweep.cc -lrocblas -lamdhip64 \
+//         -o RocblasSolutionSweep
+//   ./RocblasSolutionSweep [nbasis] [nrhs] [batch]
+//
+// If the accepted solutions include one roughly twice the double-precision
+// rate, the application can select it by index and the tuning gap is ours to
+// close.  If none does, the gap is the library's, and this output is the
+// report.
+///////////////////////////////////////////////////////////////////////////////
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <sys/time.h>
+#include <hip/hip_runtime.h>
+#include <rocblas/rocblas.h>
+
+static double usecond(void)
+{
+  struct timeval tv; gettimeofday(&tv,NULL);
+  return 1.0e6*tv.tv_sec + 1.0*tv.tv_usec;
+}
+#define HIP_CHECK(x)  do { hipError_t e=(x); if(e!=hipSuccess){ printf("HIP error %s at line %d\n",hipGetErrorString(e),__LINE__); exit(1);} } while(0)
+
+struct Result { int index; double gflops; double gbs; };
+
+// One shape, one datatype, every solution index the library accepts.
+static void sweep(rocblas_handle handle,
+		  rocblas_datatype type, const char *name, size_t elembytes,
+		  int M,int N,int K,int BATCH,int maxindex,int ncall)
+{
+  // Device data.  Contents are irrelevant to the timing; the shapes are not.
+  void *A,*B,*C;
+  HIP_CHECK(hipMalloc(&A,(size_t)M*K*BATCH*elembytes));
+  HIP_CHECK(hipMalloc(&B,(size_t)K*N*BATCH*elembytes));
+  HIP_CHECK(hipMalloc(&C,(size_t)M*N*BATCH*elembytes));
+  HIP_CHECK(hipMemset(A,0,(size_t)M*K*BATCH*elembytes));
+  HIP_CHECK(hipMemset(B,0,(size_t)K*N*BATCH*elembytes));
+  HIP_CHECK(hipMemset(C,0,(size_t)M*N*BATCH*elembytes));
+
+  // Pointer arrays, as the batched interface wants them, on the device.
+  std::vector<void *> hA(BATCH),hB(BATCH),hC(BATCH);
+  for(int b=0;b<BATCH;b++){
+    hA[b] = (char *)A + (size_t)b*M*K*elembytes;
+    hB[b] = (char *)B + (size_t)b*K*N*elembytes;
+    hC[b] = (char *)C + (size_t)b*M*N*elembytes;
+  }
+  void **dA,**dB,**dC;
+  HIP_CHECK(hipMalloc(&dA,BATCH*sizeof(void*)));
+  HIP_CHECK(hipMalloc(&dB,BATCH*sizeof(void*)));
+  HIP_CHECK(hipMalloc(&dC,BATCH*sizeof(void*)));
+  HIP_CHECK(hipMemcpy(dA,&hA[0],BATCH*sizeof(void*),hipMemcpyHostToDevice));
+  HIP_CHECK(hipMemcpy(dB,&hB[0],BATCH*sizeof(void*),hipMemcpyHostToDevice));
+  HIP_CHECK(hipMemcpy(dC,&hC[0],BATCH*sizeof(void*),hipMemcpyHostToDevice));
+
+  // alpha = beta = 1: the stencil accumulates into C, so C is read as well
+  // as written and the traffic below counts it twice.
+  double alpha_d[2] = {1.0,0.0}, beta_d[2] = {1.0,0.0};
+  float  alpha_f[2] = {1.0f,0.0f}, beta_f[2] = {1.0f,0.0f};
+  const void *alpha = (type==rocblas_datatype_f64_c) ? (const void *)alpha_d : (const void *)alpha_f;
+  const void *beta  = (type==rocblas_datatype_f64_c) ? (const void *)beta_d  : (const void *)beta_f;
+
+  double flops = 8.0*M*N*K*BATCH;                                  // complex madd
+  double bytes = (double)elembytes*((double)M*K + (double)K*N + 2.0*(double)M*N)*BATCH;
+
+  auto run = [&](rocblas_gemm_algo algo,int solution)->double
+  {
+    rocblas_status st = rocblas_gemm_batched_ex(handle,
+			   rocblas_operation_none, rocblas_operation_none,
+			   M,N,K, alpha,
+			   (const void *)dA, type, M,
+			   (const void *)dB, type, K, beta,
+			   (const void *)dC, type, M,
+			   (void *)dC,       type, M,
+			   BATCH, type, algo, solution, 0);
+    if ( st != rocblas_status_success ) return -1.0;
+    HIP_CHECK(hipDeviceSynchronize());
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      rocblas_gemm_batched_ex(handle,
+			   rocblas_operation_none, rocblas_operation_none,
+			   M,N,K, alpha,
+			   (const void *)dA, type, M,
+			   (const void *)dB, type, K, beta,
+			   (const void *)dC, type, M,
+			   (void *)dC,       type, M,
+			   BATCH, type, algo, solution, 0);
+    }
+    HIP_CHECK(hipDeviceSynchronize());
+    return (usecond()-t0)/ncall;
+  };
+
+  printf("\n=== %s : M=%d N=%d K=%d batch=%d ===\n",name,M,N,K,BATCH);
+
+  double tdef = run(rocblas_gemm_algo_standard,0);
+  printf("  default heuristic          : %8.1f Gflop/s  %8.1f GB/s  (%.3f ms)\n",
+	 flops/tdef/1.0e3, bytes/tdef/1.0e3, tdef/1.0e3);
+
+  // The extended call only accepts indices the library has itself listed for
+  // this problem, so ask it for the list rather than guessing.  If this does
+  // not compile, check the argument order in rocblas.h: the enumeration call
+  // has moved an argument or two between releases.
+  rocblas_int nsol = 0;
+  rocblas_status qst = rocblas_gemm_batched_ex_get_solutions(handle,
+			   rocblas_operation_none, rocblas_operation_none,
+			   M,N,K, alpha,
+			   (const void *)dA, type, M,
+			   (const void *)dB, type, K, beta,
+			   (const void *)dC, type, M,
+			   (void *)dC,       type, M,
+			   BATCH, type, rocblas_gemm_flags_none,
+			   NULL, &nsol);
+  std::vector<rocblas_int> sols;
+  if ( qst == rocblas_status_success && nsol > 0 ) {
+    sols.resize(nsol);
+    rocblas_gemm_batched_ex_get_solutions(handle,
+			   rocblas_operation_none, rocblas_operation_none,
+			   M,N,K, alpha,
+			   (const void *)dA, type, M,
+			   (const void *)dB, type, K, beta,
+			   (const void *)dC, type, M,
+			   (void *)dC,       type, M,
+			   BATCH, type, rocblas_gemm_flags_none,
+			   &sols[0], &nsol);
+  } else {
+    // Older library: fall back to probing indices.
+    for(int s=1;s<=maxindex;s++) sols.push_back(s);
+  }
+
+  std::vector<Result> ok;
+  for(size_t i=0;i<sols.size();i++){
+    double t = run(rocblas_gemm_algo_solution_index,sols[i]);
+    if ( t < 0 ) continue;                      // library rejects it for this problem
+    ok.push_back({(int)sols[i],flops/t/1.0e3,bytes/t/1.0e3});
+  }
+  printf("  library lists %d solution(s) for this problem\n",(int)sols.size());
+  std::sort(ok.begin(),ok.end(),[](const Result &a,const Result &b){ return a.gflops>b.gflops; });
+
+  printf("  %d solutions accepted of %d tried\n",(int)ok.size(),maxindex);
+  for(int i=0;i<(int)ok.size() && i<8;i++){
+    printf("    solution %5d            : %8.1f Gflop/s  %8.1f GB/s\n",
+	   ok[i].index,ok[i].gflops,ok[i].gbs);
+  }
+  if ( ok.size() ) {
+    printf("  best/default               : %.2f\n", ok[0].gflops*tdef*1.0e3/flops);
+  }
+
+  HIP_CHECK(hipFree(dA)); HIP_CHECK(hipFree(dB)); HIP_CHECK(hipFree(dC));
+  HIP_CHECK(hipFree(A));  HIP_CHECK(hipFree(B));  HIP_CHECK(hipFree(C));
+}
+
+int main(int argc,char **argv)
+{
+  int nbasis = (argc>1) ? atoi(argv[1]) : 60;
+  int nrhs   = (argc>2) ? atoi(argv[2]) : 12;
+  int batch  = (argc>3) ? atoi(argv[3]) : 1024;
+  int maxidx = (argc>4) ? atoi(argv[4]) : 400;
+
+  rocblas_handle handle;
+  rocblas_create_handle(&handle);
+  rocblas_set_pointer_mode(handle,rocblas_pointer_mode_host);
+
+  printf("rocBLAS solution sweep: the coarse stencil GEMM shape\n");
+  printf("nbasis %d, Nrhs %d, batch %d, solution indices 1..%d\n",nbasis,nrhs,batch,maxidx);
+  printf("(batch is the LOCAL coarse volume; x9 is the grouped call)\n");
+
+  sweep(handle,rocblas_datatype_f32_c,"complex single",8, nbasis,nrhs,nbasis,batch,maxidx,20);
+  sweep(handle,rocblas_datatype_f64_c,"complex double",16,nbasis,nrhs,nbasis,batch,maxidx,20);
+
+  rocblas_destroy_handle(handle);
+  return 0;
+}

CLAUDE.md

@@ -106,7 +106,7 @@ Tests and benchmarks that need optional fermion representations are guarded by `
 ### GPU acceleration and the view/memory-manager discipline
 ### Multigrid (`Grid/algorithms/multigrid/`)
 
-Aggregation-based algebraic multigrid for Wilson-type fermions. Key files: `CoarsenedMatrix.h` (coarse operator), `GeneralCoarsenedMatrix.h` and `GeneralCoarsenedMatrixMultiRHS.h` (general coarsening supporting multi-RHS solves), `Aggregates.h` (near-null vector construction), `Geometry.h` (coarse-grid geometry). `MultiGrid.h` is the top-level include.
+Aggregation-based algebraic multigrid for Wilson-type fermions. Key files: `GeneralCoarsenedMatrixMultiRHS.h` (`MultiGeneralCoarsenedOperator`, the coarse operator for any number of right-hand sides: coarsening by Fourier probing, batched-GEMM apply on a D+1 grid with rhs innermost and unvectorised), `MultiRHSBlockProject.h` (in `deflation/`; the batched-GEMM transfer operators), `PVdagMMultiGrid.h` (three-level non-Hermitian PVdagM chain with dense bottom) and `HDCGMultiGrid.h` (two-level Hermitian HDCG chain) with their `*Params.h` (XML-serialisable parameters), `MrhsMultiGrid.h` (mrhs V-cycle, preconditioner interface, fp64/fp32 seam), `Smoothers.h`, `MultiGridIO.h`, `Aggregates.h` (near-null vector construction), `Geometry.h` (coarse stencils: max-norm-1 boxes of Manhattan radius 1, 2 or 4), `CoarsenedMatrix.h` (the original nearest-neighbour Wilson multigrid). `deprecated/` holds the superseded coarse operators and the Aggregation-based single-RHS ADEF-2, with class names prefixed `Deprecated` (`DeprecatedGeneralCoarsenedMatrix`, `DeprecatedMultiGeneralCoarsenedMatrix`, `DeprecatedTwoLevelADEF2`); nothing in the library uses them, only older drivers in `examples/` and `tests/debug/`. `MultiGrid.h` is the top-level include.
 
 ### GPU acceleration
 
@@ -174,7 +174,7 @@ Every programme is wrapped in `Grid_init(&argc, &argv)` / `Grid_finalize()` (`Gr
 - Template structure: most classes are templated on `<_FImpl>` (fermion impl) or `<Gimpl>` (gauge impl), which encode the representation and precision. Instantiation is controlled by `--enable-fermion-instantiations`.
 - **Tensor indices are positional, not labelled.** The `Grid/tensors/` arithmetic recurses structurally over the `iScalar`/`iVector`/`iMatrix` nest: each level defines only the {scalar,vector,matrix}² products at its own level, with element types resolved by automatic type deduction, so every colour/spin/lorentz combination composes from ~200 lines (versus the pre-C++11 QDP++/PETE approach of machine-generating every case). An index's meaning derives entirely from its nesting depth counted from the outside; `iScalar` is the identity/broadcast case at every level. Never insert or remove a nesting level casually — the multiplication tables contract by position.
 - **Multigrid coarsening deepens the tensor nest by one level.** A coarse site vector is `iVector<CComplex,nbasis>`, and `innerProduct` on it returns `iScalar<CComplex>` — one level deeper than the fine block scalar. So the block-inner-product scalar type gains one `iScalar` wrapper per MG level (fine: `vTComplex`; level 2: `iScalar<vTComplex>`; see `examples/Example_pvdagm_3level.cc`). When calling `blockInnerProduct`/`blockZAXPY`/`blockOrthogonalise` on coarse fields, the coarse scalar type must match `decltype(innerProduct(siteVector(),siteVector()))` exactly; a wrong depth fails to compile (no viable `operator=` deep in the instantiation chain) rather than mis-contracting.
-- **Grids are borrowed, never owned.** `conformable` is pointer identity, so every object that interoperates must hold the *same* `GridCartesian *`; a class that minted its own grid internally could never conform with anything else. Ownership is therefore not available, and lifetime is managed by scope discipline instead of reference counting: whoever creates a grid retains it beyond every object it handed a reference to. Anything *derived* from a grid inherits this — `~PaddedCell` dereferences its `unpadded_grid`, so a `PaddedCell` cannot even be **destroyed** after its parent grid, only used. Where a consumer must let go early, it offers an explicit hand-back (`MultiGeneralCoarsenedOperatorV2::ReleaseGrid()`) to be called *before* the grid is destroyed.
+- **Grids are borrowed, never owned.** `conformable` is pointer identity, so every object that interoperates must hold the *same* `GridCartesian *`; a class that minted its own grid internally could never conform with anything else. Ownership is therefore not available, and lifetime is managed by scope discipline instead of reference counting: whoever creates a grid retains it beyond every object it handed a reference to. Anything *derived* from a grid inherits this — `~PaddedCell` dereferences its `unpadded_grid`, so a `PaddedCell` cannot even be **destroyed** after its parent grid, only used. Where a consumer must let go early, it offers an explicit hand-back (`MultiGeneralCoarsenedOperator::ReleaseGrid()`) to be called *before* the grid is destroyed.
 
 - The `RealD`/`RealF`/`ComplexD`/`ComplexF` typedefs are used everywhere; avoid raw `double`/`float`.
 - Use `GRID_ASSERT(cond)` (defined in `Grid/GridStd.h`), not bare `assert` — it prints a Grid-formatted message and aborts cleanly under MPI.

Grid/GridCore.h

@@ -56,6 +56,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <Grid/communicator/Communicator.h> 
 #include <Grid/communicator/RingAllReduce.h>
 #include <Grid/cartesian/Cartesian.h>
+#include <Grid/perfmon/HostMemory.h>
 #include <Grid/tensors/Tensors.h>      
 #include <Grid/lattice/Lattice.h>      
 #include <Grid/cshift/Cshift.h>       

Grid/algorithms/Algorithms.h

@@ -33,6 +33,7 @@ NAMESPACE_CHECK(blas);
 #include <Grid/algorithms/blas/BatchedBlas.h>
 
 NAMESPACE_CHECK(algorithms);
+#include <Grid/algorithms/SplitOperator.h>
 #include <Grid/algorithms/SparseMatrix.h>
 #include <Grid/algorithms/LinearOperator.h>
 #include <Grid/algorithms/Preconditioner.h>

Grid/algorithms/FFT.h

@@ -233,6 +233,15 @@ static void FFT_dim_execute(
   typedef typename vobj::vector_type   vector_type;
   typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
 
+#if 0
+  // ======================= ORIGINAL barrel-shift path =======================
+  // Preserved for reference.  Superseded by the transpose / all-to-all path
+  // below (the single active path for ALL P): the barrel is a P-fold redundant
+  // all-gather -- every rank assembles and transforms all Nperp lines of length
+  // G, keeping only its own L points.  The transpose partitions the Nperp
+  // perpendicular lines across the P ranks along dim, so each rank transforms
+  // only ceil(Nperp/P) lines and moves (P-1)/P of its data instead of P-1
+  // redundant copies.  See CartesianRingAllToAll in communicator/RingAllReduce.h.
   const int Ndim = grid->Nd();
   int L = grid->_ldimensions[dim];
   int G = grid->_fdimensions[dim];
@@ -363,6 +372,208 @@ static void FFT_dim_execute(
   std::cout << GridLogPerformance << "  of which shift" << t_shift/1.0e6 << " s" << std::endl;
   std::cout << GridLogPerformance << " FFT kernels " << t_fft/1.0e6    << " s" << std::endl;
   std::cout << GridLogPerformance << " FFT insert  " << t_insert/1.0e6 << " s" << std::endl;
+#endif
+
+  // ==================== transpose / all-to-all pencil FFT ====================
+  //
+  // P ranks lie along dim; each holds a local L-slab (L=ldim) of every global
+  // line (length G=fdim=L*P) and Nperp perpendicular lines.  We partition the
+  // Nperp lines across the P ranks: the rank at coord `own` owns the lines with
+  // olin/Oloc == own, gathers their full G points via a cartesian all-to-all,
+  // transforms only Oloc = ceil(Nperp/P) of them, then scatters the result back
+  // with the inverse all-to-all.
+  //
+  // The owned-line count is CEIL-padded to a multiple of P (Oloc*P >= Nperp) so
+  // the all-to-all stays SYMMETRIC (one uniform chunk) for ANY (P,Nperp) -- in
+  // particular a P carrying a factor absent from Nperp, e.g. P=3 on the T axis
+  // of 128^3x288 whose perpendicular volume is a pure power of two.  This keeps
+  // the transpose as total over decompositions as the barrel it replaces (no
+  // new geometry constraint), at a cost of <= (P-1) padded lines out of Nperp.
+  // Padding slots olin in [Nperp, Oloc*P) are never packed and never unpacked.
+  //
+  // The load-bearing identity: the all-to-all block index == cartesian coord
+  // along dim == which L-slab [c*L, c*L+L) of the global line -- so the block a
+  // rank receives carries the global-x tag needed to order the FFT input.
+  //
+  // Degenerate P: P=1 -> both all-to-alls are self-copies and the reorders are
+  // the identity (G=L), i.e. a pure local FFT.  P=2 -> each all-to-all moves
+  // half a field, two of them one field, equal to the barrel's single Cshift.
+  {
+    const int Ndim = grid->Nd();
+    int     L     = grid->_ldimensions[dim];
+    int     G     = grid->_fdimensions[dim];
+    int     Ncomp = sizeof(sobj) / sizeof(scalar);
+    int     P     = grid->_processors[dim];
+    int64_t Nperp = 1;
+    for (int d = 0; d < Ndim; d++)
+      if (d != dim) Nperp *= grid->_ldimensions[d];
+
+    int64_t Oloc  = (Nperp + P - 1) / P;        // ceil: owned (padded) lines/rank
+    int64_t chunk = (int64_t)L * Oloc * Ncomp;  // one all-to-all block (uniform)
+    int64_t nbuf  = (int64_t)P * chunk;         // == Ncomp*Oloc*G, one field's worth
+    int64_t howmany_local = (int64_t)Ncomp * Oloc;
+
+    scalar div;
+    if      (sign == FFTW_BACKWARD) div = 1.0 / G;
+    else if (sign == FFTW_FORWARD)  div = 1.0;
+    else GRID_ASSERT(0);
+
+    double t_total = -usecond();
+    double t_pack = 0, t_a2a = 0, t_reorder = 0, t_fft = 0, t_unpack = 0;
+
+    deviceVector<scalar> sbuf(nbuf);
+    deviceVector<scalar> rbuf(nbuf);
+    deviceVector<scalar> pgbuf(nbuf);           // FFTW pencil buffer, Ncomp*Oloc lines of G
+    scalar *sbuf_v  = &sbuf[0];
+    scalar *rbuf_v  = &rbuf[0];
+    scalar *pgbuf_v = &pgbuf[0];
+
+    // Pad slots (olin in [Nperp, Oloc*P)) are never packed, so they would carry
+    // garbage device memory into the FFT.  Zero them so the padded lines stay
+    // finite -- but ONLY when padding is actually present.  In the common
+    // Nperp % P == 0 case pack writes every sbuf entry bijectively, so skip the
+    // whole-buffer memset and its device sync entirely.
+    if ( (int64_t)Oloc*P != Nperp )
+      acceleratorMemSet(sbuf_v, 0, nbuf*sizeof(scalar));
+
+    const Coordinate ldims = grid->_ldimensions;
+    const Coordinate rdims = grid->_rdimensions;
+    const Coordinate sdims = grid->_simd_layout;
+    const int Nsimd = vobj::Nsimd();
+
+    // ---- 1. pack: source -> sbuf.  block = owner coord; payload xloc + L*(slot + Oloc*w)
+    t_pack -= usecond();
+    {
+      autoView(s_v, source, AcceleratorRead);
+      accelerator_for(idx, grid->oSites(), Nsimd, {
+#ifdef GRID_SIMT
+      {
+        int lane = acceleratorSIMTlane(Nsimd);
+#else
+      for (int lane = 0; lane < Nsimd; lane++) {
+#endif
+        Coordinate icoor(Ndim), ocoor(Ndim);
+        Lexicographic::CoorFromIndex(icoor, lane, sdims);
+        Lexicographic::CoorFromIndex(ocoor, idx,  rdims);
+        int64_t xloc = ocoor[dim] + icoor[dim]*rdims[dim];
+        int64_t olin = 0, str = 1;
+        for (int d = 0; d < Ndim; d++) {
+          if (d == dim) continue;
+          int64_t c = ocoor[d] + icoor[d]*rdims[d];
+          olin += str * c;
+          str  *= ldims[d];
+        }
+        int64_t own  = olin / Oloc;
+        int64_t slot = olin - own*Oloc;
+        vector_type *from = (vector_type *)&s_v[idx];
+        for (int w = 0; w < Ncomp; w++) {
+          scalar_type stmp = getlane(from[w], lane);
+          sbuf_v[ own*chunk + xloc + L*(slot + Oloc*w) ] = stmp;
+        }
+#ifdef GRID_SIMT
+      }
+#else
+      }
+#endif
+      });
+    }
+    t_pack += usecond();
+
+    // ---- 2. forward all-to-all: gather my owned lines' L-slabs from every rank
+    t_a2a -= usecond();
+    CartesianRingAllToAll(grid, sbuf_v, rbuf_v, (uint64_t)chunk, dim);
+    t_a2a += usecond();
+
+    // ---- 3. reorder rbuf -> pgbuf: contiguous G-lines (w,slot), xpos = src*L + xloc
+    t_reorder -= usecond();
+    accelerator_for(q, nbuf, 1, {
+      int64_t xpos = q % G;
+      int64_t t    = q / G;           // = w*Oloc + slot
+      int64_t slot = t % Oloc;
+      int64_t w    = t / Oloc;
+      int64_t src  = xpos / L;
+      int64_t xloc = xpos % L;
+      pgbuf_v[q] = rbuf_v[ src*chunk + xloc + L*(slot + Oloc*w) ];
+    });
+    t_reorder += usecond();
+
+    // ---- 4. FFT: Ncomp*Oloc contiguous lines of length G (istride 1, idist G)
+    {
+      FFTW_scalar *in  = (FFTW_scalar *)pgbuf_v;
+      FFTW_scalar *out = (FFTW_scalar *)pgbuf_v;
+      t_fft -= usecond();
+      FFTW<scalar>::fftw_execute_dft(p, in, out, sign);
+      t_fft += usecond();
+    }
+    flops_call = 5.0 * (double)howmany_local * G * log2(G);
+    usec  = (uint64_t)t_fft;
+    flops = flops_call;
+
+    // ---- 5a. reorder pgbuf -> sbuf: block = destination coord; xpos = dst*L + xloc
+    t_reorder -= usecond();
+    accelerator_for(j, nbuf, 1, {
+      int64_t dst  = j / chunk;
+      int64_t r    = j % chunk;
+      int64_t xloc = r % L;
+      int64_t u    = r / L;           // = slot + Oloc*w
+      int64_t slot = u % Oloc;
+      int64_t w    = u / Oloc;
+      int64_t xpos = dst*L + xloc;
+      sbuf_v[j] = pgbuf_v[ w*Oloc*G + slot*G + xpos ];
+    });
+    t_reorder += usecond();
+
+    // ---- 5b. inverse all-to-all: scatter transformed L-slabs back
+    t_a2a -= usecond();
+    CartesianRingAllToAll(grid, sbuf_v, rbuf_v, (uint64_t)chunk, dim);
+    t_a2a += usecond();
+
+    // ---- 5c. unpack rbuf -> result (x div); block = owner coord of each line
+    t_unpack -= usecond();
+    {
+      autoView(r_v, result, AcceleratorWrite);
+      accelerator_for(idx, grid->oSites(), Nsimd, {
+#ifdef GRID_SIMT
+      {
+        int lane = acceleratorSIMTlane(Nsimd);
+#else
+      for (int lane = 0; lane < Nsimd; lane++) {
+#endif
+        Coordinate icoor(Ndim), ocoor(Ndim);
+        Lexicographic::CoorFromIndex(icoor, lane, sdims);
+        Lexicographic::CoorFromIndex(ocoor, idx,  rdims);
+        int64_t xloc = ocoor[dim] + icoor[dim]*rdims[dim];
+        int64_t olin = 0, str = 1;
+        for (int d = 0; d < Ndim; d++) {
+          if (d == dim) continue;
+          int64_t c = ocoor[d] + icoor[d]*rdims[d];
+          olin += str * c;
+          str  *= ldims[d];
+        }
+        int64_t own  = olin / Oloc;
+        int64_t slot = olin - own*Oloc;
+        vector_type *to = (vector_type *)&r_v[idx];
+        for (int w = 0; w < Ncomp; w++) {
+          scalar_type stmp = div * rbuf_v[ own*chunk + xloc + L*(slot + Oloc*w) ];
+          putlane(to[w], stmp, lane);
+        }
+#ifdef GRID_SIMT
+      }
+#else
+      }
+#endif
+      });
+    }
+    t_unpack += usecond();
+    t_total  += usecond();
+
+    std::cout << GridLogPerformance << " FFT took     " << t_total/1.0e6   << " s (transpose P=" << P << ")" << std::endl;
+    std::cout << GridLogPerformance << " FFT pack     " << t_pack/1.0e6    << " s" << std::endl;
+    std::cout << GridLogPerformance << " FFT alltoall " << t_a2a/1.0e6     << " s" << std::endl;
+    std::cout << GridLogPerformance << " FFT reorder  " << t_reorder/1.0e6 << " s" << std::endl;
+    std::cout << GridLogPerformance << " FFT kernels  " << t_fft/1.0e6     << " s" << std::endl;
+    std::cout << GridLogPerformance << " FFT unpack   " << t_unpack/1.0e6  << " s" << std::endl;
+  }
 }
 
 class FFT : public FFTbase {
@@ -405,8 +616,10 @@ public:
     int64_t Nperp = 1;
     for (int d = 0; d < Ndim; d++)
       if (d != dim) Nperp *= _grid->_ldimensions[d];
+    int     P     = _grid->_processors[dim];
+    int64_t Oloc  = (Nperp + P - 1) / P;   // ceil-padded owned lines/rank (see FFT_dim_execute)
     int n[]     = {G};
-    int howmany = Ncomp * Nperp;
+    int howmany = Ncomp * (int)Oloc;
 
     deviceVector<scalar> dummy(2);
     FFTW_scalar *buf = (FFTW_scalar *)&dummy[0];
@@ -442,7 +655,9 @@ private:
       int64_t Nperp = 1;
       for (int dd = 0; dd < Ndim; dd++)
         if (dd != d) Nperp *= _grid->_ldimensions[dd];
-      int howmany = Ncomp * (int)Nperp;
+      int     P     = _grid->_processors[d];
+      int64_t Oloc  = (Nperp + P - 1) / P;   // ceil-padded owned lines/rank (see FFT_dim_execute)
+      int howmany = Ncomp * (int)Oloc;
       int n[]     = {G};
 
       deviceVector<scalar> dummy(2);

Grid/algorithms/LinearOperator.h

@@ -52,9 +52,31 @@ public:
   virtual void AdjOp  (const Field &in, Field &out) = 0; // Abstract base
   virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
   virtual void HermOp(const Field &in, Field &out)=0;
+
+  // As CheckerBoardedSparseMatrixBase::SplitClone; the returned bundle's Linop is the
+  // same kind of wrapper as this one. Collective. nullptr if not supported.
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    return nullptr;
+  }
+
   virtual ~LinearOperatorBase(){};
 };
 
+/////////////////////////////////////////////////////////////////////////////////////////////
+// The wrappers below are templates on an arbitrary Matrix, and their virtual SplitClone is
+// instantiated with the class, so it must compile for Matrix types with no SplitClone.
+/////////////////////////////////////////////////////////////////////////////////////////////
+template<class Field,class Matrix>
+SplitOperator<Field> *SplitCloneMatrix(Matrix &Mat,const Coordinate &mpi_split)
+{
+  if constexpr ( std::is_base_of<CheckerBoardedSparseMatrixBase<Field>,Matrix>::value ) {
+    return Mat.SplitClone(mpi_split);
+  } else {
+    return nullptr;
+  }
+}
+
 
 /////////////////////////////////////////////////////////////////////////////////////////////
 // By sharing the class for Sparse Matrix across multiple operator wrappers, we can share code
@@ -102,6 +124,17 @@ public:
   void HermOp(const Field &in, Field &out){
     _Mat.MdagM(in,out);
   }
+
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+    if ( split == nullptr ) {
+      return nullptr;
+    }
+    split->Linop     = new MdagMLinearOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+    split->FieldGrid = split->Matrix->Grid();
+    return split;
+  }
 };
 template<class Matrix,class Field>
 class MMdagLinearOperator : public LinearOperatorBase<Field> {
@@ -214,6 +247,29 @@ public:
   }
 };
 
+////////////////////////////////////////////////////////////////////
+// Present any LinearOperatorBase as Hermitian: Op = AdjOp = HermOp.
+// For coarsening (CoarsenOperator applies Op) an HPD operator whose
+// Op is a factor and HermOp the product, e.g. SchurDiagMooee or MdagM.
+////////////////////////////////////////////////////////////////////
+template<class Field>
+class HermOpAdaptor : public LinearOperatorBase<Field> {
+  LinearOperatorBase<Field> &_Mat;
+public:
+  HermOpAdaptor(LinearOperatorBase<Field> &Mat): _Mat(Mat){};
+  void OpDiag (const Field &in, Field &out)                   { GRID_ASSERT(0); }
+  void OpDir  (const Field &in, Field &out,int dir,int disp)  { GRID_ASSERT(0); }
+  void OpDirAll(const Field &in, std::vector<Field> &out)     { GRID_ASSERT(0); }
+  void Op     (const Field &in, Field &out){ _Mat.HermOp(in,out); }
+  void AdjOp  (const Field &in, Field &out){ _Mat.HermOp(in,out); }
+  void HermOp (const Field &in, Field &out){ _Mat.HermOp(in,out); }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+    HermOp(in,out);
+    ComplexD dot = innerProduct(in,out);
+    n1=real(dot);
+    n2=norm2(out);
+  }
+};
 
 ////////////////////////////////////////////////////////////////////
 // Wrap an already herm matrix
@@ -378,6 +434,17 @@ template<class Matrix,class Field>
       _Mat.MooeeDag(in,out);
       axpy(out,-1.0,tmp,out);
     }
+
+    virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+    {
+      SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+      if ( split == nullptr ) {
+        return nullptr;
+      }
+      split->Linop     = new SchurDiagMooeeOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+      split->FieldGrid = split->Matrix->RedBlackGrid();
+      return split;
+    }
 };
 template<class Matrix,class Field>
   class SchurDiagOneOperator :  public SchurOperatorBase<Field> {
@@ -404,6 +471,17 @@ template<class Matrix,class Field>
       _Mat.MeooeDag(out,tmp);
       axpy(out,-1.0,tmp,in);
     }
+
+    virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+    {
+      SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+      if ( split == nullptr ) {
+        return nullptr;
+      }
+      split->Linop     = new SchurDiagOneOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+      split->FieldGrid = split->Matrix->RedBlackGrid();
+      return split;
+    }
 };
 template<class Matrix,class Field>
   class SchurDiagTwoOperator :  public SchurOperatorBase<Field> {
@@ -432,6 +510,17 @@ template<class Matrix,class Field>
 
       axpy(out,-1.0,tmp,in);
     }
+
+    virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+    {
+      SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+      if ( split == nullptr ) {
+        return nullptr;
+      }
+      split->Linop     = new SchurDiagTwoOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+      split->FieldGrid = split->Matrix->RedBlackGrid();
+      return split;
+    }
 };
 
 template<class Field>
@@ -499,6 +588,17 @@ class NonHermitianSchurDiagMooeeOperator :  public NonHermitianSchurOperatorBase
     
     axpy(out, -1.0, tmp, out);
   }
+
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+    if ( split == nullptr ) {
+      return nullptr;
+    }
+    split->Linop     = new NonHermitianSchurDiagMooeeOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+    split->FieldGrid = split->Matrix->RedBlackGrid();
+    return split;
+  }
 };
     
 template<class Matrix,class Field>
@@ -529,6 +629,17 @@ class NonHermitianSchurDiagOneOperator : public NonHermitianSchurOperatorBase<Fi
     
     axpy(out, -1.0, tmp, in);
   }
+
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+    if ( split == nullptr ) {
+      return nullptr;
+    }
+    split->Linop     = new NonHermitianSchurDiagOneOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+    split->FieldGrid = split->Matrix->RedBlackGrid();
+    return split;
+  }
 };
 
 template<class Matrix, class Field>
@@ -560,6 +671,17 @@ class NonHermitianSchurDiagTwoOperator : public NonHermitianSchurOperatorBase<Fi
 
     axpy(out, -1.0, tmp, in);
   }
+
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+    if ( split == nullptr ) {
+      return nullptr;
+    }
+    split->Linop     = new NonHermitianSchurDiagTwoOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+    split->FieldGrid = split->Matrix->RedBlackGrid();
+    return split;
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -613,6 +735,17 @@ class SchurStaggeredOperator :  public SchurOperatorBase<Field> {
   virtual void MpcDagMpc(const Field &in, Field &out) {
     GRID_ASSERT(0);// Never need with staggered
   }
+
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    SplitOperator<Field> *split = SplitCloneMatrix<Field>(_Mat,mpi_split);
+    if ( split == nullptr ) {
+      return nullptr;
+    }
+    split->Linop     = new SchurStaggeredOperator<CheckerBoardedSparseMatrixBase<Field>,Field>(*split->Matrix);
+    split->FieldGrid = split->Matrix->RedBlackGrid();
+    return split;
+  }
 };
 template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
 
@@ -644,6 +777,9 @@ public:
       (*this)(in[i], out[i]);
     }
   }
+  // The identity, if a derived class says so.  A caller can then use its input
+  // where it would otherwise have materialised the output, and skip the copy.
+  virtual int isTrivial(void) { return 0; };
   virtual ~LinearFunction(){};
 };
 

Grid/algorithms/Preconditioner.h

@@ -45,6 +45,7 @@ public:
   virtual void operator()(const Field &src, Field & psi){
     psi = src;
   }
+  virtual int isTrivial(void) { return 1; };
   TrivialPrecon(void){};
 };
 

Grid/algorithms/SparseMatrix.h

@@ -78,6 +78,17 @@ public:
   virtual  void MeooeDag    (const Field &in, Field &out)=0;
   virtual  void MooeeDag    (const Field &in, Field &out)=0;
   virtual  void MooeeInvDag (const Field &in, Field &out)=0;
+
+  //////////////////////////////////////////////////////////////////////
+  // Copy of this operator on grids whose communicator is split into
+  // partitions of MPI layout mpi_split. Collective over the full
+  // communicator. nullptr if this operator cannot be cloned.
+  //////////////////////////////////////////////////////////////////////
+  virtual SplitOperator<Field> *SplitClone(const Coordinate &mpi_split)
+  {
+    return nullptr;
+  }
+
   virtual ~CheckerBoardedSparseMatrixBase() {};
 };
 

Grid/algorithms/SplitOperator.h

@@ -0,0 +1,172 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./lib/algorithms/SplitOperator.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+template<class Field> class LinearOperatorBase;
+template<class Field> class CheckerBoardedSparseMatrixBase;
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// A copy of an operator living on grids whose communicator is split into independent
+// partitions, together with those grids. Produced by SplitClone().
+//
+// Owns everything it points to. Destruction is the reverse of creation: linear operator,
+// matrix, then grids, so no object outlives a grid it references. For 4d operators the
+// fermion grids are the gauge grids and are deleted once.
+/////////////////////////////////////////////////////////////////////////////////////////////
+template<class Field>
+class SplitOperator
+{
+public:
+  GridCartesian         *GaugeGrid     = nullptr;
+  GridRedBlackCartesian *GaugeRBGrid   = nullptr;
+  GridCartesian         *FermionGrid   = nullptr;
+  GridRedBlackCartesian *FermionRBGrid = nullptr;
+
+  int Partition  = 0;  // Grid_split vector index held by this rank's partition
+  int Partitions = 1;  // number of partitions
+
+  CheckerBoardedSparseMatrixBase<Field> *Matrix    = nullptr;
+  LinearOperatorBase<Field>             *Linop     = nullptr;
+  GridBase                              *FieldGrid = nullptr;  // grid of Linop's fields
+
+  SplitOperator(void) {};
+
+  SplitOperator(const SplitOperator &) = delete;
+
+  SplitOperator &operator=(const SplitOperator &) = delete;
+
+  ~SplitOperator(void)
+  {
+    delete Linop;
+    delete Matrix;
+    if ( FermionRBGrid != GaugeRBGrid ) {
+      delete FermionRBGrid;
+    }
+    if ( FermionGrid != GaugeGrid ) {
+      delete FermionGrid;
+    }
+    delete GaugeRBGrid;
+    delete GaugeGrid;
+  }
+};
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Index of the vector that Grid_split(std::vector<Field> full, Field split) delivers to this
+// rank's partition. Grid_split orders partitions lexicographically with the first
+// dimension fastest; the split communicator's own rank (srank) uses the reversed MPI
+// convention, so the two differ whenever more than one dimension is split.
+/////////////////////////////////////////////////////////////////////////////////////////////
+inline int GridSplitVectorIndex(GridBase *full,GridBase *split)
+{
+  int nd = full->_ndimension;
+  GRID_ASSERT(split->_ndimension == nd);
+
+  Coordinate scoor(nd);
+  Coordinate ssize(nd);
+  for(int d=0;d<nd;d++){
+    scoor[d] = full->ThisProcessorCoor()[d] / split->ProcessorGrid()[d];
+    ssize[d] = full->ProcessorGrid()[d]     / split->ProcessorGrid()[d];
+  }
+  int index;
+  Lexicographic::IndexFromCoor(scoor,index,ssize);
+  return index;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Partition MPI layout for a batched solve on grid, from a request as given on the command
+// line (--batched-solver-split). Uses the trailing dimensions of grid's processor and shm
+// layouts, so it works for 4d and 5d grids. Returns grid's own processor layout (one
+// partition) when no split is requested. Asserts divisibility; warns when partitions
+// straddle nodes or when node boundaries are not visible.
+/////////////////////////////////////////////////////////////////////////////////////////////
+inline Coordinate BatchedSolverSplitLayout(GridBase *grid,
+                                           const Coordinate &request,
+                                           bool node,
+                                           int &partitions)
+{
+  int nd  = GridDefaultMpi().size();
+  int pad = grid->_ndimension - nd;
+  GRID_ASSERT(pad >= 0);
+  GRID_ASSERT(grid->ShmGrid().size() == grid->_ndimension);
+
+  Coordinate processors(nd);
+  Coordinate shm(nd);
+  for(int d=0;d<nd;d++){
+    processors[d] = grid->ProcessorGrid()[pad+d];
+    shm[d]        = grid->ShmGrid()[pad+d];
+  }
+
+  Coordinate split(nd);
+  if ( node ) {
+    split = shm;
+  } else if ( request.size() == 0 ) {
+    split = processors;
+  } else {
+    GRID_ASSERT(request.size() == nd);
+    split = request;
+  }
+
+  partitions = 1;
+  for(int d=0;d<nd;d++){
+    GRID_ASSERT( (processors[d] % split[d]) == 0 );
+    partitions *= processors[d] / split[d];
+  }
+  if ( partitions == 1 ) {
+    return split;
+  }
+
+  std::cout << GridLogMessage << "BatchedSolverSplit: partition layout " << split
+            << " of " << processors << " : " << partitions << " partitions" << std::endl;
+
+  int inside_node  = 1;
+  int whole_nodes  = 1;
+  int shm_trivial  = 1;
+  for(int d=0;d<nd;d++){
+    if ( (shm[d] % split[d]) != 0 ) {
+      inside_node = 0;
+    }
+    if ( (split[d] % shm[d]) != 0 ) {
+      whole_nodes = 0;
+    }
+    if ( shm[d] != 1 ) {
+      shm_trivial = 0;
+    }
+  }
+  if ( shm_trivial ) {
+    std::cout << GridLogWarning << "BatchedSolverSplit: shm layout is trivial (one rank per node,"
+              << " or shared memory disabled); node locality of partitions not checked" << std::endl;
+  } else if ( !inside_node && !whole_nodes ) {
+    std::cout << GridLogWarning << "BatchedSolverSplit: partitions " << split
+              << " straddle node boundaries (node layout " << shm << "); inner solves will communicate off node" << std::endl;
+  }
+  return split;
+}
+
+NAMESPACE_END(Grid);

Grid/algorithms/blas/BatchedBlas.h

@@ -995,7 +995,7 @@ public:
   ///////////////////////////////////////////////////////////////////////////////////
   // Explicit-leading-dimension complex double GEMM.  Mirror of the ComplexF
   // overload above; motivating use is the fp64 distributed recursive Schur
-  // inversion (RecursiveSchurInverse), whose operands are column windows of
+  // inversion (the distributed Schur recursion), whose operands are column windows of
   // larger row-slab allocations.
   ///////////////////////////////////////////////////////////////////////////////////
   void gemmBatched(GridBLASOperation_t OpA,
@@ -1942,6 +1942,10 @@ public:
 		Cs);
     synchronise();
 
+    // Synchronise ONCE, after the whole train of calls.  A synchronise inside
+    // the loop measures launch and completion latency, which at small shapes
+    // exceeds the kernel: at M=K=60, N=1, batch 1024 it hid a factor of two
+    // between the precisions and reported them level.
     RealD t0 = usecond();
     for(int i=0;i<ncall;i++){
       gemmBatched(M,N,K,
@@ -1950,8 +1954,8 @@ public:
 		  Bs, // k x n
 		  beta, 
 		  Cs);
-      synchronise();
     }
+    synchronise();
     RealD t1 = usecond();
     RealD bytes = 1.0*sizeof(CComplex)*(M*N*2+N*K+M*K)*BATCH;
     flops = 8.0*M*N*K*BATCH*ncall;

Grid/algorithms/blas/BatchedInverse.h

@@ -111,7 +111,9 @@ public:
   // ranks wait on it.  The _64 getrf/getrs pair is the path proven in the
   // 1-rank dense coarse-coarse setup at N=69120 (DenseCoarseMatrix.h).
   // Same in-place, column-major, lda=N contract as inverseBatched.
-  // NB: written to rocSOLVER's documented z*_64 signatures; not compiled on
+  // ComplexD and ComplexF overloads (z*_64 / c*_64), so the distributed
+  // dense inverse can run in either precision (DenseInverseScalar).
+  // NB: written to rocSOLVER's documented *_64 signatures; not compiled on
   // HIP in the air-gapped loop -- verify on first hipcc build.
   ///////////////////////////////////////////////////////////////////////////
   double lastGetrfUs = 0.0, lastGetrsUs = 0.0;   // split timing of the last inverseLU (HIP path)
@@ -145,6 +147,37 @@ public:
     inverseBatched(N, bp);
 #endif
   }
+  // fp32 overload: the Schur-recursion leaf under --enable-dense-inverse-precision=single
+  void inverseLU(int64_t N, ComplexF *A)
+  {
+#ifdef GRID_HIP
+    rocblas_handle handle = Handle();
+    deviceVector<int64_t>  ipiv((uint64_t)N);
+    deviceVector<int64_t>  info(1);
+    double t0 = usecond();
+    auto st1 = rocsolver_cgetrf_64(handle, N, N, (rocblas_float_complex *)A, N, &ipiv[0], &info[0]);
+    GRID_ASSERT(st1 == rocblas_status_success);
+    accelerator_barrier();
+    lastGetrfUs = usecond()-t0;
+    int64_t info_h = -1; acceleratorCopyFromDevice(&info[0], &info_h, sizeof(int64_t));
+    GRID_ASSERT(info_h == 0);
+    deviceVector<ComplexF> X((uint64_t)N*N);
+    { ComplexF *x = &X[0]; const int64_t NN = N;
+      accelerator_for(idx, (uint64_t)N*N, 1, { int64_t j = idx/NN, i = idx - j*NN; x[idx] = (i==j) ? ComplexF(1.0,0.0) : ComplexF(0.0,0.0); });
+      accelerator_barrier(); }
+    auto st2 = rocsolver_cgetrs_64(handle, rocblas_operation_none, N, N,
+                                   (rocblas_float_complex *)A, N, &ipiv[0],
+                                   (rocblas_float_complex *)&X[0], N);
+    GRID_ASSERT(st2 == rocblas_status_success);
+    accelerator_barrier();
+    lastGetrsUs = usecond()-t0-lastGetrfUs;
+    acceleratorCopyDeviceToDevice((void *)&X[0], (void *)A, (uint64_t)N*N*sizeof(ComplexF));
+#else
+    deviceVector<ComplexF*> bp(1); std::vector<ComplexF*> ptr(1); ptr[0] = A;
+    acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexF*));
+    inverseBatched(N, bp);
+#endif
+  }
 
   void inverseBatched(int64_t N, deviceVector<ComplexF*> &Amat)
   {

Grid/algorithms/deflation/MultiRHSBlockProject.h

@@ -93,6 +93,25 @@ public:
   deviceVector<scalar> BLAS_F;      // nrhs x fine_vol * words   -- the sources
   deviceVector<scalar> BLAS_C;      // nrhs x coarse_vol * nbasis -- the coarse coeffs
 
+  // Resident device memory this object holds.  deviceVector is not
+  // evictable, so this counts against the hard budget, not the cache.
+  // Capacity, not size: a call with fewer right-hand sides shrinks the size
+  // and frees nothing.
+  uint64_t DeviceBytes(void)
+  {
+    return (uint64_t)(BLAS_V.capacity()+BLAS_F.capacity()+BLAS_C.capacity())*sizeof(scalar);
+  }
+  // The import/export scratch grows to the largest right-hand-side count
+  // any call has used and keeps that capacity.  Setup projects the whole
+  // basis in one call, so unless it is released the scratch stays as large
+  // as the basis store for the rest of the run.  The next call regrows it
+  // to the size that call needs.
+  void ReleaseScratch(void)
+  {
+    deviceVector<scalar>().swap(BLAS_F);
+    deviceVector<scalar>().swap(BLAS_C);
+  }
+
   RealD blasNorm2(deviceVector<scalar> &blas)
   {
     scalar ss(0.0);
@@ -117,9 +136,14 @@ public:
     block_vol=0;
     coarse_vol=0;
     words=0;
-    BLAS_V.resize(0);
+    // deviceVector is a std::vector with a device allocator, so resize(0)
-    BLAS_F.resize(0);
+    // drops the SIZE and keeps the CAPACITY: it returns no device memory at
-    BLAS_C.resize(0);
+    // all.  The basis store is nbasis fine vectors (10.9 GB at nbasis 64 on
+    // a 48^3x96 Ls=24 rank), so that has to be a real free.  Swap with an
+    // empty vector, which destroys the buffer.
+    deviceVector<scalar>().swap(BLAS_V);
+    deviceVector<scalar>().swap(BLAS_F);
+    deviceVector<scalar>().swap(BLAS_C);
   }
   void Allocate(int _nbasis,GridBase *_fgrid,GridBase *_cgrid)
   {
@@ -153,25 +177,39 @@ public:
 
     BLAS_V.resize (fine_vol * words * nbasis );
   }
-  void ImportFineGridVectors(std::vector <Field > &vecs, deviceVector<scalar> &blas)
+  ////////////////////////////////////////////////////////////////////////////
+  // The fine side is templated on the INCOMING vector object, not fixed to
+  // Field.  Field fixes only the STORAGE: the BLAS scalar, the word count and
+  // the full local geometry.  Import is already a layout transformation --
+  // (oSite,lane) is unpacked to full local coordinates and regathered in
+  // block order -- so a precision change costs nothing here, and neither does
+  // a different SIMD layout.  That is what lets one projector serve an fp64
+  // outer level and an fp32 coarse sector with no second basis store.
+  //
+  // The full LOCAL dimensions must agree; the SIMD layout and the precision
+  // need not.
+  ////////////////////////////////////////////////////////////////////////////
+  template<class vobj>
+  void ImportFineGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
   {
     GRID_TRACE("ImportFineGridVectors");
     int nvec = vecs.size();
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
-    //    std::cout << GridLogMessage <<" BlockProjector importing "<<nvec<< " fine grid vectors" <<std::endl;
+    typedef typename vobj::scalar_type   fine_scalar;
-
-    GRID_ASSERT(vecs[0].Grid()==fine_grid);
 
+    GridBase *fgrid = vecs[0].Grid();
     int _ndimension = coarse_grid->_ndimension;
+    for(int d=0;d<_ndimension;d++) GRID_ASSERT(fgrid->_ldimensions[d] == fine_ldimensions[d]);
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
 
     uint64_t sz = blas.size();
 
     acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
 
-    Coordinate fine_rdimensions = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions = fgrid->_rdimensions;
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
+    Coordinate fsimd    = fgrid->_simd_layout;
     int64_t bv= block_vol;
     for(int v=0;v<vecs.size();v++){
 
@@ -181,7 +219,7 @@ public:
       auto blasData_p  = &blas[0];
       auto fineData_p  = &fineData[0];
 
-      int64_t osites = fine_grid->oSites();
+      int64_t osites = fgrid->oSites();
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
@@ -213,7 +251,7 @@ public:
 	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
 	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
 
-          scalar_object data = extractLane(lane,fineData[sf]);
+          fine_scalar_object data = extractLane(lane,fineData[sf]);
 
 	  // BLAS_F[coarse_vol][nvec][block_vol][words]
 	  int64_t site = (sc*nvec + v)*bv
@@ -221,9 +259,9 @@ public:
 
 	  //	  GRID_ASSERT(site*lwords<sz);
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-
+	  const fine_scalar *dp = (const fine_scalar *)&data;
-	  *ptr = data;
+	  for(uint64_t w=0;w<lwords;w++) blasData_p[site*lwords+w] = scalar(dp[w]);
 #ifdef GRID_SIMT
 	}
 #else
@@ -245,9 +283,11 @@ public:
   // The gather of block_vol from fine_vol, and the transpose of nrhs against
   // block_vol, are the irreducible part: this is not the identity.
   ////////////////////////////////////////////////////////////////////////////
-  void ImportFineGridMrhsVectors(Field &vec_mrhs, deviceVector<scalar> &blas)
+  template<class vobj>
+  void ImportFineGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
+    typedef typename vobj::scalar_type   fine_scalar;
 
     GridBase *fine_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
@@ -255,27 +295,31 @@ public:
     GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1);
     GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1);
     GRID_ASSERT(fine_mrhs_grid->_processors[0] == 1);
+    // Full LOCAL dimensions must agree; the SIMD layout and precision need not
     for(int d=0;d<_ndimension;d++){
-      GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]);
+      GRID_ASSERT(fine_mrhs_grid->_ldimensions[d+1] == fine_ldimensions[d]);
-      GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]);
     }
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
     int nvec = fine_mrhs_grid->_rdimensions[0];   // nrhs
 
     uint64_t sz = blas.size();
     acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
 
     Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions;
-    Coordinate fine_rdimensions      = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions(_ndimension);
+    Coordinate fsimd(_ndimension);
+    for(int d=0;d<_ndimension;d++){
+      fine_rdimensions[d] = fine_mrhs_grid->_rdimensions[d+1];
+      fsimd[d]            = fine_mrhs_grid->_simd_layout[d+1];
+    }
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
     int64_t bv= block_vol;
 
     autoView( fineData   , vec_mrhs, AcceleratorRead);
     auto blasData_p  = &blas[0];
     auto fineData_p  = &fineData[0];
 
-    int64_t osites     = fine_grid->oSites();       // D dimensional
     int64_t osites_hi  = fine_mrhs_grid->oSites();  // nvec * osites
 
     const int Nsimd = vobj::Nsimd();
@@ -312,13 +356,14 @@ public:
 	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
 	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
 
-	  scalar_object data = extractLane(lane,fineData[sfr]);
+	  fine_scalar_object data = extractLane(lane,fineData[sfr]);
 
 	  int64_t site = (sc*lnvec + v)*bv
 	               + sb;
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-	  *ptr = data;
+	  const fine_scalar *dp = (const fine_scalar *)&data;
+	  for(uint64_t w=0;w<lwords;w++) blasData_p[site*lwords+w] = scalar(dp[w]);
 #ifdef GRID_SIMT
       }
 #else
@@ -339,6 +384,7 @@ public:
   void ExportCoarseGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
 
     GridBase *coarse_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
@@ -365,7 +411,7 @@ public:
     int64_t osites_hi = coarse_mrhs_grid->oSites();  // nvec * osites
 
     const int Nsimd = vobj::Nsimd();
-    uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+    uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
     GRID_ASSERT(cwords==nbasis);
     int64_t lnvec = nvec;
 
@@ -391,8 +437,11 @@ public:
 	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
 
 	  int64_t blas_site = (sc*lnvec + v)*cwords;
-	  coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	  // Element-wise, converting: the BLAS buffer is in the FINE precision,
-	  coarse_scalar_object data = *ptr;
+	  // the coarse field in its own.
+	  coarse_scalar_object data;
+	  coarse_scalar *dp = (coarse_scalar *)&data;
+	  for(uint64_t b=0;b<cwords;b++) dp[b] = coarse_scalar(blasData_p[blas_site+b]);
 	  insertLane(lane,coarseData[scr],data);
 #ifdef GRID_SIMT
       }
@@ -405,33 +454,39 @@ public:
   ////////////////////////////////////////////////////////////////////////////
   // Reverse directions: BLAS_F -> fine mrhs field, coarse mrhs field -> BLAS_C
   ////////////////////////////////////////////////////////////////////////////
-  void ExportFineGridMrhsVectors(Field &vec_mrhs, deviceVector<scalar> &blas)
+  template<class vobj>
+  void ExportFineGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
+    typedef typename vobj::scalar_type   fine_scalar;
 
     GridBase *fine_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
 
     GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1);
     GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1);
+    // Full LOCAL dimensions must agree; the SIMD layout and precision need not
     for(int d=0;d<_ndimension;d++){
-      GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]);
+      GRID_ASSERT(fine_mrhs_grid->_ldimensions[d+1] == fine_ldimensions[d]);
-      GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]);
     }
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
     int nvec = fine_mrhs_grid->_rdimensions[0];
 
     Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions;
-    Coordinate fine_rdimensions      = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions(_ndimension);
+    Coordinate fsimd(_ndimension);
+    for(int d=0;d<_ndimension;d++){
+      fine_rdimensions[d] = fine_mrhs_grid->_rdimensions[d+1];
+      fsimd[d]            = fine_mrhs_grid->_simd_layout[d+1];
+    }
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
     int64_t bv= block_vol;
 
     autoView( fineData   , vec_mrhs, AcceleratorWrite);
     auto blasData_p  = &blas[0];
     auto fineData_p  = &fineData[0];
 
-    int64_t osites     = fine_grid->oSites();
     int64_t osites_hi  = fine_mrhs_grid->oSites();
 
     const int Nsimd = vobj::Nsimd();
@@ -468,8 +523,10 @@ public:
 	  int64_t site = (sc*lnvec + v)*bv
 	               + sb;
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-	  scalar_object data = *ptr;
+	  fine_scalar_object data;
+	  fine_scalar *dp = (fine_scalar *)&data;
+	  for(uint64_t w=0;w<lwords;w++) dp[w] = fine_scalar(blasData_p[site*lwords+w]);
 	  insertLane(lane,fineData[sfr],data);
 #ifdef GRID_SIMT
       }
@@ -483,6 +540,7 @@ public:
   void ImportCoarseGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
 
     GridBase *coarse_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
@@ -508,7 +566,7 @@ public:
     int64_t osites_hi = coarse_mrhs_grid->oSites();
 
     const int Nsimd = vobj::Nsimd();
-    uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+    uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
     GRID_ASSERT(cwords==nbasis);
     int64_t lnvec = nvec;
 
@@ -536,8 +594,8 @@ public:
 	  coarse_scalar_object data = extractLane(lane,coarseData[scr]);
 
 	  int64_t blas_site = (sc*lnvec + v)*cwords;
-	  coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	  const coarse_scalar *dp = (const coarse_scalar *)&data;
-	  *ptr = data;
+	  for(uint64_t b=0;b<cwords;b++) blasData_p[blas_site+b] = scalar(dp[b]);
 #ifdef GRID_SIMT
       }
 #else
@@ -546,21 +604,25 @@ public:
     });
   }
 
-  void ExportFineGridVectors(std::vector <Field> &vecs, deviceVector<scalar> &blas)
+  template<class vobj>
+  void ExportFineGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
   {
     GRID_TRACE("ExportFineGridVectors");
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
+    typedef typename vobj::scalar_type   fine_scalar;
 
     int nvec = vecs.size();
 
-    GRID_ASSERT(vecs[0].Grid()==fine_grid);
+    GridBase *fgrid = vecs[0].Grid();
-
     int _ndimension = coarse_grid->_ndimension;
+    // Full LOCAL dimensions must agree; the SIMD layout and precision need not
+    for(int d=0;d<_ndimension;d++) GRID_ASSERT(fgrid->_ldimensions[d] == fine_ldimensions[d]);
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
 
-    Coordinate fine_rdimensions = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions = fgrid->_rdimensions;
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
+    Coordinate fsimd    = fgrid->_simd_layout;
 
     //    std::cout << " export fine Blas norm "<<blasNorm2(blas)<<std::endl;
 
@@ -572,7 +634,7 @@ public:
       auto blasData_p  = &blas[0];
       auto fineData_p    = &fineData[0];
 
-      int64_t osites = fine_grid->oSites();
+      int64_t osites = fgrid->oSites();
       uint64_t lwords = words;
       //      std::cout << " Nsimd is "<<vobj::Nsimd() << std::endl;
       //      std::cout << " lwords is "<<lwords << std::endl;
@@ -608,9 +670,10 @@ public:
 	  int64_t site = (sc*nvec + v)*bv
 	               + sb;
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-
+	  fine_scalar_object data;
-	  scalar_object data = *ptr;
+	  fine_scalar *dp = (fine_scalar *)&data;
+	  for(uint64_t w=0;w<lwords;w++) dp[w] = fine_scalar(blasData_p[site*lwords+w]);
 
 	  insertLane(lane,fineData[sf],data);
 #ifdef GRID_SIMT
@@ -627,6 +690,7 @@ public:
     GRID_TRACE("ImportCoarseGridVectors");
     int nvec = vecs.size();
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
 
     //    std::cout << " BlockProjector importing "<<nvec<< " coarse grid vectors" <<std::endl;
 
@@ -652,7 +716,7 @@ public:
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
-      uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+      uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
       GRID_ASSERT(cwords==nbasis);
       
       accelerator_for(sc,osites,Nsimd,{
@@ -676,9 +740,8 @@ public:
 
 	    coarse_scalar_object data = extractLane(lane,coarseData[sc]);
 
-	    coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	    const coarse_scalar *dp = (const coarse_scalar *)&data;
-
+	    for(uint64_t b=0;b<cwords;b++) blasData_p[blas_site+b] = scalar(dp[b]);
-	    *ptr = data;
 #ifdef GRID_SIMT
 	}
 #else
@@ -694,6 +757,7 @@ public:
     GRID_TRACE("ExportCoarseGridVectors");
     int nvec = vecs.size();
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
     //    std::cout << GridLogMessage<<" BlockProjector exporting "<<nvec<< " coarse grid vectors" <<std::endl;
 
     GRID_ASSERT(vecs[0].Grid()==coarse_grid);
@@ -719,7 +783,7 @@ public:
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
-      uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+      uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
       GRID_ASSERT(cwords==nbasis);
       
       accelerator_for(sc,osites,Nsimd,{
@@ -740,8 +804,9 @@ public:
 	    Lexicographic::IndexFromCoor(coor_c,scl,coarse_l);
 
 	    int64_t blas_site = (scl*nvec + v)*cwords;
-	    coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	    coarse_scalar_object data;
-	    coarse_scalar_object data = *ptr;
+	    coarse_scalar *dp = (coarse_scalar *)&data;
+	    for(uint64_t b=0;b<cwords;b++) dp[b] = coarse_scalar(blasData_p[blas_site+b]);
 	    insertLane(lane,coarseData[sc],data);
 #ifdef GRID_SIMT
 	}
@@ -751,18 +816,19 @@ public:
       });
     }
   }
-  void ImportBasis(std::vector < Field > &vecs)
+  template<class fobj>
+  void ImportBasis(std::vector < Lattice<fobj> > &vecs)
   {
     //    std::cout << " BlockProjector Import basis size "<<vecs.size()<<std::endl;
     ImportFineGridVectors(vecs,BLAS_V);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(std::vector<Field> &fine,std::vector< Lattice<cobj> > & coarse)
+  void blockProject(std::vector<Lattice<fobj> > &fine,std::vector< Lattice<cobj> > & coarse)
   {
     GRID_TRACE("BlockProject");
     int nrhs=fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     //    std::cout << "blockProject nbasis " <<nbasis<<" " << _nbasis<<std::endl;
     GRID_ASSERT(nbasis==_nbasis);
     
@@ -817,12 +883,12 @@ public:
 
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(std::vector<Field> &fine,std::vector<Lattice<cobj> > & coarse)
+  void blockPromote(std::vector<Lattice<fobj> > &fine,std::vector<Lattice<cobj> > & coarse)
   {
     GRID_TRACE("BlockPromote");
     int nrhs=fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     
     BLAS_F.resize (fine_vol * words * nrhs );
@@ -876,12 +942,12 @@ public:
   // multiRHS ordered interfaces.  The GEMM is identical; only the import and
   // export differ, so those are the whole of the layout question.
   ////////////////////////////////////////////////////////////////////////////
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(Field &fine_mrhs,Lattice<cobj> &coarse_mrhs)
+  void blockProject(Lattice<fobj> &fine_mrhs,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockProjectMrhs");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -893,12 +959,12 @@ public:
     ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(Field &fine_mrhs,Lattice<cobj> &coarse_mrhs)
+  void blockPromote(Lattice<fobj> &fine_mrhs,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockPromoteMrhs");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -914,12 +980,12 @@ public:
   // Mixed orderings. A single RHS fine operator produces a vector of fine
   // fields with no packing; the coarse side is still wanted in mrhs order.
   ////////////////////////////////////////////////////////////////////////////
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(std::vector<Field> &fine,Lattice<cobj> &coarse_mrhs)
+  void blockProject(std::vector<Lattice<fobj> > &fine,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockProjectMixed");
     int nrhs = fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -931,12 +997,12 @@ public:
     ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(Field &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
+  void blockProject(Lattice<fobj> &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
   {
     GRID_TRACE("BlockProjectMixed");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse.size()==nrhs);
 
@@ -948,12 +1014,12 @@ public:
     ExportCoarseGridVectors(coarse,BLAS_C);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(std::vector<Field> &fine,Lattice<cobj> &coarse_mrhs)
+  void blockPromote(std::vector<Lattice<fobj> > &fine,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockPromoteMixed");
     int nrhs = fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -965,12 +1031,12 @@ public:
     ExportFineGridVectors(fine,BLAS_F);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(Field &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
+  void blockPromote(Lattice<fobj> &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
   {
     GRID_TRACE("BlockPromoteMixed");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse.size()==nrhs);
 

Grid/algorithms/deflation/MultiRHSDeflation.h

@@ -59,6 +59,7 @@ public:
 
   typedef typename Field::scalar_type   scalar;
   typedef typename Field::scalar_object scalar_object;
+  typedef typename Field::vector_object vobj;
 
   int nev;
   std::vector<RealD> eval;
@@ -80,10 +81,18 @@ public:
     grid=nullptr;
     vol=0;
     words=0;
-    BLAS_E.resize(0);
+    // deviceVector is a std::vector with a device allocator: resize(0) drops
-    BLAS_R.resize(0);
+    // the size and keeps the capacity, returning no device memory.  Swapping
-    BLAS_C.resize(0);
+    // with an empty vector destroys the buffer.
-    BLAS_G.resize(0);
+    deviceVector<scalar>().swap(BLAS_E);
+    deviceVector<scalar>().swap(BLAS_R);
+    deviceVector<scalar>().swap(BLAS_C);
+    deviceVector<scalar>().swap(BLAS_G);
+  }
+  // Resident (non-evictable) device memory held by this deflator.
+  uint64_t DeviceBytes(void)
+  {
+    return (BLAS_E.capacity()+BLAS_R.capacity()+BLAS_C.capacity()+BLAS_G.capacity())*sizeof(scalar);
   }
   void Allocate(int _nev,GridBase *_grid)
   {
@@ -123,31 +132,98 @@ public:
       ImportEigenVector(evec[_ev0+e],_eval[_ev0+e],e);
     }
   }
+  /////////////////////////////////////////////////////////////////////////
+  // Sources as a vector of D-dimensional fields: each is one contiguous
+  // column of BLAS_R already, so import/export are straight copies.
+  // The eigenvectors may live on a D grid or on the Nrhs=1 D+1 grid of the
+  // same local volume; the two are the same bytes, so only the site count
+  // is checked, not grid identity.
+  /////////////////////////////////////////////////////////////////////////
   void DeflateSources(std::vector<Field> &source,std::vector<Field> & guess)
   {
     int nrhs = source.size();
     GRID_ASSERT(source.size()==guess.size());
-    GRID_ASSERT(grid == guess[0].Grid());
+    GRID_ASSERT(grid->lSites() == guess[0].Grid()->lSites());
     conformable(guess[0],source[0]);
 
     int64_t vw = vol * words;
-
-    RealD t0 = usecond();
     BLAS_R.resize(nrhs * vw); // cost free if size doesn't change
     BLAS_G.resize(nrhs * vw); // cost free if size doesn't change
     BLAS_C.resize(nev * nrhs);// cost free if size doesn't change
 
-    /////////////////////////////////////////////
-    // Copy in the multi-rhs sources
-    /////////////////////////////////////////////
-    //    for(int r=0;r<nrhs;r++){
-    //      std::cout << " source["<<r<<"] = "<<norm2(source[r])<<std::endl;
-    //    }
     for(int r=0;r<nrhs;r++){
       int64_t offset = r*vw;
       autoView(v,source[r],AcceleratorRead);
       acceleratorCopyDeviceToDevice(&v[0],&BLAS_R[offset],sizeof(scalar_object)*vol);
     }
+    DeflateBLAS(nrhs);
+    for(int r=0;r<nrhs;r++){
+      int64_t offset = r*vw;
+      autoView(v,guess[r],AcceleratorWrite);
+      acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
+    }
+  }
+  /////////////////////////////////////////////////////////////////////////
+  // Sources as ONE D+1 dimensional multiRHS field, rhs innermost
+  // (dimension 0, undistributed) on an UNVECTORISED coarse space, so the
+  // site index is rhs + nrhs*x with x the D-dimensional site.  The GEMM
+  // wants x contiguous per rhs, so the import is a permutation, one fused
+  // pass over the field; likewise the export.  No slices.
+  /////////////////////////////////////////////////////////////////////////
+  void DeflateSources(const Field &source_mrhs, Field &guess_mrhs)
+  {
+    conformable(source_mrhs,guess_mrhs);
+    GridBase *hi = source_mrhs.Grid();
+    GRID_ASSERT(hi->_ndimension    == grid->_ndimension+1);
+    GRID_ASSERT(hi->_processors[0] == 1);
+    GRID_ASSERT(hi->_simd_layout[0]== 1);
+    GRID_ASSERT(vobj::Nsimd()      == 1);
+    int nrhs = hi->_fdimensions[0];
+    GRID_ASSERT(hi->lSites() == nrhs*vol);
+
+    int64_t vw = vol * words;
+    BLAS_R.resize(nrhs * vw);
+    BLAS_G.resize(nrhs * vw);
+    BLAS_C.resize(nev * nrhs);
+
+    ImportSourcesMrhs(source_mrhs,nrhs);
+    DeflateBLAS(nrhs);
+    ExportGuessMrhs(guess_mrhs,nrhs);
+  }
+  void ImportSourcesMrhs(const Field &source_mrhs,int nrhs)
+  {
+    const int64_t lwords = words, lnrhs = nrhs, vw = vol*words;
+    autoView(v,source_mrhs,AcceleratorRead);
+    auto vp = &v[0];
+    scalar *R = &BLAS_R[0];
+    accelerator_for(scr, lnrhs*vol, 1, {
+      int64_t r = scr % lnrhs;
+      int64_t x = scr / lnrhs;
+      const scalar *s = (const scalar *)&vp[scr];
+      for(int64_t w=0;w<lwords;w++) R[r*vw + x*lwords + w] = s[w];
+    });
+  }
+  void ExportGuessMrhs(Field &guess_mrhs,int nrhs)
+  {
+    const int64_t lwords = words, lnrhs = nrhs, vw = vol*words;
+    autoView(v,guess_mrhs,AcceleratorWrite);
+    auto vp = &v[0];
+    scalar *G = &BLAS_G[0];
+    accelerator_for(scr, lnrhs*vol, 1, {
+      int64_t r = scr % lnrhs;
+      int64_t x = scr / lnrhs;
+      scalar *g = (scalar *)&vp[scr];
+      for(int64_t w=0;w<lwords;w++) g[w] = G[r*vw + x*lwords + w];
+    });
+  }
+  /////////////////////////////////////////////////////////////////////////
+  // BLAS_R (nrhs columns of vol*words) -> BLAS_G, through the imported
+  // eigenbasis: C = E^dag R / lambda, G = E C.
+  /////////////////////////////////////////////////////////////////////////
+  void DeflateBLAS(int nrhs)
+  {
+    int64_t vw = vol * words;
+    RealD t0 = usecond();
 
   /*
    * in Fortran column major notation (cuBlas order)
@@ -195,7 +271,7 @@ public:
     acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
     grid->GlobalSumVector(&HOST_C[0],nev*nrhs);
     for(int e=0;e<nev;e++){
-      RealD lam(1.0/eval[e]);
+      scalar lam(1.0/eval[e]);          // in the coarse scalar: fp32 coarse spaces too
       for(int r=0;r<nrhs;r++){
 	int off = e+nev*r;
 	HOST_C[off]=HOST_C[off] * lam;
@@ -216,15 +292,6 @@ public:
 		     scalar(0.0),
 		     Gd);
     BLAS.synchronise();
-
-    ///////////////////////////////////////
-    // Copy out the multirhs
-    ///////////////////////////////////////
-    for(int r=0;r<nrhs;r++){
-      int64_t offset = r*vw;
-      autoView(v,guess[r],AcceleratorWrite);
-      acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
-    }
     RealD t1 = usecond();
     std::cout << GridLogMessage << "MultiRHSDeflation for "<<nrhs<<" sources with "<<nev<<" eigenvectors took " << (t1-t0)/1e3 <<" ms"<<std::endl;
   }

Grid/algorithms/iterative/AdefGeneric.h

@@ -480,104 +480,6 @@ class TwoLevelCG : public LinearFunction<Field>
 
 };
   
-template<class Field, class CoarseField, class Aggregation>
-class TwoLevelADEF2 : public TwoLevelCG<Field>
-{
- public:
-  ///////////////////////////////////////////////////////////////////////////////////
-  // Need something that knows how to get from Coarse to fine and back again
-  //  void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
-  //  void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
-  ///////////////////////////////////////////////////////////////////////////////////
-  GridBase *coarsegrid;
-  Aggregation &_Aggregates;                    
-  LinearFunction<CoarseField> &_CoarseSolver;
-  LinearFunction<CoarseField> &_CoarseSolverPrecise;
-  ///////////////////////////////////////////////////////////////////////////////////
-  
-  // more most opertor functions
-  TwoLevelADEF2(RealD tol,
-		Integer maxit,
-		LinearOperatorBase<Field>    &FineLinop,
-		LinearFunction<Field>        &Smoother,
-		LinearFunction<CoarseField>  &CoarseSolver,
-		LinearFunction<CoarseField>  &CoarseSolverPrecise,
-		Aggregation &Aggregates
-		) :
-      TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,Aggregates.FineGrid),
-      _CoarseSolver(CoarseSolver),
-      _CoarseSolverPrecise(CoarseSolverPrecise),
-      _Aggregates(Aggregates)
-  {
-    coarsegrid = Aggregates.CoarseGrid;
-  };
-
-  virtual void PcgM1(Field & in, Field & out)
-  {
-    GRID_TRACE("MultiGridPreconditioner ");
-    // [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
-
-    Field tmp(this->grid);
-    Field Min(this->grid);
-    CoarseField PleftProj(this->coarsegrid);
-    CoarseField PleftMss_proj(this->coarsegrid);
-
-    this->SmoothTimer.Start();
-    this->_Smoother(in,Min);
-    this->SmoothTimer.Stop();
-    this->SmoothCalls++;
-
-    this->MatrixTimer.Start();
-    this->_FineLinop.HermOp(Min,out);
-    this->MatrixTimer.Stop();
-    this->MatrixCalls++;
-    axpy(tmp,-1.0,out,in);          // tmp  = in - A Min
-
-    this->ProjectTimer.Start();
-    this->_Aggregates.ProjectToSubspace(PleftProj,tmp);
-    this->ProjectTimer.Stop();
-    this->ProjectCalls++;
-    this->CoarseTimer.Start();
-    this->_CoarseSolver(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
-    this->CoarseTimer.Stop();
-    this->CoarseCalls++;
-    this->PromoteTimer.Start();
-    this->_Aggregates.PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
-    this->PromoteTimer.Stop();
-    this->PromoteCalls++;
-
-    axpy(out,1.0,Min,tmp); // Min+tmp
-  }
-
-  virtual void Vstart(Field & x,const Field & src)
-  {
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart "<<std::endl;
-    ///////////////////////////////////
-    // Choose x_0 such that 
-    // x_0 = guess +  (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
-    //                               = [1 - Ass_inv A] Guess + Assinv src
-    //                               = P^T guess + Assinv src 
-    //                               = Vstart  [Tang notation]
-    // This gives:
-    // W^T (src - A x_0) = src_s - A guess_s - r_s
-    //                   = src_s - (A guess)_s - src_s  + (A guess)_s 
-    //                   = 0 
-    ///////////////////////////////////
-    Field r(this->grid);
-    Field mmp(this->grid);
-    CoarseField PleftProj(this->coarsegrid);
-    CoarseField PleftMss_proj(this->coarsegrid);
-
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart projecting "<<std::endl;
-    this->_Aggregates.ProjectToSubspace(PleftProj,src);     
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart coarse solve "<<std::endl;
-    this->_CoarseSolverPrecise(PleftProj,PleftMss_proj); // Ass^{-1} r_s
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart promote "<<std::endl;
-    this->_Aggregates.PromoteFromSubspace(PleftMss_proj,x);  
-
-  }
-
-};
 
   
 template<class Field>

Grid/algorithms/iterative/AdefMrhs.h

@@ -27,9 +27,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #pragma once
 
+#include <Grid/algorithms/multigrid/MrhsPreconditioner.h>
 
   /*
-   * Compared to Tang-2009:  P=Pleft. P^T = PRight Q=MssInv. 
+   * Compared to Tang-2009:  P=Pleft. P^T = PRight Q=MssInv.
    * Script A = SolverMatrix 
    * Script P = Preconditioner
    *
@@ -42,58 +43,64 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 NAMESPACE_BEGIN(Grid);
 
 
+//////////////////////////////////////////////////////////////////////
+// Two-level CG family on a vector of right-hand sides, with the
+// preconditioner (M1 and Vstart) as an MrhsPreconditioner object:
+//   fPcg           flexible PCG, one Krylov per rhs, shared M1 call
+//   PrecBlockCGrQ  preconditioned BlockCGrQ (arXiv:2409.03904 s2.2);
+//                  assumes a stationary preconditioner; not the
+//                  production choice (its linalg grows as nrhs^2)
+// Also a LinearFunction<Field>: one rhs is the nrhs=1 case.
+//////////////////////////////////////////////////////////////////////
+enum class MrhsCGAlgorithm { fPcg, PrecBlockCGrQ };
+
 template<class Field>
-class TwoLevelCGmrhs
+class TwoLevelCGmrhs : public LinearFunction<Field>
 {
  public:
+  using LinearFunction<Field>::operator();
   RealD   Tolerance;
   Integer MaxIterations;
   GridBase *grid;
+  MrhsCGAlgorithm Algorithm;
 
-  // Fine operator, Smoother, CoarseSolver
+  LinearOperatorBase<Field>    &_FineLinop;
-  LinearOperatorBase<Field>   &_FineLinop;
+  MrhsPreconditioner<Field>    &_Precon;
-  LinearFunction<Field>   &_Smoother;
+  MultiRHSBlockCGLinalg<Field>  _BlockCGLinalg;
-  MultiRHSBlockCGLinalg<Field> _BlockCGLinalg;
 
-  GridStopWatch ProjectTimer;
-  GridStopWatch PromoteTimer;
-  GridStopWatch DeflateTimer;
-  GridStopWatch CoarseTimer;
-  GridStopWatch FineTimer;
-  GridStopWatch SmoothTimer;
-  GridStopWatch InsertTimer;
-
-  /*
-    Field rrr;
-  Field sss;
-  Field qqq;
-  Field zzz;
-  */  
-  // more most opertor functions
   TwoLevelCGmrhs(RealD tol,
 		 Integer maxit,
-		 LinearOperatorBase<Field>   &FineLinop,
+		 LinearOperatorBase<Field> &FineLinop,
-		 LinearFunction<Field>       &Smoother,
+		 MrhsPreconditioner<Field> &Precon,
-		 GridBase *fine) : 
+		 GridBase *fine,
-    Tolerance(tol), 
+		 MrhsCGAlgorithm alg = MrhsCGAlgorithm::fPcg) :
+    Tolerance(tol),
     MaxIterations(maxit),
+    Algorithm(alg),
     _FineLinop(FineLinop),
-    _Smoother(Smoother)
+    _Precon(Precon)
-    /*
-    rrr(fine),
-    sss(fine),
-    qqq(fine),
-    zzz(fine)
-*/
   {
-    grid       = fine;
+    grid = fine;
   };
-  
+
-  // Vector case
+  // mrhs entry point
   virtual void operator() (std::vector<Field> &src, std::vector<Field> &x)
   {
-    SolveSingleSystem(src,x);
+    if ( Algorithm == MrhsCGAlgorithm::PrecBlockCGrQ ) SolvePrecBlockCG(src,x);
-	// SolvePrecBlockCG(src,x);
+    else                                              SolveSingleSystem(src,x);
+  }
+  // LinearFunction entry points
+  virtual void operator() (const Field &src, Field &x)
+  {
+    std::vector<Field> S(1,src);
+    std::vector<Field> X(1,x);
+    (*this)(S,X);
+    x = X[0];
+  }
+  virtual void operator() (const std::vector<Field> &src, std::vector<Field> &x)
+  {
+    std::vector<Field> S(src);
+    (*this)(S,x);
   }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -220,7 +227,7 @@ class TwoLevelCGmrhs
     //////////////////////////
     // x0 = Vstart -- possibly modify guess
     //////////////////////////
-    Vstart(X,src);
+    _Precon.Vstart(X,src);
 
     //////////////////////////
     // R = B-AX
@@ -234,7 +241,7 @@ class TwoLevelCGmrhs
     //////////////////////////////////
     // Compute MZ = M1 Z = M1 B - M1 A x0
     //////////////////////////////////
-    PcgM1(Z,MZ);  
+    _Precon(Z,MZ);  
 
     //////////////////////////////////
     // QC = Z
@@ -248,13 +255,7 @@ class TwoLevelCGmrhs
 
     std::cout << GridLogMessage<<"PrecBlockCGrQ vec computed initial residual and QR fact " <<std::endl;
 
-    ProjectTimer.Reset();
+    _Precon.ResetTimers();
-    PromoteTimer.Reset();
-    DeflateTimer.Reset();
-    CoarseTimer.Reset();
-    SmoothTimer.Reset();
-    FineTimer.Reset();
-    InsertTimer.Reset();
 
     GridStopWatch M1Timer;
     GridStopWatch M2Timer;
@@ -278,10 +279,9 @@ class TwoLevelCGmrhs
       // MZ  = M1 Z <==== the Multigrid preconditioner
       ////////////////////
       M1Timer.Start();
-      PcgM1(Z,MZ);
+      _Precon(Z,MZ);
       M1Timer.Stop();
 
-      FineTimer.Start();
       ////////////////////
       // M  = [D^dag Z]^{-1} = (<Ddag MZ>_M)^{-1} inner prod, generalising Saad derivation of Precon CG
       ////////////////////
@@ -326,7 +326,6 @@ class TwoLevelCGmrhs
       ////////////////////////////
       m_rr = m_C.adjoint() * m_C;
       
-      FineTimer.Stop();
 
       RealD max_resid=0;
       RealD rrsum=0;
@@ -350,14 +349,7 @@ class TwoLevelCGmrhs
 	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Linalg  "<<LinalgTimer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : fine H  "<<M3Timer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
+	_Precon.ReportTimers("HDCG: mrhs PrecBlockCGrQ : ");
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Project "<<ProjectTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Coarse  "<<CoarseTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Fine    "<<FineTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Smooth  "<<SmoothTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Insert  "<<InsertTimer.Elapsed()<<std::endl;;
 
 	for(int rhs=0;rhs<nrhs;rhs++){
 
@@ -423,7 +415,7 @@ class TwoLevelCGmrhs
     //////////////////////////
     // x0 = Vstart -- possibly modify guess
     //////////////////////////
-    Vstart(x,src);
+    _Precon.Vstart(x,src);
 
     for(int rhs=0;rhs<nrhs;rhs++){
       // r0 = b -A x0
@@ -435,7 +427,7 @@ class TwoLevelCGmrhs
     // Compute z = M1 x
     //////////////////////////////////
     // This needs a multiRHS version for acceleration
-    PcgM1(r,z);
+    _Precon(r,z);
 
     std::vector<RealD> ssq(nrhs);
     std::vector<RealD> rsq(nrhs);
@@ -449,13 +441,7 @@ class TwoLevelCGmrhs
       //      std::cout << GridLogMessage<<"mrhs HDCG: "<<rhs<<" k=0 residual "<<rtzp[rhs]<<" rsq "<<rsq[rhs]<<"\n";
     }
 
-    ProjectTimer.Reset();
+    _Precon.ResetTimers();
-    PromoteTimer.Reset();
-    DeflateTimer.Reset();
-    CoarseTimer.Reset();
-    SmoothTimer.Reset();
-    FineTimer.Reset();
-    InsertTimer.Reset();
 
     GridStopWatch M1Timer;
     GridStopWatch M2Timer;
@@ -488,7 +474,7 @@ class TwoLevelCGmrhs
 
       // Compute z = M x (for *all* RHS)
       M1Timer.Start();
-      PcgM1(r,z);
+      _Precon(r,z);
       M1Timer.Stop();
       
       RealD max_rn=0.0;
@@ -530,14 +516,7 @@ class TwoLevelCGmrhs
 	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Linalg  "<<LinalgTimer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : fine M3 "<<M3Timer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
+	_Precon.ReportTimers("HDCG: mrhs fPcg : ");
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Project "<<ProjectTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Coarse  "<<CoarseTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Fine    "<<FineTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Smooth  "<<SmoothTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Insert  "<<InsertTimer.Elapsed()<<std::endl;;
 
 	for(int rhs=0;rhs<nrhs;rhs++){
 	  _FineLinop.HermOp(x[rhs],mmp[rhs][0]);			  
@@ -568,167 +547,114 @@ class TwoLevelCGmrhs
       std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
     }
   }
-  
 
- public:
+  RealD PcgM3(const Field & p, Field & mmp){
-
-  virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out) = 0;
-  virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src) = 0;
-  virtual void PcgM2(const Field & in, Field & out) {
-    out=in;
-  }
-
-  virtual RealD PcgM3(const Field & p, Field & mmp){
     RealD dd;
     _FineLinop.HermOp(p,mmp);
     ComplexD dot = innerProduct(p,mmp);
     dd=real(dot);
     return dd;
   }
-
 };
 
-template<class Field, class CoarseField>
+//////////////////////////////////////////////////////////////////////
-class TwoLevelADEF2mrhs : public TwoLevelCGmrhs<Field>
+// ADEF-2 as an mrhs preconditioner object (Tang, Nabben, Vuik, Erlangga):
+//   M1     = [1 - Q A] M + Q   with  Q = P A_c^{-1} P^dag  (deflated coarse solve)
+//   Vstart = Q b               the coarse-corrected start (precise coarse solve)
+// The coarse space is the D+1 multiRHS field throughout: the projector's
+// mixed overloads take the vector of fine fields straight to it, and the
+// deflator and coarse solver act on it.  No per-rhs slices anywhere.
+// The D+1 grid follows the solver's Nrhs through SetCoarseGridMrhs.
+//////////////////////////////////////////////////////////////////////
+// Projector_t defaults to a transfer operator whose STORE matches Field.
+template<class Field, class CoarseField, class Projector_t = MultiRHSBlockProject<Field> >
+class MrhsADEF2Preconditioner : public MrhsPreconditioner<Field>
 {
 public:
-  GridBase *coarsegrid;
+  GridBase                       *grid;             // fine
-  GridBase *coarsegridmrhs;
+  GridBase                       *coarsegridmrhs;   // D+1, rhs innermost
-  LinearFunction<CoarseField> &_CoarseSolverMrhs;
+  LinearOperatorBase<Field>      &_FineLinop;
-  LinearFunction<CoarseField> &_CoarseSolverPreciseMrhs;
+  LinearFunction<Field>          &_Smoother;
-  MultiRHSBlockProject<Field>    &_Projector;
+  LinearFunction<CoarseField>    &_CoarseSolver;        // in M1
-  MultiRHSDeflation<CoarseField> &_Deflator;
+  LinearFunction<CoarseField>    &_CoarseSolverPrecise; // in Vstart
+  Projector_t                    &_Projector;   // store precision need not match Field
+  MultiRHSDeflation<CoarseField> &_Deflator;            // nev==0: no deflation, zero coarse guess
 
-  
+  MrhsADEF2Preconditioner(LinearOperatorBase<Field>      &FineLinop,
-  TwoLevelADEF2mrhs(RealD tol,
+			  LinearFunction<Field>          &Smoother,
-		    Integer maxit,
+			  LinearFunction<CoarseField>    &CoarseSolver,
-		    LinearOperatorBase<Field>    &FineLinop,
+			  LinearFunction<CoarseField>    &CoarseSolverPrecise,
-		    LinearFunction<Field>        &Smoother,
+			  Projector_t                    &Projector,
-		    LinearFunction<CoarseField>  &CoarseSolverMrhs,
+			  MultiRHSDeflation<CoarseField> &Deflator,
-		    LinearFunction<CoarseField>  &CoarseSolverPreciseMrhs,
+			  GridBase *CoarseGridMrhs,
-		    MultiRHSBlockProject<Field>    &Projector,
+			  GridBase *FineGrid) :
-		    MultiRHSDeflation<CoarseField> &Deflator,
+    _FineLinop(FineLinop), _Smoother(Smoother),
-		    GridBase *_coarsemrhsgrid) :
+    _CoarseSolver(CoarseSolver), _CoarseSolverPrecise(CoarseSolverPrecise),
-    TwoLevelCGmrhs<Field>(tol, maxit,FineLinop,Smoother,Projector.fine_grid),
+    _Projector(Projector), _Deflator(Deflator)
-    _CoarseSolverMrhs(CoarseSolverMrhs),
-    _CoarseSolverPreciseMrhs(CoarseSolverPreciseMrhs),
-    _Projector(Projector),
-    _Deflator(Deflator)
   {
-    coarsegrid = Projector.coarse_grid;
+    // The fine grid is given, NOT taken from the projector: the projector's
-    coarsegridmrhs = _coarsemrhsgrid;// Thi could be in projector
+    // grid carries its STORE's layout, which need not be this Field's (an
+    // fp32 preconditioner may project through an fp64 store, or the reverse).
+    grid           = FineGrid;
+    coarsegridmrhs = CoarseGridMrhs;
   };
+  // Store matches Field: the projector's grid IS this Field's grid.
+  MrhsADEF2Preconditioner(LinearOperatorBase<Field>      &FineLinop,
+			  LinearFunction<Field>          &Smoother,
+			  LinearFunction<CoarseField>    &CoarseSolver,
+			  LinearFunction<CoarseField>    &CoarseSolverPrecise,
+			  Projector_t                    &Projector,
+			  MultiRHSDeflation<CoarseField> &Deflator,
+			  GridBase *CoarseGridMrhs) :
+    MrhsADEF2Preconditioner(FineLinop,Smoother,CoarseSolver,CoarseSolverPrecise,
+			    Projector,Deflator,CoarseGridMrhs,Projector.fine_grid) {};
 
-  // Override Vstart
+  void SetCoarseGridMrhs(GridBase *CoarseGridMrhs) { coarsegridmrhs = CoarseGridMrhs; }
-  virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src)
+
+  // The coarse solver's starting guess: deflated, or zero
+  void CoarseGuess(CoarseField &src,CoarseField &guess)
   {
-    int nrhs=x.size();
-    ///////////////////////////////////
-    // Choose x_0 such that 
-    // x_0 = guess +  (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
-    //                               = [1 - Ass_inv A] Guess + Assinv src
-    //                               = P^T guess + Assinv src 
-    //                               = Vstart  [Tang notation]
-    // This gives:
-    // W^T (src - A x_0) = src_s - A guess_s - r_s
-    //                   = src_s - (A guess)_s - src_s  + (A guess)_s 
-    //                   = 0 
-    ///////////////////////////////////
-    std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
-    std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
-    CoarseField PleftProjMrhs(this->coarsegridmrhs);
-    CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
-
-    this->_Projector.blockProject(src,PleftProj);
-    this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
-      InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
-    }
-    
-    this->_CoarseSolverPreciseMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} r_s
-
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
-    }
-    this->_Projector.blockPromote(x,PleftMss_proj);
-  }
-
-  virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out){
-
-    int nrhs=in.size();
-
-    // [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
-    std::vector<Field> tmp(nrhs,this->grid);
-    std::vector<Field> Min(nrhs,this->grid);
-
-    std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
-    std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
-
-    CoarseField PleftProjMrhs(this->coarsegridmrhs);
-    CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
-
-    //    this->rrr=in[0];
-
-#undef SMOOTHER_BLOCK_SOLVE
-#if SMOOTHER_BLOCK_SOLVE
-    this->SmoothTimer.Start();
-    this->_Smoother(in,Min);
-    this->SmoothTimer.Stop();
-#else
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      this->SmoothTimer.Start();
-      this->_Smoother(in[rhs],Min[rhs]);
-      this->SmoothTimer.Stop();
-    }
-#endif
-    //    this->sss=Min[0];
-    
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      
-      this->FineTimer.Start();
-      this->_FineLinop.HermOp(Min[rhs],out[rhs]);
-      axpy(tmp[rhs],-1.0,out[rhs],in[rhs]);          // resid  = in - A Min
-      this->FineTimer.Stop();
-
-    }
-
-    this->ProjectTimer.Start();
-    this->_Projector.blockProject(tmp,PleftProj);
-    this->ProjectTimer.Stop();
     this->DeflateTimer.Start();
-    this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
+    if ( _Deflator.nev > 0 ) _Deflator.DeflateSources(src,guess);
+    else                     guess = Zero();
     this->DeflateTimer.Stop();
-    this->InsertTimer.Start();
+  }
-    for(int rhs=0;rhs<nrhs;rhs++) {
+  // x_0 = Q b
-      InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
+  virtual void Vstart(std::vector<Field> &x,std::vector<Field> &src)
-      InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
+  {
-    }
+    CoarseField Csrc(coarsegridmrhs), Csol(coarsegridmrhs);
-    this->InsertTimer.Stop();
+    this->ProjectTimer.Start();  _Projector.blockProject(src,Csrc);  this->ProjectTimer.Stop();
+    CoarseGuess(Csrc,Csol);
+    this->CoarseTimer.Start();   _CoarseSolverPrecise(Csrc,Csol);    this->CoarseTimer.Stop();
+    this->PromoteTimer.Start();  _Projector.blockPromote(x,Csol);    this->PromoteTimer.Stop();
+  }
+  // [1 - Q A] M in + Q in = Min + Q [in - A Min]
+  virtual void operator()(std::vector<Field> &in,std::vector<Field> &out)
+  {
+    int nrhs=in.size();
+    std::vector<Field> tmp(nrhs,grid), Min(nrhs,grid);
+    CoarseField Csrc(coarsegridmrhs), Csol(coarsegridmrhs);
 
-    this->CoarseTimer.Start();
+    this->SmoothTimer.Start();
-    this->_CoarseSolverMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} [in - A Min]_s
+    for(int r=0;r<nrhs;r++) _Smoother(in[r],Min[r]);
-    this->CoarseTimer.Stop();
+    this->SmoothTimer.Stop();
 
-    this->InsertTimer.Start();
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
-    }
-    this->InsertTimer.Stop();
-    this->PromoteTimer.Start();
-    this->_Projector.blockPromote(tmp,PleftMss_proj);// tmp= Q[in - A Min]  
-    this->PromoteTimer.Stop();
     this->FineTimer.Start();
-    //    this->qqq=tmp[0];
+    for(int r=0;r<nrhs;r++){
-    for(int rhs=0;rhs<nrhs;rhs++) {
+      _FineLinop.HermOp(Min[r],out[r]);
-      axpy(out[rhs],1.0,Min[rhs],tmp[rhs]); // Min+tmp
+      axpy(tmp[r],-1.0,out[r],in[r]);          // in - A Min
     }
-    //    this->zzz=out[0];
+    this->FineTimer.Stop();
+
+    this->ProjectTimer.Start();  _Projector.blockProject(tmp,Csrc);  this->ProjectTimer.Stop();
+    CoarseGuess(Csrc,Csol);
+    this->CoarseTimer.Start();   _CoarseSolver(Csrc,Csol);           this->CoarseTimer.Stop();
+    this->PromoteTimer.Start();  _Projector.blockPromote(tmp,Csol);  this->PromoteTimer.Stop();
+
+    this->FineTimer.Start();
+    for(int r=0;r<nrhs;r++) axpy(out[r],1.0,Min[r],tmp[r]);       // Min + Q[in - A Min]
     this->FineTimer.Stop();
   }
 };
 
-
 NAMESPACE_END(Grid);
-
-

Grid/algorithms/iterative/ConjugateGradientMixedPrecBatched.h

@@ -51,6 +51,20 @@ public:
   LinearFunction<FieldF> *guesser;
   bool updateResidual;
   
+  // Inner solves on independent partitions of the communicator (--batched-solver-split).
+  // BatchedSplit is the partition MPI layout, empty for no split; BatchedSplitNode selects
+  // one partition per node. Fixed at construction, default from the command line.
+  const Coordinate BatchedSplit;
+  const bool       BatchedSplitNode;
+
+  // Copy of Linop_f on the partitions, made once at construction and owned; nullptr when
+  // there is no split or the operator cannot be split. It holds the gauge field as it was at
+  // construction: a solver must not outlive a change of the operator's gauge field. The
+  // outer defect correction uses Linop_d, so a stale copy would cost convergence, not
+  // correctness.
+  SplitOperator<FieldF> *split = nullptr;
+  int                    Partitions = 1;
+
   MixedPrecisionConjugateGradientBatched(RealD tol, 
           Integer maxinnerit, 
           Integer maxouterit, 
@@ -58,10 +72,37 @@ public:
           GridBase* _sp_grid, 
           LinearOperatorBase<FieldF> &_Linop_f, 
           LinearOperatorBase<FieldD> &_Linop_d,
-          bool _updateResidual=true) :
+          bool _updateResidual=true,
+          const Coordinate &_split=GridDefaultBatchedSolverSplit(),
+          bool _split_node=GridDefaultBatchedSolverSplitNode()) :
     Linop_f(_Linop_f), Linop_d(_Linop_d),
     Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), MaxPatchupIterations(maxpatchit), SinglePrecGrid(_sp_grid),
-    OuterLoopNormMult(100.), guesser(NULL), updateResidual(_updateResidual) { };
+    OuterLoopNormMult(100.), guesser(NULL), updateResidual(_updateResidual),
+    BatchedSplit(_split), BatchedSplitNode(_split_node)
+  {
+    Coordinate layout = BatchedSolverSplitLayout(SinglePrecGrid,BatchedSplit,BatchedSplitNode,Partitions);
+    if ( Partitions > 1 ) {
+      double t0 = usecond();
+      split = Linop_f.SplitClone(layout);
+      double t1 = usecond();
+      if ( split == nullptr ) {
+        std::cout << GridLogMessage << "MixedPrecisionConjugateGradientBatched: operator cannot be split; serial inner solves" << std::endl;
+      } else {
+        std::cout << GridLogMessage << "MixedPrecisionConjugateGradientBatched: split clone " << (t1-t0)/1.0e6 << " s" << std::endl;
+        HostMemoryReport(SinglePrecGrid,GridLogMessage,"MixedPrecisionConjugateGradientBatched: after split clone");
+      }
+    }
+  };
+
+  // Owns split
+  MixedPrecisionConjugateGradientBatched(const MixedPrecisionConjugateGradientBatched &) = delete;
+
+  MixedPrecisionConjugateGradientBatched &operator=(const MixedPrecisionConjugateGradientBatched &) = delete;
+
+  ~MixedPrecisionConjugateGradientBatched(void)
+  {
+    delete split;
+  }
 
   void useGuesser(LinearFunction<FieldF> &g){
     guesser = &g;
@@ -91,6 +132,8 @@ public:
 
     GridStopWatch InnerCGtimer;
     GridStopWatch PrecChangeTimer;
+    GridStopWatch OuterResidualTimer;
+    GridStopWatch PatchupTimer;
     
     int cb = src_d_in[0].Checkerboard();
     
@@ -130,18 +173,27 @@ public:
     ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
     CG_f.ErrorOnNoConverge = false;
     
+    SplitTimers splitTimers;
+    if ( split != nullptr && split->Partitions > NBatch ) {
+      std::cout << GridLogWarning << "MixedPrecisionConjugateGradientBatched: " << split->Partitions << " partitions for "
+                << NBatch << " right-hand sides; the extra partitions only solve zero padding" << std::endl;
+    }
+    
     Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
       
     for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
       std::cout << GridLogMessage << std::endl;
       std::cout << GridLogMessage << "Outer iteration " << outer_iter << std::endl;
+      HostMemoryReport(DoublePrecGrid,GridLogMessage,"MixedPrecisionConjugateGradientBatched: outer iteration "+std::to_string(outer_iter));
       
       bool allConverged = true;
       
       for (int i=0; i<NBatch; i++) {
         //Compute double precision rsd and also new RHS vector.
+        OuterResidualTimer.Start();
         Linop_d.HermOp(sol_d[i], tmp_d);
         norm[i] = axpy_norm(src_d[i], -1., tmp_d, src_d_in[i]); //src_d is residual vector
+        OuterResidualTimer.Stop();
         
         std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer iteration " << outer_iter <<" solve " << i << " residual "<< norm[i] << " target "<< stop[i] <<std::endl;
 
@@ -169,12 +221,18 @@ public:
         (*guesser)(src_f, sol_f);
       }
 
+      if ( split != nullptr ) {
+        InnerSplitSolves(*split, CG_f, src_f, sol_f, TotalInnerIterations, InnerCGtimer, splitTimers);
+      }
+
       for (int i=0; i<NBatch; i++) {
         //Inner CG
-        InnerCGtimer.Start();
+        if ( split == nullptr ) {
-        CG_f(Linop_f, src_f[i], sol_f[i]);
+          InnerCGtimer.Start();
-        InnerCGtimer.Stop();
+          CG_f(Linop_f, src_f[i], sol_f[i]);
-        TotalInnerIterations[i] += CG_f.IterationsToComplete;
+          InnerCGtimer.Stop();
+          TotalInnerIterations[i] += CG_f.IterationsToComplete;
+        }
         
         //Convert sol back to double and add to double prec solution
         PrecChangeTimer.Start();
@@ -185,16 +243,18 @@ public:
       }
 
     }
-    
+
     //Final trial CG
     std::cout << GridLogMessage << std::endl;
     std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Starting final patch-up double-precision solve"<<std::endl;
     
+    PatchupTimer.Start();
     for (int i=0; i<NBatch; i++) {
       ConjugateGradient<FieldD> CG_d(Tolerance, MaxPatchupIterations);
       CG_d(Linop_d, src_d_in[i], sol_d[i]);
       TotalFinalStepIterations[i] += CG_d.IterationsToComplete;
     }
+    PatchupTimer.Stop();
 
     TotalTimer.Stop();
 
@@ -204,8 +264,118 @@ public:
     }
     std::cout << GridLogMessage << std::endl;
     std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
+    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer residual " << OuterResidualTimer.Elapsed() << " Patch-up " << PatchupTimer.Elapsed() << std::endl;
+    if ( split != nullptr ) {
+      std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Grid_split " << splitTimers.Split.Elapsed() << " (" << splitTimers.SplitCalls << " calls)"
+               << " Grid_unsplit " << splitTimers.Unsplit.Elapsed() << " (" << splitTimers.UnsplitCalls << " calls)"
+               << " Staging copies " << splitTimers.Staging.Elapsed()
+               << " Host to device " << splitTimers.H2D.Elapsed()
+               << " Iteration count sum " << splitTimers.Reduce.Elapsed() << std::endl;
+    }
+    double accounted = PrecChangeTimer.useconds() + InnerCGtimer.useconds() + OuterResidualTimer.useconds()
+                     + PatchupTimer.useconds() + splitTimers.Split.useconds()
+                     + splitTimers.Unsplit.useconds() + splitTimers.Staging.useconds() + splitTimers.H2D.useconds()
+                     + splitTimers.Reduce.useconds();
+    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Unaccounted " << (TotalTimer.useconds()-accounted)/1.0e6 << " s" << std::endl;
     
   }
+
+private:
+
+  // Coarse breakdown of the split path outside the inner CG
+  struct SplitTimers {
+    GridStopWatch Split;
+    GridStopWatch Unsplit;
+    GridStopWatch Staging;
+    GridStopWatch H2D;
+    GridStopWatch Reduce;
+    int SplitCalls   = 0;
+    int UnsplitCalls = 0;
+  };
+
+  ////////////////////////////////////////////////////////////////////////////////////////
+  // One inner solve per partition, in groups of Partitions right-hand sides. The last
+  // group is zero-padded; a zero source returns at once from CG. Collective.
+  ////////////////////////////////////////////////////////////////////////////////////////
+  void InnerSplitSolves(SplitOperator<FieldF> &split,
+                        ConjugateGradient<FieldF> &CG_f,
+                        std::vector<FieldF> &src_f,
+                        std::vector<FieldF> &sol_f,
+                        std::vector<Integer> &TotalInnerIterations,
+                        GridStopWatch &InnerCGtimer,
+                        SplitTimers &timers)
+  {
+    int NBatch = src_f.size();
+    int P      = split.Partitions;
+    int cb     = src_f[0].Checkerboard();
+
+    FieldF s_src(split.FieldGrid);
+    FieldF s_sol(split.FieldGrid);
+    std::vector<FieldF> group_src(P,SinglePrecGrid);
+    std::vector<FieldF> group_sol(P,SinglePrecGrid);
+    std::vector<uint64_t> iters(P);
+
+    for(int g=0;g<NBatch;g+=P){
+
+      // Gather the group; zero-pad past the end of the batch
+      timers.Staging.Start();
+      for(int p=0;p<P;p++){
+        group_src[p].Checkerboard() = cb;
+        group_sol[p].Checkerboard() = cb;
+        if ( g+p < NBatch ) {
+          group_src[p] = src_f[g+p];
+          group_sol[p] = sol_f[g+p];
+        } else {
+          group_src[p] = Zero();
+          group_sol[p] = Zero();
+        }
+      }
+      timers.Staging.Stop();
+
+      // The initial guess (e.g. from the guesser) travels with the source
+      timers.Split.Start();
+      Grid_split(group_src,s_src);
+      Grid_split(group_sol,s_sol);
+      timers.Split.Stop();
+      timers.SplitCalls += 2;
+
+      // Grid_split leaves its output on the host; move it now so the copy is not in the CG time
+      timers.H2D.Start();
+      {
+        autoView(s_src_v, s_src, AcceleratorRead);
+        autoView(s_sol_v, s_sol, AcceleratorRead);
+      }
+      timers.H2D.Stop();
+
+      InnerCGtimer.Start();
+      CG_f(*split.Linop,s_src,s_sol);
+      InnerCGtimer.Stop();
+
+      timers.Unsplit.Start();
+      Grid_unsplit(group_sol,s_sol);
+      timers.Unsplit.Stop();
+      timers.UnsplitCalls += 1;
+
+      // One iteration count per partition, contributed by the partition's rank 0 only
+      timers.Reduce.Start();
+      for(int p=0;p<P;p++){
+        iters[p] = 0;
+      }
+      if ( split.FieldGrid->ThisRank() == 0 ) {
+        iters[split.Partition] = CG_f.IterationsToComplete;
+      }
+      SinglePrecGrid->GlobalSumVector(&iters[0],P);
+      timers.Reduce.Stop();
+
+      // Includes the host to device copy of group_sol left by Grid_unsplit
+      timers.Staging.Start();
+      for(int p=0;p<P && g+p<NBatch;p++){
+        sol_f[g+p] = group_sol[p];
+        TotalInnerIterations[g+p] += iters[p];
+      }
+      timers.Staging.Stop();
+    }
+  }
 };
 
 NAMESPACE_END(Grid);

Grid/algorithms/iterative/PowerMethod.h

@@ -43,4 +43,69 @@ template<class Field> class PowerMethod
     return evalMaxApprox;
   }
 };
+
+// Non-Hermitian sibling of PowerMethod.  Drives Op() -- the operator the
+// caller actually applies -- rather than HermOp(), so it returns |lambda_max|
+// of a NON-Hermitian operator and reports the spectral-edge diagnostics the
+// Hermitian PowerMethod cannot:
+//   step 0 : |A v|/|v| on the (random) start src -- a one-sample lower bound
+//            on sigma_max(A).
+//   step k : |A v_k| -> |lambda_max| as v_k -> the dominant eigenvector; the
+//            complex Rayleigh quotient <v,Av> gives its phase (real => on the
+//            axis, a non-converging oscillation => a conjugate pair of equal
+//            modulus at the top).
+//   return : |lambda_max|, with a SUMMARY line flagging non-normality when the
+//            step-0 sigma_max lower bound sits well above the converged
+//            |lambda_max| -- the case where the numerical range extends beyond
+//            the spectrum and a spectrum-based smoother/Chebyshev bound is
+//            unsafe (the field of values must be used instead).
+template<class Field> class PowerMethodNonHermitian
+{
+ public:
+
+  template<typename T> static RealD normalise(T& v)
+  {
+    RealD nn = sqrt(norm2(v));
+    v = v * (1.0/nn);
+    return nn;
+  }
+
+  RealD operator()(LinearOperatorBase<Field> &Op, const Field &src)
+  {
+    GridBase *grid = src.Grid();
+    Field v(grid), Av(grid);
+    v = src;
+
+    RealD ratio = 0.0, ratio0 = 0.0;
+    ComplexD rq(0.0);
+    const int _MAX_ITER_EST_ = 200;
+
+    for (int i=0;i<_MAX_ITER_EST_;i++) {
+      normalise(v);                       // v is now unit
+      Op.Op(v,Av);
+      ratio = sqrt(norm2(Av));            // |A v| = |lambda_max| in the limit
+      rq    = innerProduct(v,Av);         // complex Rayleigh quotient
+
+      if ( i==0 ) {
+        ratio0 = ratio;
+        std::cout << GridLogMessage << "PowerMethodNonHermitian: step 0 (random v): |Av|/|v| = "
+                  << ratio << "   [lower bound on sigma_max]" << std::endl;
+      }
+      if ( (i%10==0) || (i==_MAX_ITER_EST_-1) )
+        std::cout << GridLogMessage << "PowerMethodNonHermitian: step " << i << " |Av|/|v| = " << ratio
+                  << "  Rayleigh (" << real(rq) << "," << imag(rq) << ")" << std::endl;
+
+      v = Av;
+    }
+
+    std::cout << GridLogMessage << "PowerMethodNonHermitian: |lambda_max| ~ " << ratio
+              << "  Rayleigh (" << real(rq) << "," << imag(rq) << ")"
+              << "  phase " << atan2(imag(rq),real(rq)) << " rad"
+              << "  step-0/converged = " << ratio0/ratio
+              << (ratio0/ratio > 1.2 ? "   ** non-normal: sigma_max well above |lambda_max| **"
+                                     : "   (near-normal)")
+              << std::endl;
+    return ratio;
+  }
+};
 }

Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h

@@ -1,6 +1,6 @@
 /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    Grid physics library, www.github.com/paboyle/Grid
 
     Source file: ./lib/algorithms/iterative/PrecGeneralisedConjugateResidual.h
 
@@ -28,213 +28,48 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 /*  END LEGAL */
 #ifndef GRID_PREC_GCR_H
 #define GRID_PREC_GCR_H
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
 
-///////////////////////////////////////////////////////////////////////////////////////////////////////
-//VPGCR Abe and Zhang, 2005.
-//INTERNATIONAL JOURNAL OF NUMERICAL ANALYSIS AND MODELING
-//Computing and Information Volume 2, Number 2, Pages 147-161
-//NB. Likely not original reference since they are focussing on a preconditioner variant.
-//    but VPGCR was nicely written up in their paper
-///////////////////////////////////////////////////////////////////////////////////////////////////////
 NAMESPACE_BEGIN(Grid);
 
-#define GCRLogLevel std::cout << GridLogMessage <<std::string(level,'\t')<< " Level "<<level<<" " 
+///////////////////////////////////////////////////////////////////////////////////////////////////////
-
+// Hermitian spelling of PrecGeneralisedConjugateResidualNonHermitian.
+//
+// The algorithm is the same; only the operator entry point differs.  The solver
+// drives Linop.Op(), while a caller of this class means "invert HermOp", so the
+// operator is wrapped in HermOpAdaptor.  The wrap is what makes the two classes
+// one: for an operator whose Op() already IS its HermOp() (HermitianLinearOperator,
+// Gamma5R5HermitianLinearOperator, HermOpAdaptor) it changes nothing, and for one
+// where they differ (MdagMLinearOperator: Op()=M, HermOp()=MdagM) it preserves the
+// Hermitian meaning the caller asked for.
+//
+// A non-zero initial guess is respected.  Callers that guarantee x0 = 0 can call
+// SetZeroGuess(1) to skip the first r0 = src - A x0 apply.
+///////////////////////////////////////////////////////////////////////////////////////////////////////
 template<class Field>
 class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
-public:                                                
+  // Declaration order is initialisation order: the adaptor is complete before
+  // the solver binds its reference to it.
+  HermOpAdaptor<Field>                                HermLinop;
+  PrecGeneralisedConjugateResidualNonHermitian<Field> GCR;
+public:
   using LinearFunction<Field>::operator();
-  RealD   Tolerance;
-  Integer MaxIterations;
-  int verbose;
-  int mmax;
-  int nstep;
-  int steps;
-  int level;
-  GridStopWatch PrecTimer;
-  GridStopWatch MatTimer;
-  GridStopWatch LinalgTimer;
 
-  LinearFunction<Field>     &Preconditioner;
+  PrecGeneralisedConjugateResidual(RealD tol,Integer maxit,LinearOperatorBase<Field> &Linop,
-  LinearOperatorBase<Field> &Linop;
+                                   LinearFunction<Field> &Prec,int mmax,int nstep)
+    : HermLinop(Linop),
+      GCR(tol,maxit,HermLinop,Prec,mmax,nstep) {};
 
-  void Level(int lv) { level=lv; };
+  void operator() (const Field &src, Field &psi) { GCR(src,psi); };
 
-  PrecGeneralisedConjugateResidual(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) : 
+  void Name(std::string n)                       { GCR.Name(n); };
-    Tolerance(tol), 
+  void Level(int n)                              { GCR.Level(n); };
-    MaxIterations(maxit),
+  void SetZeroGuess(int z)                       { GCR.SetZeroGuess(z); };
-    Linop(_Linop),
+  int  Steps(void) const                         { return GCR.Steps(); };
-    Preconditioner(Prec),
+  void LogCoefficients(int l)                    { GCR.LogCoefficients(l); };
-    mmax(_mmax),
+  void SetCoefficientRecorder(GCRCoefficients *r){ GCR.SetCoefficientRecorder(r); };
-    nstep(_nstep)
+  void ReleaseHistory(void)                      { GCR.ReleaseHistory(); };
-  { 
-    level=1;
-    verbose=1;
-  };
-
-  void operator() (const Field &src, Field &psi){
-
-    psi=Zero();
-    RealD cp, ssq,rsq;
-    ssq=norm2(src);
-    rsq=Tolerance*Tolerance*ssq;
-      
-    Field r(src.Grid());
-
-    PrecTimer.Reset();
-    MatTimer.Reset();
-    LinalgTimer.Reset();
-
-    GridStopWatch SolverTimer;
-    SolverTimer.Start();
-
-    steps=0;
-    for(int k=0;k<MaxIterations;k++){
-
-      cp=GCRnStep(src,psi,rsq);
-
-      GCRLogLevel <<"PGCR("<<mmax<<","<<nstep<<") "<< steps <<" steps cp = "<<cp<<" target "<<rsq <<std::endl;
-
-      if(cp<rsq) {
-
-	SolverTimer.Stop();
-
-	Linop.HermOp(psi,r);
-	axpy(r,-1.0,src,r);
-	RealD tr = norm2(r);
-	GCRLogLevel<<"PGCR: Converged on iteration " <<steps
-		 << " computed residual "<<sqrt(cp/ssq)
-		 << " true residual "    <<sqrt(tr/ssq)
-		 << " target "           <<Tolerance <<std::endl;
-
-	GCRLogLevel<<"PGCR Time elapsed: Total  "<< SolverTimer.Elapsed() <<std::endl;
-	/*
-	  GCRLogLevel<<"PGCR Time elapsed: Precon "<<   PrecTimer.Elapsed() <<std::endl;
-	  GCRLogLevel<<"PGCR Time elapsed: Matrix "<<    MatTimer.Elapsed() <<std::endl;
-	  GCRLogLevel<<"PGCR Time elapsed: Linalg "<< LinalgTimer.Elapsed() <<std::endl;
-	*/
-	return;
-      }
-
-    }
-    GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
-    //    GRID_ASSERT(0);
-  }
-
-  RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
-
-    RealD cp;
-    RealD a, b;
-    RealD zAz, zAAz;
-    RealD rq;
-
-    GridBase *grid = src.Grid();
-
-    Field r(grid);
-    Field z(grid);
-    Field tmp(grid);
-    Field ttmp(grid);
-    Field Az(grid);
-
-    ////////////////////////////////
-    // history for flexible orthog
-    ////////////////////////////////
-    std::vector<Field> q(mmax,grid);
-    std::vector<Field> p(mmax,grid);
-    std::vector<RealD> qq(mmax);
-      
-    GCRLogLevel<< "PGCR nStep("<<nstep<<")"<<std::endl;
-
-    //////////////////////////////////
-    // initial guess x0 is taken as nonzero.
-    // r0=src-A x0 = src
-    //////////////////////////////////
-    MatTimer.Start();
-    Linop.HermOpAndNorm(psi,Az,zAz,zAAz); 
-    MatTimer.Stop();
-    
-
-    LinalgTimer.Start();
-    r=src-Az;
-    LinalgTimer.Stop();
-    GCRLogLevel<< "PGCR true residual r = src - A psi   "<<norm2(r) <<std::endl;
-    
-    /////////////////////
-    // p = Prec(r)
-    /////////////////////
-
-    PrecTimer.Start();
-    Preconditioner(r,z);
-    PrecTimer.Stop();
-
-    MatTimer.Start();
-    Linop.HermOpAndNorm(z,Az,zAz,zAAz); 
-    MatTimer.Stop();
-
-    LinalgTimer.Start();
-
-    //p[0],q[0],qq[0] 
-    p[0]= z;
-    q[0]= Az;
-    qq[0]= zAAz;
-    
-    cp =norm2(r);
-    LinalgTimer.Stop();
-
-    for(int k=0;k<nstep;k++){
-
-      steps++;
-
-      int kp     = k+1;
-      int peri_k = k %mmax;
-      int peri_kp= kp%mmax;
-
-      LinalgTimer.Start();
-      rq= real(innerProduct(r,q[peri_k])); // what if rAr not real?
-      a = rq/qq[peri_k];
-
-      axpy(psi,a,p[peri_k],psi);         
-
-      cp = axpy_norm(r,-a,q[peri_k],r);
-      LinalgTimer.Stop();
-
-      GCRLogLevel<< "PGCR step["<<steps<<"]  resid " << cp << " target " <<rsq<<std::endl; 
-
-      if((k==nstep-1)||(cp<rsq)){
-	return cp;
-      }
-
-
-      PrecTimer.Start();
-      Preconditioner(r,z);// solve Az = r
-      PrecTimer.Stop();
-
-      MatTimer.Start();
-      Linop.HermOpAndNorm(z,Az,zAz,zAAz);
-      MatTimer.Stop();
-
-      LinalgTimer.Start();
-
-      q[peri_kp]=Az;
-      p[peri_kp]=z;
-
-      int northog = ((kp)>(mmax-1))?(mmax-1):(kp);  // if more than mmax done, we orthog all mmax history.
-      for(int back=0;back<northog;back++){
-
-	int peri_back=(k-back)%mmax;   	  GRID_ASSERT((k-back)>=0);
-
-	b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
-	p[peri_kp]=p[peri_kp]+b*p[peri_back];
-	q[peri_kp]=q[peri_kp]+b*q[peri_back];
-
-      }
-      qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
-      LinalgTimer.Stop();
-    }
-    GRID_ASSERT(0); // never reached
-    return cp;
-  }
 };
+
 NAMESPACE_END(Grid);
-#undef GCRLogLevel
 #endif

Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h

@@ -43,8 +43,51 @@ NAMESPACE_BEGIN(Grid);
 
 template<class Field>
 class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
-public:                                                
+public:
   using LinearFunction<Field>::operator();
+
+  PrecGeneralisedConjugateResidualNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) :
+    Tolerance(tol),
+    MaxIterations(maxit),
+    mmax(_mmax),
+    nstep(_nstep),
+    Preconditioner(Prec),
+    Linop(_Linop)
+  {
+    Level(1);
+    verbose=1;
+    trivial_prec = Preconditioner.isTrivial();
+  };
+
+  // The name is also the trace range's, so a profile separates the four
+  // instances (Fouter, Fsmoother, Couter, Csmoother) that otherwise nest
+  // indistinguishably as one shared range.
+  void Name(std::string _name) { name = _name; trace_step = name + " PGCR_step"; };
+
+  void Level(int n) { Name("Level " + std::to_string(n)); level = n; }
+
+  void SetZeroGuess(int z) { ZeroGuess = z; };
+
+  // Steps taken by the last solve.
+  int  Steps(void) const { return steps; };
+
+  // Coefficient logging: one line per step with the step length a_k and the
+  // orthogonalisation coefficients b_{k,j}.  These are the data from which a
+  // FIXED polynomial smoother can be harvested: if they are stable from call
+  // to call, the adaptive GCR can be replaced by a stationary p(A) with the
+  // same applies and no reductions.  Off by default; boss rank prints.
+  void LogCoefficients(int l) { LogCoeffs = l; };
+
+  // Optional recorder of the per-step coefficients (means over calls), for
+  // replay by GCRReplaySmoother (Smoothers.h).  Records only; no effect on
+  // the iteration.
+  void SetCoefficientRecorder(GCRCoefficients *r) { Recorder = r; if(r) r->mmax = mmax; };
+
+  // Free the persistent history (e.g. when this solver is replaced by a
+  // replayed polynomial): re-made on the next call if ever needed again.
+  void ReleaseHistory(void) { q.clear(); p.clear(); qq.clear(); hist_grid = nullptr; };
+
+private:
   RealD   Tolerance;
   RealD   SSQ;
   Integer MaxIterations;
@@ -57,6 +100,7 @@ public:
   GridStopWatch MatTimer;
   GridStopWatch LinalgTimer;
   std::string name;
+  std::string trace_step = "PGCR_step";   // name + step; see Name()
   int ZeroGuess  = 0;  // caller contract: guess is always zero => first-cycle r0 = src, skip the apply
   // persistent GCR history (see GCRnStep)
   GridBase           *hist_grid = nullptr;
@@ -64,42 +108,14 @@ public:
   std::vector<Field>  p;
   std::vector<RealD>  qq;
   int FirstCycle = 0;
+  int  LogCoeffs = 0;
+  int  trivial_prec;      // Preconditioner is the identity: p == r, copies fold away
+  GCRCoefficients *Recorder = nullptr;
 
   LinearFunction<Field>     &Preconditioner;
   LinearOperatorBase<Field> &Linop;
 
-  void Name(std::string _name) { name = _name; };
+public:
-
-  void Level(int n) { Name("Level " + std::to_string(n)); level = n; }
-
-  void SetZeroGuess(int z) { ZeroGuess = z; };
-  // Coefficient logging: one line per step with the step length a_k and the
-  // orthogonalisation coefficients b_{k,j}.  These are the data from which a
-  // FIXED polynomial smoother can be harvested: if they are stable from call
-  // to call, the adaptive GCR can be replaced by a stationary p(A) with the
-  // same applies and no reductions.  Off by default; boss rank prints.
-  int  LogCoeffs = 0;
-  void LogCoefficients(int l) { LogCoeffs = l; };
-  // Optional recorder of the per-step coefficients (means over calls), for
-  // replay by GCRReplaySmoother (Smoothers.h).  Records only; no effect on
-  // the iteration.
-  GCRCoefficients *Recorder = nullptr;
-  void SetCoefficientRecorder(GCRCoefficients *r) { Recorder = r; if(r) r->mmax = mmax; };
-  // Free the persistent history (e.g. when this solver is replaced by a
-  // replayed polynomial): re-made on the next call if ever needed again.
-  void ReleaseHistory(void) { q.clear(); p.clear(); qq.clear(); hist_grid = nullptr; };
-
-  PrecGeneralisedConjugateResidualNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) : 
-    Tolerance(tol), 
-    MaxIterations(maxit),
-    Linop(_Linop),
-    Preconditioner(Prec),
-    mmax(_mmax),
-    nstep(_nstep)
-  {
-    Level(1);
-    verbose=1;
-  };
 
   void operator() (const Field &src, Field &psi){
 
@@ -147,6 +163,7 @@ public:
     //    GRID_ASSERT(0);
   }
 
+private:
   RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
 
     RealD cp;
@@ -185,18 +202,20 @@ public:
     // contract (enforced here), so r0 = src exactly; skip the apply.
     // Restart cycles (psi!=0) always do the full computation.
     //////////////////////////////////
-    if (ZeroGuess && FirstCycle) {
+    // The residual is READ through rp and only materialised in r when it is
-      psi = Zero();
+    // first written, at the k=0 update.  On a zero-guess first cycle r0 IS src,
-      LinalgTimer.Start();
+    // so no copy is made; a restart cycle computes r and rp points at it.
-      r = src;
+    const Field *rp = &src;
-      LinalgTimer.Stop();
+    GRID_ASSERT(nstep>=1);   // psi is first written at k=0; see zero_start below
-    } else {
+    int zero_start = (ZeroGuess && FirstCycle);
+    if ( !zero_start ) {
       MatTimer.Start();
       Linop.Op(psi,Az);
       MatTimer.Stop();
       LinalgTimer.Start();
       r=src-Az;
       LinalgTimer.Stop();
+      rp = &r;
     }
     FirstCycle=0;
 
@@ -204,9 +223,11 @@ public:
     // p = Prec(r)
     /////////////////////
 
-    // p[0] = Prec(r), q[0] = A p[0], written straight into the history slots
+    // p[0] = Prec(r), q[0] = A p[0], written straight into the history slots.
+    // This copy is forced even for a trivial preconditioner: p[0] is history and
+    // must survive while the residual evolves.
     PrecTimer.Start();
-    Preconditioner(r,p[0]);
+    Preconditioner(*rp,p[0]);
     PrecTimer.Stop();
 
     MatTimer.Start();
@@ -217,12 +238,12 @@ public:
 
     qq[0]= norm2(q[0]);
 
-    cp =norm2(r);
+    cp =norm2(*rp);
     LinalgTimer.Stop();
     GCRLogLevel<< "PGCR true residual "<< sqrt(cp/SSQ)     <<std::endl;
 
     for(int k=0;k<nstep;k++){
-      GRID_TRACE("PGCR_step");
+      GRID_TRACE(trace_step.c_str());
       steps++;
 
       int kp     = k+1;
@@ -230,13 +251,32 @@ public:
       int peri_kp= kp%mmax;
 
       LinalgTimer.Start();
-      rq= innerProduct(q[peri_k],r); // what if rAr not real?
+      rq= innerProduct(q[peri_k],*rp); // what if rAr not real?
       a = rq/qq[peri_k];
       if ( Recorder ) Recorder->RecordA(k,a);
 
-      axpy(psi,a,p[peri_k],psi);         
+      // On a zero-guess first step psi is still unwritten, and the update would
+      // be a*p added to zero: assign it instead, so psi is never zeroed.
+      if ( zero_start && (k==0) ) psi = a*p[peri_k];
+      else                        axpy(psi,a,p[peri_k],psi);
 
-      cp = axpy_norm(r,-a,q[peri_k],r);
+#ifdef GRID_GCR_RESIDUAL_RECURRENCE
+      // |r - a q|^2 = |r|^2 - |<q,r>|^2/|q|^2 with a = <q,r>/|q|^2, so the new
+      // residual norm follows from scalars already in hand and the reduction in
+      // axpy_norm is not needed.  cp then DRIFTS rather than being measured; the
+      // true residual is recomputed at restart and at convergence, which bounds
+      // the exposure.  The drift only matters for an instance driven to a tight
+      // tolerance -- the outer solver -- and there it is the same computed-vs-true
+      // residual gap that CG lives with; the smoother and coarse instances run at
+      // fixed work or loose tolerance and are indifferent to it.
+      // Explicit re/im: ComplexD is thrust::complex under HIP.
+      axpy(r,-a,q[peri_k],*rp);
+      cp = cp - (real(rq)*real(rq)+imag(rq)*imag(rq))/qq[peri_k];
+      if ( cp < 0.0 ) cp = 0.0;          // drift can undershoot; keeps the log finite
+#else
+      cp = axpy_norm(r,-a,q[peri_k],*rp);
+#endif
+      rp = &r;   // the residual now lives in r; src is untouched from here on
       LinalgTimer.Stop();
       if ( LogCoeffs ) {
         GCRLogLevel<<"coeff["<<k<<"] a=("<<real(a)<<","<<imag(a)<<")"
@@ -250,13 +290,21 @@ public:
       }
 
       // New direction written straight into its history slot: p = Prec(r), q = A p.
-      PrecTimer.Start();
+      // For a trivial preconditioner p[kp] would only be a copy of the residual,
-      Preconditioner(r,p[peri_kp]);
+      // so the operator is applied to the residual directly and the copy folds
-      PrecTimer.Stop();
+      // into the orthogonalisation pass below.
-
+      if ( trivial_prec ) {
-      MatTimer.Start();
+        MatTimer.Start();
-      Linop.Op(p[peri_kp],q[peri_kp]);
+        Linop.Op(*rp,q[peri_kp]);
-      MatTimer.Stop();
+        MatTimer.Stop();
+      } else {
+        PrecTimer.Start();
+        Preconditioner(*rp,p[peri_kp]);
+        PrecTimer.Stop();
+        MatTimer.Start();
+        Linop.Op(p[peri_kp],q[peri_kp]);
+        MatTimer.Stop();
+      }
 
       LinalgTimer.Start();
 
@@ -285,16 +333,19 @@ public:
         if ( LogCoeffs ) bs<<" b["<<back<<"]="<<bcoef[back];
       }
       if ( Recorder && northog ) Recorder->RecordB(k,bcoef);
+      // axpyMultiNorm returns |q|^2 from the update pass, so qq is complete here:
+      // a separate norm2 would be a second reduction and a second GlobalSum.
       if ( northog ) {
-	axpyMulti(p[peri_kp],bcoef,pwin);
+        if ( trivial_prec ) axpyMultiFrom(p[peri_kp],*rp,bcoef,pwin);
-	qq[peri_kp]=axpyMultiNorm(q[peri_kp],bcoef,qwin);
+        else                axpyMulti(p[peri_kp],bcoef,pwin);
+        qq[peri_kp]=axpyMultiNorm(q[peri_kp],bcoef,qwin);
       } else {
-	qq[peri_kp]=norm2(q[peri_kp]);
+        if ( trivial_prec ) p[peri_kp] = *rp;   // no pass to fold the copy into
+        qq[peri_kp]=norm2(q[peri_kp]);
       }
       if ( LogCoeffs && northog ) {
         GCRLogLevel<<"coeff["<<k<<"]"<<bs.str()<<std::endl;
       }
-      qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
       LinalgTimer.Stop();
     }
     GRID_ASSERT(0); // never reached

Grid/algorithms/multigrid/Aggregates.h

@@ -592,7 +592,7 @@ public:
     }
   }
   virtual void RefineSubspaceHDCG(LinearOperatorBase<FineField> &hermop,
-				  TwoLevelADEF2mrhs<FineField,CoarseVector> & theHDCG,
+				  TwoLevelCGmrhs<FineField> & theHDCG,
 				  int nrhs)
   {
     std::vector<FineField> src_mrhs(nrhs,FineGrid);

Grid/algorithms/multigrid/BlockCyclic.h

@@ -21,12 +21,25 @@ Author: Peter Boyle <pboyle@bnl.gov>
 
 NAMESPACE_BEGIN(Grid);
 
+///////////////////////////////////////////////////////////////////////////////
+// Scalar of the distributed dense inversion: block-cyclic redistribution,
+// SUMMA, the Schur recursion and its LU leaf.  A configure-time choice
+// (--enable-dense-inverse-precision=double|single), INDEPENDENT of the
+// coarse-space precision: the matrix elements arrive in the coarse scalar
+// and the apply slab is fp32 either way; only the factorisation's
+// arithmetic and footprint follow this type.
+///////////////////////////////////////////////////////////////////////////////
+#ifdef GRID_DENSE_INVERSE_SINGLE
+typedef ComplexF DenseInverseScalar;
+#else
+typedef ComplexD DenseInverseScalar;
+#endif
+
 ///////////////////////////////////////////////////////////////////////////////
 // BlockCyclicLayout: the index arithmetic of a 2D block-cyclic distribution
 // of an N x N matrix over a Pr x Pc logical process grid with block size nb.
 //
-// This is stage 1 of the 2D distributed dense inverse
+// This is stage 1 of the 2D distributed dense inverse.  It is deliberately
-// (documentation/DistributedDenseInverse2D.tex).  It is deliberately
 // COMMUNICATOR-FREE: every mapping is a static pure function of
 // (N, nb, Pr, Pc), so the whole layout is exhaustively unit-testable on one
 // rank with no MPI in the loop (Test_blockcyclic).  A thin instance layer

Grid/algorithms/multigrid/BlockCyclicRedistribute.h

@@ -23,11 +23,47 @@ Author: Peter Boyle <pboyle@bnl.gov>
 
 NAMESPACE_BEGIN(Grid);
 
+///////////////////////////////////////////////////////////////////////////////
+// My rows of a distributed dense matrix in the 1D rank-major row layout:
+// rank r owns contiguous global rows [rowStart[r], rowStart[r+1]) of an
+// N x N matrix, stored rows x cols column major, ld = rows; element (i,j)
+// at data[i + j*ld].  This is the layout the stencil->dense import
+// produces and the apply slab consumes; the 2D inverse slots between them
+// through the redistribution below.
+///////////////////////////////////////////////////////////////////////////////
+class BlockRows
+{
+public:
+  deviceVector<DenseInverseScalar> data;
+  int64_t                rows;
+  int64_t                cols;
+  int64_t                ld;
+
+  BlockRows()
+  {
+    rows = 0;
+    cols = 0;
+    ld   = 0;
+  }
+  void Resize(int64_t r, int64_t c)
+  {
+    rows = r;
+    cols = c;
+    ld   = r;
+    data.resize((uint64_t)r*c);
+  }
+  DenseInverseScalar *ColumnWindow(int64_t col0)
+  {
+    GRID_ASSERT( col0 >= 0 );
+    GRID_ASSERT( col0 <= cols );
+    return &data[(uint64_t)col0*ld];
+  }
+};
+
 ///////////////////////////////////////////////////////////////////////////////
 // Stage 4 of the 2D distributed dense inverse: redistribution between the
-// 1D rank-major row layout (BlockRows: rank r owns contiguous global rows
+// 1D rank-major row layout (BlockRows above) and the 2D block-cyclic
-// [rowStart[r], rowStart[r+1]) of an N x N matrix, stored rows x N column
+// layout.
-// major with ld = rows) and the 2D block-cyclic layout.
 //
 // This is what lets the EXISTING stencil->dense import, its certificate,
 // the fp32 slab conversion and the apply path all remain byte-for-byte
@@ -92,10 +128,10 @@ public:
   //   buffer(a,b) = elem(rows[a], cols[b]),  a fastest.
   /////////////////////////////////////////////////////////////////////////
   static void MoveEdge(int toBuffer,
-                       ComplexD *mat, int64_t ld,
+                       DenseInverseScalar *mat, int64_t ld,
                        const std::vector<int64_t> &roff,  // per-row offset in mat
                        const std::vector<int64_t> &coff,  // per-col offset in mat
-                       ComplexD *buf)
+                       DenseInverseScalar *buf)
   {
     int64_t nr = roff.size();
     int64_t nc = coff.size();
@@ -157,7 +193,7 @@ public:
   /////////////////////////////////////////////////////////////////////////
   static void Redistribute(int dir, GridBase *grid,
                            const std::vector<int64_t> &rowStart,
-                           ComplexD *rows1d, int64_t myrows,
+                           DenseInverseScalar *rows1d, int64_t myrows,
                            BlockCyclicMatrix &A)
   {
     BlockCyclicLayout &L = A.layout;
@@ -170,7 +206,7 @@ public:
     int64_t ld1  = myrows ? myrows : 1;
 
     std::vector<int64_t> rows, cols, roff, coff;
-    deviceVector<ComplexD> sbuf(1), rbuf(1);
+    deviceVector<DenseInverseScalar> sbuf(1), rbuf(1);
 
     ///////////////////////////////////////////////////////////////////////
     // Self edge: purely local, via a bounce buffer (shares all the code).
@@ -225,7 +261,7 @@ public:
       }
       grid->SendToRecvFrom((void *)&sbuf[0], partner,
                            (void *)&rbuf[0], partner,
-                           nmax*sizeof(ComplexD));
+                           nmax*sizeof(DenseInverseScalar));
       if ( nin ){
         if ( dir > 0 ) { Offsets2D(L, irow, icol, roff, coff);
                          MoveEdge(0, &A.data[0], L.mloc, roff, coff, &rbuf[0]); }
@@ -236,12 +272,16 @@ public:
   }
 
   static void RowsToCyclic(GridBase *grid, const std::vector<int64_t> &rowStart,
-                           ComplexD *rows1d, int64_t myrows, BlockCyclicMatrix &A)
+                           DenseInverseScalar *rows1d, int64_t myrows, BlockCyclicMatrix &A)
-  { Redistribute(+1, grid, rowStart, rows1d, myrows, A); }
+  {
+    Redistribute(+1, grid, rowStart, rows1d, myrows, A);
+  }
 
   static void CyclicToRows(GridBase *grid, const std::vector<int64_t> &rowStart,
-                           BlockCyclicMatrix &A, ComplexD *rows1d, int64_t myrows)
+                           BlockCyclicMatrix &A, DenseInverseScalar *rows1d, int64_t myrows)
-  { Redistribute(-1, grid, rowStart, rows1d, myrows, A); }
+  {
+    Redistribute(-1, grid, rowStart, rows1d, myrows, A);
+  }
 };
 
 NAMESPACE_END(Grid);

Grid/algorithms/multigrid/BlockCyclicSchurInverse.h

@@ -28,12 +28,11 @@ NAMESPACE_BEGIN(Grid);
 // Stage 3 of the 2D distributed dense inverse: the recursive Schur
 // complement on a block-cyclic matrix, in place.
 //
-// The nine-step algebra is IDENTICAL to RecursiveSchurInverse (1D); what
+// The recursion splits the GLOBAL INDEX RANGE at the block boundary
-// changes is the decomposition.  The recursion splits the GLOBAL INDEX
+// nearest the midpoint -- not the rank range as the retired 1D
-// RANGE at the block boundary nearest the midpoint -- not the rank range --
+// RecursiveSchurInverse did -- so every rank owns part of every sub-block
-// so every rank owns part of every sub-block at every depth, and the
+// at every depth, and the ownership gating (inI/inJ, dummy operands,
-// ownership gating (inI/inJ, dummy operands, zero-width rank ranges) of the
+// zero-width rank ranges) a 1D scheme needs has no analogue here.
-// 1D scheme has no analogue here: it is simply gone.
 //
 //   I = [c0,m)   J = [m,c1)      (block-aligned, m the mid block boundary)
 //   1. recurse I  :  A11 -> A11inv                        (in place)
@@ -64,8 +63,7 @@ NAMESPACE_BEGIN(Grid);
 // LEAF.  A leaf is a single diagonal block, and block (b,b) of a
 // block-cyclic layout lives ENTIRELY on rank (b%Pr, b%Pc).  The leaf
 // inversion is therefore purely local -- pack the strided block dense,
-// GridBLASInverse, unpack -- with NO communication and no assembly.  The
+// GridBLASInverse, unpack -- with NO communication and no assembly.
-// leaf-assembly transport question of the 1D scheme does not arise.
 // Successive leaves cycle over ranks, so leaf work is naturally spread.
 //
 // COMMUNICATION.  Every transfer in the whole inversion is a
@@ -75,12 +73,9 @@ NAMESPACE_BEGIN(Grid);
 // one optional exception: it performs reductions, and is only ever called
 // explicitly by a caller who wants the numbers.
 //
-// NUMERICS.  No pivoting, exactly as the 1D scheme: every A11 and every
+// NUMERICS.  No pivoting: every A11 and every Schur complement met on the
-// Schur complement met on the way down must be non-singular.  The growth
+// way down must be non-singular.  The growth telemetry stands in for
-// telemetry stands in for pivoting; note the recursion splits differently
+// pivoting.
-// from the 1D rank-range tree, so DIFFERENT sub-blocks are inverted and
-// telemetry values are NOT comparable with the 1D implementation's --
-// re-baseline, do not compare.
 ///////////////////////////////////////////////////////////////////////////////
 
 class BlockCyclicSchurInverse
@@ -92,14 +87,16 @@ public:
   // Telemetry: accumulated LOCALLY, no comms unless ReportTelemetry().
   double                  telLeafMaxInv;
   uint64_t                nLeaf;
-  // BIG LEAVES (SCHUR2D_LEAF_SPAN=s, default 1 = off).  Below span s blocks a
+  // BIG LEAVES.  Below span s blocks a sub-block lives on <= s of the Pr
-  // sub-block lives on <= s of the Pr process rows / s of the Pc columns; the
+  // process rows / s of the Pc columns; when s is small relative to the grid
-  // SUMMA rings then run on a few ranks while the rest block in their next
+  // the SUMMA rings run on a few ranks while the rest block in their next
-  // SendToRecvFrom (histogram 2026-08-27: 93% of ring time in the 3.7 MB
+  // SendToRecvFrom.  Instead: gather the (s*nb)^2 sub-block to one rank,
-  // single-block panels of exactly these levels).  Instead: gather the
+  // invert locally (inverseLU), scatter back.  Fires only while
-  // (s*nb)^2 sub-block to one rank, invert locally, scatter back.
+  // span < min(Pr,Pc) -- when the whole grid participates the rings are not
-  int                     leafSpan = -1;
+  // degenerate and gathering would only concentrate memory (and at the top
-  int                     leafLU   = -1;  // SCHUR2D_LEAF_LU=1: big-leaf inverse via GridBLASInverse::inverseLU (blocked getrf_64 + identity getrs_64) instead of getri_batched
+  // of the tree would gather the whole matrix).  Larger s concentrates more
+  // memory on the root; smaller s loses ring parallelism.
+  int                     leafSpan = 9;
   uint64_t                nBigLeaf = 0;  int64_t maxBigW = 0;
   double                  tBigGather = 0, tBigInv = 0, tBigScatter = 0;
   uint64_t                nNode;
@@ -118,7 +115,7 @@ public:
   // Window copy-scale:  Dst[i0:i1, j0:j1] = alpha * Src[same window].
   // Both share one layout, so the local bands coincide; pure local kernel.
   ///////////////////////////////////////////////////////////////////////////
-  void WindowCopyScale(ComplexD alpha,
+  void WindowCopyScale(DenseInverseScalar alpha,
                        BlockCyclicMatrix &Src, BlockCyclicMatrix &Dst,
                        int64_t i0, int64_t i1, int64_t j0, int64_t j1)
   {
@@ -130,8 +127,8 @@ public:
     L.ColRange(j0,j1, lj0,lj1);
     int64_t m = li1-li0, n = lj1-lj0;
     if ( !(m && n) ) return;
-    ComplexD *src = Src.LocalWindow(li0,lj0);
+    DenseInverseScalar *src = Src.LocalWindow(li0,lj0);
-    ComplexD *dst = Dst.LocalWindow(li0,lj0);
+    DenseInverseScalar *dst = Dst.LocalWindow(li0,lj0);
     int64_t ldS = Src.layout.mloc;
     int64_t ldD = L.mloc;
     tCopy -= usecond();
@@ -167,10 +164,10 @@ public:
     GRID_ASSERT( lc1-lc0 == w );
 
     // Pack the strided block dense (inverseBatched assumes lda == w).
-    deviceVector<ComplexD> dense((uint64_t)w*w);
+    deviceVector<DenseInverseScalar> dense((uint64_t)w*w);
     {
-      ComplexD *src = A.LocalWindow(lr0,lc0);
+      DenseInverseScalar *src = A.LocalWindow(lr0,lc0);
-      ComplexD *dst = &dense[0];
+      DenseInverseScalar *dst = &dense[0];
       int64_t   ld  = L.mloc;
       accelerator_for(idx, (uint64_t)(w*w), 1, {
         int64_t jj = idx / w;
@@ -179,15 +176,15 @@ public:
       });
     }
     {
-      deviceVector<ComplexD*> bp(1);
+      deviceVector<DenseInverseScalar*> bp(1);
-      std::vector<ComplexD*>  ptr(1);
+      std::vector<DenseInverseScalar*>  ptr(1);
       ptr[0] = &dense[0];
-      acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD*));
+      acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(DenseInverseScalar*));
       INV.inverseBatched(w, bp);
     }
     {
-      ComplexD *src = &dense[0];
+      DenseInverseScalar *src = &dense[0];
-      ComplexD *dst = A.LocalWindow(lr0,lc0);
+      DenseInverseScalar *dst = A.LocalWindow(lr0,lc0);
       int64_t   ld  = L.mloc;
       accelerator_for(idx, (uint64_t)(w*w), 1, {
         int64_t jj = idx / w;
@@ -197,8 +194,8 @@ public:
     }
     // Growth telemetry, local only.
     {
-      std::vector<ComplexD> h((uint64_t)w*w);
+      std::vector<DenseInverseScalar> h((uint64_t)w*w);
-      acceleratorCopyFromDevice(&dense[0], &h[0], h.size()*sizeof(ComplexD));
+      acceleratorCopyFromDevice(&dense[0], &h[0], h.size()*sizeof(DenseInverseScalar));
       double mx = 0.0;
       for(auto &z : h){
         double re=z.real(), im=z.imag();
@@ -218,7 +215,11 @@ public:
   // SendToRecvFrom with root (symmetric byte count: the reverse direction
   // carries a same-size dummy -- a leaf-local cost, accepted for simplicity).
   ///////////////////////////////////////////////////////////////////////////
-  static int64_t FirstBlock(int64_t b0, int p, int Pg){ int64_t r = ((b0 % Pg) <= p) ? b0 - (b0 % Pg) + p : b0 - (b0 % Pg) + Pg + p; return r; }
+  static int64_t FirstBlock(int64_t b0, int p, int Pg)
+  {
+    int64_t r = ((b0 % Pg) <= p) ? b0 - (b0 % Pg) + p : b0 - (b0 % Pg) + Pg + p;
+    return r;
+  }
   void BigLeaf(BlockCyclicMatrix &A, int64_t b0, int64_t b1)
   {
     GRID_TRACE("SchurBigLeaf");
@@ -235,23 +236,23 @@ public:
     int64_t lr0,lr1,lc0,lc1; L.RowRange(c0,c1,lr0,lr1); L.ColRange(c0,c1,lc0,lc1);
     const int64_t mq = lr1-lr0, nq = lc1-lc0;
 
-    deviceVector<ComplexD> dense;            // root only: W x W column major
+    deviceVector<DenseInverseScalar> dense;            // root only: W x W column major
-    deviceVector<ComplexD> piece, dummy;     // piece: my mq x nq contiguous; dummy: reverse-direction filler
+    deviceVector<DenseInverseScalar> piece, dummy;     // piece: my mq x nq contiguous; dummy: reverse-direction filler
     if ( me == root ) dense.resize((uint64_t)W*W);
 
-    auto pack_piece = [&](ComplexD *dst, int64_t m, int64_t n, int64_t r0, int64_t cc0){
+    auto pack_piece = [&](DenseInverseScalar *dst, int64_t m, int64_t n, int64_t r0, int64_t cc0){
-      ComplexD *src = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
+      DenseInverseScalar *src = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
       accelerator_for(idx,(uint64_t)(m*n),1,{ int64_t jj=idx/m, ii=idx-jj*m; dst[ii+jj*m] = src[ii+jj*ld]; });
     };
-    auto unpack_piece = [&](ComplexD *src, int64_t m, int64_t n, int64_t r0, int64_t cc0){
+    auto unpack_piece = [&](DenseInverseScalar *src, int64_t m, int64_t n, int64_t r0, int64_t cc0){
-      ComplexD *dst = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
+      DenseInverseScalar *dst = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
       accelerator_for(idx,(uint64_t)(m*n),1,{ int64_t jj=idx/m, ii=idx-jj*m; dst[ii+jj*ld] = src[ii+jj*m]; });
     };
     // root: piece of rank q  <->  dense, via the closed-form block map
-    auto root_place = [&](ComplexD *pc, int64_t m, int64_t n, int q, int to_dense){
+    auto root_place = [&](DenseInverseScalar *pc, int64_t m, int64_t n, int q, int to_dense){
       const int pq=q/Pc, cq=q%Pc;
       const int64_t brq0=FirstBlock(b0,pq,Pr), bcq0=FirstBlock(b0,cq,Pc);
-      ComplexD *dn = &dense[0]; const int64_t WW=W, NB=nb, PR=Pr, PC=Pc, C0=c0;
+      DenseInverseScalar *dn = &dense[0]; const int64_t WW=W, NB=nb, PR=Pr, PC=Pc, C0=c0;
       accelerator_for(idx,(uint64_t)(m*n),1,{
         int64_t jj=idx/m, ii=idx-jj*m;
         int64_t gr = (brq0 + (ii/NB)*PR)*NB + ii%NB - C0;
@@ -269,49 +270,40 @@ public:
     tBigGather -= usecond();
     if ( mq*nq ) { piece.resize((uint64_t)mq*nq); dummy.resize((uint64_t)mq*nq); pack_piece(&piece[0],mq,nq,lr0,lc0); accelerator_barrier(); }
     if ( me == root ) {
-      deviceVector<ComplexD> stage;
+      deviceVector<DenseInverseScalar> stage;
       for(int q=0;q<Pr*Pc;q++){
         int64_t m,n; piece_dims(q,m,n); if ( !(m*n) ) continue;
         if ( q == root ) { root_place(&piece[0],m,n,q,1); continue; }
         if ( stage.size() < (uint64_t)(m*n) ) stage.resize((uint64_t)m*n);
-        deviceVector<ComplexD> junk((uint64_t)m*n);
+        deviceVector<DenseInverseScalar> junk((uint64_t)m*n);
-        grid->SendToRecvFrom((void *)&junk[0], q, (void *)&stage[0], q, (uint64_t)m*n*sizeof(ComplexD));
+        grid->SendToRecvFrom((void *)&junk[0], q, (void *)&stage[0], q, (uint64_t)m*n*sizeof(DenseInverseScalar));
         root_place(&stage[0],m,n,q,1);
       }
       accelerator_barrier();
     } else if ( mq*nq ) {
-      grid->SendToRecvFrom((void *)&piece[0], root, (void *)&dummy[0], root, (uint64_t)mq*nq*sizeof(ComplexD));
+      grid->SendToRecvFrom((void *)&piece[0], root, (void *)&dummy[0], root, (uint64_t)mq*nq*sizeof(DenseInverseScalar));
     }
     tBigGather += usecond();
 
-    // ---- invert on root ----
+    // ---- invert on root: blocked getrf_64 + identity getrs_64 ----
     tBigInv -= usecond();
-    if ( leafLU < 0 ) leafLU = getenv("SCHUR2D_LEAF_LU") ? atoi(getenv("SCHUR2D_LEAF_LU")) : 0;
+    if ( me == root ) INV.inverseLU(W, &dense[0]);
-    if ( me == root ) {
-      if ( leafLU ) {
-        INV.inverseLU(W, &dense[0]);
-      } else {
-        deviceVector<ComplexD*> bp(1); std::vector<ComplexD*> ptr(1); ptr[0] = &dense[0];
-        acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD*));
-        INV.inverseBatched(W, bp);
-      }
-    }
     tBigInv += usecond();
 
     // ---- scatter ----
     tBigScatter -= usecond();
     if ( me == root ) {
-      deviceVector<ComplexD> stage;
+      deviceVector<DenseInverseScalar> stage;
       for(int q=0;q<Pr*Pc;q++){
         int64_t m,n; piece_dims(q,m,n); if ( !(m*n) ) continue;
         if ( q == root ) { root_place(&piece[0],m,n,q,0); accelerator_barrier(); continue; }
         if ( stage.size() < (uint64_t)(m*n) ) stage.resize((uint64_t)m*n);
-        deviceVector<ComplexD> junk((uint64_t)m*n);
+        deviceVector<DenseInverseScalar> junk((uint64_t)m*n);
         root_place(&stage[0],m,n,q,0); accelerator_barrier();
-        grid->SendToRecvFrom((void *)&stage[0], q, (void *)&junk[0], q, (uint64_t)m*n*sizeof(ComplexD));
+        grid->SendToRecvFrom((void *)&stage[0], q, (void *)&junk[0], q, (uint64_t)m*n*sizeof(DenseInverseScalar));
       }
     } else if ( mq*nq ) {
-      grid->SendToRecvFrom((void *)&dummy[0], root, (void *)&piece[0], root, (uint64_t)mq*nq*sizeof(ComplexD));
+      grid->SendToRecvFrom((void *)&dummy[0], root, (void *)&piece[0], root, (uint64_t)mq*nq*sizeof(DenseInverseScalar));
     }
     if ( mq*nq ) { unpack_piece(&piece[0],mq,nq,lr0,lc0); accelerator_barrier(); }
     tBigScatter += usecond();
@@ -330,8 +322,7 @@ public:
     int64_t span = b1-b0;
     GRID_ASSERT( span >= 1 );
     if ( span == 1 ) { Leaf(A, b0); return; }
-    if ( leafSpan < 0 ) leafSpan = getenv("SCHUR2D_LEAF_SPAN") ? atoi(getenv("SCHUR2D_LEAF_SPAN")) : 1;
+    if ( span <= leafSpan && span < std::min(L.Pr,L.Pc) ) { BigLeaf(A, b0, b1); return; }
-    if ( span <= leafSpan ) { BigLeaf(A, b0, b1); return; }
     GRID_TRACE("SchurNode");
     nNode++;
 
@@ -339,7 +330,7 @@ public:
     int64_t c0 = b0*L.nb;
     int64_t m  = bm*L.nb;
     int64_t c1 = std::min(L.N, b1*L.nb);
-    ComplexD one (1.0,0.0), mone(-1.0,0.0), zero(0.0,0.0);
+    DenseInverseScalar one (1.0,0.0), mone(-1.0,0.0), zero(0.0,0.0);
 
     // 1. A11 -> A11inv
     SchurNode(A,Bt,Ct,Tt,Ut, b0,bm);
@@ -376,150 +367,11 @@ public:
   // PUBLIC ENTRY.  In-place inverse of the whole matrix.  Scratch (4x the
   // matrix footprint) is allocated here and released on return.
   ///////////////////////////////////////////////////////////////////////////
-  ///////////////////////////////////////////////////////////////////////////
-  // POWER - CLOCK - GROUND.  Before the inverse runs, print the preconditions
-  // that have differed between fast (62 s) and slow (133-141 s) runs of the
-  // SAME inverse, and measure ONE SendToRecvFrom to the ACTUAL ring partners
-  // at three sizes, device and host buffers.  Same code in the harness and in
-  // production, so the two processes are compared on the identical primitive
-  // before any explanation of the SUMMA rings is attempted.
-  // SCHUR2D_PROBE=0 disables (costs ~0.1-0.5 s).
-  ///////////////////////////////////////////////////////////////////////////
-  void Probe(BlockCyclicMatrix &A)
-  {
-    BlockCyclicLayout &L = A.layout;
-    GridBase *grid = A.grid;
-    int me = grid->ThisRank();
-    // --- banner ---
-    int thr = -1;
-#ifdef GRID_COMMS_MPI3
-    MPI_Query_thread(&thr);
-#endif
-    const char *omp = getenv("OMP_NUM_THREADS");
-    MemoryStatus ms = MemoryManager::GetFootprint();
-    std::cout << GridLogMessage << "Schur2D PROBE banner: MPI thread level " << thr
-              << " (0 single,1 funneled,2 serialized,3 multiple)  OMP_NUM_THREADS=" << (omp?omp:"unset")
-              << "  MemoryManager device bytes " << ms.DeviceBytes/1.0e9 << " GB (LRU " << ms.DeviceLRUBytes/1.0e9
-              << " GB, cap " << ms.DeviceMaxBytes/1.0e9 << " GB)"
-              << "  grid " << L.Pr << "x" << L.Pc << " nb " << L.nb << std::endl;
-#ifdef GRID_HIP
-    if ( me==0 ) acceleratorMem();
-#endif
-    // --- ring partners exactly as SUMMA uses them ---
-    int prow=L.prow, pcol=L.pcol, Pr=L.Pr, Pc=L.Pc;
-    struct Ring { const char *name; int dest, src; };
-    Ring rings[2] = { {"ringA(row, q+-1)", prow*Pc + (pcol+1)%Pc,       prow*Pc + (pcol-1+Pc)%Pc},
-                      {"ringB(col, p+-1)", ((prow+1)%Pr)*Pc + pcol,     ((prow-1+Pr)%Pr)*Pc + pcol} };
-    // 2/3/4 MB added 2026-08-27: the SUMMA histogram put 93% of ring time in
-    // [2,4) MB messages at 0.3 GB/s while >=4 MB ran at 11-13 GB/s.
-    const int NSZ = 6;
-    uint64_t sizes[NSZ] = { 64ull*1024, 1024ull*1024, 2048ull*1024, 3072ull*1024, 4096ull*1024, 8ull*1024*1024 };
-    uint64_t maxb = sizes[NSZ-1];
-    deviceVector<char> dsend(maxb), drecv(maxb);
-    std::vector<char>  hsend(maxb), hrecv(maxb);
-    for(int r=0;r<2;r++){
-      if ( (r==0 && Pc==1) || (r==1 && Pr==1) ) continue;
-      int off = grid->IsOffNode(rings[r].dest);
-      for(int si=0;si<NSZ;si++){
-        uint64_t bytes = sizes[si];
-        // warm one, time five, both memory spaces
-        grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes);
-        double t0=usecond();
-        for(int i=0;i<5;i++) grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes);
-        double td=(usecond()-t0)/5.0;
-        grid->SendToRecvFrom(&hsend[0], rings[r].dest, &hrecv[0], rings[r].src, bytes);
-        t0=usecond();
-        for(int i=0;i<5;i++) grid->SendToRecvFrom(&hsend[0], rings[r].dest, &hrecv[0], rings[r].src, bytes);
-        double th=(usecond()-t0)/5.0;
-        // spread over ranks
-        RealD dmax=td, dmin=-td; grid->GlobalMax(dmax); grid->GlobalMax(dmin); dmin=-dmin;
-        std::cout << GridLogMessage << "Schur2D PROBE " << rings[r].name << (off?" OFF-node":" on-node")
-                  << " " << bytes/1024 << " KB: device " << td << " us (" << bytes/td/1.0e3 << " GB/s) [min/max over ranks " << dmin << "/" << dmax << " us]"
-                  << "  host " << th << " us (" << bytes/th/1.0e3 << " GB/s)" << std::endl;
-      }
-    }
-    ///////////////////////////////////////////////////////////////////////
-    // ONE-GCD LEAF MICROBENCHMARK (boss only).  The big-leaf inverse at
-    // W=4320 measured 0.53 s with BOTH getri_batched and getrf_64+getrs_64
-    // (2026-08-27) -- ~0.4 TF/s on a GCD that runs zgemm at ~15.  Time the
-    // three primitives in isolation on a well-conditioned dense matrix so the
-    // leaf's rate can be compared with the machine's, and getrf split from
-    // getrs.  Sizes: the W of span 4 / 9 / 18 leaves on accelerator builds;
-    // tiny on CPU builds (Eigen would take minutes at 4320).
-    ///////////////////////////////////////////////////////////////////////
-    if ( me == 0 ) {
-#if defined(GRID_HIP) || defined(GRID_CUDA) || defined(GRID_SYCL)
-      std::vector<int64_t> Ws({1920, 4320, 8640});
-#else
-      std::vector<int64_t> Ws({240, 480});
-#endif
-      for(int64_t W : Ws){
-        deviceVector<ComplexD> M((uint64_t)W*W), C((uint64_t)W*W);
-        { ComplexD *m = &M[0]; const int64_t WW = W;   // diagonally dominant: (i==j ? W : 0) + cos/sin noise
-          accelerator_for(idx,(uint64_t)W*W,1,{ int64_t j=idx/WW, i=idx-j*WW; double x=0.37*i+0.61*j;
-            m[idx] = ComplexD((i==j)?(double)WW:0.0,0.0) + ComplexD(std::cos(x),std::sin(1.3*x)); });
-          accelerator_barrier(); }
-        double flopLU = 8.0/3.0*(double)W*W*W;          // complex LU  ~ (4 real flops per complex mult-add) * (2/3 n^3)
-        double flopGEMM = 8.0*(double)W*W*W;            // complex GEMM
-        // 1. getri_batched (batch 1)
-        double tb;
-        { deviceVector<ComplexD*> bp(1); std::vector<ComplexD*> ptr(1); ptr[0]=&M[0];
-          acceleratorCopyToDevice(&ptr[0],&bp[0],sizeof(ComplexD*));
-          double t0=usecond(); INV.inverseBatched(W,bp); tb=usecond()-t0; }
-        // 2. inverseLU (getrf_64 + identity getrs_64), timed inside
-        double tl; { double t0=usecond(); INV.inverseLU(W,&M[0]); tl=usecond()-t0; }
-        // 3. one zgemm W x W x W for the machine rate
-        double tg;
-        { deviceVector<ComplexD*> ap(1),bp(1),cp(1); std::vector<ComplexD*> ptr(1);
-          ptr[0]=&M[0]; acceleratorCopyToDevice(&ptr[0],&ap[0],sizeof(ComplexD*)); acceleratorCopyToDevice(&ptr[0],&bp[0],sizeof(ComplexD*));
-          ptr[0]=&C[0]; acceleratorCopyToDevice(&ptr[0],&cp[0],sizeof(ComplexD*));
-          double t0=usecond();
-          SUMMA.BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,(int)W,(int)W,(int)W,ComplexD(1.0,0.0),ap,(int)W,bp,(int)W,ComplexD(0.0,0.0),cp,(int)W);
-          SUMMA.BLAS.synchronise(); tg=usecond()-t0; }
-        std::cout << GridLogMessage << "Schur2D PROBE leaf W=" << W
-                  << ": getri_batched " << tb/1.0e6 << " s (" << flopLU/tb/1.0e6 << " TF/s)"
-                  << "  getrf_64+getrs_64 " << tl/1.0e6 << " s (getrf " << INV.lastGetrfUs/1.0e6 << " getrs " << INV.lastGetrsUs/1.0e6 << ")"
-                  << "  zgemm " << tg/1.0e6 << " s (" << flopGEMM/tg/1.0e6 << " TF/s)" << std::endl;
-      }
-    }
-    ///////////////////////////////////////////////////////////////////////
-    // The SUMMA's conditions, one at a time, at 8 MB on ring B:
-    //  (a) LARGE persistent buffers (the rings use ~0.5 GB Abuf/Bbuf), sending
-    //      from offset 0 and from deep inside the region;
-    //  (b) a pack kernel + accelerator_barrier immediately before each
-    //      message, as the SUMMA does.
-    // Isolated 8 MB messages ran at 11-20 GB/s while the SUMMA averaged 2.1;
-    // whichever variant drops to ~2 GB/s names the condition.
-    ///////////////////////////////////////////////////////////////////////
-    {
-      int r = (Pr>1) ? 1 : 0;
-      uint64_t bytes = sizes[NSZ-1];
-      uint64_t big = 512ull*1024*1024;
-      deviceVector<char> bsend(big), brecv(big);
-      for(int variant=0; variant<3; variant++){
-        uint64_t so = (variant==1) ? big-bytes : 0;               // deep offset in the large region
-        char *sp=&bsend[so], *rp=&brecv[so];
-        grid->SendToRecvFrom(sp, rings[r].dest, rp, rings[r].src, bytes);
-        double t0=usecond();
-        for(int i=0;i<5;i++){
-          if ( variant==2 ) { accelerator_for(k, bytes/8, 1, { ((uint64_t *)sp)[k] = (uint64_t)k; }); accelerator_barrier(); }
-          grid->SendToRecvFrom(sp, rings[r].dest, rp, rings[r].src, bytes);
-        }
-        double t=(usecond()-t0)/5.0;
-        RealD tmax=t, tmin=-t; grid->GlobalMax(tmax); grid->GlobalMax(tmin); tmin=-tmin;
-        const char *vn[3]={"512MB buffer, offset 0","512MB buffer, offset 504MB","pack kernel + barrier before each send"};
-        std::cout << GridLogMessage << "Schur2D PROBE " << rings[r].name << " 8192 KB device, " << vn[variant] << ": "
-                  << t << " us (" << bytes/t/1.0e3 << " GB/s) [min/max over ranks " << tmin << "/" << tmax << " us]" << std::endl;
-      }
-    }
-  }
-
   void Invert(BlockCyclicMatrix &A)
   {
     BlockCyclicLayout &L = A.layout;
     GRID_ASSERT( L.N >= 1 );
     int64_t nblocks = (L.N + L.nb - 1)/L.nb;
-    if ( !(getenv("SCHUR2D_PROBE") && atoi(getenv("SCHUR2D_PROBE"))==0) ) Probe(A);
 
     BlockCyclicMatrix Bt(A.grid, L.N, L.nb, L.Pr, L.Pc);
     BlockCyclicMatrix Ct(A.grid, L.N, L.nb, L.Pr, L.Pc);
@@ -550,8 +402,8 @@ public:
               << std::endl;
     if ( nBigLeaf ) {
       RealD ti = tBigInv/1.0e6; grid->GlobalMax(ti);     // inverse runs on the root of each leaf: report the max over ranks
-      std::cout << GridLogMessage << "BlockCyclicSchurInverse: BIG LEAVES (SCHUR2D_LEAF_SPAN=" << leafSpan
+      std::cout << GridLogMessage << "BlockCyclicSchurInverse: BIG LEAVES (span " << leafSpan
-                << (leafLU>0 ? ", SCHUR2D_LEAF_LU=1: getrf_64+getrs_64" : ", getri_batched") << "): " << nBigLeaf
+                << ", inverseLU): " << nBigLeaf
                 << " leaves, max W " << maxBigW
                 << "  boss secs: gather " << tBigGather/1.0e6 << " scatter " << tBigScatter/1.0e6
                 << "  inverse (max over ranks) " << ti << std::endl;
@@ -576,8 +428,7 @@ public:
     // same sequence of sizes).  Time is wall time inside SendToRecvFrom, so it
     // includes waiting for the partner -- a bucket whose GB/s is far below the
     // probe's for the same size is wait, not wire.
-    std::cout << GridLogMessage << "BlockCyclicSumma ring histogram (boss):  size-bucket  msgs  GB  xfer-secs  GB/s  %time"
+    std::cout << GridLogMessage << "BlockCyclicSumma ring histogram (boss):  size-bucket  msgs  GB  xfer-secs  GB/s  %time" << std::endl;
-              << (SUMMA.handshake>0 ? "  handshake-secs (partner wait, excluded from xfer)" : "") << std::endl;
     std::streamsize oldprec = std::cout.precision();
     for(int b=0;b<SUMMA.NHIST;b++){
       if ( !SUMMA.histN[b] ) continue;
@@ -590,7 +441,6 @@ public:
                 << std::setw(9) << std::setprecision(3) << sec
                 << std::setw(9) << std::setprecision(3) << (sec>0 ? g/sec : 0.0)
                 << std::setw(8) << std::setprecision(3) << (ring>0 ? 100.0*sec/ring : 0.0);
-      if ( SUMMA.handshake>0 ) std::cout << std::setw(12) << std::setprecision(3) << SUMMA.histHsUs[b]/1.0e6;
       std::cout << std::endl;
     }
     std::cout.precision(oldprec);

Grid/algorithms/multigrid/BlockCyclicSumma.h

@@ -71,7 +71,7 @@ class BlockCyclicMatrix
 public:
   GridBase              *grid;      // borrowed, never owned
   BlockCyclicLayout      layout;
-  deviceVector<ComplexD> data;      // column major, ld = layout.mloc
+  deviceVector<DenseInverseScalar> data;      // column major, ld = layout.mloc
 
   BlockCyclicMatrix(GridBase *g, int64_t N, int64_t nb, int Pr, int Pc)
     : grid(g),
@@ -82,7 +82,7 @@ public:
     data.resize( sz ? sz : 1 );
   }
 
-  ComplexD *LocalWindow(int64_t li, int64_t lj)
+  DenseInverseScalar *LocalWindow(int64_t li, int64_t lj)
   {
     return &data[0] + li + lj*layout.mloc;
   }
@@ -93,32 +93,32 @@ public:
   // oracle only; production data enters through the direct block-cyclic
   // import, never through these.
   /////////////////////////////////////////////////////////////////////////
-  void ImportGlobal(const std::vector<ComplexD> &G)
+  void ImportGlobal(const std::vector<DenseInverseScalar> &G)
   {
     int64_t N = layout.N;
     GRID_ASSERT( (int64_t)G.size() == N*N );
-    std::vector<ComplexD> h((uint64_t)layout.mloc*layout.nloc, ComplexD(0.0,0.0));
+    std::vector<DenseInverseScalar> h((uint64_t)layout.mloc*layout.nloc, DenseInverseScalar(0.0,0.0));
     for(int64_t j=0;j<N;j++){
       for(int64_t i=0;i<N;i++){
         if ( layout.Owns(i,j) ) h[layout.LocalOffset(i,j)] = G[i + j*N];
       }
     }
     if ( h.size() )
-      acceleratorCopyToDevice(&h[0], &data[0], h.size()*sizeof(ComplexD));
+      acceleratorCopyToDevice(&h[0], &data[0], h.size()*sizeof(DenseInverseScalar));
   }
-  void ExportGlobal(std::vector<ComplexD> &G)
+  void ExportGlobal(std::vector<DenseInverseScalar> &G)
   {
     int64_t N = layout.N;
-    G.assign((uint64_t)N*N, ComplexD(0.0,0.0));
+    G.assign((uint64_t)N*N, DenseInverseScalar(0.0,0.0));
-    std::vector<ComplexD> h((uint64_t)layout.mloc*layout.nloc);
+    std::vector<DenseInverseScalar> h((uint64_t)layout.mloc*layout.nloc);
     if ( h.size() )
-      acceleratorCopyFromDevice(&data[0], &h[0], h.size()*sizeof(ComplexD));
+      acceleratorCopyFromDevice(&data[0], &h[0], h.size()*sizeof(DenseInverseScalar));
     for(int64_t j=0;j<N;j++){
       for(int64_t i=0;i<N;i++){
         if ( layout.Owns(i,j) ) G[i + j*N] = h[layout.LocalOffset(i,j)];
       }
     }
-    if ( N ) grid->GlobalSumVector((ComplexD *)&G[0], (int)(N*N)); // zero-fill: exact
+    if ( N ) grid->GlobalSumVector((DenseInverseScalar *)&G[0], (int)(N*N)); // zero-fill: exact
   }
 };
 
@@ -139,31 +139,27 @@ public:
   // per message: measured 1.26 GB/s/rank on 13 MB ring messages (production,
   // GRID_ALLOC_NCACHE_LARGE=64) against 62 s total for the same inverse when
   // hipMalloc returned a stable address.
-  deviceVector<ComplexD> Abuf;
+  deviceVector<DenseInverseScalar> Abuf;
-  deviceVector<ComplexD> Bbuf;
+  deviceVector<DenseInverseScalar> Bbuf;
   double   tAlloc=0, tPack=0, tRingA=0, tRingB=0, tGemm=0;
   uint64_t bytesRing=0, nRingMsg=0, nMultiply=0, nGemm=0;
   // Per-message-size histogram (bucket = floor(log2 bytes)): count, bytes,
   // microseconds -- decomposes the ring time by packet size so a low average
   // GB/s can be attributed (many small latency-bound messages vs slow large
-  // ones vs partner-wait).  The 8 MB probe runs at 11-20 GB/s; SUMMA averaged 2.
+  // ones vs partner-wait).  Time is wall time inside SendToRecvFrom, so a
+  // bucket far below the wire rate for its size is wait, not wire.
   static const int NHIST=48;
-  uint64_t histN[NHIST]={0}, histBytes[NHIST]={0}; double histUs[NHIST]={0}, histHsUs[NHIST]={0};
+  uint64_t histN[NHIST]={0}, histBytes[NHIST]={0}; double histUs[NHIST]={0};
-  // SUMMA_HANDSHAKE=1: a 4-byte SendToRecvFrom with the same partner
+  void HistAdd(uint64_t bytes, double us){ int b=0; while((bytes>>b)>1) b++; histN[b]++; histBytes[b]+=bytes; histUs[b]+=us; }
-  // immediately before each ring message, timed separately (histHsUs).
+  void ResetTelemetry(void){ tAlloc=tPack=tRingA=tRingB=tGemm=0; bytesRing=nRingMsg=nMultiply=nGemm=0; for(int b=0;b<NHIST;b++){histN[b]=histBytes[b]=0; histUs[b]=0;} }
-  // Handshake time = partner-arrival skew; the remainder = transfer.  Splits
-  // the [2,4) MB bucket's 12 ms/msg (2026-08-27) into wait vs wire.
-  int handshake = -1; int hsTx=0, hsRx=0;
-  void HistAdd(uint64_t bytes, double us, double hs=0.0){ int b=0; while((bytes>>b)>1) b++; histN[b]++; histBytes[b]+=bytes; histUs[b]+=us; histHsUs[b]+=hs; }
-  void ResetTelemetry(void){ tAlloc=tPack=tRingA=tRingB=tGemm=0; bytesRing=nRingMsg=nMultiply=nGemm=0; for(int b=0;b<NHIST;b++){histN[b]=histBytes[b]=0; histUs[b]=histHsUs[b]=0;} }
 
   static int Overlap(int64_t a0,int64_t a1,int64_t b0,int64_t b1)
   { return (a0 < b1) && (b0 < a1); }
 
-  void Multiply(ComplexD alpha,
+  void Multiply(DenseInverseScalar alpha,
                 BlockCyclicMatrix &A,
                 BlockCyclicMatrix &B,
-                ComplexD beta,
+                DenseInverseScalar beta,
                 BlockCyclicMatrix &C,
                 int64_t i0, int64_t i1,
                 int64_t j0, int64_t j1,
@@ -219,14 +215,13 @@ public:
     const uint64_t slotB1 = (uint64_t)nb*nloc_j;         // one panel
     const uint64_t slotB  = (uint64_t)S*slotB1;
     nMultiply++;
-    if ( handshake < 0 ) handshake = getenv("SUMMA_HANDSHAKE") ? atoi(getenv("SUMMA_HANDSHAKE")) : 0;
     tAlloc -= usecond();
     if ( Abuf.size() < std::max<uint64_t>(slotA*Pc,1) ) Abuf.resize( std::max<uint64_t>(slotA*Pc,1) );
     if ( Bbuf.size() < std::max<uint64_t>(slotB*Pr,1) ) Bbuf.resize( std::max<uint64_t>(slotB*Pr,1) );
     tAlloc += usecond();
 
-    deviceVector<ComplexD *> ap(1), bp(1), cp(1);
+    deviceVector<DenseInverseScalar *> ap(1), bp(1), cp(1);
-    std::vector<ComplexD *>  ptr(1);
+    std::vector<DenseInverseScalar *>  ptr(1);
 
     int firstblock = 1;
     for(int64_t r0=kb0; r0<kb1; r0+=Pc){       // rounds of Pc k-blocks
@@ -243,8 +238,8 @@ public:
           int64_t lc0, lc1;
           L.ColRange(s*nb, std::min(N,(s+1)*nb), lc0, lc1);
           GRID_ASSERT( lc1-lc0 == nb_s );
-          ComplexD *src = A.LocalWindow(li0, lc0);
+          DenseInverseScalar *src = A.LocalWindow(li0, lc0);
-          ComplexD *dst = &Abuf[0] + slotA*pcol;
+          DenseInverseScalar *dst = &Abuf[0] + slotA*pcol;
           int64_t   ld  = A.layout.mloc;
           int64_t   m   = mloc_i;
           accelerator_for(idx, (uint64_t)(m*nb_s), 1, {
@@ -259,8 +254,8 @@ public:
           GRID_ASSERT( lr1-lr0 == nb_s );
           int64_t idxs = (s - r0 - ((prow - r0%Pr + Pr) % Pr)) / Pr; // my panel # in round
           GRID_ASSERT( idxs >= 0 ); GRID_ASSERT( idxs < S );
-          ComplexD *src = B.LocalWindow(lr0, lj0);
+          DenseInverseScalar *src = B.LocalWindow(lr0, lj0);
-          ComplexD *dst = &Bbuf[0] + slotB*prow + slotB1*idxs;
+          DenseInverseScalar *dst = &Bbuf[0] + slotB*prow + slotB1*idxs;
           int64_t   ld  = B.layout.mloc;
           int64_t   nn  = nloc_j;
           accelerator_for(idx, (uint64_t)(nb_s*nn), 1, {
@@ -286,13 +281,12 @@ public:
         for(int t=1;t<Pc;t++){
           int cs = (pcol - t + 1 + Pc*Pc) % Pc;
           int cr = (pcol - t     + Pc*Pc) % Pc;
-          double ths = 0.0, tm = usecond();
+          double tm = usecond();
-          if ( handshake ) { grid->SendToRecvFrom((void *)&hsTx, dest, (void *)&hsRx, src, sizeof(int)); ths = usecond()-tm; tm = usecond(); }
           grid->SendToRecvFrom((void *)(&Abuf[0]+slotA*cs), dest,
                                (void *)(&Abuf[0]+slotA*cr), src,
-                               slotA*sizeof(ComplexD));
+                               slotA*sizeof(DenseInverseScalar));
-          HistAdd(slotA*sizeof(ComplexD), usecond()-tm, ths);
+          HistAdd(slotA*sizeof(DenseInverseScalar), usecond()-tm);
-          bytesRing += slotA*sizeof(ComplexD); nRingMsg++;
+          bytesRing += slotA*sizeof(DenseInverseScalar); nRingMsg++;
         }
         tRingA += usecond();
       }
@@ -307,13 +301,12 @@ public:
         for(int t=1;t<Pr;t++){
           int rs = (prow - t + 1 + Pr*Pr) % Pr;
           int rr = (prow - t     + Pr*Pr) % Pr;
-          double ths = 0.0, tm = usecond();
+          double tm = usecond();
-          if ( handshake ) { grid->SendToRecvFrom((void *)&hsTx, dest, (void *)&hsRx, src, sizeof(int)); ths = usecond()-tm; tm = usecond(); }
           grid->SendToRecvFrom((void *)(&Bbuf[0]+slotB*rs), dest,
                                (void *)(&Bbuf[0]+slotB*rr), src,
-                               slotB*sizeof(ComplexD));
+                               slotB*sizeof(DenseInverseScalar));
-          HistAdd(slotB*sizeof(ComplexD), usecond()-tm, ths);
+          HistAdd(slotB*sizeof(DenseInverseScalar), usecond()-tm);
-          bytesRing += slotB*sizeof(ComplexD); nRingMsg++;
+          bytesRing += slotB*sizeof(DenseInverseScalar); nRingMsg++;
         }
         tRingB += usecond();
       }
@@ -329,15 +322,15 @@ public:
         int cA = (int)(s%Pc);
         int rB = (int)(s%Pr);
         int64_t idxs = (s - r0 - ((rB - r0%Pr + Pr) % Pr)) / Pr;
-        ComplexD beta_use = firstblock ? beta : ComplexD(1.0,0.0);
+        DenseInverseScalar beta_use = firstblock ? beta : DenseInverseScalar(1.0,0.0);
         firstblock = 0;
 
         ptr[0] = &Abuf[0] + slotA*cA;
-        acceleratorCopyToDevice(&ptr[0], &ap[0], sizeof(ComplexD *));
+        acceleratorCopyToDevice(&ptr[0], &ap[0], sizeof(DenseInverseScalar *));
         ptr[0] = &Bbuf[0] + slotB*rB + slotB1*idxs;
-        acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD *));
+        acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(DenseInverseScalar *));
         ptr[0] = C.LocalWindow(li0, lj0);
-        acceleratorCopyToDevice(&ptr[0], &cp[0], sizeof(ComplexD *));
+        acceleratorCopyToDevice(&ptr[0], &cp[0], sizeof(DenseInverseScalar *));
 
         BLAS.gemmBatched(GridBLAS_OP_N, GridBLAS_OP_N,
                          (int)mloc_i, (int)nloc_j, (int)nb_s,

Grid/algorithms/multigrid/DenseCoarseMatrix.h

@@ -29,7 +29,6 @@ Author: Peter Boyle <pboyle@bnl.gov>
 
 #include <Grid/algorithms/blas/BatchedBlas.h>
 #include <Grid/algorithms/blas/BatchedInverse.h>
-#include <Grid/algorithms/multigrid/RecursiveSchurInverse.h>
 #include <Grid/algorithms/multigrid/BlockCyclicSchurInverse.h>
 #include <Grid/algorithms/multigrid/BlockCyclicRedistribute.h>
 
@@ -39,48 +38,34 @@ NAMESPACE_BEGIN(Grid);
 
 //////////////////////////////////////////////////////////////////////////////////////
 // DenseCoarseMatrix: a coarsened operator treated as a DENSE matrix -- explicit,
-// row-distributed A^{-1} of a GeneralCoarsenedMatrix.  Library-grade successor of
+// row-distributed A^{-1} of a coarsened operator.
-// the example-local DistributedDenseInverse (Example_pvdagm_mrhs_3level_dense.cc,
-// FROZEN as the regression baseline).
 //
-// What is new versus the example class:
 //  - Stencil -> dense DIRECT IMPORT.  The coarse operator IS the dense matrix
 //    unrolled: Dense[(s,a),(s+shift_p,b)] += A[p][s]_{a,b}.  Rows of my sites are
-//    assembled from purely LOCAL _A[p] data: no operator applies, no comms -- the
+//    assembled from purely LOCAL _A[p] data: no operator applies, no comms.
-//    O(N) probe assembly (93 s at N=69120) is retired.  ACCUMULATE (+=) because on
+//    ACCUMULATE (+=) because on short axes distinct shifts wrap to the same
-//    short axes distinct shifts wrap to the same neighbour.  An IMPORT CERTIFICATE
+//    neighbour.  An IMPORT CERTIFICATE compares the dense apply against Op.M on a
-//    compares the dense apply against Op.M on a NON-CONSTANT vector (a constant one
+//    NON-CONSTANT vector (a constant one cannot see a shift-sign error).
-//    cannot see a shift-sign error); DENSE_IMPORT_SIGN=-1 flips the convention
+//
-//    without recompiling.
+//  - Inversion is END-TO-END fp64 through the 2D block-cyclic recursive Schur
+//    complement (BlockCyclicSchurInverse): fp64 rank-major import ->
+//    RowsToCyclic -> in-place recursion (pure point-to-point SUMMA rings and
+//    local leaves; bitwise reproducible) -> CyclicToRows -> ONE terminal
+//    rounding into the fp32 apply slab.  Distributed at every N and P.
+//
 //  - Split-K apply through GridBLAS.gemmBatched with EXPLICIT leading dimensions
 //    (arXiv:2409.03904 fig 11): the tiny-output/huge-K GEMM Y = slab^T X becomes
-//    DENSE_SPLITK chunk-GEMMs by pointer offset into the resident slab (lda = N),
+//    SPLITK chunk-GEMMs by pointer offset into the resident slab (lda = N),
-//    partials reduced in one accelerator_for.  Platform-agnostic: deviceVector +
+//    partials reduced in one accelerator_for.  The source vector is assembled by
-//    GridBLAS run the SAME code on HIP/CUDA/SYCL and CPU(Eigen).
+//    a cartesian ring ALLGATHER of device buffers (pure P2P; ~8x fewer bytes
-//  - deviceVector everywhere in the apply path; the ONE surviving naked-HIP block
+//    than a padded allreduce, and no collective size cliffs).  Platform-agnostic:
-//    is the boss inversion buffer (quarantined below, documented).
+//    deviceVector + GridBLAS run the SAME code on HIP/CUDA/SYCL and CPU(Eigen).
 //
-// Setup: SLAB_FILE=<stem> loads per-rank <stem>.<rank> (header-guarded N/nrows/
+// VERIFY ||A Ainv x - x||/||x|| certifies the DEVICE slab + split-K path at the
-// nbasis -- the interchange format shared with the frozen example; the STEM must
+// end of Import, since the single-RHS apply routes through the same core.
-// encode cfg/mass/blocking/nbasis, only the header is guarded).  Absent: direct
-// import -> import certificate -> chunked zero-fill+GlobalSum gather streamed to
-// the boss GCD -> cgetrf_64 (ILP64) -> rows of A^{-1} via blocked identity
-// cgetrs_64 + broadcast, each rank keeping the rows of its own sites -> save.
-// VERIFY ||A Ainv x - x||/||x|| runs in BOTH paths (and now certifies the DEVICE
-// slab + split-K path, since the single-RHS apply routes through the same core).
 //
-// Env: SLAB_FILE  DENSE_SPLITK (default 32, snapped to a divisor of N)
+// Tensor-depth agnostic: site scalar objects treated as contiguous coarse
-//      DENSE_DEVICE_SUM  DENSE_IMPORT_SIGN  DENSE_APPLY_PROFILE  DENSE_CC_CHECK
+// scalars (ComplexF or ComplexD, following the coefficient precision)
-//      DENSE_SCHUR (0/absent: single-GCD gather-invert; 1: distributed
-//      recursive Schur; 2: AUDIT -- run BOTH on the same imported A, report
-//      the slab difference, keep the Schur result)  DENSE_PANEL_BYTES
-//
-// The DENSE_SCHUR=1 path is the RecursiveSchurInverse distributed
-// factorisation: it lifts the fp32 N ~ 90k boss-HBM ceiling (the CC-grid
-// 256-rank SIMD cap remains -- separate issue).  Internal only: slab layout,
-// apply path, SLAB_FILE format and VERIFY are identical in every mode.
-//
-// Tensor-depth agnostic: site scalar objects treated as contiguous ComplexD
 // (iScalar wrappers add no data), so any MG level's coarse operator imports.
 //////////////////////////////////////////////////////////////////////////////////////
 //
@@ -101,6 +86,10 @@ public:
   typedef typename vobj::scalar_object  sobj;
   typedef typename CoarseMatrix::vector_object Mvobj;
   typedef typename Mvobj::scalar_object        Msobj;
+  // Scalar of the coarse site objects (ComplexF or ComplexD): the apply slab
+  // is always fp32 and the inversion always fp64, but the SOURCE of both --
+  // the coarse operator's matrix elements -- carries this precision.
+  typedef typename GridTypeMapper<CComplex>::scalar_type CoarseScalar;
 
   GridBase *grid;
   int      nd;
@@ -111,15 +100,15 @@ public:
   std::vector<int64_t>    myGsite;  // global lex site index of my site ss
   std::vector<ComplexF>   slab;     // nrows x N row-major: A during setup, rows of A^{-1} after
 
-  static const int64_t CHUNKROWS = 1024;  // getrs harvest block (trsm efficiency + fewer broadcasts)
   static const int MRHS_MAX = 32;
+  static const int SPLITK   = 32;   // requested split-K chunk count, snapped DOWN to a divisor of N
 
   // Apply machinery: resident slab + persistent buffers + AOT split-K pointers.
   GridBLAS BLAS;
   deviceVector<ComplexF>  dSlab;
   deviceVector<ComplexF>  dX;       // N x MRHS_MAX
   deviceVector<ComplexF>  dY;       // nrows x MRHS_MAX
-  deviceVector<ComplexF>  dG;       // N x MRHS_MAX lex-major staging for the allgather (devSum==4)
+  deviceVector<ComplexF>  dG;       // N x MRHS_MAX lex-major staging for the allgather
   deviceVector<int>       dLex2Rank;// lex index of a process coordinate -> its rank (allgather block order -> row-block order)
   deviceVector<int64_t>   dRm2G;    // rank-major index (rank*nrows + ss*nbasis + b) -> global column (gsite*nbasis + b) of x / the slab
   int                     myLex;
@@ -130,19 +119,25 @@ public:
   std::vector<ComplexF>   hX;
   std::vector<ComplexF>   hY;
   int NK;                           // split-K chunk count (divides N)
-  int devSum;
+
-  double schurAuditRel;             // DENSE_SCHUR=2: rel slab diff single-vs-schur (-1 = not run)
+  // Resident device memory: the apply slab and its staging.  deviceVector is
+  // not evictable, so this counts against the hard budget, not the cache.
+  uint64_t DeviceBytes(void)
+  {
+    return (uint64_t)(dSlab.capacity()+dX.capacity()+dY.capacity()+dG.capacity()+dPartial.capacity())*sizeof(ComplexF)
+         + (uint64_t)dLex2Rank.capacity()*sizeof(int)
+         + (uint64_t)dRm2G.capacity()*sizeof(int64_t);
+  }
 
   DenseCoarseMatrix(GridBase *g)
     : grid(g)
   {
-    GRID_ASSERT( sizeof(sobj)  == nbasis*sizeof(ComplexD) );
+    GRID_ASSERT( sizeof(sobj)  == nbasis*sizeof(CoarseScalar) );
-    GRID_ASSERT( sizeof(Msobj) == nbasis*nbasis*sizeof(ComplexD) );
+    GRID_ASSERT( sizeof(Msobj) == nbasis*nbasis*sizeof(CoarseScalar) );
     nd     = grid->_ndimension;
     N      = grid->gSites() * nbasis;
     lsites = grid->lSites();
     nrows  = (int64_t)lsites * nbasis;
-    schurAuditRel = -1.0;
 
     std::cout << GridLogMessage << "DenseCoarseMatrix: N = " << N
               << " (" << grid->gSites() << " sites x " << nbasis << ")"
@@ -180,47 +175,12 @@ public:
   void Import(CoarseOp &Op)
   {
     double t0 = usecond();
-    ////////////////////////////////////////////////////////////////////
+    {
-    // 0. Slab cache: SLAB_FILE=<stem> -> per-rank raw file <stem>.<rank>.
+      CheckMrhsSlices(Op);    // the batched apply keeps its slices apart
-    //    SAME format as the frozen example (interchange compatible).
-    ////////////////////////////////////////////////////////////////////
-    bool loaded = false;
-    char *sfile = getenv("SLAB_FILE");
-    std::string slabfile;
-    if (sfile) {
-      slabfile = std::string(sfile) + "." + std::to_string(grid->ThisRank());
-      FILE *f = fopen(slabfile.c_str(),"rb");
-      if (f) {
-        int64_t hdr[4] = {0,0,0,0};
-        GRID_ASSERT( fread(hdr,sizeof(int64_t),4,f) == 4 );
-        GRID_ASSERT( hdr[0] == (int64_t)0x44454E5345 );  // magic "DENSE"
-        GRID_ASSERT( hdr[1] == N && hdr[2] == (int64_t)nrows && hdr[3] == (int64_t)nbasis );
-        uint64_t nelem = (uint64_t)nrows * N;
-        GRID_ASSERT( fread(&slab[0], sizeof(ComplexF), nelem, f) == nelem );
-        fclose(f);
-        loaded = true;
-        std::cout << GridLogMessage << "DenseCoarseMatrix: slab loaded from "
-                  << slabfile << " -- skipping import/factor/solve" << std::endl;
-      } else {
-        std::cout << GridLogMessage << "DenseCoarseMatrix: slab cache " << slabfile
-                  << " absent -- full setup, will write it" << std::endl;
-      }
-    }
-    if (!loaded) {
       ImportDense(Op);        // slab <- my rows of A   (LOCAL, no comms)
       ImportCertificate(Op);  // dense apply == Op.M, before inversion
       InvertDense(Op);        // slab <- my rows of A^{-1}
       double t1 = usecond();
-      if (sfile) {
-        FILE *f = fopen(slabfile.c_str(),"wb");
-        GRID_ASSERT(f != nullptr);
-        int64_t hdr[4] = { (int64_t)0x44454E5345, N, (int64_t)nrows, (int64_t)nbasis };
-        GRID_ASSERT( fwrite(hdr,sizeof(int64_t),4,f) == 4 );
-        uint64_t nelem = (uint64_t)nrows * N;
-        GRID_ASSERT( fwrite(&slab[0], sizeof(ComplexF), nelem, f) == nelem );
-        fclose(f);
-        std::cout << GridLogMessage << "DenseCoarseMatrix: slab written to " << slabfile << std::endl;
-      }
       std::cout << GridLogMessage << "DenseCoarseMatrix: import+invert took "
                 << (t1-t0)/1.0e6 << " s" << std::endl;
     }
@@ -233,11 +193,9 @@ public:
       dSlab.resize((uint64_t)nrows*N);
       acceleratorCopyToDevice(&slab[0],&dSlab[0],sbytes);
 
-      // DENSE_SPLITK: requested chunk count, snapped DOWN to a divisor of N.
+      // Split-K chunk count, snapped DOWN to a divisor of N.
-      int req = getenv("DENSE_SPLITK") ? atoi(getenv("DENSE_SPLITK")) : 32;
-      if (req < 1) req = 1;
       NK = 1;
-      for(int j=1;j<=req;j++) if ( (N % j) == 0 ) NK = j;
+      for(int j=1;j<=SPLITK;j++) if ( (N % j) == 0 ) NK = j;
       int64_t Kc = N / NK;
 
       dX.resize((uint64_t)N*MRHS_MAX);
@@ -255,12 +213,7 @@ public:
       for(int j=0;j<NK;j++) h[j] = &dPartial[0] + (uint64_t)j*nrows*MRHS_MAX;  // compact, ldc=nrows
       acceleratorCopyToDevice(&h[0],&cptrs[0],NK*sizeof(ComplexF*));
 
-      devSum = getenv("DENSE_DEVICE_SUM") ? atoi(getenv("DENSE_DEVICE_SUM")) : 0;
+      {
-      const char *sumName[5] = {"host allreduce","DEVICE-buffer allreduce (GPU-aware MPI)",
-                                "DEVICE cartesian ring allreduce (P2P)","DEVICE flat ring allreduce (P2P)",
-                                "DEVICE cartesian ring ALLGATHER (P2P, ~8x fewer bytes than the padded allreduce)"};
-      GRID_ASSERT(devSum>=0 && devSum<=4);
-      if ( devSum==4 ) {
         dG.resize((uint64_t)N*MRHS_MAX);
         // allgather delivers blocks in lexicographic-coordinate order; the row
         // blocks of x are in RANK order.  Same table as BuildRankMajorMap.
@@ -273,8 +226,7 @@ public:
         GRID_ASSERT( l2r[myLex] == grid->ThisRank() );
         // x and the slab columns are in GLOBAL-SITE order (hX[myGsite*nbasis+b]);
         // the gathered blocks are in RANK-MAJOR order (rank*nrows + ss*nbasis + b).
-        // On one rank the two coincide, which is how the laptop passed while
+        // The two coincide only on one rank, so scatter through the inverse map.
-        // Frontier VERIFYed 0.9965 (2026-08-27).  Scatter through the inverse map.
         std::vector<int64_t> g2rm; BuildRankMajorMap(g2rm);
         std::vector<int64_t> rm2g(N); for(int64_t g=0; g<N; g++) rm2g[g2rm[g]] = g;
         for(int ss=0; ss<lsites; ss++) GRID_ASSERT( rm2g[(int64_t)grid->ThisRank()*nrows + (int64_t)ss*nbasis] == myGsite[ss]*nbasis );
@@ -282,8 +234,8 @@ public:
         acceleratorCopyToDevice(&rm2g[0], &dRm2G[0], N*sizeof(int64_t));
       }
       std::cout << GridLogMessage << "DenseCoarseMatrix: slab resident on device ("
-                << sbytes/1024./1024. << " MB/rank), split-K NK=" << NK << " (Kc=" << Kc << "); "
+                << sbytes/1024./1024. << " MB/rank), split-K NK=" << NK << " (Kc=" << Kc
-                << sumName[devSum] << std::endl;
+                << "); DEVICE cartesian ring ALLGATHER (P2P)" << std::endl;
     }
 
     ////////////////////////////////////////////////////////////////////
@@ -291,7 +243,7 @@ public:
     ////////////////////////////////////////////////////////////////////
     {
       Field x(grid); Field y(grid); Field z(grid);
-      x = ComplexD(1.0,0.0);
+      x = CoarseScalar(1.0,0.0);
       double ta = usecond();
       (*this)(x, y);
       double tb = usecond();
@@ -323,27 +275,67 @@ public:
     GRID_ASSERT(mgrid->_ndimension == nd+1);
     int nr = mgrid->_fdimensions[0];
 
+    // Every slice carries the same vector; slice 0 is the answer.  The
+    // slices are not a check here -- see CheckMrhsSlices, which does that
+    // once on a field chosen for the purpose.
+    Field min(mgrid), mout(mgrid);
+    for(int r=0;r<nr;r++) InsertSliceFast(in,min,r,0);
+    Op.M(min,mout);
+    ExtractSliceFast(out,mout,0,0);
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Do the multiRHS slices stay independent?  Slice r carries (r+1) times
+  // one RANDOM field, so linearity says the results scale likewise, and a
+  // slot reading another slot's data is an O(1) error.
+  //
+  // The field is random on purpose.  Run on the output of the inverse, as
+  // this check once was, it measures the cancellation in A applied to
+  // A^-1 x instead: at the production point that is 1.3e-4 in an fp32
+  // sector with an fp32 inversion, so the check then fires on any change
+  // of summation order while saying nothing about slice independence.
+  // Once per import, not per apply.
+  ////////////////////////////////////////////////////////////////////
+  template<class CoarseOp>
+  void CheckMrhsSlices(CoarseOp &Op)
+  {
+    if ( Op.Grid() == grid ) return;               // single rhs: nothing to mix
+
+    GridBase *mgrid = Op.Grid();
+    GRID_ASSERT(mgrid->_ndimension == nd+1);
+    int nr = mgrid->_fdimensions[0];
+    if ( nr < 2 ) return;
+
+    Field in(grid);
+    GridParallelRNG rng(grid); rng.SeedFixedIntegers(std::vector<int>({7,8,9,10}));
+    random(rng,in);
+
     Field min(mgrid), mout(mgrid);
     for(int r=0;r<nr;r++){
       Field scaled(grid);
-      scaled = ComplexD(r+1.0,0.0)*in;
+      scaled = CoarseScalar(r+1.0,0.0)*in;
       InsertSliceFast(scaled,min,r,0);
     }
-
     Op.M(min,mout);
 
-    ExtractSliceFast(out,mout,0,0);
+    Field s0(grid);
+    ExtractSliceFast(s0,mout,0,0);
+    RealD worst=0.0;
     for(int r=1;r<nr;r++){
       Field sr(grid),d(grid);
       ExtractSliceFast(sr,mout,r,0);
-      d = sr - ComplexD(r+1.0,0.0)*out;
+      d = sr - CoarseScalar(r+1.0,0.0)*s0;
       RealD rel = std::sqrt(norm2(d)/norm2(sr));
-      if ( rel >= 1.0e-6 ) {
+      worst = std::max(worst,rel);
-        std::cout << GridLogMessage << "DenseCoarseMatrix: oracle rhs "<<r
+      const RealD otol = (sizeof(CoarseScalar)==sizeof(ComplexF)) ? 1.0e-4 : 1.0e-6;
-                  <<" inconsistent with rhs 0, rel "<<rel<<std::endl;
+      if ( rel >= otol ) {
+        std::cout << GridLogMessage << "DenseCoarseMatrix: mrhs slice "<<r
+                  <<" inconsistent with slice 0, rel "<<rel<<std::endl;
       }
-      GRID_ASSERT( rel < 1.0e-6 );
+      GRID_ASSERT( rel < otol );
     }
+    std::cout << GridLogMessage << "DenseCoarseMatrix: mrhs slices independent to "
+              << worst << " over " << nr << " right hand sides" << std::endl;
   }
 
   ////////////////////////////////////////////////////////////////////
@@ -355,55 +347,31 @@ public:
   {
     double t = -usecond();
     Coordinate gdims = grid->GlobalDimensions();
-    int sign = getenv("DENSE_IMPORT_SIGN") ? atoi(getenv("DENSE_IMPORT_SIGN")) : 1;
-    GRID_ASSERT( sign==1 || sign==-1 );
 
     uint64_t nelem = (uint64_t)nrows * N;
     thread_for(i, nelem, { slab[i] = ComplexF(0.0,0.0); });
 
     for(int p=0; p<Op.Geometry().npoint; p++){
       Coordinate shift = Op.Geometry().shifts[p];
-      // _A[p] is PADDED after ExchangeCoarseLinks (end of CoarsenOperator):
+      // ExtractMatrix hands back the unpadded matrix field, which is what the
-      // extract the unpadded field before peeking with unpadded coordinates
+      // unpadded coordinates below index.
-      // (exactly as MultiGeneralCoarsenedMatrix::CopyMatrix does).
       CoarseMatrix Aun(grid);  Op.ExtractMatrix(p,Aun);
-      if ( getenv("DENSE_IMPORT_DEBUG") ) {
-        // Peek-path vs field-norm audit: sum |peekLocalSite|^2 must match norm2
-        double pk = 0.0;
-        autoView(Adbg, Aun, CpuRead);
-        for(int ss=0; ss<lsites; ss++){
-          Msobj m;
-          peekLocalSite(m, Adbg, myLcoor[ss]);
-          ComplexD *md = (ComplexD *)&m;
-          for(int i=0; i<nbasis*nbasis; i++) pk += md[i].real()*md[i].real() + md[i].imag()*md[i].imag();
-        }
-        RealD gpk = pk;
-        grid->GlobalSumVector(&gpk, 1);
-        std::cout << GridLogMessage << "DenseCoarseMatrix: DEBUG p=" << p
-                  << " norm2(_A[p]) " << norm2(Aun)
-                  << " norm2(Extract) " << norm2(Aun)
-                  << " sum|peek|^2 " << gpk << std::endl;
-      }
       autoView(Av, Aun, CpuRead);
       thread_for(ss, lsites, {
         Coordinate ncoor(nd);
         for(int d=0; d<nd; d++){
-          int64_t g = grid->_lstart[d] + myLcoor[ss][d] + sign*shift[d];
+          int64_t g = grid->_lstart[d] + myLcoor[ss][d] + shift[d];
           ncoor[d] = (int)((g % gdims[d] + gdims[d]) % gdims[d]);
         }
         int64_t nsite;
         Lexicographic::IndexFromCoor(ncoor, nsite, gdims);
         Msobj m;
         peekLocalSite(m, Av, myLcoor[ss]);
-        ComplexD *md = (ComplexD *)&m;
+        CoarseScalar *md = (CoarseScalar *)&m;
         // The operator contracts out(s,b) = sum_a A[p](s)(a,b) in(nbr,a)
         // (GeneralCoarsenedMatrix.h Mult kernel): the stored site matrix
         // acts TRANSPOSED, so element (a,b) lands at dense row (s,b),
-        // column (nbr,a).  BUG LEDGER 2026-08-14: the original mapping
+        // column (nbr,a).
-        // wrote (s,a),(nbr,b) -- caught by the IMPORT CERTIFICATE on its
-        // FIRST fresh-import exercise (Test_schur_dense_coarse); every
-        // production slab predates this path (probe-import SLAB_FILEs),
-        // so no production output is suspect.
         for(int b=0; b<nbasis; b++){
           ComplexF *row = &slab[(uint64_t)(ss*nbasis+b)*N + nsite*nbasis];
           for(int a=0; a<nbasis; a++)
@@ -468,7 +436,7 @@ public:
       sobj s;
       for(int b=0; b<nbasis; b++){
         double ph = 0.37*(double)(myGsite[ss]*nbasis+b);
-        ((ComplexD *)&s)[b] = ComplexD(std::cos(ph),std::sin(0.61*ph));
+        ((CoarseScalar *)&s)[b] = CoarseScalar(std::cos(ph),std::sin(0.61*ph));
       }
       pokeLocalSite(s, x, myLcoor[ss]);
     }
@@ -477,7 +445,7 @@ public:
     for(int ss=0; ss<lsites; ss++){
       sobj s;
       peekLocalSite(s, x, myLcoor[ss]);
-      for(int b=0; b<nbasis; b++) xh[ myGsite[ss]*nbasis + b ] = ((ComplexD *)&s)[b];
+      for(int b=0; b<nbasis; b++) xh[ myGsite[ss]*nbasis + b ] = ComplexD(((CoarseScalar *)&s)[b]);
     }
     grid->GlobalSumVector(&xh[0], (int)N);
     std::vector<ComplexD> yh(nrows);
@@ -489,7 +457,7 @@ public:
     });
     for(int ss=0; ss<lsites; ss++){
       sobj s;
-      for(int b=0; b<nbasis; b++) ((ComplexD *)&s)[b] = yh[ss*nbasis+b];
+      for(int b=0; b<nbasis; b++) ((CoarseScalar *)&s)[b] = CoarseScalar(yh[ss*nbasis+b]);
       pokeLocalSite(s, Dx, myLcoor[ss]);
     }
     ApplyOracle(Op, x, Ax);
@@ -499,250 +467,15 @@ public:
               << rel << std::endl;
     if ( rel >= 1.0e-3 ) {
       std::cout << GridLogMessage << "DenseCoarseMatrix: IMPORT CERTIFICATE FAILED. If O(1), the "
-                << "stencil shift-sign convention is opposite: rerun with DENSE_IMPORT_SIGN=-1"
+                << "stencil shift-sign convention of the coarse operator has changed: "
+                << "the import in ImportDense/ImportDenseForInversion must change with it"
                 << std::endl;
     }
     GRID_ASSERT(rel < 1.0e-3);
   }
 
   ////////////////////////////////////////////////////////////////////
-  // 3. Invert dispatcher.  slab holds my rows of A on entry, my rows
+  // 3a. Global column -> rank-major column map, computed LOCALLY.
-  //    of A^{-1} on exit, in every mode.
-  //      DENSE_SCHUR absent/0 : single-GCD gather-invert (the oracle)
-  //      DENSE_SCHUR=1        : distributed recursive Schur
-  //      DENSE_SCHUR=2        : AUDIT -- both on the same A; report the
-  //                             slab difference; keep the Schur result
-  //                             (so VERIFY certifies the new path).
-  ////////////////////////////////////////////////////////////////////
-  template<class CoarseOp>
-  void InvertDense(CoarseOp &Op)
-  {
-    char *sc  = getenv("DENSE_SCHUR");
-    int  mode = sc ? atoi(sc) : 0;
-
-    if ( mode == 0 )
-    {
-      InvertDenseSingle();
-      return;
-    }
-    if ( mode == 1 )
-    {
-      InvertDenseSchur(Op);
-      return;
-    }
-    GRID_ASSERT( mode == 2 );
-    std::vector<ComplexF> Aimp(slab);       // imported A
-    InvertDenseSingle();
-    std::vector<ComplexF> ref(slab);        // Ainv, single path
-    slab = Aimp;
-    InvertDenseSchur(Op);                   // slab = Ainv, Schur path
-
-    // NaN-PROOF comparison: max() masks NaN, so count non-finite
-    // entries in each result explicitly.
-    double  mx = 0.0;
-    double  mr = 0.0;
-    int64_t badschur  = 0;
-    int64_t badsingle = 0;
-    for(uint64_t i=0; i<(uint64_t)nrows*N; i++)
-    {
-      double as = abs(ComplexD(slab[i]));
-      double ar = abs(ComplexD(ref[i]));
-      if ( !std::isfinite(as) ) badschur++;
-      if ( !std::isfinite(ar) ) badsingle++;
-      if ( std::isfinite(as) && std::isfinite(ar) )
-      {
-        mx = std::max(mx, (double)abs(ComplexD(slab[i]) - ComplexD(ref[i])));
-        mr = std::max(mr, ar);
-      }
-    }
-    RealD gmx = mx;
-    RealD gmr = mr;
-    RealD gbs = (RealD)badschur;
-    RealD gbr = (RealD)badsingle;
-    grid->GlobalMax(gmx);
-    grid->GlobalMax(gmr);
-    grid->GlobalSumVector(&gbs, 1);
-    grid->GlobalSumVector(&gbr, 1);
-    schurAuditRel = gmx/gmr;
-    std::cout << GridLogMessage << "DenseCoarseMatrix: DENSE_SCHUR=2 AUDIT "
-              << "max|Ainv_schur - Ainv_single| = " << gmx
-              << "  relative " << schurAuditRel
-              << "  non-finite: schur " << (int64_t)gbs << " single " << (int64_t)gbr
-              << "  (two fp32 roundings of the same inverse; expect ~ growth * eps32)"
-              << std::endl;
-    GRID_ASSERT( gbs == 0 );
-    GRID_ASSERT( gbr == 0 );
-  }
-
-  ////////////////////////////////////////////////////////////////////
-  // 3a. Single-GCD invert: chunked zero-fill+GlobalSum gather of A
-  //    streamed to the boss GCD, cgetrf_64 (ILP64), rows of A^{-1} by
-  //    blocked identity cgetrs_64 + broadcast; each rank keeps its own
-  //    rows (in `slab`, overwriting A).  Proven path; the SCHUR oracle.
-  ////////////////////////////////////////////////////////////////////
-  void InvertDenseSingle(void)
-  {
-    double t1 = usecond();
-    int boss = grid->IsBoss();
-    std::vector<ComplexF> Afull;
-#ifdef GRID_HIP
-    // QUARANTINED naked HIP: the boss-only N^2 inversion buffer (34GB at
-    // N=65536) must come from raw HBM; EvictAll flushes the device-copy
-    // layer to make the window.  (FreePool of the allocator free-list
-    // awaits the type-dispatched fix.)  Confined to setup; the apply path
-    // is pure Grid primitives.
-    rocblas_float_complex *dA = nullptr;
-    rocblas_float_complex *dB = nullptr;
-    int64_t *dIpiv = nullptr;
-    uint64_t Abytes = (uint64_t)N * N * sizeof(ComplexF);
-    MemoryManager::EvictAll();
-    if (boss) {
-      auto aerr = hipMalloc((void **)&dA, Abytes);
-      if (aerr != hipSuccess) {
-        std::cout << GridLogMessage << "DenseCoarseMatrix: hipMalloc of "
-                  << Abytes/1024./1024./1024. << " GB FAILED -- reduce --device-mem" << std::endl;
-        GRID_ASSERT(aerr == hipSuccess);
-      }
-      std::cout << GridLogMessage << "DenseCoarseMatrix: device inversion buffer allocated ("
-                << Abytes/1024./1024./1024. << " GB)" << std::endl;
-    }
-#else
-    if (boss) Afull.resize((uint64_t)N * N);
-#endif
-    {
-      std::unordered_map<int64_t,int> rowmap;   // global row -> my slab row
-      for(int ss=0; ss<lsites; ss++)
-        for(int a=0; a<nbasis; a++)
-          rowmap[ myGsite[ss]*nbasis + a ] = ss*nbasis + a;
-
-      std::vector<ComplexF> chunk((uint64_t)CHUNKROWS * N);
-      for(int64_t row0=0; row0<N; row0+=CHUNKROWS){
-        int64_t nrow = std::min(CHUNKROWS, N-row0);
-        uint64_t nelem = (uint64_t)nrow * N;
-        for(uint64_t i=0;i<nelem;i++) chunk[i]=ComplexF(0.0,0.0);
-        for(int64_t r=row0; r<row0+nrow; r++){
-          auto it = rowmap.find(r);
-          if (it != rowmap.end()) {
-            uint64_t src = (uint64_t)(it->second) * N;
-            uint64_t dst = (uint64_t)(r-row0) * N;
-            for(int64_t j=0;j<N;j++) chunk[dst+j] = slab[src+j];
-          }
-        }
-        grid->GlobalSumVector(&chunk[0], (int)nelem);
-        if (boss) {
-#ifdef GRID_HIP
-          GRID_ASSERT( hipMemcpy((char *)dA + (uint64_t)row0*N*sizeof(ComplexF),
-                                 &chunk[0], nelem*sizeof(ComplexF),
-                                 hipMemcpyHostToDevice) == hipSuccess );
-#else
-          uint64_t dst = (uint64_t)row0 * N;
-          for(uint64_t i=0;i<nelem;i++) Afull[dst+i] = chunk[i];
-#endif
-        }
-      }
-    }
-    double t2 = usecond();
-    std::cout << GridLogMessage << "DenseCoarseMatrix: gather to boss took "
-              << (t2-t1)/1.0e6 << " s" << std::endl;
-
-    ////////////////////////////////////////////////////////////////////
-    // Factor in place on the boss (fp32, ILP64).  Row-major buffer handed
-    // to column-major LAPACK => LU of A^T.
-    ////////////////////////////////////////////////////////////////////
-    if (boss) {
-#ifdef GRID_HIP
-      std::cout << GridLogMessage << "DenseCoarseMatrix: rocSOLVER cgetrf_64 (ILP64 LU) N=" << N
-                << " in place on resident device buffer" << std::endl;
-      rocblas_handle handle = GridBLASInverse::Handle();
-      int64_t *dInfo;
-      GRID_ASSERT( hipMalloc((void **)&dIpiv, N*sizeof(int64_t)) == hipSuccess );
-      GRID_ASSERT( hipMalloc((void **)&dInfo, sizeof(int64_t))   == hipSuccess );
-      auto st1 = rocsolver_cgetrf_64(handle, (int64_t)N, (int64_t)N, dA, (int64_t)N, dIpiv, dInfo);
-      GRID_ASSERT( hipDeviceSynchronize() == hipSuccess );
-      int64_t info_h = -1;
-      GRID_ASSERT( hipMemcpy(&info_h, dInfo, sizeof(int64_t),
-                             hipMemcpyDeviceToHost) == hipSuccess );
-      std::cout << GridLogMessage << "DenseCoarseMatrix: cgetrf_64 status " << (int)st1
-                << " info = " << (int)info_h << std::endl;
-      GRID_ASSERT(st1 == rocblas_status_success);
-      GRID_ASSERT(info_h == 0);
-      GRID_ASSERT( hipFree(dInfo) == hipSuccess );
-      GRID_ASSERT( hipMalloc((void **)&dB, (uint64_t)CHUNKROWS*N*sizeof(ComplexF)) == hipSuccess );
-      // dA holds the LU of A^T; rows of A^{-1} are produced blockwise below via
-      // cgetrs_64 on identity-column blocks: A^T X = E => X columns = rows of
-      // A^{-1}, in exactly the linear layout the harvest expects.
-#else
-      // Eigen fallback: small local CPU tests only.
-      std::cout << GridLogMessage << "DenseCoarseMatrix: Eigen fallback inversion N=" << N
-                << (N > 10000 ? "  (WARNING: SLOW; use the HIP/rocSOLVER path)" : "")
-                << std::endl;
-      typedef Eigen::Matrix<std::complex<float>,Eigen::Dynamic,Eigen::Dynamic,Eigen::RowMajor> MatF;
-      Eigen::Map<MatF> A(reinterpret_cast<std::complex<float>*>(&Afull[0]), N, N);
-      MatF Ainv = A.inverse();
-      A = Ainv;
-#endif
-    }
-    double t3 = usecond();
-    std::cout << GridLogMessage << "DenseCoarseMatrix: factorisation took "
-              << (t3-t2)/1.0e6 << " s" << std::endl;
-
-    ////////////////////////////////////////////////////////////////////
-    // Blocked solve + broadcast: rows of A^{-1} chunk by chunk; each
-    // rank keeps the rows of its own sites (ownership-aligned).
-    ////////////////////////////////////////////////////////////////////
-    {
-      std::unordered_map<int64_t,int> rowmap;
-      for(int ss=0; ss<lsites; ss++)
-        for(int a=0; a<nbasis; a++)
-          rowmap[ myGsite[ss]*nbasis + a ] = ss*nbasis + a;
-
-      std::vector<ComplexF> chunk((uint64_t)CHUNKROWS * N);
-      for(int64_t row0=0; row0<N; row0+=CHUNKROWS){
-        int64_t nrow = std::min(CHUNKROWS, N-row0);
-        uint64_t nelem = (uint64_t)nrow * N;
-        if (boss) {
-#ifdef GRID_HIP
-          // Identity block E: column j = e_{row0+j}; solve A^T X = E so X's
-          // columns are rows [row0,row0+nrow) of A^{-1}.
-          for(uint64_t i=0;i<nelem;i++) chunk[i] = ComplexF(0.0,0.0);
-          for(int64_t j=0;j<nrow;j++) chunk[(uint64_t)j*N + (uint64_t)(row0+j)] = ComplexF(1.0,0.0);
-          GRID_ASSERT( hipMemcpy(dB, &chunk[0], nelem*sizeof(ComplexF), hipMemcpyHostToDevice) == hipSuccess );
-          auto strs = rocsolver_cgetrs_64(GridBLASInverse::Handle(), rocblas_operation_none,
-                                          (int64_t)N, (int64_t)nrow,
-                                          dA, (int64_t)N, dIpiv, dB, (int64_t)N);
-          GRID_ASSERT(strs == rocblas_status_success);
-          GRID_ASSERT( hipDeviceSynchronize() == hipSuccess );
-          GRID_ASSERT( hipMemcpy(&chunk[0], dB, nelem*sizeof(ComplexF), hipMemcpyDeviceToHost) == hipSuccess );
-#else
-          uint64_t src = (uint64_t)row0 * N;
-          for(uint64_t i=0;i<nelem;i++) chunk[i] = Afull[src+i];
-#endif
-        }
-        grid->Broadcast(0, &chunk[0], nelem*sizeof(ComplexF));
-        for(int64_t r=row0; r<row0+nrow; r++){
-          auto it = rowmap.find(r);
-          if (it != rowmap.end()) {
-            uint64_t dst = (uint64_t)(it->second) * N;
-            uint64_t src = (uint64_t)(r-row0) * N;
-            for(int64_t j=0;j<N;j++) slab[dst+j] = chunk[src+j];
-          }
-        }
-      }
-    }
-#ifdef GRID_HIP
-    if (boss) {
-      if (dA)    GRID_ASSERT( hipFree(dA)    == hipSuccess );
-      if (dB)    GRID_ASSERT( hipFree(dB)    == hipSuccess );
-      if (dIpiv) GRID_ASSERT( hipFree(dIpiv) == hipSuccess );
-    }
-#endif
-    double t4 = usecond();
-    std::cout << GridLogMessage << "DenseCoarseMatrix: blocked getrs solve+scatter took "
-              << (t4-t3)/1.0e6 << " s" << std::endl;
-  }
-
-  ////////////////////////////////////////////////////////////////////
-  // 3b. Global column -> rank-major column map, computed LOCALLY.
   //    Rank-major ordering: rank q's rows/columns are the contiguous
   //    block [q*nrows, (q+1)*nrows), ordered by q's local site index
   //    (uniform local volumes make ownership arithmetic exact).
@@ -789,26 +522,25 @@ public:
   }
 
   ////////////////////////////////////////////////////////////////////
-  // 3c. Direct stencil -> fp64 rank-major import of MY ROWS of A (the
+  // 3b. Direct stencil -> rank-major import of MY ROWS of A in the
-  //    end-to-end fp64 path: the stencil source IS ComplexD; nothing is
+  //    inversion precision (DenseInverseScalar): the stencil source is
-  //    rounded through fp32 on the way into the inversion).  Same
+  //    read once, straight into the buffer the Schur recursion factorises,
+  //    never via the fp32 apply slab.  Same
   //    loop/sign/accumulate/transposed-contraction discipline as
   //    ImportDense; output is column-major rows x N with columns in
   //    rank-major order (g2rm).
-  //    ALWAYS-ON CERTIFICATE: the fp64 import, rounded, must agree with
+  //    ALWAYS-ON CERTIFICATE: this import, rounded, must agree with
   //    the fp32 slab entry at the corresponding global column, over the
   //    WHOLE of my rows (few ulp: wrapped-shift collisions accumulate in
   //    different precision order).  NaN-proof: non-finite entries are
   //    counted explicitly since max() silently masks NaN.
   ////////////////////////////////////////////////////////////////////
   template<class CoarseOp>
-  void ImportDenseFP64(CoarseOp &Op, BlockRows &S, std::vector<int64_t> &g2rm)
+  void ImportDenseForInversion(CoarseOp &Op, BlockRows &S, std::vector<int64_t> &g2rm)
   {
     Coordinate gdims = grid->GlobalDimensions();
-    int sign = getenv("DENSE_IMPORT_SIGN") ? atoi(getenv("DENSE_IMPORT_SIGN")) : 1;
-    GRID_ASSERT( sign==1 || sign==-1 );
 
-    std::vector<ComplexD> h((uint64_t)nrows*N, ComplexD(0.0,0.0));
+    std::vector<DenseInverseScalar> h((uint64_t)nrows*N, DenseInverseScalar(0.0,0.0));
     for(int p=0; p<Op.Geometry().npoint; p++)
     {
       Coordinate shift = Op.Geometry().shifts[p];
@@ -818,14 +550,14 @@ public:
         Coordinate ncoor(nd);
         for(int d=0; d<nd; d++)
         {
-          int64_t g = grid->_lstart[d] + myLcoor[ss][d] + sign*shift[d];
+          int64_t g = grid->_lstart[d] + myLcoor[ss][d] + shift[d];
           ncoor[d] = (int)((g % gdims[d] + gdims[d]) % gdims[d]);
         }
         int64_t nsite;
         Lexicographic::IndexFromCoor(ncoor, nsite, gdims);
         Msobj m;
         peekLocalSite(m, Av, myLcoor[ss]);
-        ComplexD *md = (ComplexD *)&m;
+        CoarseScalar *md = (CoarseScalar *)&m;
         // Transposed contraction as ImportDense: (a,b) lands at
         // row (s,b), column (nbr,a); column index in rank-major order.
         for(int a=0; a<nbasis; a++)
@@ -833,7 +565,7 @@ public:
           int64_t jj = g2rm[ nsite*nbasis + a ];
           for(int b=0; b<nbasis; b++)
           {
-            h[(uint64_t)(ss*nbasis+b) + (uint64_t)jj*nrows] += md[a*nbasis+b];
+            h[(uint64_t)(ss*nbasis+b) + (uint64_t)jj*nrows] += DenseInverseScalar(md[a*nbasis+b]);
           }
         }
       });
@@ -846,7 +578,7 @@ public:
     {
       for(int64_t gcol=0; gcol<N; gcol++)
       {
-        ComplexD d64 = h[(uint64_t)(i + g2rm[gcol]*nrows)];
+        ComplexD d64 = ComplexD(h[(uint64_t)(i + g2rm[gcol]*nrows)]);
         ComplexF f32 = slab[(uint64_t)i*N + gcol];
         double dev = abs(ComplexD(f32) - d64);
         if ( !std::isfinite(dev) ) nbad++;
@@ -857,25 +589,25 @@ public:
     RealD gbad = (RealD)nbad;
     grid->GlobalMax(gmx);
     grid->GlobalSumVector(&gbad, 1);
-    std::cout << GridLogMessage << "DenseCoarseMatrix: fp64 import certificate "
+    std::cout << GridLogMessage << "DenseCoarseMatrix: inversion-source import certificate "
-              << "max|A64 - A32| = " << gmx
+              << "max|A_inv - A_slab| = " << gmx
               << "  non-finite entries " << (int64_t)gbad << std::endl;
     GRID_ASSERT( gbad == 0 );
     GRID_ASSERT( gmx < 1.0e-5 );
 
     S.Resize(nrows, N);
-    acceleratorCopyToDevice(&h[0], &S.data[0], (uint64_t)nrows*N*sizeof(ComplexD));
+    acceleratorCopyToDevice(&h[0], &S.data[0], (uint64_t)nrows*N*sizeof(DenseInverseScalar));
   }
 
   ////////////////////////////////////////////////////////////////////
-  // 3d. Distributed recursive Schur invert, END-TO-END fp64 (decision
+  // 3c. The inverse: distributed recursive Schur, end to end in the
-  //    2026-08-14): stencil (ComplexD) -> fp64 rank-major import ->
+  //    inversion precision (DenseInverseScalar, a configure-time choice):
-  //    fp64 recursion -> ONE terminal rounding into the fp32 apply
+  //    stencil -> rank-major import -> recursion -> ONE terminal rounding
-  //    slab.  Everything downstream (device residency, split-K apply,
+  //    into the fp32 apply slab.  Everything downstream (device
-  //    VERIFY, SLAB_FILE) is untouched.
+  //    residency, split-K apply, VERIFY) is fp32 regardless.
   ////////////////////////////////////////////////////////////////////
   template<class CoarseOp>
-  void InvertDenseSchur(CoarseOp &Op)
+  void InvertDense(CoarseOp &Op)
   {
     double t1 = usecond();
     int P  = grid->ProcessorCount();
@@ -902,29 +634,20 @@ public:
     }
 
     BlockRows S;
-    ImportDenseFP64(Op, S, g2rm);
+    ImportDenseForInversion(Op, S, g2rm);
 
     ////////////////////////////////////////////////////////////////
-    // DENSE_SCHUR2D=1 : invert via the 2D block-cyclic recursion
+    // The 2D block-cyclic recursion (BlockCyclicSchurInverse).
-    // (BlockCyclicSchurInverse) instead of the 1D rank-range one.
+    // Everything in it -- redistribution, SUMMA rings, leaves -- is
-    // The SAME imported rank-major rows S go in and come back, so the
+    // point-to-point SendToRecvFrom; no collectives at all.  Block size
-    // import certificate above and the slab rounding / VERIFY below are
+    // nb = rows-per-rank makes the redistribution edges maximally
-    // identical for both paths: a clean A/B on one imported operator.
+    // regular.
-    //
-    // Everything in the 2D path -- redistribution, SUMMA rings, leaf --
-    // is point-to-point SendToRecvFrom; no collectives at all.
-    // DENSE_NB overrides the block size (default: rows-per-rank, which
-    // makes the redistribution edges maximally regular).
     ////////////////////////////////////////////////////////////////
-    int use2d = getenv("DENSE_SCHUR2D") ? atoi(getenv("DENSE_SCHUR2D")) : 0;
-    int64_t panelBytes = getenv("DENSE_PANEL_BYTES") ? atol(getenv("DENSE_PANEL_BYTES"))
-                                                     : (int64_t)1024*1024*1024;   // 1D path only
     double t2, t3;
-    if ( use2d )
     {
       int Pr,Pc;
       BlockCyclicLayout::ChooseProcessGrid(P, Pr, Pc);
-      int64_t nb = getenv("DENSE_NB") ? atol(getenv("DENSE_NB")) : nrows;
+      int64_t nb = nrows;
       GRID_ASSERT( nb >= 1 );
       std::cout << GridLogMessage << "DenseCoarseMatrix: 2D SCHUR invert, process grid "
                 << Pr << " x " << Pc << "  nb " << nb
@@ -938,20 +661,12 @@ public:
       t3 = usecond();
       RSI2.ReportTelemetry(grid);
     }
-    else
-    {
-      RecursiveSchurInverse RSI(grid, N, rowStart, panelBytes);
-      t2 = usecond();
-      RSI.Invert(S);
-      t3 = usecond();
-      RSI.ReportTelemetry();
-    }
 
     // The single terminal rounding: fp64 inverse -> fp32 apply slab
     // (row-major, global columns)
     {
-      std::vector<ComplexD> h((uint64_t)nrows*N);
+      std::vector<DenseInverseScalar> h((uint64_t)nrows*N);
-      acceleratorCopyFromDevice(&S.data[0], &h[0], (uint64_t)nrows*N*sizeof(ComplexD));
+      acceleratorCopyFromDevice(&S.data[0], &h[0], (uint64_t)nrows*N*sizeof(DenseInverseScalar));
       thread_for(gcol, N, {
         int64_t jj = g2rm[gcol];
         for(int64_t i=0; i<nrows; i++)
@@ -961,29 +676,29 @@ public:
       });
     }
     double t4 = usecond();
-    std::cout << GridLogMessage << "DenseCoarseMatrix: SCHUR fp64 distributed invert took "
+    std::cout << GridLogMessage << "DenseCoarseMatrix: SCHUR "
-              << (t4-t1)/1.0e6 << " s (recursion " << (t3-t2)/1.0e6 << " s), panelBytes "
+              << (sizeof(DenseInverseScalar)==sizeof(ComplexF) ? "fp32" : "fp64")
-              << panelBytes << std::endl;
+              << " distributed invert took "
+              << (t4-t1)/1.0e6 << " s (recursion " << (t3-t2)/1.0e6 << " s)" << std::endl;
   }
 
   ////////////////////////////////////////////////////////////////////
   // CORE apply on packed data: hX[N x nr] (zero-filled, my sites only)
-  // -> allreduce -> split-K GEMM against the resident slab -> reduce
+  // -> ring allgather -> split-K GEMM against the resident slab ->
-  // partials -> hY[nrows x nr] (column major).  Platform-agnostic:
+  // reduce partials -> hY[nrows x nr] (column major).  Platform-agnostic:
   // deviceVector + GridBLAS (Eigen fallback on CPU builds).
-  // fp32 allreduce is EXACT: zero-fill assembly gives every element
+  // tprof (optional): per-phase microseconds {allgather, H2D, gemm+reduce,
-  // exactly one contributing rank.
+  // D2H}, printed by the caller on GridLogPerformance.
   ////////////////////////////////////////////////////////////////////
   void SlabApplyPacked(int nr, double *tprof)
   {
     GRID_TRACE("DenseSlabApply");
     GRID_ASSERT(nr <= MRHS_MAX);
-    uint64_t nX = (uint64_t)N * nr;
     uint64_t nY = (uint64_t)nrows * nr;
     int64_t  Kc = N / NK;
     double t1 = usecond();
     double t2, t3;
-    if (devSum==4) {
+    {
       // ALLGATHER: x is not a reduction -- every rank owns the rows of x at
       // global columns myGsite[ss]*nbasis+b (scattered by site coordinate, NOT
       // a contiguous block) and needs all of it.  Only MY rows go host->device
@@ -1011,31 +726,6 @@ public:
         });
       }
       t3 = usecond();
-    } else if (devSum) {
-      { GRID_TRACE("DenseH2D");
-        acceleratorCopyToDevice(&hX[0],&dX[0],nX*sizeof(ComplexF));
-      }
-      t2 = usecond();
-      { GRID_TRACE("DenseAllreduce");
-        // DENSE_DEVICE_SUM=1 : device-buffer MPI_Allreduce (Cray MPICH aborts
-        //                      above ~8 MB: 12 RHS at N=138240 is 13.3 MB)
-        // DENSE_DEVICE_SUM=2 : CartesianRingAllReduce, P2P only, no size cliff
-        // DENSE_DEVICE_SUM=3 : flat RingAllReduce, P2P only
-        // DENSE_DEVICE_SUM=4 : CartesianRingAllGather (branch above)
-        if      (devSum==2) CartesianRingAllReduce(grid,(ComplexF *)&dX[0],nX);
-        else if (devSum==3) RingAllReduce(grid,(ComplexF *)&dX[0],nX);
-        else                grid->GlobalSumVector((ComplexF *)&dX[0], (int)nX);
-      }
-      t3 = usecond();
-    } else {
-      { GRID_TRACE("DenseAllreduce");
-        grid->GlobalSumVector(&hX[0], (int)nX);
-      }
-      t2 = usecond();
-      { GRID_TRACE("DenseH2D");
-        acceleratorCopyToDevice(&hX[0],&dX[0],nX*sizeof(ComplexF));
-      }
-      t3 = usecond();
     }
     // Y = op(slab,T) . X : row-major slab (nrows x N) == col-major A^T
     // (N x nrows, lda=N) => transpose gives the nrows x N operator.
@@ -1065,8 +755,8 @@ public:
     }
     double t5 = usecond();
     if (tprof) {
-      tprof[0] = devSum ? (t3-t2) : (t2-t1);   // allreduce
+      tprof[0] = t3-t2;                        // allgather
-      tprof[1] = devSum ? (t2-t1) : (t3-t2);   // H2D
+      tprof[1] = t2-t1;                        // H2D
       tprof[2] = t4-t3;                        // gemm+reduce
       tprof[3] = t5-t4;                        // D2H
     }
@@ -1085,7 +775,7 @@ public:
         sobj s;
         peekLocalSite(s, src, myLcoor[ss]);
         for(int b=0; b<nbasis; b++)
-          hX[ myGsite[ss]*nbasis + b ] = ComplexF(((ComplexD *)&s)[b]);
+          hX[ myGsite[ss]*nbasis + b ] = ComplexF(((CoarseScalar *)&s)[b]);
       }
     }
     SlabApplyPacked(1, nullptr);
@@ -1093,7 +783,7 @@ public:
       for(int ss=0; ss<lsites; ss++){
         sobj s;
         for(int b=0; b<nbasis; b++)
-          ((ComplexD *)&s)[b] = ComplexD(hY[ss*nbasis + b]);
+          ((CoarseScalar *)&s)[b] = CoarseScalar(hY[ss*nbasis + b]);
         pokeLocalSite(s, psi, myLcoor[ss]);
       }
     }
@@ -1130,7 +820,7 @@ public:
           sobj s;
           peekLocalSite(s, src[rr], myLcoor[ss]);
           for(int b=0; b<nbasis; b++)
-            hX[ (uint64_t)rr*N + myGsite[ss]*nbasis + b ] = ComplexF(((ComplexD *)&s)[b]);
+            hX[ (uint64_t)rr*N + myGsite[ss]*nbasis + b ] = ComplexF(((CoarseScalar *)&s)[b]);
         }
       }
     }
@@ -1140,7 +830,7 @@ public:
         for(int ss=0; ss<lsites; ss++){
           sobj s;
           for(int b=0; b<nbasis; b++)
-            ((ComplexD *)&s)[b] = ComplexD(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);
+            ((CoarseScalar *)&s)[b] = CoarseScalar(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);
           pokeLocalSite(s, psi[rr], myLcoor[ss]);
         }
       }
@@ -1183,7 +873,7 @@ public:
           sobj s;
           peekLocalSite(s, iv, c6);
           for(int b=0; b<nbasis; b++)
-            hX[(uint64_t)rr*N + myGsite[ss]*nbasis + b] = ComplexF(((ComplexD *)&s)[b]);
+            hX[(uint64_t)rr*N + myGsite[ss]*nbasis + b] = ComplexF(((CoarseScalar *)&s)[b]);
         }
       }
     }
@@ -1200,7 +890,7 @@ public:
           c6[0] = rr;
           sobj s;
           for(int b=0; b<nbasis; b++)
-            ((ComplexD *)&s)[b] = ComplexD(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);  // Y col-major
+            ((CoarseScalar *)&s)[b] = CoarseScalar(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);  // Y col-major
           pokeLocalSite(s, ov, c6);
         }
       }
@@ -1208,16 +898,14 @@ public:
     double t6 = usecond();
     std::cout << GridLogMessage << "DenseCoarseMatrix: apply6D " << nr << " rhs took "
               << (t6-t0)/1000.0 << " ms" << std::endl;
-    if ( getenv("DENSE_APPLY_PROFILE") ) {
+    std::cout << GridLogPerformance << "DenseCoarseMatrix: apply6D profile:"
-      std::cout << GridLogMessage << "DenseCoarseMatrix: apply6D profile:"
+              << " pack "        << (t1-t0)/1000.0
-                << " pack "        << (t1-t0)/1000.0
+              << "  allgather "  << tprof[0]/1000.0
-                << (devSum==4 ? "  allgather " : "  allreduce ")  << tprof[0]/1000.0
+              << "  H2D "        << tprof[1]/1000.0
-                << "  H2D "        << tprof[1]/1000.0
+              << "  gemm+reduce "<< tprof[2]/1000.0
-                << "  gemm+reduce "<< tprof[2]/1000.0
+              << "  D2H "        << tprof[3]/1000.0
-                << "  D2H "        << tprof[3]/1000.0
+              << "  unpack "     << (t6-t5)/1000.0
-                << "  unpack "     << (t6-t5)/1000.0
+              << "  ms" << std::endl;
-                << "  ms" << std::endl;
-    }
   }
 };
 

Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHSV2.h

@@ -1,870 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/GeneralCoarsenedMatrixMultiRHS.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <pboyle@bnl.gov>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#pragma once
-
-
-NAMESPACE_BEGIN(Grid);
-
-
-// Fine Object == (per site) type of fine field
-// nbasis      == number of deflation vectors
-template<class Fobj,class CComplex,int nbasis>
-class MultiGeneralCoarsenedOperatorV2 : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
-public:
-  typedef typename CComplex::scalar_object SComplex;
-  typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
-  typedef MultiGeneralCoarsenedOperatorV2<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
-
-  typedef iVector<CComplex,nbasis >           siteVector;
-  typedef iMatrix<CComplex,nbasis >           siteMatrix;
-  typedef iVector<SComplex,nbasis >           calcVector;
-  typedef iMatrix<SComplex,nbasis >           calcMatrix;
-  typedef Lattice<iScalar<CComplex> >         CoarseComplexField;
-  typedef Lattice<siteVector>                 CoarseVector;
-  typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
-  typedef iMatrix<CComplex,nbasis >  Cobj;
-  typedef iVector<CComplex,nbasis >  Cvec;
-  typedef Lattice< CComplex >   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<Fobj >        FineField;
-  typedef CoarseVector Field;
-
-  // Block operations on the fine vectors carry the fine layout, which need
-  // not be the coarse one
-  typedef decltype(innerProduct(Fobj(),Fobj())) FineInner;
-  typedef Lattice<FineInner>                    FineComplexField;
-  typedef Lattice<FineInner>                    BlockComplexField;
-
-  ////////////////////
-  // Data members
-  //
-  // Nrhs independent: the D dimensional coarse grid, the geometry, the padded
-  // cell that supplies the stencil grid, the stencil, and the matrix elements.
-  //
-  // Nrhs dependent: the D+1 grid, its padded cell, and the BLAS B/C buffers
-  // with their pointer tables. Owned by SetNRHS().
-  ////////////////////
-  GridCartesian *       _CoarseGrid;        // D dimensional
-  NonLocalStencilGeometry geom;
-  NonLocalStencilGeometry geom_srhs;
-  PaddedCell CellD;                         // D dimensional, supplies stencil grid
-  GeneralLocalStencil Stencil;              // D dimensional
-
-  int                   _Nrhs;
-  GridCartesian *       _CoarseGridMulti;   // D+1 dimensional, SetNRHS
-  PaddedCell *          CellMulti;          // D+1 dimensional, SetNRHS
-
-  deviceVector<calcVector> BLAS_B;
-  deviceVector<calcVector> BLAS_C;
-  std::vector<deviceVector<calcMatrix> > BLAS_A;
-
-  std::vector<deviceVector<ComplexD *> > BLAS_AP;
-  std::vector<deviceVector<ComplexD *> > BLAS_BP;
-  deviceVector<ComplexD *>               BLAS_CP;
-
-  ///////////////////////
-  // Interface
-  ///////////////////////
-  GridBase      * Grid(void)           { CheckGridSet(); return _CoarseGridMulti; };
-  GridCartesian * CoarseGrid(void)     { CheckGridSet(); return _CoarseGridMulti; };
-  GridCartesian * CoarseGridD(void)    { return _CoarseGrid; };        // lower dimensional grid
-  int             Nrhs(void)           { CheckGridSet(); return _Nrhs; };
-
-  void CheckGridSet(void)
-  {
-    if ( _CoarseGridMulti == nullptr ) {
-      std::cout << GridLogError
-		<< "MultiGeneralCoarsenedOperatorV2: the multiRHS grid has not been set."
-		<< std::endl;
-      std::cout << GridLogError
-		<< "  Call SetGrid(CoarseGridMulti) with the D+1 dimensional grid your"
-		<< std::endl;
-      std::cout << GridLogError
-		<< "  coarse vectors live on, before Grid(), Nrhs() or M()."
-		<< std::endl;
-      GRID_ASSERT(_CoarseGridMulti != nullptr);
-    }
-  }
-
-  //////////////////////////////////////////////////////////////////////////
-  // Bilingual accessors, matching GeneralCoarsenedMatrix. Note Grid() is the
-  // D+1 multiRHS grid here, so a consumer wanting the space the elements live
-  // on must ask for CoarseGridD().
-  //////////////////////////////////////////////////////////////////////////
-  NonLocalStencilGeometry & Geometry(void)      { return geom_srhs; };
-  void ExtractMatrix(int p,CoarseMatrix &A)     { BLAStoGrid(A,BLAS_A[p]); };
-
-  // I/O on the operator matrices, via the BLAS layout array. The parameter is
-  // a vector over the geometry points; the body indexes A[p].
-  void SetMatrix (int p,std::vector<CoarseMatrix> & A)
-  {
-    GRID_ASSERT(A.size()==geom_srhs.npoint);
-    GridtoBLAS(A[p],BLAS_A[p]);
-  }
-  void GetMatrix (int p,std::vector<CoarseMatrix> & A)
-  {
-    GRID_ASSERT(A.size()==geom_srhs.npoint);
-    BLAStoGrid(A[p],BLAS_A[p]);
-  }
-  void CopyMatrix (GeneralCoarseOp &_Op)
-  {
-    for(int p=0;p<geom.npoint;p++){
-      auto Aup = _Op.Cell.Extract(_Op._A[p]);
-      //Unpadded
-      GridtoBLAS(Aup,BLAS_A[p]);
-    }
-  }
-  /*
-  void CheckMatrix (GeneralCoarseOp &_Op)
-  {
-    std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
-    for(int p=0;p<geom.npoint;p++){
-      //Unpadded
-      auto Aup = _Op.Cell.Extract(_Op._A[p]);
-      auto Ack = Aup;
-      BLAStoGrid(Ack,BLAS_A[p]);
-      std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
-      std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
-    }
-    std::cout <<"************* "<<std::endl;
-  }
-  */
-  
-  ///////////////////////////////////////////////////////////////////////////
-  // Constructor takes the D dimensional coarse grid. Everything built here
-  // is independent of Nrhs, in particular the matrix elements, which must
-  // survive a change of Nrhs untouched.
-  ///////////////////////////////////////////////////////////////////////////
-  MultiGeneralCoarsenedOperatorV2(NonLocalStencilGeometry &_geom,GridCartesian *CoarseGrid) :
-    _CoarseGrid(CoarseGrid),
-    geom_srhs(_geom),
-    geom(CoarseGrid,_geom.hops,_geom.skip),
-    CellD(geom.Depth(),CoarseGrid),
-    Stencil(CellD.grids.back(),geom.shifts), // D dimensional padded cell stencil
-    _Nrhs(-1),
-    _CoarseGridMulti(nullptr),
-    CellMulti(nullptr)
-  {
-    int32_t unpadded_sites = _CoarseGrid->lSites();
-
-    /////////////////////////////////////////////////
-    // Matrix elements and their pointer table
-    /////////////////////////////////////////////////
-    BLAS_A.resize(geom.npoint);
-    BLAS_AP.resize(geom.npoint);
-    for(int p=0;p<geom.npoint;p++){
-      BLAS_A[p].resize (unpadded_sites); // no ghost zone, npoint elements
-      BLAS_AP[p].resize(unpadded_sites);
-    }
-
-    // Site identity mapping for A
-    for(int p=0;p<geom.npoint;p++){
-      for(int ss=0;ss<unpadded_sites;ss++){
-	ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
-	acceleratorPut(BLAS_AP[p][ss],ptr);
-      }
-    }
-  }
-
-  virtual ~MultiGeneralCoarsenedOperatorV2()
-  {
-    ReleaseGrid();
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Free everything SetGrid allocated. The D+1 grid is borrowed from the
-  // caller and is never deleted here. Safe to call repeatedly and before
-  // the destructor.
-  ///////////////////////////////////////////////////////////////////////////
-  void ReleaseGrid(void)
-  {
-    if ( CellMulti != nullptr ) { delete CellMulti; CellMulti = nullptr; }
-
-    _CoarseGridMulti = nullptr;  // borrowed, not owned
-    _Nrhs            = -1;
-
-    BLAS_B.resize(0);
-    BLAS_C.resize(0);
-    for(int p=0;p<BLAS_BP.size();p++){
-      BLAS_BP[p].resize(0);
-    }
-    BLAS_BP.resize(0);
-    BLAS_CP.resize(0);
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Everything that depends on Nrhs. Idempotent; lazy called on demand.
-  //
-  // The stencil lives on the D dimensional padded grid. PaddedCell only pads
-  // a dimension when it is distributed, and the rhs direction never is, so
-  // the D+1 padded grid is exactly Nrhs copies of the D dimensional padded
-  // grid with rhs innermost. The neighbour offset therefore carries an Nrhs
-  // factor, in the same way the Nsimd factor is carried.
-  ///////////////////////////////////////////////////////////////////////////
-  void SetGrid(GridCartesian *CoarseGridMulti)
-  {
-    GRID_ASSERT(CoarseGridMulti != nullptr);
-
-    if ( CoarseGridMulti == _CoarseGridMulti ) return; // idempotent on identity
-
-    ReleaseGrid();
-
-    /////////////////////////////////////////////////
-    // The D+1 grid is supplied and owned by the caller. Two operators over
-    // the same coarse space must share one grid object or their fields will
-    // not conform, so this is never manufactured internally.
-    /////////////////////////////////////////////////
-    int nd = _CoarseGrid->_ndimension;
-
-    GRID_ASSERT(CoarseGridMulti->_ndimension == nd+1);
-    GRID_ASSERT(CoarseGridMulti->_processors[0] == 1);   // rhs is not distributed
-    for(int d=0;d<nd;d++){
-      GRID_ASSERT(CoarseGridMulti->_fdimensions[d+1] == _CoarseGrid->_fdimensions[d]);
-      GRID_ASSERT(CoarseGridMulti->_processors [d+1] == _CoarseGrid->_processors [d]);
-      GRID_ASSERT(CoarseGridMulti->_simd_layout[d+1] == _CoarseGrid->_simd_layout[d]);
-    }
-
-    _CoarseGridMulti = CoarseGridMulti;
-    _Nrhs            = CoarseGridMulti->_fdimensions[0];
-    GRID_ASSERT(_Nrhs>=1);
-
-    int nrhs = _Nrhs;
-
-    CellMulti = new PaddedCell(geom.Depth(),_CoarseGridMulti);
-
-    int32_t padded_sites   = CellD.grids.back()->lSites(); // D dimensional
-    int32_t unpadded_sites = _CoarseGrid->lSites();        // D dimensional
-
-    // The neighbour offset multiplication by nrhs is exact only if the D+1
-    // padded volume is nrhs copies of the D dimensional one. Check it.
-    GRID_ASSERT(CellMulti->grids.back()->lSites() == nrhs*padded_sites);
-    GRID_ASSERT(_CoarseGridMulti->lSites()        == nrhs*unpadded_sites);
-
-    /////////////////////////////////////////////////
-    // Device data vector storage
-    /////////////////////////////////////////////////
-    BLAS_B.resize(nrhs *padded_sites);   // includes ghost zone
-    BLAS_C.resize(nrhs *unpadded_sites); // no ghost zone
-    BLAS_BP.resize(geom.npoint);
-    for(int p=0;p<geom.npoint;p++){
-      BLAS_BP[p].resize(unpadded_sites);
-    }
-    BLAS_CP.resize(unpadded_sites);
-
-    // Site identity mapping for C
-    for(int ss=0;ss<unpadded_sites;ss++){
-      ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
-      acceleratorPut(BLAS_CP[ss],ptr);
-    }
-
-    // Neighbour table is more complicated
-    int32_t j=0; // Interior point counter (unpadded)
-    for(int32_t s=0;s<padded_sites;s++){ // D volume, padded
-      int ghost_zone=0;
-      for(int32_t point = 0 ; point < geom.npoint; point++){
-	int i=s*geom.npoint+point;
-	if( Stencil._entries[i]._wrap ) { // stencil is indexed by the oSite of the D dim grid
-	  ghost_zone=1; // If general stencil wrapped in any direction, wrap=1
-	}
-      }
-
-      if( ghost_zone==0) {
-	for(int32_t point = 0 ; point < geom.npoint; point++){
-	  int i=s*geom.npoint+point;
- 	  int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite, D dim
-	  nbr = nbr*nrhs;                                              // D -> D+1, rhs innermost
-	  GRID_ASSERT(nbr<BLAS_B.size());
-	  ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
-	  acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
-	}
-	j++;
-      }
-    }
-    GRID_ASSERT(j==unpadded_sites);
-  }
-  template<class vobj> void GridtoBLAS(const Lattice<vobj> &from,deviceVector<typename vobj::scalar_object> &to)
-  {
-  typedef typename vobj::scalar_object sobj;
-  typedef typename vobj::scalar_type scalar_type;
-  typedef typename vobj::vector_type vector_type;
-
-  GridBase *Fg = from.Grid();
-  GRID_ASSERT(!Fg->_isCheckerBoarded);
-  int nd = Fg->_ndimension;
-
-  to.resize(Fg->lSites());
-
-  Coordinate LocalLatt = Fg->LocalDimensions();
-  size_t nsite = 1;
-  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
-
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  // do the index calc on the GPU
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  Coordinate f_ostride = Fg->_ostride;
-  Coordinate f_istride = Fg->_istride;
-  Coordinate f_rdimensions = Fg->_rdimensions;
-
-  autoView(from_v,from,AcceleratorRead);
-  auto to_v = &to[0];
-
-  const int words=sizeof(vobj)/sizeof(vector_type);
-  accelerator_for(idx,nsite,1,{
-      
-      Coordinate from_coor, base;
-      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
-      for(int i=0;i<nd;i++){
-	from_coor[i] = base[i];
-      }
-      int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*(from_coor[d]%f_rdimensions[d]);
-      int from_lane = 0; for(int d=0;d<nd;d++) from_lane+=f_istride[d]*(from_coor[d]/f_rdimensions[d]);
-
-      const vector_type* from = (const vector_type *)&from_v[from_oidx];
-      scalar_type* to = (scalar_type *)&to_v[idx];
-      
-      scalar_type stmp;
-      for(int w=0;w<words;w++){
-	stmp = getlane(from[w], from_lane);
-	to[w] = stmp;
-      }
-    });
-  }    
-  template<class vobj> void BLAStoGrid(Lattice<vobj> &grid,deviceVector<typename vobj::scalar_object> &in)
-  {
-  typedef typename vobj::scalar_object sobj;
-  typedef typename vobj::scalar_type scalar_type;
-  typedef typename vobj::vector_type vector_type;
-
-  GridBase *Tg = grid.Grid();
-  GRID_ASSERT(!Tg->_isCheckerBoarded);
-  int nd = Tg->_ndimension;
-  
-  GRID_ASSERT(in.size()==Tg->lSites());
-
-  Coordinate LocalLatt = Tg->LocalDimensions();
-  size_t nsite = 1;
-  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
-
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  // do the index calc on the GPU
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  Coordinate t_ostride = Tg->_ostride;
-  Coordinate t_istride = Tg->_istride;
-  Coordinate t_rdimensions = Tg->_rdimensions;
-
-  autoView(to_v,grid,AcceleratorWrite);
-  auto from_v = &in[0];
-
-  const int words=sizeof(vobj)/sizeof(vector_type);
-  accelerator_for(idx,nsite,1,{
-      
-      Coordinate to_coor, base;
-      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
-      for(int i=0;i<nd;i++){
-	to_coor[i] = base[i];
-      }
-      int to_oidx = 0; for(int d=0;d<nd;d++) to_oidx+=t_ostride[d]*(to_coor[d]%t_rdimensions[d]);
-      int to_lane = 0; for(int d=0;d<nd;d++) to_lane+=t_istride[d]*(to_coor[d]/t_rdimensions[d]);
-
-      vector_type* to = (vector_type *)&to_v[to_oidx];
-      scalar_type* from = (scalar_type *)&from_v[idx];
-      
-      scalar_type stmp;
-      for(int w=0;w<words;w++){
-	stmp=from[w];
-	putlane(to[w], stmp, to_lane);
-      }
-    });
-  }
-  ///////////////////////////////////////////////////////////////////////////
-  // Shared by both CoarsenOperator variants
-  //
-  //     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
-  //       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
-  //       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
-  //       = M_{kl} A_ji^{b.b+l}
-  //
-  //     Where q_k = delta_k . (2*M_PI/global_nb[mu])
-  //     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
-  ///////////////////////////////////////////////////////////////////////////
-  void CoarsenFourierMatrix(GridBase *CoarseGrid,Eigen::MatrixXcd &invMkl)
-  {
-    const int npoint = geom_srhs.npoint;
-    Coordinate clatt = CoarseGrid->GlobalDimensions();
-    int Nd = CoarseGrid->Nd();
-
-    Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
-    ComplexD ci(0.0,1.0);
-    for(int k=0;k<npoint;k++){ // Loop over momenta
-      for(int l=0;l<npoint;l++){ // Loop over nbr relative
-	ComplexD phase(0.0,0.0);
-	for(int mu=0;mu<Nd;mu++){
-	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
-	}
-	phase=exp(phase*ci);
-	Mkl(k,l) = phase;
-      }
-    }
-    invMkl = Mkl.inverse();
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // blockOrthogonalise and blockZAXPY are block operations on the fine
-  // vectors, using a coarse shaped field only as an index set. They need a
-  // grid carrying the fine SIMD layout, which the coarse space no longer
-  // does. Constructed local to the caller so it cannot be mistaken for the
-  // coarse grid.
-  ///////////////////////////////////////////////////////////////////////////
-  void CoarsenBlockGridLayout(GridBase *grid,GridBase *CoarseGrid,
-			      Coordinate &latt,Coordinate &simd,Coordinate &mpi)
-  {
-    int nd = CoarseGrid->_ndimension;
-    latt.resize(nd); simd.resize(nd); mpi.resize(nd);
-    for(int d=0;d<nd;d++){
-      latt[d] = CoarseGrid->_fdimensions[d];
-      simd[d] = grid->_simd_layout[d];
-      mpi [d] = CoarseGrid->_processors[d];
-    }
-  }
-
-  // D+1 coarse grid holding the batch, rhs innermost and unvectorised
-  void CoarsenBatchGridLayout(GridBase *CoarseGrid,int batch,
-			      Coordinate &latt,Coordinate &simd,Coordinate &mpi)
-  {
-    latt.resize(1,batch); simd.resize(1,1); mpi.resize(1,1);
-    latt[0]=batch; simd[0]=1; mpi[0]=1;
-    for(int d=0;d<CoarseGrid->_ndimension;d++){
-      latt.push_back(CoarseGrid->_fdimensions[d]);
-      simd.push_back(CoarseGrid->_simd_layout[d]);
-      mpi .push_back(CoarseGrid->_processors[d]);
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // The Fourier inverse needs the phase in the coarse layout and the basis
-  // phasing needs it in the fine layout; each is built from its own
-  // coordinates rather than transferred.
-  ///////////////////////////////////////////////////////////////////////////
-  void CoarsenPhases(GridBase *grid,GridBase *CoarseGrid,GridCartesian *BlockGrid,
-		     std::vector<CoarseComplexField> &pha,
-		     std::vector<FineComplexField> &phaF)
-  {
-    const int npoint = geom_srhs.npoint;
-    Coordinate clatt = CoarseGrid->GlobalDimensions();
-    int Nd = CoarseGrid->Nd();
-    ComplexD ci(0.0,1.0);
-
-    typedef typename CComplex::scalar_type SComplex;
-    FineComplexField one(grid); one=SComplex(1.0);
-    FineComplexField zz(grid);  zz = Zero();
-    BlockComplexField pha_blk (BlockGrid);
-    BlockComplexField blk_coor(BlockGrid);
-
-    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
-      CoarseComplexField coor(CoarseGrid);
-      pha[p] =Zero();
-      pha_blk=Zero();
-      for(int mu=0;mu<Nd;mu++){
-	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	LatticeCoordinate(coor,mu);
-	pha[p]  = pha[p]  + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
-	LatticeCoordinate(blk_coor,mu);
-	pha_blk = pha_blk + (TwoPiL * geom_srhs.shifts[p][mu]) * blk_coor;
-      }
-      pha[p] =exp(pha[p] *ci);
-      pha_blk=exp(pha_blk*ci);
-
-      blockZAXPY(phaF[p],pha_blk,one,zz);
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Remove the bulk phase from the batch of coarse projections and
-  // accumulate the Fourier inverse into A. Both variants reach here with
-  // TmpProj in the same batch coarse order, so this is shared verbatim.
-  ///////////////////////////////////////////////////////////////////////////
-  void CoarsenAccumulate(int p,int i0,int nbv,int batch,
-			 Eigen::MatrixXcd &invMkl,
-			 std::vector<CoarseComplexField> &pha,
-			 CoarseComplexField &phaB,
-			 CoarseVector &TmpProj,
-			 std::vector<CoarseMatrix> &_A,
-			 GridBase *CoarseGrid)
-  {
-    typedef typename CComplex::scalar_type SComplex;
-    const int npoint = geom_srhs.npoint;
-
-    for(int b=0;b<batch;b++) InsertSliceFast(pha[p],phaB,b,0);
-    TmpProj = conjugate(phaB)*TmpProj;
-
-    int osites=CoarseGrid->oSites();
-    for(int k=0;k<npoint;k++){
-      SComplex sc(invMkl(p,k).real(),invMkl(p,k).imag());
-      CComplex coef(sc);
-      autoView( A_v  , _A[k], AcceleratorWrite);
-      autoView( TP_v , TmpProj, AcceleratorRead);
-      accelerator_for(sss, osites, 1, {
-	  for(int b=0;b<nbv;b++){
-	    for(int j=0;j<nbasis;j++){
-	      A_v[sss](i0+b,j) = A_v[sss](i0+b,j) + coef*TP_v[b+batch*sss](j);
-	    }
-	  }
-      });
-    }
-  }
-
-  void CoarsenReport(RealD tphase,RealD tphaseBZ,RealD tslice,
-		     RealD tmat,RealD tproj,RealD tinv)
-  {
-    std::cout << GridLogMessage<<"CoarsenOperator phase  "<<tphase<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator slice  "<<tslice <<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator mat    "<<tmat <<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator proj   "<<tproj<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator inv    "<<tinv<<" us"<<std::endl;
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Coarsen a NATIVELY multiRHS fine operator.
-  //
-  // linop acts on the D+1 dimensional fine grid FineGridMulti, with the batch
-  // of phased basis vectors carried in the rhs direction. A single RHS
-  // operator can be promoted with MrhsPromotedOperator, but that pays an
-  // ExtractSlice/InsertSlice pair per rhs; prefer the single RHS variant
-  // below in that case.
-  ///////////////////////////////////////////////////////////////////////////
-  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
-		       GridCartesian *FineGridMulti,
-		       std::vector<FineField> &Subspace,
-		       GridBase *CoarseGrid)
-  {
-    RealD tproj=0.0, tmat=0.0, tphase=0.0, tphaseBZ=0.0, tslice=0.0, tinv=0.0;
-
-    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs (multiRHS fine operator)"<< std::endl;
-
-    GRID_ASSERT(Subspace.size()==nbasis);
-    GridBase *grid = Subspace[0].Grid();
-
-    GRID_ASSERT(FineGridMulti->_ndimension == grid->_ndimension+1);
-    GRID_ASSERT(FineGridMulti->_processors[0] == 1);
-    for(int d=0;d<grid->_ndimension;d++){
-      GRID_ASSERT(FineGridMulti->_fdimensions[d+1] == grid->_fdimensions[d]);
-      GRID_ASSERT(FineGridMulti->_processors [d+1] == grid->_processors [d]);
-    }
-    int batch = FineGridMulti->_fdimensions[0];
-
-    Coordinate blatt,bsimd,bmpi;
-    CoarsenBlockGridLayout(grid,CoarseGrid,blatt,bsimd,bmpi);
-    GridCartesian BlockGrid(blatt,bsimd,bmpi);
-
-    BlockComplexField InnerProd(&BlockGrid);
-    blockOrthogonalise(InnerProd,Subspace);
-
-    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
-    Projector.Allocate(nbasis,grid,CoarseGrid);
-    Projector.ImportBasis(Subspace);
-
-    const int npoint = geom_srhs.npoint;
-
-    Eigen::MatrixXcd invMkl;
-    CoarsenFourierMatrix(CoarseGrid,invMkl);
-
-    FineField phaV(grid);
-    std::vector<FineComplexField>   phaF(npoint,grid);
-    std::vector<CoarseComplexField> pha (npoint,CoarseGrid);
-
-    tphase=-usecond();
-    CoarsenPhases(grid,CoarseGrid,&BlockGrid,pha,phaF);
-    tphase+=usecond();
-
-    std::vector<CoarseMatrix> _A;
-    _A.resize(npoint,CoarseGrid);
-    for(int k=0;k<npoint;k++) _A[k] = Zero();
-
-    Coordinate cmlatt,cmsimd,cmmpi;
-    CoarsenBatchGridLayout(CoarseGrid,batch,cmlatt,cmsimd,cmmpi);
-    GridCartesian CoarseBatchGrid(cmlatt,cmsimd,cmmpi);
-
-    CoarseVector       TmpProj(&CoarseBatchGrid);
-    CoarseComplexField phaB(&CoarseBatchGrid);
-
-    FineField hi_in (FineGridMulti);
-    FineField hi_out(FineGridMulti);
-    FineField zzF(grid); zzF = Zero();
-
-    for(int i0=0;i0<nbasis;i0+=batch){ // Loop over batches of basis vectors
-
-      int nbv = MIN(batch,nbasis-i0);
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i0<<"/"<<nbasis<< std::endl;
-
-      for(int p=0;p<npoint;p++){ // Loop over momenta
-
-	// One phase, applied to the whole batch. Tail slices are zeroed so
-	// the operator never sees undefined data.
-	for(int b=0;b<nbv;b++){
-	  tphaseBZ-=usecond();
-	  phaV = phaF[p]*Subspace[i0+b];
-	  tphaseBZ+=usecond();
-	  tslice-=usecond();
-	  InsertSliceFast(phaV,hi_in,b,0);
-	  tslice+=usecond();
-	}
-	tslice-=usecond();
-	for(int b=nbv;b<batch;b++){
-	  InsertSliceFast(zzF,hi_in,b,0);
-	}
-	tslice+=usecond();
-
-	tmat-=usecond();
-	linop.Op(hi_in,hi_out);
-	tmat+=usecond();
-
-	tproj-=usecond();
-	Projector.blockProject(hi_out,TmpProj);
-	tproj+=usecond();
-
-	tinv-=usecond();
-	CoarsenAccumulate(p,i0,nbv,batch,invMkl,pha,phaB,TmpProj,_A,CoarseGrid);
-	tinv+=usecond();
-      }
-    }
-
-    for(int p=0;p<npoint;p++){
-      GridtoBLAS(_A[p],BLAS_A[p]);
-    }
-    CoarsenReport(tphase,tphaseBZ,tslice,tmat,tproj,tinv);
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Coarsen a SINGLE RHS fine operator.
-  //
-  // No multiRHS packing: the operator is applied once per phased basis
-  // vector and the batch is assembled on the coarse side by the mixed
-  // blockProject, which takes a vector of fine fields and writes the batch
-  // coarse field the accumulate expects. Only nbv applications per momentum,
-  // so a batch that does not divide nbasis wastes nothing, and the live fine
-  // storage is batch fields rather than two D+1 fields of extent batch.
-  ///////////////////////////////////////////////////////////////////////////
-  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
-		       std::vector<FineField> &Subspace,
-		       GridBase *CoarseGrid,
-		       int batch)
-  {
-    RealD tproj=0.0, tmat=0.0, tphase=0.0, tphaseBZ=0.0, tslice=0.0, tinv=0.0;
-
-    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs (single RHS fine operator)"<< std::endl;
-
-    GRID_ASSERT(Subspace.size()==nbasis);
-    GRID_ASSERT(batch>=1);
-    GridBase *grid = Subspace[0].Grid();
-
-    Coordinate blatt,bsimd,bmpi;
-    CoarsenBlockGridLayout(grid,CoarseGrid,blatt,bsimd,bmpi);
-    GridCartesian BlockGrid(blatt,bsimd,bmpi);
-
-    BlockComplexField InnerProd(&BlockGrid);
-    blockOrthogonalise(InnerProd,Subspace);
-
-    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
-    Projector.Allocate(nbasis,grid,CoarseGrid);
-    Projector.ImportBasis(Subspace);
-
-    const int npoint = geom_srhs.npoint;
-
-    Eigen::MatrixXcd invMkl;
-    CoarsenFourierMatrix(CoarseGrid,invMkl);
-
-    FineField phaV(grid);
-    std::vector<FineComplexField>   phaF(npoint,grid);
-    std::vector<CoarseComplexField> pha (npoint,CoarseGrid);
-
-    tphase=-usecond();
-    CoarsenPhases(grid,CoarseGrid,&BlockGrid,pha,phaF);
-    tphase+=usecond();
-
-    std::vector<CoarseMatrix> _A;
-    _A.resize(npoint,CoarseGrid);
-    for(int k=0;k<npoint;k++) _A[k] = Zero();
-
-    Coordinate cmlatt,cmsimd,cmmpi;
-    CoarsenBatchGridLayout(CoarseGrid,batch,cmlatt,cmsimd,cmmpi);
-    GridCartesian CoarseBatchGrid(cmlatt,cmsimd,cmmpi);
-
-    CoarseVector       TmpProj(&CoarseBatchGrid);
-    CoarseComplexField phaB(&CoarseBatchGrid);
-
-    std::vector<FineField> MphaV(batch,grid);
-
-    for(int i0=0;i0<nbasis;i0+=batch){ // Loop over batches of basis vectors
-
-      int nbv = MIN(batch,nbasis-i0);
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i0<<"/"<<nbasis<< std::endl;
-
-      for(int p=0;p<npoint;p++){ // Loop over momenta
-
-	for(int b=0;b<nbv;b++){
-	  tphaseBZ-=usecond();
-	  phaV = phaF[p]*Subspace[i0+b];
-	  tphaseBZ+=usecond();
-	  tmat-=usecond();
-	  linop.Op(phaV,MphaV[b]);
-	  tmat+=usecond();
-	}
-	// The accumulate reads only the first nbv slices, but the projector
-	// sees the whole vector, so the tail must not be undefined.
-	for(int b=nbv;b<batch;b++) MphaV[b] = Zero();
-
-	tproj-=usecond();
-	Projector.blockProject(MphaV,TmpProj);
-	tproj+=usecond();
-
-	tinv-=usecond();
-	CoarsenAccumulate(p,i0,nbv,batch,invMkl,pha,phaB,TmpProj,_A,CoarseGrid);
-	tinv+=usecond();
-      }
-    }
-
-    for(int p=0;p<npoint;p++){
-      GridtoBLAS(_A[p],BLAS_A[p]);
-    }
-    CoarsenReport(tphase,tphaseBZ,tslice,tmat,tproj,tinv);
-  }
-  void Mdag(const CoarseVector &in, CoarseVector &out)
-  {
-    this->M(in,out);
-  }
-  void M (const CoarseVector &in, CoarseVector &out)
-  {
-    //    std::cout << GridLogMessage << "New Mrhs coarse"<<std::endl;
-    conformable(CoarseGrid(),in.Grid());
-    conformable(in.Grid(),out.Grid());
-    out.Checkerboard() = in.Checkerboard();
-
-    RealD t_tot;
-    RealD t_exch;
-    RealD t_GtoB;
-    RealD t_BtoG;
-    RealD t_mult;
-
-    CheckGridSet();
-    if ( in.Grid() != _CoarseGridMulti ) {
-      std::cout << GridLogError
-		<< "MultiGeneralCoarsenedOperatorV2::M called with a field on a"
-		<< std::endl;
-      std::cout << GridLogError
-		<< "  different grid object from the one given to SetGrid(). Two"
-		<< std::endl;
-      std::cout << GridLogError
-		<< "  grids of identical shape do not conform; share one object."
-		<< std::endl;
-      GRID_ASSERT(in.Grid() == _CoarseGridMulti);
-    }
-
-    GRID_TRACE("CoarseV2Mult");
-    t_tot=-usecond();
-    CoarseVector tin=in;
-    t_exch=-usecond();
-    // lambda scope so the roctx range covers exactly the exchange; the
-    // PaddedCellFwd/BwdMPI markers inside it then nest properly.
-    CoarseVector pin = [&](){ GRID_TRACE("CoarseV2Exchange");
-                              return CellMulti->ExchangePeriodic(tin); }(); //padded input
-    t_exch+=usecond();
-
-    CoarseVector pout(pin.Grid());
-
-    int npoint = geom.npoint;
-    typedef calcMatrix* Aview;
-    typedef LatticeView<Cvec> Vview;
-      
-    const int Nsimd = CComplex::Nsimd();
-
-    int64_t nrhs  =pin.Grid()->GlobalDimensions()[0];
-    GRID_ASSERT(nrhs>=1);
-
-    RealD flops,bytes;
-    int64_t osites=in.Grid()->oSites(); // unpadded
-    int64_t unpadded_vol = CoarseGrid()->lSites()/nrhs;
-    
-    flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
-    bytes = 1.0*osites*sizeof(siteMatrix)*npoint/pin.Grid()->GlobalDimensions()[0]
-          + 2.0*osites*sizeof(siteVector)*npoint;
-    
-
-    t_GtoB=-usecond();
-    { GRID_TRACE("CoarseV2GridToBLAS");
-      GridtoBLAS(pin,BLAS_B);
-    }
-    t_GtoB+=usecond();
-
-    GridBLAS BLAS;
-
-    t_mult=-usecond();
-    { GRID_TRACE("CoarseV2StencilGEMM");
-    for(int p=0;p<geom.npoint;p++){
-      RealD c = 1.0;
-      if (p==0) c = 0.0;
-      ComplexD beta(c);
-
-      BLAS.gemmBatched(nbasis,nrhs,nbasis,
-		       ComplexD(1.0),
-		       BLAS_AP[p], 
-		       BLAS_BP[p], 
-		       ComplexD(c), 
-		       BLAS_CP);
-    }
-    BLAS.synchronise();
-    }
-    t_mult+=usecond();
-
-    t_BtoG=-usecond();
-    { GRID_TRACE("CoarseV2BLASToGrid");
-      BLAStoGrid(out,BLAS_C);
-    }
-    t_BtoG+=usecond();
-    t_tot+=usecond();
-    /*
-    std::cout << GridLogMessage << "New Mrhs coarse DONE "<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult exch "<<t_exch<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult mult "<<t_mult<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult GtoB  "<<t_GtoB<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult BtoG  "<<t_BtoG<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult tot  "<<t_tot<<" us"<<std::endl;
-    */
-    //    std::cout << GridLogMessage<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse Kernel flops "<< flops<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse Kernel flop/s "<< flops/t_mult<<" mflop/s"<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse Kernel bytes/s "<< bytes/t_mult/1000<<" GB/s"<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse overall flops/s "<< flops/t_tot<<" mflop/s"<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse total bytes   "<< bytes/1e6<<" MB"<<std::endl;
-  };
-  virtual  void Mdiag    (const Field &in, Field &out){ GRID_ASSERT(0);};
-  virtual  void Mdir     (const Field &in, Field &out,int dir, int disp){assert(0);};
-  virtual  void MdirAll  (const Field &in, std::vector<Field> &out){assert(0);};
-};
-  
-NAMESPACE_END(Grid);

Grid/algorithms/multigrid/HDCGMultiGrid.h

@@ -0,0 +1,632 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/HDCGMultiGrid.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+#include <Grid/algorithms/iterative/ImplicitlyRestartedBlockLanczos.h>
+#include <Grid/algorithms/iterative/ImplicitlyRestartedBlockLanczosCoarse.h>
+#include <Grid/algorithms/multigrid/PVdagMMultiGrid.h>
+#include <Grid/algorithms/multigrid/HDCGMultiGridParams.h>
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////////////////////
+// The two-level mrhs HDCG (arXiv:1402.2585, mrhs form arXiv:2409.03904) as
+// objects, the shape of PVdagMMultiGrid.h:
+//
+//   MGCoarseGrids / MGFineGridsF   shared with the PVdagM chain
+//   HDCGCoarsening                 everything that is a function of the gauge
+//                                  field: the raw near-null basis (and its
+//                                  refinement), the transfer operators, the
+//                                  Galerkin coarse operator (Hermitian),
+//                                  the coarse eigenvectors for deflation
+//   HDCGSolver                     the solve chain on a borrowed coarsening:
+//                                  smoother, deflated coarse solve, ADEF-2
+//                                  preconditioner (fp64 or fp32 behind the
+//                                  seam), outer mrhs fPcg / BlockCGrQ
+//
+// The Hermitian operator is the Schur-preconditioned M^dag M on one
+// checkerboard; the near-null basis and the fine fields live on the
+// checkerboarded grid.  The coarse space is unvectorised and ALWAYS the D+1
+// multiRHS field; Nrhs 1 is the extent-1 case.  Grids outlive the
+// coarsening outlives the solver.
+//////////////////////////////////////////////////////////////////////////////////////
+template<class Fobj,class CComplex,int nbasis>
+class HDCGCoarsening {
+public:
+  typedef Lattice<Fobj>                                          FineField;
+  typedef MultiGeneralCoarsenedOperator<Fobj,CComplex,nbasis>  CoarseOperator;
+  typedef typename CoarseOperator::CoarseVector                  CoarseVector;
+  typedef typename GridTypeMapper<Fobj>::SinglePrecision         FobjF;
+  typedef Lattice<FobjF>                                         FineFieldF;
+
+  MGCoarseGrids                    &Grids;     // borrowed
+  MGFineGridsF                     &GridsF;    // borrowed
+  GridBase                         *FrbGrid;   // borrowed: the checkerboarded fine grid
+  int                               cb;
+  HDCGMultiGridParams               Params;
+  NextToNextToNextToNearestStencilGeometry5D geom;   // the coarse stencil of the Schur operator
+  CoarseOperator                    CoarseOp;
+  //////////////////////////////////////////////////////////////////////
+  // ONE fine transfer operator.  Its STORE follows the coarse sector --
+  // the coarse space is what it feeds -- while its import and export
+  // accept EITHER fine precision, because they are already a layout
+  // transformation and a scalar conversion inside one costs nothing.
+  //////////////////////////////////////////////////////////////////////
+  static const bool CoarseIsSingle =
+    ( sizeof(typename GridTypeMapper<CComplex>::scalar_type) == sizeof(ComplexF) );
+  typedef typename std::conditional<CoarseIsSingle,FineFieldF,FineField>::type ProjectorField;
+  typedef MultiRHSBlockProject<ProjectorField>                        ProjectorL1_t;
+
+  ProjectorL1_t                     MrhsProjectorL1;
+  std::vector<FineField>            rawNull;         // RAW fine near-null basis
+  std::vector<CoarseVector>         evec;            // coarse eigenvectors, on Coarse5d
+  std::vector<RealD>                eval;
+  MultiRHSDeflation<CoarseVector>   Deflator;        // nev==0 until CoarseLanczos
+  GridCartesian                    *CMrhs;           // transient solve grid, owned
+  int                               nrhs;
+
+  HDCGCoarsening(MGCoarseGrids &_Grids, MGFineGridsF &_GridsF, GridBase *_FrbGrid, int _cb,
+                 const HDCGMultiGridParams &P)
+    : Grids(_Grids), GridsF(_GridsF), FrbGrid(_FrbGrid), cb(_cb), Params(P),
+      geom(_Grids.Coarse5d),
+      CoarseOp(geom,_Grids.Coarse5d),
+      CMrhs(nullptr),
+      nrhs(-1)
+  {
+    Deflator.Deallocate();
+    // The coarse stencil is a max-norm-1 box (Geometry.h) and the Schur
+    // operator M^dag M reaches four fine hops, so a block shorter than 4
+    // leaves couplings outside the stencil: the Fourier extraction then
+    // folds them into the retained points and the Galerkin certificate
+    // reads it.  Warn here; the certificate is the guard.
+    for(int d=0;d<4;d++){
+      if ( Params.Setup.Block1[d] < 4 )
+        std::cout << GridLogWarning << "HDCGCoarsening: Block1[" << d << "] = " << Params.Setup.Block1[d]
+                  << " < 4: the Schur operator's range exceeds the coarse stencil; "
+                  << "expect a Galerkin defect" << std::endl;
+    }
+  };
+  ~HDCGCoarsening()
+  {
+    CoarseOp.ReleaseGrid();       // borrowers let go before their grids die
+    if ( CMrhs ) delete CMrhs;
+  }
+
+  int FileExists(const std::string &fname)
+  {
+    uint64_t exists=0;
+    if ( fname.length() ) {
+      if ( Grids.FGrid->IsBoss() ){ std::ifstream f(fname); exists=f.good()?1:0; }
+      Grids.FGrid->GlobalSum(exists);
+    }
+    return (int)exists;
+  }
+  FineField FineNoise(GridParallelRNG &RNG)
+  {
+    FineField v(FrbGrid); v.Checkerboard()=cb; gaussian(RNG,v);
+    return v;
+  }
+  // Chebyshev upper bound: the parameter, or 1.1 x a power-method estimate
+  RealD UpperBound(RealD param, LinearOperatorBase<FineField> &HermOp, GridParallelRNG &RNG)
+  {
+    if ( param > 0.0 ) return param;
+    FineField src = FineNoise(RNG);
+    PowerMethod<FineField> PM;
+    RealD lam = PM(HermOp,src);
+    std::cout << GridLogMessage << "HDCGCoarsening: fine power method " << lam
+              << ", Chebyshev upper bound " << 1.1*lam << std::endl;
+    return 1.1*lam;
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // The RAW basis: a refined basis on disk wins, then a raw basis on
+  // disk, else built by the chosen method (and saved if named).  CG
+  // refinement happens here; HDCG refinement needs a solver, see
+  // HDCGRefineSubspace below.  The Aggregation is scaffolding for the
+  // Chebyshev filters only: they run entirely on the fine grid.
+  ////////////////////////////////////////////////////////////////////
+  void GetSubspace(GridParallelRNG &RNG, LinearOperatorBase<FineField> &HermOp)
+  {
+    const HDCGSubspaceParams &S = Params.Subspace;
+    rawNull.clear();
+    rawNull.reserve(nbasis);
+    for(int k=0;k<nbasis;k++){ rawNull.push_back(FineField(FrbGrid)); rawNull[k].Checkerboard()=cb; }
+
+    if ( FileExists(S.RefinedSubspaceFile) ) {
+      std::cout << GridLogMessage << "HDCGCoarsening: loading REFINED subspace " << S.RefinedSubspaceFile << std::endl;
+      loadSubspace(rawNull, S.RefinedSubspaceFile, S.FileControl);
+      return;
+    }
+    if ( FileExists(Params.Setup.SubspaceFile) ) {
+      std::cout << GridLogMessage << "HDCGCoarsening: loading subspace " << Params.Setup.SubspaceFile << std::endl;
+      loadSubspace(rawNull, Params.Setup.SubspaceFile, S.FileControl);
+    } else {
+      Aggregation<Fobj,CComplex,nbasis> Agg(Grids.Coarse5d,FrbGrid,cb);
+      if ( S.Method == HDCGSubspaceMethod::ChebyshevNew ) {
+        RealD hi = UpperBound(S.ChebyHi,HermOp,RNG);
+        std::cout << GridLogMessage << "HDCGCoarsening: CreateSubspaceChebyshevNew hi " << hi << std::endl;
+        Agg.CreateSubspaceChebyshevNew(RNG,HermOp,hi);
+      } else if ( S.Method == HDCGSubspaceMethod::Chebyshev ) {
+        RealD hi = UpperBound(S.ChebyHi,HermOp,RNG);
+        std::cout << GridLogMessage << "HDCGCoarsening: CreateSubspaceChebyshev hi " << hi
+                  << " lo " << S.ChebyLo << " order " << S.ChebyOrder << std::endl;
+        Agg.CreateSubspaceChebyshev(RNG,HermOp,nbasis,hi,S.ChebyLo,S.ChebyOrder);
+      } else {
+        std::cout << GridLogMessage << "HDCGCoarsening: subspace from fine eigenvectors " << S.FineEvecFile
+                  << " summed " << S.FineEvecSample << " per vector" << std::endl;
+        LoadFineEvecSum(Agg.subspace,S.FineEvecFile,S.FineEvecSample,S.FileControl);
+      }
+      for(int k=0;k<nbasis;k++) rawNull[k]=Agg.subspace[k];
+      if ( Params.Setup.SubspaceFile.length() )
+        saveSubspace(rawNull, Params.Setup.SubspaceFile, S.FileControl);
+    }
+    if ( S.Refine == HDCGRefineMethod::CG ) {
+      std::cout << GridLogMessage << "HDCGCoarsening: CG refinement shift " << S.RefineShift
+                << " tol " << S.RefineTol << " maxit " << S.RefineMaxIt << std::endl;
+      Aggregation<Fobj,CComplex,nbasis> Agg(Grids.Coarse5d,FrbGrid,cb);
+      for(int k=0;k<nbasis;k++) Agg.subspace[k]=rawNull[k];
+      Agg.RefineSubspace(HermOp,S.RefineShift,S.RefineTol,S.RefineMaxIt);
+      for(int k=0;k<nbasis;k++) rawNull[k]=Agg.subspace[k];
+      if ( S.RefinedSubspaceFile.length() )
+        saveSubspace(rawNull, S.RefinedSubspaceFile, S.FileControl);
+    }
+  }
+  // Basis vector b = precisionChange of the sum of `sample` consecutive fp32 records
+  void LoadFineEvecSum(std::vector<FineField> &sub, const std::string &file, int sample, int control)
+  {
+#ifdef HAVE_LIME
+    emptyUserRecord record;
+    FineFieldF tmp(GridsF.FrbGridF), sum(GridsF.FrbGridF);
+    tmp.Checkerboard()=cb; sum.Checkerboard()=cb;
+    ScidacReader RD;
+    RD.open(file);
+    for(int b=0;b<nbasis;b++){
+      sum = Zero();
+      for(int n=0;n<sample;n++){
+        RD.readScidacFieldRecord(tmp,record,control);
+        sum = sum + tmp;
+      }
+      precisionChange(sub[b],sum);
+    }
+    RD.close();
+#else
+    GRID_ASSERT(0);
+#endif
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // HDCG refinement of the raw basis (the paper's loop): blocks of
+  // nrhs vectors, normalised, solved with the refining solver, and
+  // renormalised.  The solver was built on the SHIFTED operator and
+  // the coarsening of it; the caller re-coarsens afterwards.
+  ////////////////////////////////////////////////////////////////////
+  void RefineSubspaceHDCG(LinearOperatorBase<FineField> &HermOp, TwoLevelCGmrhs<FineField> &Refiner, int nr)
+  {
+    std::vector<FineField> src(nr,FrbGrid), res(nr,FrbGrid);
+    FineField tmp(FrbGrid); tmp.Checkerboard()=cb;
+    for(int r=0;r<nr;r++){ src[r].Checkerboard()=cb; res[r].Checkerboard()=cb; }
+    for(int b=0;b<nbasis;b+=nr){
+      int nb = std::min(nbasis-b,nr);
+      for(int r=0;r<nb;r++){
+        RealD scale = std::pow(norm2(rawNull[b+r]),-0.5);
+        src[r] = rawNull[b+r]*scale;
+      }
+      for(int r=nb;r<nr;r++) src[r] = src[0];   // pad the block; results discarded
+      for(int r=0;r<nr;r++)  res[r] = Zero();
+      HermOp.HermOp(src[0],tmp);
+      std::cout << GridLogMessage << "HDCGCoarsening: refine block " << b << " before <n|A|n> " << norm2(tmp) << std::endl;
+      Refiner(src,res);
+      for(int r=0;r<nb;r++){
+        RealD scale = std::pow(norm2(res[r]),-0.5);
+        rawNull[b+r] = res[r]*scale;
+      }
+      HermOp.HermOp(rawNull[b],tmp);
+      std::cout << GridLogMessage << "HDCGCoarsening: refine block " << b << " after  <n|A|n> " << norm2(tmp) << std::endl;
+    }
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // The Galerkin coarse operator and transfer operators from the RAW
+  // basis.  CoarsenOperator block-orthonormalises its input in place,
+  // so a working copy is taken and the RAW vectors survive.  The
+  // operator handed in must apply the Hermitian operator through Op
+  // (HermOpAdaptor).
+  //
+  // The RAW basis has done its work once this returns, so it is freed --
+  // it is nbasis fine vectors, the single largest resident block of the
+  // setup.  retain_basis keeps it, for the two callers that coarsen more
+  // than once: the HDCG subspace refinement below, and a fixed-basis
+  // rebuild on a changed gauge field (HMC).  Setup.RetainSubspace keeps
+  // it for the whole life of the object.
+  ////////////////////////////////////////////////////////////////////
+  void Coarsen(LinearOperatorBase<FineField> &HermOp, int retain_basis=0)
+  {
+    GRID_ASSERT( rawNull.size() == nbasis );
+    std::vector<FineField> sub(rawNull);
+
+    CoarseOp.SetGrid(Grids.CoarseBatch);
+    std::cout << GridLogMessage << "HDCGCoarsening: CoarsenOperator, batch " << Grids.batch << std::endl;
+    // CoarsenOperator orthonormalises sub in place and imports it into OUR
+    // transfer operator, which it then uses.  One basis store, imported once.
+    // The projector's grid carries the STORE's SIMD layout: the fp32 fine
+    // red-black grid when the coarse sector is fp32.
+    MrhsProjectorL1.Allocate(nbasis,
+                             CoarseIsSingle ? (GridBase *)GridsF.FrbGridF : (GridBase *)FrbGrid,
+                             Grids.Coarse5d);
+    CoarseOp.CoarsenOperator(HermOp,sub,Grids.Coarse5d,Grids.batch,MrhsProjectorL1);
+    if ( !retain_basis && !Params.Setup.RetainSubspace ) DiscardBasis();
+    MrhsProjectorL1.ReleaseScratch();
+    nrhs = -1;                     // the batch grid is set; SetNrhs retargets
+    ReportDeviceFootprint("after Coarsen");
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Retarget the coarse operator to Nrhs: the D+1 grid is owned here
+  // and every borrower re-captures it after this call.
+  ////////////////////////////////////////////////////////////////////
+  void SetNrhs(int nr)
+  {
+    if ( nr == nrhs ) return;
+    CoarseOp.ReleaseGrid();
+    if ( CMrhs ) { delete CMrhs; CMrhs=nullptr; }
+    Coordinate cml({nr,1,Grids.clatt[0],Grids.clatt[1],Grids.clatt[2],Grids.clatt[3]});
+    CMrhs = new GridCartesian(cml,Grids.cmsimd,Grids.cmmpi);
+    CoarseOp.SetGrid(CMrhs);
+    nrhs = nr;
+    std::cout << GridLogMessage << "HDCGCoarsening: coarse operator at Nrhs " << nr << std::endl;
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Coarse eigenvectors for the deflated coarse solve.  Block Lanczos
+  // runs at the solve's Nrhs on the D+1 operator; plain Lanczos at
+  // Nrhs 1 (the extent-1 D+1 grid), its vectors then sliced to the D
+  // grid so both feed the deflator the same way.  Loaded from EvecFile
+  // when present, saved there when named.
+  ////////////////////////////////////////////////////////////////////
+  void CoarseLanczos(int nr)
+  {
+    const HDCGLanczosParams &L = Params.Lanczos;
+    evec.clear(); eval.clear();
+    Deflator.Deallocate();
+    if ( L.Method == HDCGLanczosMethod::None ) {
+      std::cout << GridLogMessage << "HDCGCoarsening: no coarse deflation" << std::endl;
+      return;
+    }
+    if ( FileExists(L.EvecFile) ) {
+      std::cout << GridLogMessage << "HDCGCoarsening: loading coarse eigenpairs " << L.EvecFile << std::endl;
+      loadEigenpairs(evec,eval,L.EvecFile,Grids.Coarse5d,Params.Subspace.FileControl);
+    } else {
+      int Nconv=0;
+      SetNrhs( L.Method == HDCGLanczosMethod::Block ? nr : 1 );
+      HermitianLinearOperator<CoarseOperator,CoarseVector> HermOpC(CoarseOp);
+      GridParallelRNG cRNG(CMrhs); cRNG.SeedFixedIntegers(std::vector<int>({31,32,33,34}));
+      RealD hi = L.ChebyHi;
+      if ( hi <= 0.0 ) {
+        CoarseVector s(CMrhs); random(cRNG,s);
+        PowerMethod<CoarseVector> PM;
+        hi = 1.1*PM(HermOpC,s);
+        std::cout << GridLogMessage << "HDCGCoarsening: coarse Chebyshev upper bound " << hi << std::endl;
+      }
+      Chebyshev<CoarseVector> Cheby(L.ChebyLo,hi,L.ChebyOrder);
+
+      if ( L.Method == HDCGLanczosMethod::Block ) {
+        std::cout << GridLogMessage << "HDCGCoarsening: block Lanczos, Nrhs " << nr
+                  << " Nstop " << L.Nstop << " Nk " << L.Nk << " Nm " << L.Nm << std::endl;
+        ImplicitlyRestartedBlockLanczosCoarse<CoarseVector>
+          IRL(HermOpC,Grids.Coarse5d,CMrhs,nr,Cheby,L.Nstop,1,nr,L.Nk,L.Nm,L.Tol,L.MaxIt);
+        evec.resize(L.Nm,Grids.Coarse5d);
+        eval.resize(L.Nm);
+        std::vector<CoarseVector> src(nr,Grids.Coarse5d);
+        GridParallelRNG dRNG(Grids.Coarse5d); dRNG.SeedFixedIntegers(std::vector<int>({41,42,43,44}));
+        for(int r=0;r<nr;r++) random(dRNG,src[r]);
+        IRL.calc(eval,evec,src,Nconv,LanczosType::irbl);
+        evec.resize(Nconv,Grids.Coarse5d);                 // only the converged pairs
+        eval.resize(Nconv);
+      } else {
+        std::cout << GridLogMessage << "HDCGCoarsening: plain Lanczos at Nrhs 1, Nstop " << L.Nstop
+                  << " Nk " << L.Nk << " Nm " << L.Nm << std::endl;
+        PlainHermOp<CoarseVector>    Op(HermOpC);
+        FunctionHermOp<CoarseVector> OpCheby(Cheby,HermOpC);
+        ImplicitlyRestartedLanczos<CoarseVector> IRL(OpCheby,Op,L.Nstop,L.Nk,L.Nm,L.Tol,L.MaxIt);
+        std::vector<CoarseVector> evec1(L.Nm,CMrhs);
+        std::vector<RealD>        eval1(L.Nm);
+        CoarseVector src(CMrhs); random(cRNG,src);
+        IRL.calc(eval1,evec1,src,Nconv);
+        evec.resize(Nconv,Grids.Coarse5d);
+        eval.resize(Nconv);
+        for(int e=0;e<Nconv;e++){ ExtractSliceFast(evec[e],evec1[e],0,0); eval[e]=eval1[e]; }
+      }
+      // The Lanczos routines set their outputs only on reaching Nstop;
+      // a partial result carries no usable eigenpairs.
+      if ( Nconv < L.Nstop ) {
+        std::cout << GridLogWarning << "HDCGCoarsening: coarse Lanczos converged " << Nconv << " < Nstop " << L.Nstop
+                  << " modes (Chebyshev window [" << L.ChebyLo << "," << hi << "] must hold at least Nstop eigenvalues"
+                  << " and Nm must resolve them); coarse solve undeflated" << std::endl;
+        evec.clear(); eval.clear();
+        return;
+      }
+      if ( L.EvecFile.length() ) saveEigenpairs(evec,eval,L.EvecFile,Params.Subspace.FileControl);
+    }
+    const int nev = evec.size();
+    Deflator.ImportEigenBasis(evec,eval);
+    // The Lattice-layout copies have been imported into the BLAS store and
+    // saved if they were going to be; the solve reads only the BLAS store.
+    evec.clear(); evec.shrink_to_fit();
+    std::cout << GridLogMessage << "HDCGCoarsening: " << nev << " coarse eigenvectors imported for deflation"
+              << (nev ? " (lowest "+std::to_string(eval[0])+", highest "+std::to_string(eval[nev-1])+")" : "")
+              << std::endl;
+    ReportDeviceFootprint("after CoarseLanczos");
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Certificates at Nrhs 1 through the solve's own transfer operators:
+  // Galerkin ||A_c x - P^dag A P x|| / ||A_c x||  (the rounding level of
+  // the coarsening: fp64 coarse ~1e-13, fp32 ~1e-6), and Hermiticity
+  // |<x,A_c y> - <A_c x,y>| / |<x,A_c y>|, which CG relies on.
+  ////////////////////////////////////////////////////////////////////
+  RealD CertifyCoarsening(LinearOperatorBase<FineField> &HermOp)
+  {
+    SetNrhs(1);
+    CoarseVector xc(CMrhs), yc(CMrhs), Acx(CMrhs), Acy(CMrhs), PtAPx(CMrhs);
+    GridParallelRNG cRNG(CMrhs); cRNG.SeedFixedIntegers(std::vector<int>({21,22,23,24}));
+    random(cRNG,xc);
+    random(cRNG,yc);
+
+    CoarseOp.M(xc,Acx);                                     // A_c x
+    CoarseOp.M(yc,Acy);                                     // A_c y
+
+    std::vector<FineField> Px(1,FrbGrid), APx(1,FrbGrid);
+    Px[0].Checkerboard()=cb; APx[0].Checkerboard()=cb;
+    MrhsProjectorL1.blockPromote(Px,xc);                      // P x
+    HermOp.HermOp(Px[0],APx[0]);                            // A P x
+    MrhsProjectorL1.blockProject(APx,PtAPx);                  // P^dag A P x
+
+    CoarseVector d(CMrhs); d = Acx - PtAPx;
+    RealD rel = std::sqrt(norm2(d)/norm2(Acx));
+    ComplexD xAy = TensorRemove(innerProduct(xc,Acy));
+    ComplexD Axy = TensorRemove(innerProduct(Acx,yc));
+    RealD herm = abs(xAy-Axy)/abs(xAy);
+    std::cout << GridLogMessage << "HDCGCoarsening: GALERKIN CERTIFICATE ||A_c x - P^dag A P x||/||A_c x|| = " << rel << std::endl;
+    std::cout << GridLogMessage << "HDCGCoarsening: HERMITICITY CERTIFICATE |<x,A_c y> - <A_c x,y>|/|<x,A_c y>| = " << herm << std::endl;
+    return rel;
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  // The device memory this coarsening holds that the memory manager
+  // CANNOT evict: BLAS stores, which are plain device allocations.
+  // Lattice fields are omitted on purpose -- they are evictable, so they
+  // cost eviction traffic, not failure.  What is reported is what has to
+  // fit alongside the manager's cache, the comms buffers and the shm
+  // segment, which is the budget an out-of-memory run actually breaks.
+  //////////////////////////////////////////////////////////////////////
+  void ReportDeviceFootprint(const std::string &when)
+  {
+    uint64_t p1 = MrhsProjectorL1.DeviceBytes();
+    uint64_t o1 = CoarseOp.DeviceBytes();
+    uint64_t df = Deflator.DeviceBytes();
+    const double G = 1024.*1024.*1024.;
+    std::cout << GridLogMessage << "HDCGCoarsening (" << when << "): resident device memory/rank "
+              << (p1+o1+df)/G << " GB = transfer L1 " << p1/G
+              << " + operator L1 " << o1/G
+              << " + deflation " << df/G << " GB" << std::endl;
+    std::cout << GridLogMessage << "HDCGCoarsening (" << when << "): not evictable; must fit alongside the "
+              << MemoryManager::DeviceMaxBytes/G
+              << " GB manager cache, the comms buffers and the shm segment" << std::endl;
+  }
+
+  void DiscardBasis(void)
+  {
+    rawNull.clear(); rawNull.shrink_to_fit();
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// The solve chain on a borrowed coarsening.  Both fine precisions are
+// built; the outer picks the fp32 preconditioner through the seam when
+// Setup.FinePrecision is fp32.  OperatorShift is 0 in production; the
+// HDCG-refinement solver runs on the shifted operator.
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class MatrixF,class Coarsening>
+class HDCGSolver {
+public:
+  typedef typename Coarsening::FineField      FineField;
+  typedef typename Coarsening::FineFieldF     FineFieldF;
+  typedef typename Coarsening::CoarseOperator CoarseOperator;
+  typedef typename Coarsening::CoarseVector   CoarseVector;
+
+  Coarsening          &C;
+  HDCGMultiGridParams  Params;
+  int                  nrhs;
+  int                  _regrid;     // FIRST member-like init: SetNrhs before members capture grids
+  RealD                OperatorShift;
+
+  // fp64 fine chain
+  SchurDiagMooeeOperator<Matrix,FineField>    HermOpEO;
+  ShiftedHermOpLinearOperator<FineField>      HermOpShifted;
+  HermOpAdaptor<FineField>                    FineOp;         // Op = HermOp: the operator of the solve
+  ShiftedHermOpLinearOperator<FineField>      SmootherOp;
+  CGSmoother<FineField>                       SmootherCG;
+  ChebyshevSmoother<FineField>                SmootherCheby;
+  // coarse
+  HermitianLinearOperator<CoarseOperator,CoarseVector> LinOpC;
+  ConjugateGradient<CoarseVector>             CoarseCG;
+  DoNothingGuesser<CoarseVector>              NoGuess;
+  HPDSolver<CoarseVector>                     CoarseDeflatedCG;
+  ChebyshevInverter<CoarseVector>             CoarseCheby;
+  MrhsADEF2Preconditioner<FineField,CoarseVector,typename Coarsening::ProjectorL1_t> ADEF2;
+  // fp32 fine chain, behind the seam; the coarse solve is shared
+  SchurDiagMooeeOperator<MatrixF,FineFieldF>  HermOpEOF;
+  ShiftedHermOpLinearOperator<FineFieldF>     HermOpShiftedF;
+  HermOpAdaptor<FineFieldF>                   FineOpF;
+  ShiftedHermOpLinearOperator<FineFieldF>     SmootherOpF;
+  CGSmoother<FineFieldF>                      SmootherCGF;
+  ChebyshevSmoother<FineFieldF>               SmootherChebyF;
+  MrhsADEF2Preconditioner<FineFieldF,CoarseVector,typename Coarsening::ProjectorL1_t> ADEF2F;
+  MrhsMixedPrecPreconditioner<FineField,FineFieldF> PreconSeam;
+  TwoLevelCGmrhs<FineField>                   Outer;
+
+  LinearFunction<FineField>  &PickSmoother(void) {
+    return (Params.Smoother.Type==HDCGSmootherType::CG)
+      ? static_cast<LinearFunction<FineField>&>(SmootherCG)
+      : static_cast<LinearFunction<FineField>&>(SmootherCheby);
+  }
+  LinearFunction<FineFieldF> &PickSmootherF(void) {
+    return (Params.Smoother.Type==HDCGSmootherType::CG)
+      ? static_cast<LinearFunction<FineFieldF>&>(SmootherCGF)
+      : static_cast<LinearFunction<FineFieldF>&>(SmootherChebyF);
+  }
+  LinearFunction<CoarseVector> &PickCoarseSolve(void) {
+    return (Params.CoarseSolver.Method==HDCGCoarseMethod::DeflatedCG)
+      ? static_cast<LinearFunction<CoarseVector>&>(CoarseDeflatedCG)
+      : static_cast<LinearFunction<CoarseVector>&>(CoarseCheby);
+  }
+  template<class F, class Op, class ShiftedOp>
+  static LinearOperatorBase<F> &Unshifted(RealD shift, Op &op, ShiftedOp &shifted) {
+    return (shift==0.0) ? static_cast<LinearOperatorBase<F>&>(op) : static_cast<LinearOperatorBase<F>&>(shifted);
+  }
+
+  HDCGSolver(Matrix &Ddwf, MatrixF &DdwfF, Coarsening &_C, const HDCGMultiGridParams &P, int nr, RealD shift=0.0)
+    : C(_C), Params(P), nrhs(nr),
+      _regrid((C.SetNrhs(nr),0)),
+      OperatorShift(shift),
+      HermOpEO(Ddwf),
+      HermOpShifted(HermOpEO,shift),
+      FineOp(Unshifted<FineField>(shift,HermOpEO,HermOpShifted)),
+      SmootherOp(FineOp,P.Smoother.Shift),
+      SmootherCG(P.Smoother.Order,SmootherOp),
+      SmootherCheby(P.Smoother.ChebyLo,P.Smoother.ChebyHi,P.Smoother.Order,FineOp),
+      LinOpC(C.CoarseOp),
+      CoarseCG(P.CoarseSolver.Tol,P.CoarseSolver.MaxIt,false),
+      CoarseDeflatedCG(LinOpC,CoarseCG,NoGuess),
+      CoarseCheby(P.CoarseSolver.ChebyLo,P.CoarseSolver.ChebyHi,P.CoarseSolver.ChebyOrder,LinOpC),
+      ADEF2(FineOp,PickSmoother(),PickCoarseSolve(),PickCoarseSolve(),C.MrhsProjectorL1,C.Deflator,C.CMrhs,C.FrbGrid),
+      HermOpEOF(DdwfF),
+      HermOpShiftedF(HermOpEOF,shift),
+      FineOpF(Unshifted<FineFieldF>(shift,HermOpEOF,HermOpShiftedF)),
+      SmootherOpF(FineOpF,P.Smoother.Shift),
+      SmootherCGF(P.Smoother.Order,SmootherOpF),
+      SmootherChebyF(P.Smoother.ChebyLo,P.Smoother.ChebyHi,P.Smoother.Order,FineOpF),
+      ADEF2F(FineOpF,PickSmootherF(),PickCoarseSolve(),PickCoarseSolve(),C.MrhsProjectorL1,C.Deflator,C.CMrhs,C.GridsF.FrbGridF),
+      PreconSeam(ADEF2F,C.FrbGrid,C.GridsF.FrbGridF,nr),
+      Outer(P.Outer.Tol,P.Outer.MaxIt,FineOp,
+            (P.Setup.FinePrecision==MGPrecision::fp32)
+              ? static_cast<MrhsPreconditioner<FineField>&>(PreconSeam)
+              : static_cast<MrhsPreconditioner<FineField>&>(ADEF2),
+            C.FrbGrid,
+            (P.Outer.Algorithm==HDCGOuterAlgorithm::PrecBlockCGrQ) ? MrhsCGAlgorithm::PrecBlockCGrQ : MrhsCGAlgorithm::fPcg)
+  {
+    // BlockCGrQ assumes a stationary preconditioner: the fixed-iteration CG
+    // smoother is fixed WORK but a nonlinear map, and a coarse CG solved to a
+    // tolerance is not stationary either.  Chebyshev on both is.
+    if ( Params.Outer.Algorithm==HDCGOuterAlgorithm::PrecBlockCGrQ &&
+         ( Params.CoarseSolver.Method!=HDCGCoarseMethod::Chebyshev ||
+           Params.Smoother.Type     !=HDCGSmootherType::Chebyshev ) )
+      std::cout << GridLogWarning << "HDCGSolver: PrecBlockCGrQ with a non-stationary preconditioner "
+                << "(needs Smoother.Type Chebyshev AND CoarseSolver.Method Chebyshev); expect stagnation"
+                << std::endl;
+    std::cout << GridLogMessage << "HDCGSolver: Nrhs " << nr
+              << ", operator shift " << shift
+              << ", smoother " << (Params.Smoother.Type==HDCGSmootherType::CG ? "CG" : "Chebyshev")
+              << " order " << Params.Smoother.Order
+              << ", coarse " << (Params.CoarseSolver.Method==HDCGCoarseMethod::DeflatedCG ? "deflated CG" : "Chebyshev")
+              << " (deflation vectors " << C.Deflator.nev << ")"
+              << ", preconditioner fine level "
+              << (Params.Setup.FinePrecision==MGPrecision::fp32 ? "fp32 (seam at the outer CG)" : "fp64")
+              << ", outer " << (Params.Outer.Algorithm==HDCGOuterAlgorithm::PrecBlockCGrQ ? "PrecBlockCGrQ" : "fPcg")
+              << " EXACT fp64" << std::endl;
+  }
+
+  // One right-hand side through the LinearFunction interface (nrhs must be 1)
+  void Solve(const FineField &src, FineField &sol)
+  {
+    GRID_ASSERT( nrhs == 1 );
+    sol.Checkerboard() = src.Checkerboard();
+    GridStopWatch w; w.Start();
+    Outer(src,sol);
+    w.Stop();
+    FineField Ax(src.Grid()); Ax.Checkerboard()=src.Checkerboard();
+    FineOp.HermOp(sol,Ax);
+    Ax=Ax-src;
+    std::cout << GridLogMessage << "HDCGSolver: single rhs total " << w.Elapsed()
+              << "; FINAL Nrhs 1 (LinearFunction): true residual = " << std::sqrt(norm2(Ax)/norm2(src)) << std::endl;
+  }
+  void Solve(std::vector<FineField> &src, std::vector<FineField> &sol)
+  {
+    GRID_ASSERT( (int)src.size() == nrhs );
+    for(int r=0;r<nrhs;r++) sol[r].Checkerboard() = src[r].Checkerboard();
+    MemoryManager::EvictAll();
+    MemoryManager::DropCache();
+
+    GridStopWatch w; w.Start();
+    Outer(src,sol);
+    w.Stop();
+    std::cout << GridLogMessage << "HDCGSolver: Nrhs "<<nrhs<<" total " << w.Elapsed()
+              << "  (per RHS: " << w.useconds()/1.0e6/nrhs << " s)" << std::endl;
+    {
+      FineField Ax(src[0].Grid()); Ax.Checkerboard()=src[0].Checkerboard();
+      RealD worst=0.0;
+      for(int r=0;r<nrhs;r++){
+        FineOp.HermOp(sol[r],Ax);
+        Ax=Ax-src[r];
+        RealD rn=std::sqrt(norm2(Ax)/norm2(src[r]));
+        std::cout << GridLogMessage << "FINAL Nrhs "<<nrhs<<": rhs["<<r<<"] true residual = " << rn << std::endl;
+        worst=std::max(worst,rn);
+      }
+      std::cout << GridLogMessage << "FINAL Nrhs "<<nrhs<<": worst-case residual = " << worst << std::endl;
+    }
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// HDCG refinement of the basis (the paper's loop): coarsen the SHIFTED
+// operator, deflate it, solve each raw basis vector on it with a
+// short-tolerance HDCG, keep the normalised solutions.  A no-op unless
+// Subspace.Refine is HDCG and no refined basis was loaded.  The caller
+// then coarsens the unshifted operator as usual.
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class MatrixF,class Coarsening>
+void HDCGRefineSubspace(Coarsening &C, Matrix &Ddwf, MatrixF &DdwfF,
+                        LinearOperatorBase<typename Coarsening::FineField> &HermOp, int nr)
+{
+  typedef typename Coarsening::FineField FineField;
+  const HDCGSubspaceParams &S = C.Params.Subspace;
+  if ( S.Refine != HDCGRefineMethod::HDCG ) return;
+  if ( C.FileExists(S.RefinedSubspaceFile) ) return;
+
+  std::cout << GridLogMessage << "HDCGRefineSubspace: shift " << S.RefineShift << " tol " << S.RefineTol
+            << " maxit " << S.RefineMaxIt << " smoother order " << S.RefineSmootherOrder << std::endl;
+  ShiftedHermOpLinearOperator<FineField> Shifted(HermOp,S.RefineShift);
+  HermOpAdaptor<FineField>               ShiftedOp(Shifted);
+  C.Coarsen(ShiftedOp,1);   // refinement below still needs the RAW basis
+  C.CoarseLanczos(nr);
+
+  HDCGMultiGridParams PR = C.Params;
+  PR.Outer.Tol      = S.RefineTol;
+  PR.Outer.MaxIt    = S.RefineMaxIt;
+  PR.Smoother.Order = S.RefineSmootherOrder;
+  HDCGSolver<Matrix,MatrixF,Coarsening> Refiner(Ddwf,DdwfF,C,PR,nr,S.RefineShift);
+  C.RefineSubspaceHDCG(HermOp,Refiner.Outer,nr);
+
+  if ( S.RefinedSubspaceFile.length() )
+    saveSubspace(C.rawNull, S.RefinedSubspaceFile, S.FileControl);
+}
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/HDCGMultiGridParams.h

@@ -0,0 +1,177 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/HDCGMultiGridParams.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////////////////////
+// Parameters of the two-level mrhs HDCG (Hermitian, Schur-preconditioned)
+// multigrid, HDCGMultiGrid.h.  Serialisable: read from XML, template written
+// on a missing file.  Setup (blocking, batch, basis file, fine precision) is
+// the same struct as the PVdagM chain's; Block2 is only used when a
+// coarse-coarse level exists but must still divide the coarse lattice.
+//////////////////////////////////////////////////////////////////////////////////////
+
+GRID_SERIALIZABLE_ENUM(HDCGSubspaceMethod, undef, ChebyshevNew, 1, Chebyshev, 2, FineEvecs, 3);
+GRID_SERIALIZABLE_ENUM(HDCGRefineMethod,   undef, None, 1, CG, 2, HDCG, 3);
+GRID_SERIALIZABLE_ENUM(HDCGLanczosMethod,  undef, None, 1, Block, 2, Plain, 3);
+GRID_SERIALIZABLE_ENUM(HDCGCoarseMethod,   undef, DeflatedCG, 1, Chebyshev, 2);
+GRID_SERIALIZABLE_ENUM(HDCGSmootherType,   undef, CG, 1, Chebyshev, 2);
+GRID_SERIALIZABLE_ENUM(HDCGOuterAlgorithm, undef, fPcg, 1, PrecBlockCGrQ, 2);
+
+// Near-null basis.  ChebyshevNew filters noise through the Hermitian operator
+// with the upper bound only; Chebyshev takes lo/hi/order; FineEvecs sums
+// FineEvecSample consecutive fp32 fine eigenvector records per basis vector.
+// A ChebyHi <= 0 means 1.1 x the power-method estimate.  FileControl is the
+// scidac binary-IO control word of the basis files (0 for pre-2026 HDCG files).
+struct HDCGSubspaceParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGSubspaceParams,
+                                  HDCGSubspaceMethod, Method,
+                                  RealD,       ChebyHi,
+                                  RealD,       ChebyLo,
+                                  int,         ChebyOrder,
+                                  std::string, FineEvecFile,
+                                  int,         FineEvecSample,
+                                  int,         FileControl,
+                                  HDCGRefineMethod, Refine,
+                                  RealD,       RefineShift,         // CG: the shift of the refining CG; HDCG: of the refining operator
+                                  RealD,       RefineTol,
+                                  int,         RefineMaxIt,
+                                  int,         RefineSmootherOrder, // HDCG refine only
+                                  std::string, RefinedSubspaceFile);
+  HDCGSubspaceParams()
+    : Method(HDCGSubspaceMethod::ChebyshevNew), ChebyHi(0.0), ChebyLo(0.01), ChebyOrder(500),
+      FineEvecFile(""), FineEvecSample(1),
+      FileControl(BinaryIO::BINARYIO_LEXICOGRAPHIC|BinaryIO::BINARYIO_AGGREGATE),
+      Refine(HDCGRefineMethod::None), RefineShift(1.0e-3), RefineTol(1.0e-3), RefineMaxIt(500),
+      RefineSmootherOrder(11), RefinedSubspaceFile("") {};
+};
+
+// Coarse eigenvectors for the deflated coarse solve: block Lanczos at the
+// solve's Nrhs, or plain Lanczos at Nrhs 1, on the same coarse operator.
+// ChebyHi <= 0 means 1.1 x the power-method estimate on the coarse operator.
+struct HDCGLanczosParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGLanczosParams,
+                                  HDCGLanczosMethod, Method,
+                                  int,   Nstop,
+                                  int,   Nk,
+                                  int,   Nm,
+                                  RealD, ChebyLo,
+                                  RealD, ChebyHi,
+                                  int,   ChebyOrder,
+                                  RealD, Tol,
+                                  int,   MaxIt,
+                                  std::string, EvecFile);
+  HDCGLanczosParams()
+    : Method(HDCGLanczosMethod::Block), Nstop(32), Nk(32), Nm(128),
+      ChebyLo(0.01), ChebyHi(0.0), ChebyOrder(201), Tol(1.0e-4), MaxIt(100), EvecFile("") {};
+};
+
+struct HDCGCoarseSolverParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGCoarseSolverParams,
+                                  HDCGCoarseMethod, Method,
+                                  RealD, Tol,            // DeflatedCG
+                                  int,   MaxIt,
+                                  RealD, ChebyLo,        // Chebyshev polynomial inverse
+                                  RealD, ChebyHi,
+                                  int,   ChebyOrder);
+  HDCGCoarseSolverParams()
+    : Method(HDCGCoarseMethod::DeflatedCG), Tol(5.0e-2), MaxIt(3000),
+      ChebyLo(1.0e-2), ChebyHi(40.0), ChebyOrder(120) {};
+};
+
+// CG: Order iterations of CG on the operator shifted by Shift.
+// Chebyshev: order Order polynomial on [ChebyLo,ChebyHi] of the unshifted operator.
+struct HDCGSmootherParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGSmootherParams,
+                                  HDCGSmootherType, Type,
+                                  RealD, Shift,
+                                  int,   Order,
+                                  RealD, ChebyLo,
+                                  RealD, ChebyHi);
+  HDCGSmootherParams() : Type(HDCGSmootherType::CG), Shift(2.0), Order(7), ChebyLo(2.0), ChebyHi(92.0) {};
+};
+
+struct HDCGOuterParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGOuterParams,
+                                  RealD, Tol,
+                                  int,   MaxIt,
+                                  HDCGOuterAlgorithm, Algorithm);
+  HDCGOuterParams() : Tol(1.0e-8), MaxIt(500), Algorithm(HDCGOuterAlgorithm::fPcg) {};
+};
+
+struct HDCGMultiGridParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGMultiGridParams,
+                                  MGSetupParams,          Setup,
+                                  HDCGSubspaceParams,     Subspace,
+                                  HDCGLanczosParams,      Lanczos,
+                                  HDCGCoarseSolverParams, CoarseSolver,
+                                  HDCGSmootherParams,     Smoother,
+                                  HDCGOuterParams,        Outer);
+  HDCGMultiGridParams() {};
+};
+
+inline void CheckValidity(const HDCGMultiGridParams &P)
+{
+  GRID_ASSERT( P.Setup.Block1.size() == 4 );
+  GRID_ASSERT( P.Setup.Block2.size() == 4 );
+  GRID_ASSERT( P.Setup.CoarsenBatch  >= 1 );
+  GRID_ASSERT( P.Setup.FinePrecision != MGPrecision::undef );
+  GRID_ASSERT( P.Subspace.Method     != HDCGSubspaceMethod::undef );
+  GRID_ASSERT( P.Subspace.Refine     != HDCGRefineMethod::undef );
+  GRID_ASSERT( P.Subspace.FineEvecSample >= 1 );
+  GRID_ASSERT( P.Lanczos.Method      != HDCGLanczosMethod::undef );
+  GRID_ASSERT( P.Lanczos.Nstop <= P.Lanczos.Nk && P.Lanczos.Nk <= P.Lanczos.Nm );
+  GRID_ASSERT( P.CoarseSolver.Method != HDCGCoarseMethod::undef );
+  GRID_ASSERT( P.CoarseSolver.Tol     > 0.0 );
+  GRID_ASSERT( P.Smoother.Type       != HDCGSmootherType::undef );
+  GRID_ASSERT( P.Smoother.Order       > 0 );
+  GRID_ASSERT( P.Outer.Tol            > 0.0 );
+  GRID_ASSERT( P.Outer.Algorithm     != HDCGOuterAlgorithm::undef );
+}
+
+// As ReadPVdagMMultiGridParams: a missing file gets a template written next
+// to it and false returned; an empty filename means defaults.
+inline bool ReadHDCGMultiGridParams(HDCGMultiGridParams &P, const std::string &xmlfile)
+{
+  if ( xmlfile.length() ) {
+    bool good;
+    { std::ifstream f(xmlfile); good = f.good(); }
+    if ( !good ) {
+      std::cout << GridLogMessage << "HDCGMultiGridParams: " << xmlfile << " does not exist" << std::endl;
+      if ( GlobalSharedMemory::WorldRank == 0 ) {
+        XmlWriter WR(xmlfile+".templ");
+        write(WR, "HDCGMultiGridParams", P);
+        std::cout << GridLogMessage << "HDCGMultiGridParams: template written to "
+                  << xmlfile << ".templ" << std::endl;
+      }
+      return false;
+    }
+    XmlReader RD(xmlfile);
+    read(RD, "HDCGMultiGridParams", P);
+  }
+  CheckValidity(P);
+  std::cout << GridLogMessage << "HDCGMultiGridParams ("
+            << (xmlfile.length() ? xmlfile : std::string("defaults")) << "):" << std::endl;
+  std::cout << P << std::endl;
+  return true;
+}
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/MrhsMultiGrid.h

@@ -0,0 +1,433 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/MrhsMultiGrid.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+#include <Grid/algorithms/deflation/MultiRHSBlockProject.h>
+#include <Grid/algorithms/multigrid/MrhsPreconditioner.h>
+
+NAMESPACE_BEGIN(Grid);
+
+
+//////////////////////////////////////////////////////////////////////
+// Single-polynomial mrhs PGCR: one step length and one set of
+// orthogonalisation coefficients SHARED across the right-hand sides
+// (vnorm2/vinnerProduct sum over rhs).  For DWF the dense spectrum
+// makes sharing near-free and the summed reductions amortise; see the
+// mrhs-HDCG paper (arXiv:2409.03904).  Orthogonalisation is classical
+// Gram-Schmidt, batched per rhs (rankInnerProductMulti/axpyMulti) with
+// ONE GlobalSumVector for all coefficients.
+//
+// OnStep is a programmatic per-step hook for algorithmic studies
+// (e.g. driving the GCR coefficient recorder); it is not a consumer
+// parameter.
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class MrhsPGCRNonHermitian {
+public:
+  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
+  int ZeroGuess = 0; int FirstCycle = 0;
+  std::string name = "Level 1";
+  // Trace range names carry the instance name, so a profile separates the
+  // solvers that otherwise nest as one shared range.
+  std::string trace_op = "MrhsPGCR::vOp", trace_orthog = "MrhsPGCR orthog";
+  LinearOperatorBase<Field> &Linop;
+  MrhsLinearFunction<Field> &Preconditioner;
+  std::function<void(int)> OnStep;      // called with the outer step count after every step
+  
+  void Level(int lv){
+    name = "Level " + std::to_string(lv);
+    level=lv;
+  }
+
+  void Name(std::string n){
+    name = n;
+    trace_op = name+" MrhsPGCR::vOp";
+    trace_orthog = name+" MrhsPGCR orthog";
+  }
+
+  void SetZeroGuess(int z)
+  {
+    ZeroGuess=z;
+  }
+
+  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
+    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep)
+  {
+    level=1;
+  }
+
+  static RealD vnorm2(std::vector<Field> &x)
+  {
+    RealD s=0;
+    for(auto &f:x) s+=norm2(f);
+    return s;
+  }
+
+  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){
+    ComplexD s(0);
+    for(int r=0;r<(int)x.size();r++) {
+      s+=innerProduct(x[r],y[r]);
+    }
+    return s;
+  }
+
+  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y)
+  {
+    for(int r=0;r<(int)z.size();r++) {
+      axpy(z[r],a,x[r],y[r]);
+    }
+  }
+
+  void vOp(std::vector<Field> &in,std::vector<Field> &out)
+  {
+    GRID_TRACE(trace_op.c_str());
+    for(int r=0;r<(int)in.size();r++) {
+      Linop.Op(in[r],out[r]);
+    }
+  }
+  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
+    RealD cp,ssq,rsq;
+    int nrhs=src.size();
+    GridBase *grid=src[0].Grid();
+    ssq=vnorm2(src);
+    rsq=Tolerance*Tolerance*ssq;
+
+    std::vector<Field> r(nrhs,grid);
+    GridStopWatch T;
+    T.Start();
+    steps=0;
+    FirstCycle=1;
+    for(int k=0;k<MaxIterations;k++){
+      cp=GCRnStep(src,psi,rsq);
+      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps
+	       <<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
+      if(cp<rsq){
+        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
+        RealD tr=vnorm2(r);
+        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
+                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
+        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
+        return;
+      }
+    }
+    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
+  }
+  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq)
+  {
+    RealD cp;
+    ComplexD a,rq;
+    int nrhs=src.size();
+    GridBase *grid=src[0].Grid();
+
+    std::vector<Field> r(nrhs,grid),Az(nrhs,grid);   // Az: restart residual scratch only
+    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
+    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
+    std::vector<RealD> qq(mmax);
+
+    if (ZeroGuess && FirstCycle) {
+      for(int rr=0;rr<nrhs;rr++){
+	psi[rr]=Zero();
+	r[rr]=src[rr];
+      }
+    } else {
+      vOp(psi,Az);
+      for(int rr=0;rr<nrhs;rr++) {
+	r[rr]=src[rr]-Az[rr];
+      }
+    }
+
+    FirstCycle=0;
+    // p[0]=Prec(r), q[0]=A p[0], produced directly in the history slots (no copies)
+
+    Preconditioner(r,p[0]);
+    vOp(p[0],q[0]);
+    qq[0]=vnorm2(q[0]);
+    cp=vnorm2(r);
+    
+    for(int k=0;k<nstep;k++){
+      steps++;
+      int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
+      
+      if ( OnStep ) OnStep(steps);
+      
+      rq=vinnerProduct(q[peri_k],r);
+      a=rq/qq[peri_k];
+      vaxpy(psi,a,p[peri_k],psi);
+      vaxpy(r,-a,q[peri_k],r);
+      cp=vnorm2(r);
+
+      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
+
+      if((k==nstep-1)||(cp<rsq)) return cp;
+
+      // New direction straight into its history slot: p=Prec(r), q=A p.
+      Preconditioner(r,p[peri_kp]);
+
+      vOp(p[peri_kp],q[peri_kp]);
+
+      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
+
+      {
+        GRID_TRACE(trace_orthog.c_str());
+        // Classical Gram-Schmidt: all coefficients against the UN-updated new q
+        // (independent, batchable), then apply.  Complex coefficient: the
+        // operator is non-Hermitian, real(<q_j,Aq>) alone left q's non-orthogonal.
+        // Batched per rhs (one fused kernel + one reduction each), the shared
+        // coefficient summed over rhs on the host, ONE GlobalSumVector.
+        std::vector<ComplexD> bcoef(northog,ComplexD(0.0)), part;
+        for(int rr=0;rr<nrhs;rr++){
+          std::vector<const Field*> qwin(northog);
+          for(int back=0;back<northog;back++){
+	    int peri_back=(k-back)%mmax;
+	    GRID_ASSERT((k-back)>=0);
+	    qwin[back]=&q[peri_back][rr];
+	  }
+
+          rankInnerProductMulti(part,qwin,q[peri_kp][rr]);
+	  
+          for(int back=0;back<northog;back++) bcoef[back]+=part[back];
+        }
+
+        if(northog) grid->GlobalSumVector(&bcoef[0],northog);
+
+        for(int back=0;back<northog;back++){
+	  int peri_back=(k-back)%mmax;
+	  bcoef[back]=-bcoef[back]/qq[peri_back];
+	}
+        for(int rr=0;rr<nrhs;rr++){
+          std::vector<const Field*> qwin(northog), pwin(northog);
+          for(int back=0;back<northog;back++){
+	    int peri_back=(k-back)%mmax;
+	    qwin[back]=&q[peri_back][rr];
+	    pwin[back]=&p[peri_back][rr];
+	  }
+          axpyMulti(p[peri_kp][rr],bcoef,pwin);
+          axpyMulti(q[peri_kp][rr],bcoef,qwin);
+        }
+      }
+      qq[peri_kp]=vnorm2(q[peri_kp]);
+    }
+    GRID_ASSERT(0); return cp;
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// Dense L3 solve on the packed D+1 coarse-coarse field.  Adapts any
+// class exposing ApplyBatch6D (DenseCoarseMatrix) to a LinearFunction.
+//////////////////////////////////////////////////////////////////////
+template<class DenseType, class CoarseCoarseField>
+class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
+public:
+  DenseType &_Dense;
+  int _nrhs;
+  MrhsDenseCCSolve(DenseType &D, int nrhs) : _Dense(D), _nrhs(nrhs) {}
+  using LinearFunction<CoarseCoarseField>::operator();
+  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
+    _Dense.ApplyBatch6D(in, out, _nrhs);
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// L2->L3 half V-cycle on the D+1 coarse field: coarse-coarse correction
+// through the mixed blockProject, then post-smooth.
+//////////////////////////////////////////////////////////////////////
+template<class CoarseField, class CoarseCoarseField>
+class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
+public:
+  LinearOperatorBase<CoarseField>          &_CoarseOp;
+  LinearFunction<CoarseField>              &_CoarseSmoother;
+  MultiRHSBlockProject<CoarseField>        &_Projector;
+  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve;
+  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
+  int _nrhs;
+  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
+                           LinearFunction<CoarseField> &CoarseSmoother,
+                           MultiRHSBlockProject<CoarseField> &Projector,
+                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
+                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
+    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
+      _CoarseCoarseSolve(CoarseCoarseSolve),
+      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
+  using LinearFunction<CoarseField>::operator();
+  virtual void operator()(const CoarseField &in, CoarseField &out) {
+    GRID_TRACE("MGCoarseVcycle");
+    CoarseField vec1(in.Grid());
+    CoarseField vec2(in.Grid());
+    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
+    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
+    // The cycle starts from x0 = in, one unit-step Richardson iteration ahead of
+    // a zero start; measured, it pays for its apply by cutting Couter steps 22%.
+    // x0 is never materialised in out: the operator is applied to in directly and
+    // the copy folds into the add that lands the coarse-coarse correction.
+    {
+      GRID_TRACE("MGCoarseResidual");
+      _CoarseOp.Op(in,vec1);  sub(vec1,in,vec1);
+    }
+    // restrict, through the mixed blockProject: D+1 coarse in, D+1 cc out
+    {
+      GRID_TRACE("MGCCProject");
+      _Projector.blockProject(vec1,CCsrc);
+    }
+    {
+      GRID_TRACE("MGCCSolve");
+      _CoarseCoarseSolve(CCsrc,CCsol);
+    }
+    {
+      GRID_TRACE("MGCCPromote");
+      _Projector.blockPromote(vec1,CCsol);
+      add(out,in,vec1);
+    }
+    {
+      GRID_TRACE("MGCoarseResidual2");
+      _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
+    }
+    {
+      GRID_TRACE("MGCoarseSmooth");
+      _CoarseSmoother(vec1,vec2);
+      add(out,out,vec2);
+    }
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// L1->L2 mrhs V-cycle.  The whole V-cycle is preconditioner: its fine
+// residuals and the smoother may run with sloppy halos; the caller
+// (the outer Krylov) gets the exact operator back on exit.  SetSloppy
+// is wired by the composer to PVdagMLinearOperator::SloppyComms (a
+// no-op by default), replacing the file-scope global the example used.
+//////////////////////////////////////////////////////////////////////
+// Projector_t defaults to a transfer operator whose STORE matches FineField,
+// which is the case whenever the coarse sector and the fine level share a
+// precision; pass it explicitly when they differ.
+template<class FineField, class MrhsCoarseVector, class FineSmoother,
+         class Projector_t = MultiRHSBlockProject<FineField> >
+class MrhsTwoLevelMG : public MrhsPreconditioner<FineField> {
+public:
+  typedef MrhsCoarseVector CoarseVector;
+  LinearOperatorBase<FineField>   &_FineOperator;
+  FineSmoother                    &_PostSmoother;
+  Projector_t                     &_Projector;   // store precision need not match FineField
+  LinearFunction<CoarseVector>    &_CoarseSolve;
+  GridBase *_CoarseGrid, *_CoarseGridMrhs;
+  std::function<void(int)> SetSloppy = [](int){};
+  int SloppyComms = 0;                 // value passed to SetSloppy on entry
+
+  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
+                 Projector_t &Projector, LinearFunction<CoarseVector> &CoarseSolve,
+                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
+    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
+      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
+
+  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out)
+  {
+    GRID_TRACE("MGVcycle");
+    SetSloppy(SloppyComms);
+    int nrhs=in.size(); GridBase *fgrid=in[0].Grid();
+    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
+    // x0 = in, never materialised in out: the operator is applied to in directly
+    // and the copy folds into the add that lands the coarse correction.
+    {
+      GRID_TRACE("MGFineResidual");
+      for(int r=0;r<nrhs;r++){ _FineOperator.Op(in[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
+    }
+    // fine vector -> D+1 coarse, via the mixed blockProject
+    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
+    {
+      GRID_TRACE("MGProject");
+      _Projector.blockProject(vec1,CsrcMrhs);
+    }
+    CsolMrhs=Zero();
+    {
+      GRID_TRACE("MGCoarseSolve");
+      _CoarseSolve(CsrcMrhs,CsolMrhs);
+    }
+    {
+      GRID_TRACE("MGPromote");
+      _Projector.blockPromote(vec1,CsolMrhs);
+      for(int r=0;r<nrhs;r++) add(out[r],in[r],vec1[r]);
+    }
+    {
+      GRID_TRACE("MGFineResidual2");
+      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
+    }
+    {
+      GRID_TRACE("MGPostSmooth");
+      for(int r=0;r<nrhs;r++){
+        _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]);
+      }
+    }
+    SetSloppy(0);
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// The fp64/fp32 seam of the solve chain.  The outer Krylov hands fp64
+// residuals to its preconditioner; this adapter converts them to fp32,
+// runs an fp32 preconditioner (the whole V-cycle), and converts the
+// correction back.  Two precisionChange per outer step per rhs, on
+// workspaces built once.  The outer operator never sees fp32.
+//
+// The fp32 scratch is allocated ON FIRST USE, not in the constructor: the
+// solver builds this seam whichever precision is selected, and at Nrhs 12
+// on a 48^3x96 Ls=24 rank the scratch is 2 GB of device memory.  A seam
+// that is never called must cost nothing.  Same idiom as the GCR history
+// vectors: re-made only if the grid or the rhs count changes.
+//////////////////////////////////////////////////////////////////////
+template<class FieldD, class FieldF>
+class MrhsMixedPrecPreconditioner : public MrhsPreconditioner<FieldD> {
+public:
+  MrhsPreconditioner<FieldF> &_Inner;
+  precisionChangeWorkspace    _ws_d2f;   // out fp32, in fp64
+  precisionChangeWorkspace    _ws_f2d;   // out fp64, in fp32
+  GridBase                   *_gridF;
+  std::vector<FieldF>         _in_f, _out_f;
+
+  MrhsMixedPrecPreconditioner(MrhsPreconditioner<FieldF> &Inner, GridBase *gridD, GridBase *gridF, int nrhs)
+    : _Inner(Inner), _ws_d2f(gridF,gridD), _ws_f2d(gridD,gridF), _gridF(gridF) {}
+
+  void Scratch(int nrhs){
+    if ( (int)_in_f.size() == nrhs ) return;
+    _in_f.clear();  _in_f.reserve(nrhs);
+    _out_f.clear(); _out_f.reserve(nrhs);
+    for(int r=0;r<nrhs;r++){ _in_f.emplace_back(_gridF); _out_f.emplace_back(_gridF); }
+  }
+
+  virtual void operator()(std::vector<FieldD> &in, std::vector<FieldD> &out)
+  {
+    GRID_TRACE("MGPrecisionSeam");
+    int nrhs=in.size(); Scratch(nrhs);
+    for(int r=0;r<nrhs;r++) precisionChange(_in_f[r],in[r],_ws_d2f);
+    _Inner(_in_f,_out_f);
+    for(int r=0;r<nrhs;r++) precisionChange(out[r],_out_f[r],_ws_f2d);
+  }
+  // The start and the timers are the inner preconditioner's
+  virtual void Vstart(std::vector<FieldD> &x, std::vector<FieldD> &src)
+  {
+    GRID_TRACE("MGPrecisionSeamVstart");
+    int nrhs=src.size(); Scratch(nrhs);
+    for(int r=0;r<nrhs;r++) precisionChange(_in_f[r],src[r],_ws_d2f);
+    _Inner.Vstart(_out_f,_in_f);
+    for(int r=0;r<nrhs;r++) precisionChange(x[r],_out_f[r],_ws_f2d);
+  }
+  virtual void ResetTimers(void)                       { _Inner.ResetTimers(); }
+  virtual void ReportTimers(const std::string &prefix) { _Inner.ReportTimers(prefix); }
+};
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/MrhsPreconditioner.h

@@ -0,0 +1,73 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/MrhsPreconditioner.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////
+// The mrhs interfaces.  This header is in the Algorithms.h include
+// chain (AdefMrhs.h needs it); the concrete mrhs classes are in
+// MrhsMultiGrid.h, reached through PVdagMMultiGrid.h / HDCGMultiGrid.h.
+//
+// MrhsLinearFunction: vector-of-fields in, vector out.  The outer level
+// carries mrhs as std::vector<Field>; below it mrhs is PACKED into a
+// single D+1 field (rhs = dim 0) and the coarse classes are plain
+// LinearFunctions on that.
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class MrhsLinearFunction {
+public:
+  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
+  virtual ~MrhsLinearFunction(){};
+};
+
+//////////////////////////////////////////////////////////////////////
+// An mrhs preconditioner for the two-level CG family (AdefMrhs.h):
+// operator() is M1; Vstart chooses x_0 from the source (Tang's ADEF-2
+// start x_0 = Q b; the default is the zero start).  The timers are the
+// M1 breakdown the outer solver reports.
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class MrhsPreconditioner : public MrhsLinearFunction<Field> {
+public:
+  GridStopWatch ProjectTimer, PromoteTimer, DeflateTimer, CoarseTimer, FineTimer, SmoothTimer;
+  virtual void Vstart(std::vector<Field> &x, std::vector<Field> &src)
+  {
+    for(int r=0;r<(int)x.size();r++) x[r] = Zero();
+  }
+  virtual void ResetTimers(void)
+  {
+    ProjectTimer.Reset(); PromoteTimer.Reset(); DeflateTimer.Reset();
+    CoarseTimer.Reset();  FineTimer.Reset();    SmoothTimer.Reset();
+  }
+  virtual void ReportTimers(const std::string &prefix)
+  {
+    std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
+    std::cout<<GridLogMessage<<prefix<<"Project "<<ProjectTimer.Elapsed()<<std::endl;
+    std::cout<<GridLogMessage<<prefix<<"Promote "<<PromoteTimer.Elapsed()<<std::endl;
+    std::cout<<GridLogMessage<<prefix<<"Deflate "<<DeflateTimer.Elapsed()<<std::endl;
+    std::cout<<GridLogMessage<<prefix<<"Coarse  "<<CoarseTimer.Elapsed()<<std::endl;
+    std::cout<<GridLogMessage<<prefix<<"Fine    "<<FineTimer.Elapsed()<<std::endl;
+    std::cout<<GridLogMessage<<prefix<<"Smooth  "<<SmoothTimer.Elapsed()<<std::endl;
+  }
+  virtual ~MrhsPreconditioner(){};
+};
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/MultiGrid.h

@@ -30,8 +30,22 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #include <Grid/algorithms/multigrid/Aggregates.h>
 #include <Grid/algorithms/multigrid/Geometry.h>
 #include <Grid/algorithms/multigrid/CoarsenedMatrix.h>
-#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrix.h>
 #include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHS.h>
-#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHSV2.h>
+// DEPRECATED: the superseded coarse operators and the Aggregation-based
+// single-RHS ADEF-2.  Nothing in the library uses them; the PVdagM and HDCG
+// chains are on MultiGeneralCoarsenedOperator (PVdagMMultiGrid.h,
+// HDCGMultiGrid.h).  Kept so the pre-2026 drivers in tests/debug and
+// examples still build; removing this block is the deletion gate for them.
+#include <Grid/algorithms/multigrid/deprecated/GeneralCoarsenedMatrix.h>
+#include <Grid/algorithms/multigrid/deprecated/GeneralCoarsenedMatrixMultiRHS.h>
+#include <Grid/algorithms/multigrid/deprecated/TwoLevelADEF2.h>
 #include <Grid/algorithms/multigrid/MrhsPromotedOperator.h>
 #include <Grid/algorithms/multigrid/Smoothers.h>
+#include <Grid/algorithms/multigrid/PVdagMMultiGridParams.h>
+// PVdagMOperators.h / MrhsMultiGrid.h / PVdagMMultiGrid.h /
+// DenseCoarseMatrix.h / MultiGridIO.h / HDCGMultiGrid.h are NOT in this
+// umbrella: consumers include PVdagMMultiGrid.h or HDCGMultiGrid.h
+// explicitly (they pull the dense stack, BLAS, and the scidac I/O, which
+// is declared after Algorithms.h in Grid.h).  Keeping MrhsMultiGrid.h out
+// of the Algorithms.h chain also keeps its class names away from the
+// pre-2026 drivers that define their own copies.

Grid/algorithms/multigrid/MultiGridIO.h

@@ -0,0 +1,94 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/MultiGridIO.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////
+// Multigrid setup I/O shared by the PVdagM and HDCG chains: a set of
+// bare-vector scidac records (a near-null basis, or a set of
+// eigenvectors with their eigenvalues beside them in XML).
+//
+// The coarse OPERATOR is not written: coarsening is cheap relative
+// to producing the basis, and is redone.
+//
+// `control` is the scidac binary-IO control word.  The library default
+// is lexicographic + aggregated; pass 0 to read files written by the
+// pre-2026 HDCG drivers, which used control 0.
+//////////////////////////////////////////////////////////////////////
+static const int MultiGridIODefaultControl = BinaryIO::BINARYIO_LEXICOGRAPHIC|BinaryIO::BINARYIO_AGGREGATE;
+
+template <class Field>
+void saveSubspace(std::vector<Field> &subspace, std::string const fname,
+		  int control = MultiGridIODefaultControl)
+{
+#ifdef HAVE_LIME
+  Grid::emptyUserRecord record;
+  Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
+  SW.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++) {
+    SW.writeScidacFieldRecord(subspace[k], record, 0, control);
+  }
+  SW.close();
+#endif
+}
+template <class Field>
+void loadSubspace(std::vector<Field> &subspace, std::string const fname,
+		  int control = MultiGridIODefaultControl)
+{
+#ifdef HAVE_LIME
+  Grid::emptyUserRecord record;
+  Grid::ScidacReader SR;
+  SR.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++) {
+    SR.readScidacFieldRecord(subspace[k], record, control);
+  }
+  SR.close();
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////
+// Eigenpairs: the vectors as a subspace file, the values in
+// <fname>.evals.xml.  evec.size() fixes how many are read.
+//////////////////////////////////////////////////////////////////////
+template <class Field>
+void saveEigenpairs(std::vector<Field> &evec, std::vector<RealD> &eval, std::string const fname,
+		    int control = MultiGridIODefaultControl)
+{
+  GRID_ASSERT(evec.size()==eval.size());
+  saveSubspace(evec, fname, control);
+  if ( evec[0].Grid()->IsBoss() ) {
+    XmlWriter WR(fname+".evals.xml");
+    write(WR,"evals",eval);
+  }
+}
+// The eigenvalue file sets the count; the vectors are allocated on grid.
+template <class Field>
+void loadEigenpairs(std::vector<Field> &evec, std::vector<RealD> &eval, std::string const fname,
+		    GridBase *grid, int control = MultiGridIODefaultControl)
+{
+  XmlReader RD(fname+".evals.xml");
+  read(RD,"evals",eval);
+  evec.clear();
+  evec.resize(eval.size(), Field(grid));
+  loadSubspace(evec, fname, control);
+}
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/PVdagMMultiGrid.h

@@ -0,0 +1,682 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/PVdagMMultiGrid.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+#include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
+#include <Grid/algorithms/multigrid/MultiGridIO.h>
+#include <Grid/algorithms/multigrid/PVdagMOperators.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////////////////////
+// The three-level mrhs PVdagM multigrid, as objects:
+//
+//   MGCoarseGrids              the derived coarse grids (owned here, since
+//                              conformability is pointer identity and someone
+//                              must hold them; declare it BEFORE anything that
+//                              borrows from it)
+//   PVdagMMultiGridCoarsening  ALL the coarsening information for one gauge
+//                              configuration: the raw near-null basis, the
+//                              transfer operators it induces, the Galerkin
+//                              coarse operator at each level, and the dense
+//                              bottom inverse.  Everything here is a function
+//                              of the gauge field -- the state HMC must
+//                              rebuild or maintain as U evolves -- and
+//                              everything is solver-independent.
+//   PVdagMMultiGridSolver      the solve chain composed on a borrowed
+//                              coarsening: smoothers, coarse Krylov, V-cycle,
+//                              outer mrhs PGCR.
+//
+// Scope discipline: grids outlive the coarsening outlives the solver.
+//////////////////////////////////////////////////////////////////////////////////////
+
+//////////////////////////////////////////////////////////////////////
+// Subspace I/O is in MultiGridIO.h (shared with the HDCG chain).
+// Loaded vectors are RAW -- deliberately NOT re-orthogonalised:
+// CoarsenOperator block orthonormalises in place, and projecting a
+// block-orthonormal vector onto its own block-orthonormalised
+// aggregation gives e_k with the near-null content silently gone.
+// GramGuard below catches that.
+//////////////////////////////////////////////////////////////////////
+
+//////////////////////////////////////////////////////////////////////
+// ||<v|v> - I||_F over a set of coarse vectors.  Small means the raw
+// near-null content survived the projection.
+//////////////////////////////////////////////////////////////////////
+template<class CoarseField>
+RealD GramDefect(std::vector<CoarseField> &v)
+{
+  RealD s2=0.0;
+  for(int i=0;i<(int)v.size();i++){
+    for(int j=0;j<(int)v.size();j++){
+      ComplexD sij=TensorRemove(innerProduct(v[i],v[j]));
+      ComplexD d=sij-(i==j?ComplexD(1.0):ComplexD(0.0));
+      s2+=real(d)*real(d)+imag(d)*imag(d);
+    }
+  }
+  return std::sqrt(s2);
+}
+
+// On a leak every image collapses to the block unit e_k, the Gram becomes
+// N*I, and the defect lands at (N-1)*sqrt(nbasis) -- orders above the ~0.2
+// of a content-preserving projection.  Trip well below that so a mis-set
+// threshold costs a log line rather than the run.
+template<class CoarseField>
+void GramGuard(const std::string &name,std::vector<CoarseField> &v,GridBase *grid)
+{
+  RealD defect = GramDefect(v);
+  RealD N      = (RealD)grid->gSites();
+  RealD leak   = (N-1.0)*std::sqrt((RealD)v.size());
+  RealD trip   = std::sqrt(N);
+  std::cout << GridLogMessage << "GUARD: ||<"<<name<<"|"<<name<<"> - I||_F = " << defect
+            << "   (e_k leak would be " << leak << ", trip at " << trip << ")" << std::endl;
+  GRID_ASSERT( defect < trip );
+}
+
+//////////////////////////////////////////////////////////////////////
+// The derived grids of the three-level chain, owned in one place.
+// The coarse space is UNVECTORISED (sComplex scalar, simd {1,..,1});
+// the 5D/6D grids are built directly so the SIMD layout is ours.
+// rhs/batch is dim 0 of the 6D grids, undistributed -- no divisibility
+// constraint on nrhs.
+//////////////////////////////////////////////////////////////////////
+class MGCoarseGrids {
+public:
+  GridCartesian *FGrid;                     // borrowed
+  int Ls;
+  int batch;
+  Coordinate clatt;                         // 4d coarse lattice
+  Coordinate cclatt;                        // 4d coarse-coarse lattice
+  Coordinate c5simd, c5mpi;                 // 5D coarse simd/mpi
+  Coordinate cmsimd, cmmpi;                 // 6D coarse simd/mpi
+  // owned:
+  GridCartesian *Coarse5d;
+  GridCartesian *CoarseBatch;               // 6D at the coarsening batch
+  GridCartesian *CoarseCoarse5d;
+  GridCartesian *CoarseCoarseBatch;
+
+  MGCoarseGrids(GridCartesian *_FGrid, const MGSetupParams &P)
+    : FGrid(_FGrid)
+  {
+    Coordinate fdims = FGrid->FullDimensions();      // {Ls, x,y,z,t}
+    Coordinate fmpi  = FGrid->_processors;
+    GRID_ASSERT( fdims.size() == 5 );
+    Ls    = fdims[0];
+    batch = P.CoarsenBatch;
+
+    clatt.resize(4); cclatt.resize(4);
+    for(int d=0;d<4;d++){
+      GRID_ASSERT( fdims[d+1] % P.Block1[d] == 0 );
+      clatt[d] = fdims[d+1] / P.Block1[d];
+    }
+    for(int d=0;d<4;d++){
+      GRID_ASSERT( clatt[d] % P.Block2[d] == 0 );
+      cclatt[d] = clatt[d] / P.Block2[d];
+    }
+    std::cout << GridLogMessage << "MGCoarseGrids: Block1 " << P.Block1 << "  coarse lattice        " << clatt  << std::endl;
+    std::cout << GridLogMessage << "MGCoarseGrids: Block2 " << P.Block2 << "  coarse-coarse lattice " << cclatt << std::endl;
+
+    Coordinate c5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    c5simd = Coordinate({1,1,1,1,1});
+    c5mpi  = Coordinate({1,fmpi[1],fmpi[2],fmpi[3],fmpi[4]});
+    Coarse5d = new GridCartesian(c5latt,c5simd,c5mpi);
+
+    cmsimd = Coordinate({1,1,1,1,1,1});
+    cmmpi  = Coordinate({1,1,fmpi[1],fmpi[2],fmpi[3],fmpi[4]});
+    Coordinate cblatt({batch,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    CoarseBatch = new GridCartesian(cblatt,cmsimd,cmmpi);
+
+    Coordinate cc5latt({1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+    CoarseCoarse5d = new GridCartesian(cc5latt,c5simd,c5mpi);
+
+    Coordinate ccblatt({batch,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+    CoarseCoarseBatch = new GridCartesian(ccblatt,cmsimd,cmmpi);
+  }
+  ~MGCoarseGrids()
+  {
+    delete CoarseCoarseBatch;
+    delete CoarseCoarse5d;
+    delete CoarseBatch;
+    delete Coarse5d;
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// The fp32 fine grids: same lattice and decomposition as the fp64 fine
+// grid, vComplexF SIMD layout.  For the fp32 fine level inside the
+// preconditioner (the fp32 fermion operators and the fp32 transfer
+// operator borrow these).  Owned here; declare BEFORE the coarsening.
+//////////////////////////////////////////////////////////////////////
+class MGFineGridsF {
+public:
+  GridCartesian         *UGridF;
+  GridRedBlackCartesian *UrbGridF;
+  GridCartesian         *FGridF;
+  GridRedBlackCartesian *FrbGridF;
+
+  MGFineGridsF(GridCartesian *FGrid)
+  {
+    Coordinate fdims = FGrid->FullDimensions();      // {Ls, x,y,z,t}
+    Coordinate fmpi  = FGrid->_processors;
+    GRID_ASSERT( fdims.size() == 5 );
+    int Ls = fdims[0];
+    Coordinate latt4({fdims[1],fdims[2],fdims[3],fdims[4]});
+    Coordinate mpi4 ({fmpi[1], fmpi[2], fmpi[3], fmpi[4]});
+    UGridF   = SpaceTimeGrid::makeFourDimGrid(latt4,GridDefaultSimd(Nd,vComplexF::Nsimd()),mpi4);
+    UrbGridF = SpaceTimeGrid::makeFourDimRedBlackGrid(UGridF);
+    FGridF   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGridF);
+    FrbGridF = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGridF);
+  }
+  ~MGFineGridsF()
+  {
+    delete FrbGridF;
+    delete FGridF;
+    delete UrbGridF;
+    delete UGridF;
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// ALL the coarsening information for one gauge configuration.
+//
+// Owns: the RAW near-null bases (fine and coarse -- retained so the
+// coarse operators can be REBUILT on a changed gauge field with a
+// fixed basis, the cheap HMC maintenance step; DiscardBasis() frees
+// them for valence use), the two coarsened operators, the two block
+// projectors, the dense bottom inverse (raw owning pointer: null in
+// the constructor, allocated by BuildDenseBottom, deleted here), and
+// the transient per-Nrhs solve grids (created by SetNrhs, which owns
+// the ReleaseGrid -> delete -> new -> SetGrid ordering in ONE place).
+//
+// Borrows: the grid bundle.  Declare MGCoarseGrids first.
+//////////////////////////////////////////////////////////////////////
+template<class Fobj,class CComplex,int nbasis>
+class PVdagMMultiGridCoarsening {
+public:
+  typedef Lattice<Fobj>                                                 FineField;
+  typedef MultiGeneralCoarsenedOperator<Fobj,CComplex,nbasis>         CoarseOperator;
+  typedef typename CoarseOperator::CoarseVector                         CoarseVector;
+  typedef typename CoarseVector::vector_object                          CoarseSiteObj;
+  // Every coarse level carries the same site type iVector<CComplex,nbasis>:
+  // the coefficient scalar does not deepen with the level (the operator keeps
+  // its own deeper scratch type for the block inner products).  Levels are
+  // told apart by their grids, not their C++ types.
+  typedef MultiGeneralCoarsenedOperator<CoarseSiteObj,CComplex,nbasis> CoarseCoarseOperator;
+  typedef typename CoarseCoarseOperator::CoarseVector                   CoarseCoarseVector;
+  typedef DenseCoarseMatrix<CComplex,nbasis>                            DenseBottom;
+  // The fp32 fine field, derived from the fp64 one: the fp32 fine level of
+  // the preconditioner projects through the SAME basis in fp32 layout.
+  typedef typename GridTypeMapper<Fobj>::SinglePrecision                FobjF;
+  typedef Lattice<FobjF>                                                FineFieldF;
+
+  //////////////////////////////////////////////////////////////////////
+  // ONE fine transfer operator.  Its STORE follows the coarse sector --
+  // the coarse space is what it feeds -- while its import and export
+  // accept EITHER fine precision, because they are already a layout
+  // transformation and a scalar conversion inside one costs nothing.
+  // So an fp64 outer level and an fp32 V-cycle share this single object,
+  // and no second basis store, handover or retained basis is needed.
+  //////////////////////////////////////////////////////////////////////
+  static const bool CoarseIsSingle =
+    ( sizeof(typename GridTypeMapper<CComplex>::scalar_type) == sizeof(ComplexF) );
+  typedef typename std::conditional<CoarseIsSingle,FineFieldF,FineField>::type ProjectorField;
+  typedef MultiRHSBlockProject<ProjectorField>                          ProjectorL1_t;
+  typedef MultiRHSBlockProject<CoarseVector>                            ProjectorL2_t;
+
+  MGCoarseGrids                     &Grids;      // borrowed
+  MGFineGridsF                      &GridsF;     // borrowed
+  MGSetupParams                      Params;
+  NextToNearestStencilGeometry5D     geom;
+  NextToNearestStencilGeometry5D     geom2;
+  CoarseOperator                     CoarseOpPV;
+  CoarseCoarseOperator               CoarseOpL2;
+  ProjectorL1_t                      MrhsProjectorL1;  // store follows the coarse sector
+  ProjectorL2_t                      MrhsProjectorL2;
+  DenseBottom                       *DenseCC;
+  std::vector<FineField>             rawNull;    // RAW fine near-null basis
+  std::vector<CoarseVector>          rawPsi;     // RAW coarse near-null basis
+  GridCartesian                     *CMrhs;      // transient solve grids, owned
+  GridCartesian                     *CCMrhs;
+  int                                nrhs;
+
+  PVdagMMultiGridCoarsening(MGCoarseGrids &_Grids, MGFineGridsF &_GridsF, const MGSetupParams &P)
+    : Grids(_Grids),
+      GridsF(_GridsF),
+      Params(P),
+      geom (_Grids.Coarse5d),
+      geom2(_Grids.CoarseCoarse5d),
+      CoarseOpPV(geom ,_Grids.Coarse5d),
+      CoarseOpL2(geom2,_Grids.CoarseCoarse5d),
+      DenseCC(nullptr),
+      CMrhs(nullptr),
+      CCMrhs(nullptr),
+      nrhs(-1)
+  {};
+
+  ~PVdagMMultiGridCoarsening()
+  {
+    if ( DenseCC ) delete DenseCC;
+    CoarseOpPV.ReleaseGrid();     // borrowers let go before their grids die
+    CoarseOpL2.ReleaseGrid();
+    if ( CMrhs  ) delete CMrhs;
+    if ( CCMrhs ) delete CCMrhs;
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // The RAW fine basis: load from Params.SubspaceFile if it exists,
+  // else GCR inverse iteration from noise (and save if a file name
+  // was given).  The Aggregation is scaffolding for CreateSubspaceGCR
+  // only -- that runs entirely on the fine grid, so the coarse grid
+  // it holds is never dereferenced.
+  ////////////////////////////////////////////////////////////////////
+  void GetSubspace(GridParallelRNG &RNG, LinearOperatorBase<FineField> &FineOp)
+  {
+    uint64_t file_exists=0;
+    if ( Params.SubspaceFile.length() ) {
+      if ( Grids.FGrid->IsBoss() ){ std::ifstream f(Params.SubspaceFile); file_exists=f.good()?1:0; }
+      Grids.FGrid->GlobalSum(file_exists);
+    }
+    rawNull.clear();
+    rawNull.reserve(nbasis);
+    for(int k=0;k<nbasis;k++) rawNull.push_back(FineField(Grids.FGrid));
+    if ( file_exists ){
+      std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: loading subspace "
+                << Params.SubspaceFile << " (kept RAW)" << std::endl;
+      loadSubspace(rawNull, Params.SubspaceFile);
+    } else {
+      std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: GCR subspace generation" << std::endl;
+      Aggregation<Fobj,CComplex,nbasis> Agg(Grids.Coarse5d,Grids.FGrid,0);
+      Agg.CreateSubspaceGCR(RNG,FineOp,nbasis);
+      for(int k=0;k<nbasis;k++)
+	rawNull[k]=Agg.subspace[k];
+      if ( Params.SubspaceFile.length() )
+	saveSubspace(rawNull, Params.SubspaceFile);
+    }
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Both coarsenings from the retained RAW basis.  Calling this again
+  // on a CHANGED fine operator is the fixed-basis rebuild: for the
+  // PVdagM coarsening the coarse operator with a fixed basis is exact
+  // in the changed links.  CoarsenOperator block-orthonormalises its
+  // input in place, so working copies are taken and the RAW vectors
+  // survive for the next rebuild.
+  //
+  // The fine operator must be the PVdagM wrapper (it needs
+  // SloppyComms: the coarsening builds the PRECONDITIONER, so its
+  // halos follow the FineSloppyComms policy, restored EXACT on exit).
+  ////////////////////////////////////////////////////////////////////
+  template<class PVdagMOp>
+  void Coarsen(PVdagMOp &FineOp, int retain_basis=0)
+  {
+    GRID_ASSERT( rawNull.size() == nbasis );
+
+    // L1: working copy of the raw fine basis, orthonormalised in place
+    std::vector<FineField> sub(rawNull);
+
+    CoarseOpPV.SetGrid(Grids.CoarseBatch);
+
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L1 CoarsenOperator, batch "
+              << Grids.batch << std::endl;
+
+    // CoarsenOperator orthonormalises sub in place and imports it into OUR
+    // transfer operator, which it then uses.  One basis store, imported once.
+    // The projector's grid carries the STORE's SIMD layout, which is the fp32
+    // fine grid when the coarse sector is fp32; fp64 vectors convert on import.
+    MrhsProjectorL1.Allocate(nbasis,
+                             CoarseIsSingle ? (GridBase *)GridsF.FGridF : (GridBase *)Grids.FGrid,
+                             Grids.Coarse5d);
+    FineOp.SloppyComms(Params.FineSloppyComms);
+    CoarseOpPV.CoarsenOperator(FineOp,sub,Grids.Coarse5d,Grids.batch,MrhsProjectorL1);
+    FineOp.SloppyComms(0);
+    // The Lattice copy of the basis has served its purpose: the transfer
+    // operator holds it in BLAS layout from here on.
+    sub.clear(); sub.shrink_to_fit();
+
+    std::vector<CoarseVector> psi(nbasis,Grids.Coarse5d);
+    MrhsProjectorL1.blockProject(rawNull,psi);
+    GramGuard("psi_coarse",psi,Grids.Coarse5d);
+
+    rawPsi.clear();
+    rawPsi.reserve(nbasis);
+    for(int k=0;k<nbasis;k++) rawPsi.push_back(psi[k]);
+
+    // L2: the L1 operator is natively multiRHS at the batch grid
+    CoarseOpL2.SetGrid(Grids.CoarseCoarseBatch);
+    NonHermitianLinearOperator<CoarseOperator,CoarseVector> LinOpCoarse(CoarseOpPV);
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L2 CoarsenOperator, batch "
+              << Grids.batch << std::endl;
+    MrhsProjectorL2.Allocate(nbasis,Grids.Coarse5d,Grids.CoarseCoarse5d);
+    CoarseOpL2.CoarsenOperator(LinOpCoarse,Grids.CoarseBatch,psi,Grids.CoarseCoarse5d,MrhsProjectorL2);
+    {
+      std::vector<CoarseCoarseVector> psi_cc(nbasis,Grids.CoarseCoarse5d);
+      MrhsProjectorL2.blockProject(rawPsi,psi_cc); // RAW vectors in
+      GramGuard("psi_cc",psi_cc,Grids.CoarseCoarse5d);
+    }
+    // The RAW basis has done its work (the L2 null space above was its last
+    // use) and is the single largest resident block of the setup, so it is
+    // freed.  retain_basis keeps it for a caller that coarsens again from the
+    // same basis; Setup.RetainSubspace keeps it for the object's whole life,
+    // which is what a fixed-basis rebuild on a changed gauge field (HMC)
+    // needs.  A valence solve never rebuilds.
+    if ( !retain_basis && !Params.RetainSubspace ) DiscardBasis();
+    MrhsProjectorL1.ReleaseScratch();
+    MrhsProjectorL2.ReleaseScratch();
+
+    nrhs = -1;                                     // both operators left at the batch grids
+    ReportDeviceFootprint("after Coarsen");
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Dense bottom: import the L2 matrix elements, invert (2D
+  // block-cyclic, fp64), then the check Import cannot run itself --
+  // the mrhs operator applies only on a D+1 grid, so drive it at
+  // Nrhs 1 through a slice: ||A Ainv x - x||/||x||.
+  ////////////////////////////////////////////////////////////////////
+  void BuildDenseBottom(void)
+  {
+    if ( DenseCC ) delete DenseCC;
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L3 dense bottom import" << std::endl;
+    DenseCC = new DenseBottom(Grids.CoarseCoarse5d);
+    DenseCC->Import(CoarseOpL2);
+
+    Coordinate cc1latt({1,1,Grids.cclatt[0],Grids.cclatt[1],Grids.cclatt[2],Grids.cclatt[3]});
+    GridCartesian *CoarseCoarseOne = new GridCartesian(cc1latt,Grids.cmsimd,Grids.cmmpi);
+    CoarseOpL2.SetGrid(CoarseCoarseOne);
+
+    CoarseCoarseVector x(Grids.CoarseCoarse5d),y(Grids.CoarseCoarse5d),z(Grids.CoarseCoarse5d);
+    GridParallelRNG dRNG(Grids.CoarseCoarse5d); dRNG.SeedFixedIntegers(std::vector<int>({11,12,13,14}));
+    random(dRNG,x);
+
+    (*DenseCC)(x,y);                    // y = Ainv x
+
+    CoarseCoarseVector y1(CoarseCoarseOne),z1(CoarseCoarseOne);
+    InsertSliceFast(y,y1,0,0);
+    CoarseOpL2.M(y1,z1);                // z = A y
+    ExtractSliceFast(z,z1,0,0);
+
+    z = z - x;
+    RealD rel = std::sqrt(norm2(z)/norm2(x));
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L3 dense ||A Ainv x - x||/||x|| = "
+              << rel << std::endl;
+    GRID_ASSERT( rel < 1.0e-2 );
+
+    CoarseOpL2.ReleaseGrid();           // let go before the grid dies
+    delete CoarseCoarseOne;
+    nrhs = -1;
+    ReportDeviceFootprint("after BuildDenseBottom");
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Retarget both operators to a solve Nrhs.  The matrix elements are
+  // Nrhs independent and survive; only the D+1 grids and BLAS buffers
+  // change.  The ordering discipline (borrowers release BEFORE their
+  // grids are destroyed) lives here and nowhere else.
+  ////////////////////////////////////////////////////////////////////
+  void SetNrhs(int nr)
+  {
+    if ( nr == nrhs ) return;
+    CoarseOpPV.ReleaseGrid();
+    CoarseOpL2.ReleaseGrid();
+    if ( CMrhs  ) { delete CMrhs;  CMrhs =nullptr; }
+    if ( CCMrhs ) { delete CCMrhs; CCMrhs=nullptr; }
+    Coordinate cml ({nr,1,Grids.clatt[0], Grids.clatt[1], Grids.clatt[2], Grids.clatt[3]});
+    Coordinate ccml({nr,1,Grids.cclatt[0],Grids.cclatt[1],Grids.cclatt[2],Grids.cclatt[3]});
+    CMrhs  = new GridCartesian(cml, Grids.cmsimd,Grids.cmmpi);
+    CCMrhs = new GridCartesian(ccml,Grids.cmsimd,Grids.cmmpi);
+    CoarseOpPV.SetGrid(CMrhs);
+    CoarseOpL2.SetGrid(CCMrhs);
+    nrhs = nr;
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: operators at Nrhs " << nr << std::endl;
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Galerkin certificate for the L1 coarsening: ||A_c x - P^dag A P x||
+  // over ||A_c x|| on a random coarse vector, through the SAME transfer
+  // operators the solve uses.  The coarse operator is compared with its
+  // own definition, so this reads the ROUNDING level of the coarsening
+  // (fp64 coarse ~1e-13, fp32 coarse ~1e-6) independently of how the
+  // solve converges.  Drives the fine operator with EXACT halos; at more
+  // than one rank the sloppy-halo coarsening error is included in the
+  // reading.  Nrhs 1; the caller's Nrhs is restored by its own SetNrhs.
+  ////////////////////////////////////////////////////////////////////
+  template<class PVdagMOp>
+  RealD CertifyCoarsening(PVdagMOp &FineOp)
+  {
+    SetNrhs(1);
+    CoarseVector xc(CMrhs), Acx(CMrhs), PtAPx(CMrhs);
+    GridParallelRNG cRNG(CMrhs); cRNG.SeedFixedIntegers(std::vector<int>({21,22,23,24}));
+    random(cRNG,xc);
+
+    CoarseOpPV.M(xc,Acx);                                   // A_c x
+
+    std::vector<FineField> Px(1,Grids.FGrid), APx(1,Grids.FGrid);
+    MrhsProjectorL1.blockPromote(Px,xc);                      // P x
+    FineOp.SloppyComms(0);
+    FineOp.Op(Px[0],APx[0]);                                // A P x
+    MrhsProjectorL1.blockProject(APx,PtAPx);                  // P^dag A P x
+
+    CoarseVector d(CMrhs); d = Acx - PtAPx;
+    RealD rel = std::sqrt(norm2(d)/norm2(Acx));
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L1 GALERKIN CERTIFICATE "
+              << "||A_c x - P^dag A P x||/||A_c x|| = " << rel << std::endl;
+    return rel;
+  }
+
+
+  ////////////////////////////////////////////////////////////////////
+  // The device memory this coarsening holds that the memory manager
+  // CANNOT evict: BLAS stores, which are plain device allocations.
+  // Lattice fields are omitted on purpose -- they are evictable, so they
+  // cost eviction traffic, not failure.  What is reported is what has to
+  // fit alongside the manager's cache, the comms buffers and the shm
+  // segment, which is the budget an out-of-memory run actually breaks.
+  ////////////////////////////////////////////////////////////////////
+  void ReportDeviceFootprint(const std::string &when)
+  {
+    uint64_t p1 = MrhsProjectorL1.DeviceBytes();
+    uint64_t p2 = MrhsProjectorL2.DeviceBytes();
+    uint64_t o1 = CoarseOpPV.DeviceBytes();
+    uint64_t o2 = CoarseOpL2.DeviceBytes();
+    uint64_t dn = DenseCC ? DenseCC->DeviceBytes() : 0;
+    uint64_t tot = p1+p2+o1+o2+dn;
+    const double G = 1024.*1024.*1024.;
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening (" << when << "): resident device memory/rank "
+              << tot/G << " GB = transfer L1 " << p1/G << " + L2 " << p2/G
+              << " + operator L1 " << o1/G << " + L2 " << o2/G
+              << " + dense " << dn/G << " GB" << std::endl;
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening (" << when << "): not evictable; must fit alongside the "
+              << MemoryManager::DeviceMaxBytes/G
+              << " GB manager cache, the comms buffers and the shm segment" << std::endl;
+  }
+
+  ////////////////////////////////////////////////////////////////////
+  // Free the retained RAW bases (valence use: coarsening is final for
+  // this configuration and the fine basis is ~GB-scale host memory).
+  ////////////////////////////////////////////////////////////////////
+  void DiscardBasis(void)
+  {
+    rawNull.clear(); rawNull.shrink_to_fit();
+    rawPsi.clear();  rawPsi.shrink_to_fit();
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// The solve chain on a borrowed coarsening: adaptive PGCR smoothers
+// at both levels (the one correct route for this non-Hermitian
+// chain), dense bottom, mrhs V-cycle, outer shared-coefficient mrhs
+// PGCR.  Construction retargets the coarsening to this Nrhs.
+//
+// Halo policy: the whole V-cycle is preconditioner and runs with
+// sloppy halos when FineSloppyComms is set; the outer Krylov is EXACT
+// (its applications define what "converged" means), asserted by the
+// FINAL true-residual report in Solve.
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class MatrixF,class Coarsening>
+class PVdagMMultiGridSolver {
+public:
+  typedef typename Coarsening::FineField            FineField;
+  typedef typename Coarsening::FineFieldF           FineFieldF;
+  typedef typename Coarsening::CoarseOperator       CoarseOperator;
+  typedef typename Coarsening::CoarseVector         CoarseVector;
+  typedef typename Coarsening::CoarseCoarseOperator CoarseCoarseOperator;
+  typedef typename Coarsening::CoarseCoarseVector   CoarseCoarseVector;
+  typedef typename Coarsening::DenseBottom          DenseBottom;
+  typedef PrecGeneralisedConjugateResidualNonHermitian<FineField>    FineSmoother_t;
+  typedef PrecGeneralisedConjugateResidualNonHermitian<FineFieldF>   FineSmootherF_t;
+  typedef PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> CoarseKrylov_t;
+
+  Coarsening            &C;
+  PVdagMMultiGridParams  Params;
+  int                    nrhs;
+  int                    _regrid;    // FIRST member-like init: SetNrhs before members capture grids
+
+  PVdagMLinearOperator<Matrix,FineField>        PVdagM;
+  ShiftedPVdagMLinearOperator<Matrix,FineField> ShiftedPVdagM;
+  TrivialPrecon<FineField>                      simple_fine;
+  TrivialPrecon<CoarseVector>                   simpleC;
+  NonHermitianLinearOperator<CoarseOperator,CoarseVector> LinOpC;
+  ShiftedLinearOperator<CoarseVector>           ShiftedC;
+  MrhsDenseCCSolve<DenseBottom,CoarseCoarseVector> ccSolve;
+  CoarseKrylov_t                                CoarseSmootherGCR;
+  MrhsCoarseThreeLevelPrec<CoarseVector,CoarseCoarseVector> L2to3Precon;
+  CoarseKrylov_t                                L2PGCR;
+  // The fp64 fine level of the preconditioner
+  FineSmoother_t                                SmootherGCR;
+  MrhsTwoLevelMG<FineField,CoarseVector,FineSmoother_t,typename Coarsening::ProjectorL1_t> ThreeLevelPrecon;
+  // The fp32 fine level of the preconditioner, behind the fp64/fp32 seam.
+  // Both share the coarse chain (L2PGCR); the outer Krylov picks one.
+  PVdagMLinearOperator<MatrixF,FineFieldF>        PVdagMF;
+  ShiftedPVdagMLinearOperator<MatrixF,FineFieldF> ShiftedPVdagMF;
+  TrivialPrecon<FineFieldF>                       simple_fine_f;
+  FineSmootherF_t                                 SmootherGCRF;
+  MrhsTwoLevelMG<FineFieldF,CoarseVector,FineSmootherF_t,typename Coarsening::ProjectorL1_t> ThreeLevelPreconF;
+  MrhsMixedPrecPreconditioner<FineField,FineFieldF> PreconSeam;
+  MrhsPGCRNonHermitian<FineField>               L1PGCR;
+
+  PVdagMMultiGridSolver(Matrix &Ddwf, Matrix &Dpv, MatrixF &DdwfF, MatrixF &DpvF,
+                        Coarsening &_C, const PVdagMMultiGridParams &P, int nr)
+    : C(_C), Params(P), nrhs(nr),
+      _regrid((C.SetNrhs(nr),0)),              // members below capture C.CMrhs/C.CCMrhs
+      PVdagM(Ddwf,Dpv),
+      ShiftedPVdagM(P.FineSmoother.Shift,Ddwf,Dpv),
+      LinOpC(C.CoarseOpPV),
+      ShiftedC(P.CoarseSmoother.Shift,LinOpC),
+      ccSolve(*C.DenseCC,nr),
+      // Coarse smoother: one preconditioned restart of nstep GCR steps.
+      CoarseSmootherGCR(0.01,1,ShiftedC,simpleC,P.CoarseSmoother.Mmax,P.CoarseSmoother.Nstep),
+      L2to3Precon(LinOpC,CoarseSmootherGCR,C.MrhsProjectorL2,ccSolve,
+                  C.Grids.Coarse5d,C.Grids.CoarseCoarse5d,C.CCMrhs,nr),
+      // Coarse Krylov: Order/16 restarts of 16 steps.
+      L2PGCR(P.CoarseSolver.Tol,P.CoarseSolver.Order/16,LinOpC,L2to3Precon,P.CoarseSolver.Mmax,16),
+      // Fine smoother: one restart of nstep GCR steps, tolerance 0 = fixed work.
+      SmootherGCR(0.0,1,ShiftedPVdagM,simple_fine,P.FineSmoother.Mmax,P.FineSmoother.Nstep),
+      ThreeLevelPrecon(PVdagM,SmootherGCR,C.MrhsProjectorL1,L2PGCR,C.Grids.Coarse5d,C.CMrhs),
+      PVdagMF(DdwfF,DpvF),
+      ShiftedPVdagMF(P.FineSmoother.Shift,DdwfF,DpvF),
+      SmootherGCRF(0.0,1,ShiftedPVdagMF,simple_fine_f,P.FineSmoother.Mmax,P.FineSmoother.Nstep),
+      ThreeLevelPreconF(PVdagMF,SmootherGCRF,C.MrhsProjectorL1,L2PGCR,C.Grids.Coarse5d,C.CMrhs),
+      PreconSeam(ThreeLevelPreconF,Ddwf.FermionGrid(),DdwfF.FermionGrid(),nr),
+      L1PGCR(P.Outer.Tol,P.Outer.MaxIterations,PVdagM,
+             (P.Setup.FinePrecision==MGPrecision::fp32)
+               ? static_cast<MrhsLinearFunction<FineField>&>(PreconSeam)
+               : static_cast<MrhsLinearFunction<FineField>&>(ThreeLevelPrecon),
+             P.Outer.Mmax,P.Outer.Nstep)
+  {
+    GRID_ASSERT( C.DenseCC != nullptr );
+
+    CoarseSmootherGCR.Level(2);
+    CoarseSmootherGCR.Name("Csmoother");
+    CoarseSmootherGCR.SetZeroGuess(1);
+
+    L2PGCR.Level(2);
+    L2PGCR.Name("Couter");
+    L2PGCR.SetZeroGuess(1);
+
+    SmootherGCR.Level(1);
+    SmootherGCR.Name("Fsmoother");
+    SmootherGCR.SetZeroGuess(1);
+
+    SmootherGCRF.Level(1);
+    SmootherGCRF.Name("Fsmoother");
+    SmootherGCRF.SetZeroGuess(1);
+
+    L1PGCR.Level(1);
+    L1PGCR.Name("Fouter");
+    L1PGCR.SetZeroGuess(1);
+
+    // The V-cycle is preconditioner: sloppy halos inside, exact restored on exit.
+    ThreeLevelPrecon.SetSloppy    = [this](int s){ PVdagM.SloppyComms(s); };
+    ThreeLevelPrecon.SloppyComms  = Params.Setup.FineSloppyComms;
+    ThreeLevelPreconF.SetSloppy   = [this](int s){ PVdagMF.SloppyComms(s); };
+    ThreeLevelPreconF.SloppyComms = Params.Setup.FineSloppyComms;
+    PVdagM.SloppyComms(0);
+    PVdagMF.SloppyComms(0);
+    // Three stages, three arithmetics, and a wire format that follows the
+    // ARITHMETIC of the stage it serves: fp32 on an fp64 operator, bf16 on
+    // an fp32 one.  The coarsening always applies the fp64 operator, so its
+    // wire is fp32 whatever the preconditioner runs in.
+    const bool sloppy = Params.Setup.FineSloppyComms;
+    const bool pfp32  = (Params.Setup.FinePrecision==MGPrecision::fp32);
+    std::cout << GridLogMessage << "PVdagMMultiGridSolver: Nrhs " << nr << std::endl;
+    std::cout << GridLogMessage << "  outer Krylov       : fp64, halos EXACT (the true residual is measured here)" << std::endl;
+    std::cout << GridLogMessage << "  preconditioner fine: " << (pfp32 ? "fp32 (seam at the outer Krylov)" : "fp64")
+              << ", halos " << (sloppy ? (pfp32 ? "SLOPPY bf16 wire" : "SLOPPY fp32 wire") : "exact") << std::endl;
+    std::cout << GridLogMessage << "  coarsening (done)  : fp64 operator, halos "
+              << (sloppy ? "SLOPPY fp32 wire" : "exact")
+              << " -- the coarse operator does not follow FinePrecision" << std::endl;
+  }
+
+  void Solve(std::vector<FineField> &src, std::vector<FineField> &sol)
+  {
+    GRID_ASSERT( (int)src.size() == nrhs );
+    // Start the solve from a clean device: evict setup-era Lattice copies
+    // and release the allocation caches' held blocks, so the LRU cap
+    // applies to the solve's own working set (OOM policy, Frontier).
+    MemoryManager::EvictAll();
+    MemoryManager::DropCache();
+
+    GridStopWatch w; w.Start();
+    L1PGCR(src,sol);
+    w.Stop();
+    std::cout << GridLogMessage << "PVdagMMultiGridSolver: Nrhs "<<nrhs<<" total " << w.Elapsed()
+              << "  (per RHS: " << w.useconds()/1.0e6/nrhs << " s)" << std::endl;
+
+    // The outer operator is exact by policy; assert the state rather than
+    // trust it -- a preconditioner that failed to restore would surface here.
+    PVdagM.SloppyComms(0);
+    {
+      FineField Ax(src[0].Grid());
+      RealD worst=0.0;
+      for(int r=0;r<nrhs;r++){
+	PVdagM.Op(sol[r],Ax);
+	Ax=Ax-src[r];
+        RealD rn=std::sqrt(norm2(Ax)/norm2(src[r]));
+        std::cout << GridLogMessage << "FINAL Nrhs "<<nrhs<<": rhs["<<r<<"] true residual = " << rn << std::endl;
+        worst=std::max(worst,rn);
+      }
+      std::cout << GridLogMessage << "FINAL Nrhs "<<nrhs<<": worst-case residual = " << worst
+                << "   (exact-halo verification)" << std::endl;
+    }
+  }
+};
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/PVdagMMultiGridParams.h

@@ -0,0 +1,165 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/PVdagMMultiGridParams.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////////////////////
+// Serialisable parameters for the mrhs PVdagM multigrid solver chain.
+//
+// Named for the multigrid FORM it configures, as WilsonMGParams is for the
+// Wilson MG tests; an MrhsHDCGParams will compose the same generic MG*
+// sub-structs for the Hermitian chain.  Read from XML (or JSON -- the
+// serialisation macros give both) via ReadPVdagMMultiGridParams below and
+// printed at startup by the macro's operator<<, so every log identifies its
+// own run.  The constructor defaults ARE a tuned operating point, not
+// arbitrary: an unconfigured run reproduces it, so changing them changes what
+// an unconfigured run does.  Smoother mmax == nstep (full GCR history).
+//
+// There are NO environment-variable controls anywhere in this subsystem:
+// parameters live here, library-internal constants are hard defaults in their
+// classes, verbosity is the --log channels.
+// Research instruments (GCR coefficient recording/replay, Chebyshev and
+// stationary smoother variants, power iteration) are deliberately NOT part
+// of this interface: they remain programmatic, for algorithmic studies,
+// not consumer knobs.
+//////////////////////////////////////////////////////////////////////////////////////
+
+// Arithmetic precision of the fine level INSIDE the preconditioner -- the
+// smoother and the V-cycle's own fine residuals.  The outer Krylov is
+// always fp64: the fp64/fp32 seam sits at its preconditioner call.
+// Orthogonal to FineSloppyComms, which is the halo WIRE format.
+GRID_SERIALIZABLE_ENUM(MGPrecision, undef, fp64, 1, fp32, 2);
+
+// The smoother is the adaptive shifted PGCR -- the one correct route for
+// this non-Hermitian chain (stationary replay and Chebyshev were explored
+// and did not win here; Chebyshev remains the HERMITIAN chain's smoother).
+struct MGSmootherParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MGSmootherParams,
+                                  RealD, Shift,
+                                  int,   Nstep,
+                                  int,   Mmax);
+  MGSmootherParams(RealD shift=0.1, int nstep=6, int mmax=6)
+    : Shift(shift), Nstep(nstep), Mmax(mmax) {};
+};
+
+struct MGCoarseSolverParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MGCoarseSolverParams,
+                                  RealD, Tol,
+                                  int,   Order,
+                                  int,   Mmax);
+  MGCoarseSolverParams() : Tol(0.05), Order(200), Mmax(16) {};
+};
+
+struct MGOuterParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MGOuterParams,
+                                  RealD, Tol,
+                                  int,   MaxIterations,
+                                  int,   Mmax,
+                                  int,   Nstep);
+  MGOuterParams() : Tol(1.0e-8), MaxIterations(1000), Mmax(4), Nstep(8) {};
+};
+
+struct MGDenseParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MGDenseParams,
+                                  int, LeafSpan);   // big-leaf span in blocks; see BlockCyclicSchurInverse
+  MGDenseParams() : LeafSpan(9) {};
+};
+
+struct MGSetupParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MGSetupParams,
+                                  std::vector<int>, Block1,          // fine -> coarse blocking
+                                  std::vector<int>, Block2,          // coarse -> coarse-coarse blocking
+                                  int,              CoarsenBatch,
+                                  std::string,      SubspaceFile,    // scidac; empty = create from noise, no I/O
+                                  int,              FineSloppyComms, // reduced-precision halo WIRE inside the preconditioner only: fp32 on the fp64 operator, bf16 on the fp32 operator
+                                  MGPrecision,      FinePrecision,   // fine-level ARITHMETIC inside the preconditioner
+                                  int,              RetainSubspace); // keep the RAW basis after setup (nbasis fine vectors); needed ONLY for a fixed-basis rebuild on a changed gauge field (HMC).  A valence solve never rebuilds, so 0 frees it.
+  MGSetupParams()
+    : Block1({2,2,3,3}), Block2({4,4,2,4}), CoarsenBatch(9),
+      SubspaceFile(""), FineSloppyComms(1), FinePrecision(MGPrecision::fp64),
+      RetainSubspace(0) {};
+};
+
+struct PVdagMMultiGridParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(PVdagMMultiGridParams,
+                                  MGSetupParams,        Setup,
+                                  MGSmootherParams,     FineSmoother,
+                                  MGSmootherParams,     CoarseSmoother,
+                                  MGCoarseSolverParams, CoarseSolver,
+                                  MGOuterParams,        Outer,
+                                  MGDenseParams,        Dense);
+  PVdagMMultiGridParams()
+    : FineSmoother  (0.1, 6, 6),
+      CoarseSmoother(0.1, 2, 2) {};
+};
+
+inline void CheckValidity(const PVdagMMultiGridParams &P)
+{
+  GRID_ASSERT( P.Setup.Block1.size() == 4 );
+  GRID_ASSERT( P.Setup.Block2.size() == 4 );
+  GRID_ASSERT( P.Setup.CoarsenBatch  >= 1 );
+  GRID_ASSERT( P.Setup.FinePrecision != MGPrecision::undef );
+  GRID_ASSERT( P.FineSmoother.Nstep   > 0 );
+  GRID_ASSERT( P.CoarseSmoother.Nstep > 0 );
+  GRID_ASSERT( P.CoarseSolver.Tol     > 0.0 );
+  GRID_ASSERT( P.Outer.Tol            > 0.0 );
+  GRID_ASSERT( P.Dense.LeafSpan      >= 1 );
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+// Read from an XML file; on a MISSING file the boss writes <file>.templ from
+// the defaults and returns false so the caller can Grid_finalize() and exit
+// -- the produced template is the documentation of every parameter.  An
+// empty filename means "defaults": no I/O, returns true.  Always prints the
+// effective parameters, so the log describes the run.
+//
+// Drivers fetch the filename themselves, e.g.
+//   std::string pfile("");
+//   if( GridCmdOptionExists(argv,argv+argc,"--multigrid-params") )
+//     pfile = GridCmdOptionPayload(argv,argv+argc,"--multigrid-params");
+//   if( !ReadPVdagMMultiGridParams(Params, pfile) ) { Grid_finalize(); return 0; }
+//////////////////////////////////////////////////////////////////////////////////////
+inline bool ReadPVdagMMultiGridParams(PVdagMMultiGridParams &P, const std::string &xmlfile)
+{
+  if ( xmlfile.length() ) {
+    bool good;
+    { std::ifstream f(xmlfile); good = f.good(); }
+    if ( !good ) {
+      std::cout << GridLogMessage << "PVdagMMultiGridParams: " << xmlfile << " does not exist" << std::endl;
+      if ( GlobalSharedMemory::WorldRank == 0 ) {
+        XmlWriter WR(xmlfile+".templ");
+        write(WR, "PVdagMMultiGridParams", P);
+        std::cout << GridLogMessage << "PVdagMMultiGridParams: template written to "
+                  << xmlfile << ".templ" << std::endl;
+      }
+      return false;
+    }
+    XmlReader RD(xmlfile);
+    read(RD, "PVdagMMultiGridParams", P);
+  }
+  CheckValidity(P);
+  std::cout << GridLogMessage << "PVdagMMultiGridParams ("
+            << (xmlfile.length() ? xmlfile : std::string("defaults")) << "):" << std::endl;
+  std::cout << P << std::endl;
+  return true;
+}
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/PVdagMOperators.h

@@ -0,0 +1,179 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./Grid/algorithms/multigrid/PVdagMOperators.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+//////////////////////////////////////////////////////////////////////
+// A = PV^dag M (non-Hermitian).  The operator the PVdagM multigrid
+// coarsens and solves.
+//
+// SloppyComms delegates to BOTH underlying fermion operators, so one
+// call controls every fine halo in the chain -- including any shifted
+// wrapper built on the same two matrices.  Halo-precision POLICY:
+// reduced-precision (fp32 wire) halos belong in the PRECONDITIONER --
+// the smoother, the V-cycle's own residuals, the coarsening -- and
+// NEVER in the outer Krylov, whose applications define what
+// "converged" means.  The operators default to EXACT; the V-cycle
+// turns sloppiness on for its own scope only (MrhsTwoLevelMG).
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class Field>
+class PVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat;
+  Matrix &_PV;
+public:
+  PVdagMLinearOperator(Matrix &Mat,Matrix &PV):
+    _Mat(Mat),_PV(PV)
+  {};
+  void SloppyComms(int sloppy)
+  {
+    _Mat.SloppyComms(sloppy);
+    _PV.SloppyComms(sloppy);
+  }
+  void OpDiag (const Field &in, Field &out)
+  {
+    GRID_ASSERT(0);
+  }
+  void OpDir  (const Field &in, Field &out,int dir,int disp)
+  {
+    GRID_ASSERT(0);
+  }
+  void OpDirAll  (const Field &in, std::vector<Field> &out)
+  {
+    GRID_ASSERT(0);
+  };
+  void Op     (const Field &in, Field &out)
+  {
+    Field tmp(in.Grid());
+    _Mat.M(in,tmp);
+    _PV.Mdag(tmp,out);
+  }
+  void AdjOp  (const Field &in, Field &out)
+  {
+    Field tmp(in.Grid());
+    _PV.M(in,tmp);
+    _Mat.Mdag(tmp,out);
+  }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)
+  {
+    HermOp(in,out);
+    ComplexD d=innerProduct(in,out);
+    n1=real(d);
+    n2=norm2(out);
+  }
+  void HermOp(const Field &in, Field &out)
+  {
+    Field tmp(in.Grid());
+    Op(in,tmp);
+    AdjOp(tmp,out);
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// A + shift, built directly on the two matrices (fine level: avoids
+// an extra fine-field pass through a generic shifted wrapper).
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class Field>
+class ShiftedPVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat;
+  Matrix &_PV;
+public:
+  RealD shift;
+  ShiftedPVdagMLinearOperator(RealD _shift,Matrix &Mat,Matrix &PV): shift(_shift),_Mat(Mat),_PV(PV){};
+  void OpDiag (const Field &in, Field &out)
+  {
+    GRID_ASSERT(0);
+  }
+  void OpDir  (const Field &in, Field &out,int dir,int disp)
+  {
+    GRID_ASSERT(0);
+  }
+  void OpDirAll  (const Field &in, std::vector<Field> &out)
+  {
+    GRID_ASSERT(0);
+  };
+  void Op     (const Field &in, Field &out)
+  {
+    Field tmp(in.Grid());
+    _Mat.M(in,tmp);
+    _PV.Mdag(tmp,out);
+    out = out + shift*in;
+  }
+  void AdjOp  (const Field &in, Field &out)
+  {
+    Field tmp(in.Grid());
+    _PV.M(in,tmp);
+    _Mat.Mdag(tmp,out);
+    out = out + shift*in;
+  }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)
+  {
+    GRID_ASSERT(0);
+  }
+  void HermOp(const Field &in, Field &out){
+    Field tmp(in.Grid());
+    Op(in,tmp);
+    AdjOp(tmp,out);
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// Op + shift for any operator (coarse levels).
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class ShiftedLinearOperator : public LinearOperatorBase<Field> {
+  LinearOperatorBase<Field> &_Op;
+  RealD shift;
+public:
+  ShiftedLinearOperator(RealD _shift, LinearOperatorBase<Field> &Op) : _Op(Op), shift(_shift) {}
+  void OpDiag  (const Field &in, Field &out)
+  {
+    GRID_ASSERT(0);
+  }
+  void OpDir   (const Field &in, Field &out,int dir,int disp)
+  {
+    GRID_ASSERT(0);
+  }
+  void OpDirAll (const Field &in, std::vector<Field> &out)
+  {
+    GRID_ASSERT(0);
+  }
+  void Op      (const Field &in, Field &out)
+  {
+    _Op.Op(in,out);
+    out = out + shift*in;
+  }
+  void AdjOp   (const Field &in, Field &out) {
+    _Op.AdjOp(in,out);
+    out = out + shift*in;
+  }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)
+  {
+    GRID_ASSERT(0);
+  }
+  void HermOp  (const Field &in, Field &out)
+  {
+    Field tmp(in.Grid());
+    Op(in,tmp);
+    AdjOp(tmp,out);
+  }
+};
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/RecursiveSchurInverse.h

@@ -1,874 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: RecursiveSchurInverse.h
-
-    Copyright (C) 2026
-
-Author: Peter Boyle <pboyle@bnl.gov>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#pragma once
-
-#include <Grid/algorithms/blas/BatchedBlas.h>
-#include <Grid/algorithms/blas/BatchedInverse.h>
-
-#include <algorithm>
-
-NAMESPACE_BEGIN(Grid);
-
-///////////////////////////////////////////////////////////////////////////////
-// RecursiveSchurInverse: distributed dense inversion by recursive Schur
-// complement over a binary rank-range tree.
-//
-// Contract: rank r owns global rows [rowStart[r], rowStart[r+1]) of an
-// N x N matrix in rank-major ordering, and receives its rows of the
-// inverse in the same layout.  Consumes GridBase collectives, GridBLAS
-// and GridBLASInverse only; Eigen reference backends permit CPU unit
-// testing under mpirun (Test_schur_inverse).
-//
-// Arithmetic is fp64 throughout; the caller rounds once into fp32 storage.
-//
-// Execution: SPMD full-tree walk.  Every rank makes the identical call
-// sequence; data participation is ownership-gated; all collectives are
-// world-wide, so no deadlock surface exists.
-//
-// Storage: BlockRows is column-major, ld = rows; element (i,j) at
-// data[i + j*ld]; a column window is the contiguous slice at data[col0*ld].
-///////////////////////////////////////////////////////////////////////////////
-
-///////////////////////////////////////////////////////////////////////////////
-// My rows of a distributed dense matrix: rows x cols, column major, ld = rows.
-///////////////////////////////////////////////////////////////////////////////
-class BlockRows
-{
-public:
-  deviceVector<ComplexD> data;
-  int64_t                rows;
-  int64_t                cols;
-  int64_t                ld;
-
-  BlockRows()
-  {
-    rows = 0;
-    cols = 0;
-    ld   = 0;
-  }
-  void Resize(int64_t r, int64_t c)
-  {
-    rows = r;
-    cols = c;
-    ld   = r;
-    data.resize((uint64_t)r*c);
-  }
-  ComplexD *ColumnWindow(int64_t col0)
-  {
-    GRID_ASSERT( col0 >= 0 );
-    GRID_ASSERT( col0 <= cols );
-    return &data[(uint64_t)col0*ld];
-  }
-};
-
-class RecursiveSchurInverse
-{
-public:
-  GridBase             *grid;       // world collectives only
-  int64_t               N;          // global matrix dimension
-  int                   P;          // ranks
-  int                   me;         // this rank
-  std::vector<int64_t>  rowStart;   // P+1 entries: rank-major row ownership
-  int64_t               myRow0;
-  int64_t               myNrows;
-  int64_t               panelBytes; // gather panel budget (DENSE_PANEL_BYTES)
-
-  GridBLAS              BLAS;
-  GridBLASInverse       INV;
-
-  // Growth telemetry: one entry per merge node, walk order
-  std::vector<double>   telNormB;      // ||B||_F = ||A11inv A12||_F
-  std::vector<double>   telSratio;     // ||S||_F / ||A22||_F
-  double                telLeafMaxInv; // max |(leaf inverse)_ij| over leaves
-
-  // Phase timers/counters, reported by ReportTelemetry
-  double                tMemset;       // device panel zero-fill
-  double                tDeposit;      // owner rows -> panel (device kernel)
-  double                tAllreduce;    // panel collective (see tBarrier)
-  double                tBarrier;      // arrival skew, when DENSE_BARRIER_PROBE
-  double                tRepack;       // rank-major -> panel (gather path)
-  double                tGemm;         // strided gemm + synchronise
-  double                tLeaf;         // leaf inversions
-  double                tARmin;        // fastest single panel collective
-  double                tARmax;        // slowest single panel collective
-  std::vector<double>   tARall;        // every panel collective, for percentiles
-  uint64_t              bytesAllreduce;
-  uint64_t              nAllreduce;    // panel collectives
-  uint64_t              nGatherGemm;   // GatherGemm calls
-  uint64_t              nGather;       // gather-path collectives (debug gate)
-  uint64_t              nGatherV;      // ... of which MPI_Allgatherv
-  uint64_t              nBcast;        // ... of which Bcast-assembled
-
-  // Persistent grow-only device panel; assembly and collectives are
-  // device-resident.  Device builds require GPU-aware MPI.
-  deviceVector<ComplexD> dPanelBuf;
-  // Rank-major receive staging for the AllGatherV path, plus the per-panel-row
-  // owner maps that drive the repack.  Grow-only, same discipline as dPanelBuf:
-  // a fresh device allocation per collective is catastrophic (measured).
-  deviceVector<ComplexD> dRecvBuf;
-  deviceVector<int64_t>  dOwnerOff;    // panel row -> owner's first panel row
-  deviceVector<int64_t>  dOwnerRows;   // panel row -> owner's row count
-
-  // DENSE_GATHER            : transport for the panel assembly.
-  //     0 = zero-fill + GlobalSumVector  (default; moves the payload twice)
-  //     1 = MPI_Allgatherv               (KNOWN BROKEN here, see guard below)
-  //     2 = C sequential MPI_Bcast, one per contributing rank.  Bcast has no
-  //         count vector, so the zero-count shape that breaks Allgatherv
-  //         cannot arise.  Costs C collectives instead of 1 and the roots do
-  //         not transmit concurrently, so the byte cost is ~2*panel per rank
-  //         -- the same as the allreduce.  It wins only if Bcast is faster
-  //         PER BYTE than Allreduce, which is why it must be measured.
-  // DENSE_GATHER_MIN_BYTES  : panels below this stay on the allreduce path.
-  //                           Default 0 (always gather).  The Schur tree puts
-  //                           94% of its bytes in panels with ~3.7 MB per-rank
-  //                           messages, i.e. squarely bandwidth bound; only the
-  //                           two deepest levels (1.2% of bytes, 16-65 KB per
-  //                           rank) are latency bound.  This exists so that
-  //                           tail can be excluded with an env var rather than
-  //                           a rebuild, if it ever proves to matter.
-  // DENSE_BARRIER_PROBE     : Barrier() before each panel collective.
-  //   DEFAULTS ON, and it is an optimisation rather than instrumentation.
-  //   Measured at 288 ranks, N=138240: comms 380.3 s -> 314.7 s and the whole
-  //   invert 394.3 s -> 327.7 s, with the worst single collective falling
-  //   2474.9 ms -> 256.2 ms and effective rate 2.40 -> 4.27 GB/s.  Convoy
-  //   accounting alone predicts NO saving (barrier and collective both
-  //   complete at max-over-ranks), so the mechanism is that the collective
-  //   itself degrades under skewed arrival, not that skew is rebooked.
-  //   Set DENSE_BARRIER_PROBE=0 to recover the old behaviour.  It also
-  //   separates arrival skew (tBarrier) from transfer (tAllreduce).
-  int                    useGather;
-  int64_t                gatherMinBytes;
-  int                    barrierProbe;
-  // DENSE_GATHER_DEBUG=N : dump the descriptor of the first N AllGatherV
-  // calls from rank 0.  The tiling invariant below is checked ALWAYS -- it
-  // costs one O(P) host loop against a collective, and a counts array that
-  // does not tile the panel is exactly the failure we are hunting.
-  int                    gatherDebug;
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Ownership-table validation: a proper partition of [0,N).
-  // Static and communicator-free so synthetic tables unit-test directly.
-  ///////////////////////////////////////////////////////////////////////////
-  static void CheckRowStart(const std::vector<int64_t> &table, int64_t N)
-  {
-    int P = (int)table.size() - 1;
-    GRID_ASSERT( P >= 1 );
-    GRID_ASSERT( table[0] == 0 );
-    GRID_ASSERT( table[P] == N );
-    for(int r=0; r<P; r++)
-    {
-      GRID_ASSERT( table[r+1] >= table[r] );  // zero-row ranks permitted
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Build the ownership table from each rank's local row count: zero-fill
-  // allgather (the standing comms idiom) then prefix sum.  Every rank
-  // returns the identical table.
-  ///////////////////////////////////////////////////////////////////////////
-  static std::vector<int64_t> MakeRowStart(GridBase *g, int64_t myNrows)
-  {
-    int P  = g->ProcessorCount();
-    int me = g->ThisRank();
-
-    std::vector<uint64_t> counts(P, 0);
-    counts[me] = (uint64_t)myNrows;
-    g->GlobalSumVector(&counts[0], P);
-
-    std::vector<int64_t> table(P+1);
-    table[0] = 0;
-    for(int r=0; r<P; r++)
-    {
-      table[r+1] = table[r] + (int64_t)counts[r];
-    }
-    CheckRowStart(table, table[P]);
-    return table;
-  }
-
-  RecursiveSchurInverse(GridBase *g,
-                        int64_t N_,
-                        std::vector<int64_t> &rowStart_,
-                        int64_t panelBytes_)
-  {
-    grid       = g;
-    N          = N_;
-    P          = g->ProcessorCount();
-    me         = g->ThisRank();
-    rowStart   = rowStart_;
-    panelBytes = panelBytes_;
-
-    GRID_ASSERT( (int)rowStart.size() == P+1 );
-    CheckRowStart(rowStart, N);
-
-    myRow0  = rowStart[me];
-    myNrows = rowStart[me+1] - rowStart[me];
-
-    telLeafMaxInv = 0.0;
-
-    useGather      = getenv("DENSE_GATHER") ? atoi(getenv("DENSE_GATHER")) : 0;
-    gatherMinBytes = getenv("DENSE_GATHER_MIN_BYTES")
-                   ? atol(getenv("DENSE_GATHER_MIN_BYTES")) : 0;
-    barrierProbe   = getenv("DENSE_BARRIER_PROBE") ? atoi(getenv("DENSE_BARRIER_PROBE")) : 1;
-    gatherDebug    = getenv("DENSE_GATHER_DEBUG") ? atoi(getenv("DENSE_GATHER_DEBUG")) : 0;
-
-    ///////////////////////////////////////////////////////////////////////
-    // DENSE_GATHER=1 is KNOWN BROKEN on Cray MPICH and refuses to start.
-    //
-    // The gather asks MPI_Allgatherv to assemble a panel to which only the
-    // rank sub-range [rB0,rB1) contributes; every other rank passes count 0.
-    // At Schur depth d that is P/2^(d+1) contributors out of P, i.e. 9 of 288
-    // at depth 4 and ~1 of 288 at depth 7.  Measured consequences at 288
-    // ranks (tests/debug/Test_allgather):
-    //   T5  288/288 contributing, 18.4 MB on device : sub-second.
-    //   T6  144/288 contributing,  9.2 MB on device : ~53 s.
-    // and in production, 9/288 contributing at 299 MB hangs and then aborts
-    // inside PMPI_Allgatherv ("req != NULL", mpir_request.h:508).
-    //
-    // The primitive itself is sound -- Test_allgather T1-T6 pass, and T3/T4
-    // verify it BITWISE against the zero-fill+GlobalSum idiom.  What fails is
-    // MPI's handling of a collective in which almost every rank contributes
-    // nothing.  Fixing it needs either P2P (a bisection allgather over
-    // [r0,r1)) or the 2D block-cyclic layout, which removes the shape
-    // altogether by giving every rank part of every sub-block.
-    //
-    // Fail here, in the first second, rather than 100 s and 36 nodes into a
-    // job.  DENSE_GATHER_FORCE=1 proceeds anyway for debugging.
-    ///////////////////////////////////////////////////////////////////////
-    if ( (useGather==1) && !getenv("DENSE_GATHER_FORCE") )
-    {
-      std::cout << GridLogError
-                << "DENSE_GATHER=1 is disabled: MPI_Allgatherv on this MPI is"
-                << " pathological when most ranks contribute count 0 (see the"
-                << " note at RecursiveSchurInverse.h, and Test_allgather T5 vs"
-                << " T6).  Unset DENSE_GATHER, or set DENSE_GATHER_FORCE=1 to"
-                << " proceed anyway." << std::endl;
-      GRID_ASSERT(0 && "DENSE_GATHER=1 known broken: see comment above");
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // The communication primitive.
-  //
-  //   C(:, colC : colC+widthB) <-   beta  * C(:, colC : colC+widthB)
-  //                               + alpha * A(:, colA : colA+widthA) * Bsub
-  //
-  // Bsub is the widthA x widthB sub-block of a row-distributed operand
-  // owned by ranks [rB0, rB1): owner r contributes its rows of
-  // B(:, colB : colB+widthB) at sub-block row offset
-  // rowStart[r] - rowStart[rB0], gathered in panelBytes row-chunks by
-  // device zero-fill + deposit kernel + GlobalSumVector.
-  //
-  // Every rank calls; non-owners of B add zeros; ranks with A.rows == 0
-  // skip local compute but make every collective call.  Column offsets
-  // are local buffer offsets -- non-participants pass 0.
-  ///////////////////////////////////////////////////////////////////////////
-  void GatherGemm(ComplexD alpha,
-                  BlockRows &A, int64_t colA, int64_t widthA,
-                  int rB0, int rB1,
-                  BlockRows &B, int64_t colB, int64_t widthB,
-                  ComplexD beta,
-                  BlockRows &C, int64_t colC)
-  {
-    GRID_ASSERT( rB0 >= 0 );
-    GRID_ASSERT( rB1 >  rB0 );
-    GRID_ASSERT( rB1 <= P );
-
-    int64_t k = rowStart[rB1] - rowStart[rB0];
-    int64_t m = A.rows;
-    int64_t n = widthB;
-    GRID_ASSERT( widthA == k );
-    GRID_ASSERT( n >= 1 );
-
-    int owner = ( me >= rB0 ) && ( me < rB1 ) && ( B.rows > 0 );
-    int64_t myOff = 0;
-    if ( owner )
-    {
-      myOff = rowStart[me] - rowStart[rB0];
-    }
-
-    if ( m > 0 )
-    {
-      GRID_ASSERT( colA + widthA <= A.cols );
-      GRID_ASSERT( colC + widthB <= C.cols );
-      GRID_ASSERT( C.rows == m );
-    }
-
-    nGatherGemm++;
-    GRID_TRACE("GatherGemm");
-
-    if ( owner )
-    {
-      GRID_ASSERT( colB + widthB <= B.cols );
-    }
-
-    // COLUMN chunking, not row chunking.  Each rank's contribution to a
-    // column chunk is rows_r x nchunk at ld = B.rows -- i.e. exactly the
-    // contiguous window B.ColumnWindow(colB+j0) -- so the AllGatherV send
-    // needs no pack.  Row chunking would slice each column and force one.
-    // It also makes every chunk a full-k product, so beta applies directly
-    // and the cross-chunk accumulation disappears.
-    int64_t bufs = useGather ? 2 : 1;         // panel, plus receive staging
-    int64_t nc = panelBytes / ( bufs * (int64_t)sizeof(ComplexD) * k );
-    if ( nc < 1 ) nc = 1;
-    if ( nc > n ) nc = n;
-    GRID_ASSERT( k*nc < 2147483647L );        // collective counts are int
-
-    deviceVector<ComplexD> &dPanel = dPanelBuf;
-    if ( dPanel.size() < (uint64_t)k*nc ) dPanel.resize((uint64_t)k*nc);
-    deviceVector<ComplexD*> ap(1);
-    deviceVector<ComplexD*> bp(1);
-    deviceVector<ComplexD*> cp(1);
-    std::vector<ComplexD*>  ptr(1);
-
-    // Panel-row -> owner maps for the repack.  Depend only on [rB0,rB1),
-    // so they are built once per call rather than once per chunk.
-    if ( useGather )
-    {
-      if ( dRecvBuf.size()   < (uint64_t)k*nc ) dRecvBuf.resize((uint64_t)k*nc);
-      if ( dOwnerOff.size()  < (uint64_t)k )    dOwnerOff.resize((uint64_t)k);
-      if ( dOwnerRows.size() < (uint64_t)k )    dOwnerRows.resize((uint64_t)k);
-      std::vector<int64_t> hoff(k), hrows(k);
-      for(int r=rB0; r<rB1; r++)
-      {
-        int64_t off_r  = rowStart[r]   - rowStart[rB0];
-        int64_t rows_r = rowStart[r+1] - rowStart[r];
-        for(int64_t i=0;i<rows_r;i++){ hoff[off_r+i]=off_r; hrows[off_r+i]=rows_r; }
-      }
-      acceleratorCopyToDevice(&hoff[0], &dOwnerOff[0], (uint64_t)k*sizeof(int64_t));
-      acceleratorCopyToDevice(&hrows[0],&dOwnerRows[0],(uint64_t)k*sizeof(int64_t));
-      if ( owner ) GRID_ASSERT( B.rows == rowStart[me+1]-rowStart[me] );
-    }
-
-    for(int64_t j0=0; j0<n; j0+=nc)
-    {
-      int64_t nchunk = std::min(nc, n-j0);
-      uint64_t panelWords = (uint64_t)k*nchunk;
-      uint64_t panelBytesThis = panelWords*sizeof(ComplexD);
-
-      // The MODE (0/1/2), not a boolean: `useGather && cond` collapses 2 to 1
-      // and silently dispatched DENSE_GATHER=2 onto the known-broken
-      // AllGatherV path (Frontier hang, 2026-08-24).  Identical on all ranks.
-      int gatherThis = ( (int64_t)panelBytesThis >= gatherMinBytes ) ? useGather : 0;
-
-      // Arrival skew is charged to the barrier, so that tAllreduce measures
-      // transfer alone.  Diagnostic only; off by default.
-      if ( barrierProbe ) { double tb = -usecond(); grid->Barrier(); tBarrier += tb+usecond(); }
-
-      double tar = -usecond();
-      if ( gatherThis )
-      {
-        //////////////////////////////////////////////////////////////////
-        // AllGatherV: move the payload once, no arithmetic, no zero fill.
-        // Receive is rank major -- a concatenation of rows_r x nchunk
-        // column-major blocks -- which is the correct ROW order but the
-        // wrong LAYOUT, so one device repack follows.
-        //////////////////////////////////////////////////////////////////
-        std::vector<int> counts(P,0), displs(P,0);
-        for(int r=rB0; r<rB1; r++)
-        {
-          counts[r] = (int)((rowStart[r+1]-rowStart[r])*nchunk);
-          displs[r] = (int)((rowStart[r]  -rowStart[rB0])*nchunk);
-        }
-        //////////////////////////////////////////////////////////////////
-        // The counts MUST tile the panel exactly: every one of the
-        // k*nchunk words has exactly one owner, or MPI is handed an
-        // inconsistent descriptor.  Always checked; O(P) against a
-        // collective is free.
-        //////////////////////////////////////////////////////////////////
-        int64_t csum = 0; int nz = 0;
-        for(int r=0;r<P;r++){ csum += counts[r]; if ( counts[r] ) nz++; }
-        GRID_ASSERT( csum == (int64_t)panelWords );
-        GRID_ASSERT( displs[rB1-1] + counts[rB1-1] == (int)panelWords );
-        // Gate on the count of GATHERS, not of all collectives.  nAllreduce
-        // is already in the hundreds by the first gather -- the recursion is
-        // depth first and everything below the size threshold takes the
-        // allreduce path -- so gating on it prints nothing at any sane value.
-        nGather++;
-        if ( gatherDebug && ((int)nGather <= gatherDebug) && (me==0) ) {
-          std::cout << GridLogMessage << "GATHER["<<nGather-1<<"]"
-                    << "  ranks ["<<rB0<<","<<rB1<<")"
-                    << "  k "<<k<<"  n "<<n<<"  nchunk "<<nchunk
-                    << "  panelWords "<<panelWords
-                    << " ("<<panelBytesThis/1024/1024<<" MB)"
-                    << "  contributors "<<nz<<"/"<<P
-                    << "  maxcount "<<*std::max_element(counts.begin(),counts.end())
-                    << "  rank0: count "<<counts[me]<<(owner?" owner":" non-owner")
-                    << std::endl;
-        }
-        if ( gatherThis == 1 )
-        {
-          nGatherV++;
-          void *send = owner ? (void *)B.ColumnWindow(colB+j0) : (void *)&dRecvBuf[0];
-          grid->AllGatherV(send, counts[me],
-                           (void *)&dRecvBuf[0], counts, displs, sizeof(ComplexD));
-        }
-        else
-        {
-          ////////////////////////////////////////////////////////////////
-          // C sequential broadcasts into the SAME rank-major staging
-          // buffer, so the repack below is shared with the AllGatherV arm.
-          // The root must already hold its own block: B.ColumnWindow is
-          // contiguous rows_r x nchunk, and so is its slot in dRecvBuf.
-          // Every rank issues all C broadcasts in the same order, so
-          // collective matching stays positional and safe.
-          ////////////////////////////////////////////////////////////////
-          nBcast++;
-          if ( owner )
-          {
-            int64_t off_me = rowStart[me] - rowStart[rB0];
-            acceleratorCopyDeviceToDevice((void *)B.ColumnWindow(colB+j0),
-                                          (void *)&dRecvBuf[off_me*nchunk],
-                                          (uint64_t)B.rows*nchunk*sizeof(ComplexD));
-          }
-          for(int r=rB0;r<rB1;r++)
-          {
-            int64_t off_r  = rowStart[r]   - rowStart[rB0];
-            int64_t rows_r = rowStart[r+1] - rowStart[r];
-            if ( rows_r )
-              grid->Broadcast(r,(void *)&dRecvBuf[off_r*nchunk],
-                              (uint64_t)rows_r*nchunk*sizeof(ComplexD));
-          }
-        }
-        tar += usecond();
-
-        tRepack -= usecond();
-        {
-          ComplexD *pan = &dPanel[0];
-          ComplexD *rcv = &dRecvBuf[0];
-          int64_t  *ooff  = &dOwnerOff[0];
-          int64_t  *orows = &dOwnerRows[0];
-          int64_t   kk = k, ncw = nchunk;
-          accelerator_for(idx, panelWords, 1, {
-            int64_t j = idx / kk;
-            int64_t p = idx - j*kk;
-            int64_t o = ooff[p];
-            int64_t rr= orows[p];
-            pan[idx] = rcv[(uint64_t)(o*ncw + (p-o) + j*rr)];
-          });
-        }
-        tRepack += usecond();
-      }
-      else
-      {
-        //////////////////////////////////////////////////////////////////
-        // Zero fill + deposit + GlobalSumVector.  Exact because the zero
-        // fill leaves exactly one contributing rank per element, but it
-        // moves the payload twice and reduces over zeros.
-        //////////////////////////////////////////////////////////////////
-        tar += usecond();
-        tMemset -= usecond();
-        acceleratorMemSet(&dPanel[0], 0, panelBytesThis);
-        tMemset += usecond();
-        if ( owner )
-        {
-          int64_t   brows = B.rows;
-          int64_t   kk    = k;
-          int64_t   dof   = myOff;
-          ComplexD *src   = B.ColumnWindow(colB+j0);
-          ComplexD *dst   = &dPanel[0];
-          tDeposit -= usecond();
-          accelerator_for(idx, (uint64_t)(brows*nchunk), 1, {
-            int64_t j = idx / brows;
-            int64_t i = idx - j*brows;
-            dst[(uint64_t)(dof + i + j*kk)] = src[(uint64_t)(i + j*brows)];
-          });
-          tDeposit += usecond();
-        }
-        tar = -usecond();
-        grid->GlobalSumVector(&dPanel[0], (int)panelWords);
-        tar += usecond();
-      }
-      tAllreduce += tar;
-      tARmin = std::min(tARmin, tar);
-      tARmax = std::max(tARmax, tar);
-      tARall.push_back(tar);
-      bytesAllreduce += panelBytesThis;
-      nAllreduce++;
-
-      if ( m > 0 )
-      {
-        // Full-k product into this column chunk of C: beta applies directly.
-        ptr[0] = A.ColumnWindow(colA);
-        acceleratorCopyToDevice(&ptr[0], &ap[0], sizeof(ComplexD*));
-        ptr[0] = &dPanel[0];
-        acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD*));
-        ptr[0] = C.ColumnWindow(colC+j0);
-        acceleratorCopyToDevice(&ptr[0], &cp[0], sizeof(ComplexD*));
-
-        tGemm -= usecond();
-        BLAS.gemmBatched(GridBLAS_OP_N, GridBLAS_OP_N,
-                         (int)m, (int)nchunk, (int)k,
-                         alpha,    ap, (int)A.ld,
-                                   bp, (int)k,
-                         beta,     cp, (int)C.ld);
-        BLAS.synchronise();
-        tGemm += usecond();
-      }
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Local Frobenius norm-squared of a full-height column window.
-  // NO comms; callers GlobalSum the result.  Host staging, setup-scale.
-  ///////////////////////////////////////////////////////////////////////////
-  double FrobNorm2Local(BlockRows &X, int64_t col0, int64_t w)
-  {
-    if ( X.rows == 0 ) return 0.0;
-    GRID_ASSERT( col0 + w <= X.cols );
-    uint64_t len = (uint64_t)X.rows*w;
-    std::vector<ComplexD> h(len);
-    acceleratorCopyFromDevice(X.ColumnWindow(col0), &h[0], len*sizeof(ComplexD));
-    // Member real()/imag(): portable across std::complex (CPU) and
-    // thrust::complex (HIP), where std::norm does not resolve.
-    double s = 0.0;
-    for(uint64_t i=0; i<len; i++)
-    {
-      double re = h[i].real();
-      double im = h[i].imag();
-      s += re*re + im*im;
-    }
-    return s;
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // dst(:, dcol0 : dcol0+w) = - src(:, 0:w).  Both operands have ld == rows
-  // so full-height windows are contiguous: flat elementwise device copy.
-  ///////////////////////////////////////////////////////////////////////////
-  void NegateCopy(BlockRows &src, BlockRows &dst, int64_t dcol0, int64_t w)
-  {
-    GRID_ASSERT( src.rows == dst.rows );
-    GRID_ASSERT( w <= src.cols );
-    GRID_ASSERT( dcol0 + w <= dst.cols );
-    if ( src.rows == 0 ) return;
-    uint64_t len = (uint64_t)src.rows*w;
-    ComplexD *s = &src.data[0];
-    ComplexD *d = dst.ColumnWindow(dcol0);
-    accelerator_for(i, len, 1, {
-      d[i] = -s[i];
-    });
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Leaf inversion: local, in place on the contiguous diagonal window.
-  // No collectives.
-  ///////////////////////////////////////////////////////////////////////////
-  void LeafInvert(int64_t col0, int64_t width, BlockRows &Arows)
-  {
-    GRID_TRACE("SchurLeaf");
-    GRID_ASSERT( width == Arows.rows );
-    GRID_ASSERT( col0 + width <= Arows.cols );
-    int64_t  w   = width;
-    uint64_t len = (uint64_t)w*w;
-    tLeaf -= usecond();
-
-    deviceVector<ComplexD*> bp(1);
-    std::vector<ComplexD*>  ptr(1);
-    ptr[0] = Arows.ColumnWindow(col0);
-    acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD*));
-    INV.inverseBatched(w, bp);
-
-    // Telemetry: max |element| of the leaf inverse
-    {
-      std::vector<ComplexD> h(len);
-      acceleratorCopyFromDevice(Arows.ColumnWindow(col0), &h[0], len*sizeof(ComplexD));
-      double mx = 0.0;
-      for(uint64_t i=0; i<len; i++)
-      {
-        double re = h[i].real();
-        double im = h[i].imag();
-        mx = std::max(mx, re*re + im*im);
-      }
-      telLeafMaxInv = std::max(telLeafMaxInv, std::sqrt(mx));
-    }
-    tLeaf += usecond();
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // The recursion (plan 3.5 / 4B.3).  Inverts the diagonal block of the
-  // rank-major matrix spanned by ranks [r0, r1), living in every member
-  // rank's column window [col0, col0+width) -- IN PLACE.
-  //
-  // SPMD: every rank calls with IDENTICAL (r0, r1, width) and its own
-  // local (col0, Arows); ranks outside [r0, r1) participate in the
-  // collectives only (dummy operands, zero contributions).  The collective
-  // sequence -- 5 GatherGemm calls + 3 scalar GlobalSums per merge node --
-  // is identical on every rank by construction.
-  //
-  //   I = [r0, mid)   J = [mid, r1)   widths WI, WJ
-  //   1. recurse I:      A11 -> A11inv
-  //   2. B = A11inv.A12                     (I rows)
-  //   3. C = A21.A11inv                     (J rows)
-  //   4. S = A22 - A21.B    in place        (J rows)   [alpha=-1, beta=1]
-  //   5. recurse J:      S -> Sinv
-  //   6. T = Sinv.C                         (J rows)
-  //   7. U = B.Sinv                         (I rows)
-  //   8. X11 = A11inv + U.C  in place       (I rows)   [beta=1]
-  //   9. X12 = -U, X21 = -T  local negates; X22 = Sinv already in place
-  ///////////////////////////////////////////////////////////////////////////
-  void SchurNode(int r0, int r1, int64_t col0, int64_t width, BlockRows &Arows)
-  {
-    int span = r1 - r0;
-    GRID_ASSERT( span >= 1 );
-    GRID_ASSERT( width == rowStart[r1] - rowStart[r0] );
-
-    if ( span == 1 )
-    {
-      if ( ( me == r0 ) && ( myNrows > 0 ) )
-      {
-        LeafInvert(col0, width, Arows);
-      }
-      return;
-    }
-
-    int     mid = ( r0 + r1 ) / 2;
-    int64_t WI  = rowStart[mid] - rowStart[r0];
-    int64_t WJ  = rowStart[r1]  - rowStart[mid];
-
-    // Zero-width child ranges (all ranks of a half owning no rows) are a
-    // KNOWN LIMITATION: fail loudly rather than divide mysteriously.
-    GRID_ASSERT( WI > 0 );
-    GRID_ASSERT( WJ > 0 );
-
-    int inI = ( me >= r0  ) && ( me < mid );
-    int inJ = ( me >= mid ) && ( me < r1  );
-
-    ComplexD one  ( 1.0,0.0);
-    ComplexD mone (-1.0,0.0);
-    ComplexD zero ( 0.0,0.0);
-
-    BlockRows dummy;
-
-    // 1. A11 -> A11inv
-    SchurNode(r0, mid, col0, WI, Arows);
-
-    // 2. B = A11inv . A12   (I rows; gather A12 from I owners)
-    BlockRows Bbuf;
-    if ( inI ) Bbuf.Resize(myNrows, WJ);
-    {
-      BlockRows &Aop = inI ? Arows : dummy;
-      BlockRows &Cop = inI ? Bbuf  : dummy;
-      int64_t    cA  = inI ? col0  : 0;
-      GatherGemm(one, Aop, cA, WI,
-                 r0, mid,
-                 Arows, col0+WI, WJ,
-                 zero, Cop, 0);
-    }
-    double nB = FrobNorm2Local(Bbuf, 0, inI ? WJ : 0);
-    grid->GlobalSumVector(&nB, 1);
-    telNormB.push_back(std::sqrt(nB));
-
-    // 3. C = A21 . A11inv   (J rows; gather A11inv from I owners)
-    BlockRows Cbuf;
-    if ( inJ ) Cbuf.Resize(myNrows, WI);
-    {
-      BlockRows &Aop = inJ ? Arows : dummy;
-      BlockRows &Cop = inJ ? Cbuf  : dummy;
-      int64_t    cA  = inJ ? col0  : 0;
-      GatherGemm(one, Aop, cA, WI,
-                 r0, mid,
-                 Arows, col0, WI,
-                 zero, Cop, 0);
-    }
-
-    // 4. S = A22 - A21 . B  in place on my A22 window (J rows)
-    double nA22 = FrobNorm2Local( inJ ? Arows : dummy, inJ ? col0+WI : 0, inJ ? WJ : 0 );
-    grid->GlobalSumVector(&nA22, 1);
-    {
-      BlockRows &Aop = inJ ? Arows   : dummy;
-      BlockRows &Cop = inJ ? Arows   : dummy;
-      int64_t    cA  = inJ ? col0    : 0;
-      int64_t    cC  = inJ ? col0+WI : 0;
-      GatherGemm(mone, Aop, cA, WI,
-                 r0, mid,
-                 Bbuf, 0, WJ,
-                 one, Cop, cC);
-    }
-    double nS = FrobNorm2Local( inJ ? Arows : dummy, inJ ? col0+WI : 0, inJ ? WJ : 0 );
-    grid->GlobalSumVector(&nS, 1);
-    telSratio.push_back( std::sqrt(nS) / ( std::sqrt(nA22) + 1.0e-300 ) );
-
-    // 5. S -> Sinv
-    SchurNode(mid, r1, col0+WI, WJ, Arows);
-
-    // 6. T = Sinv . C       (J rows; gather C from J owners)
-    BlockRows Tbuf;
-    if ( inJ ) Tbuf.Resize(myNrows, WI);
-    {
-      BlockRows &Aop = inJ ? Arows   : dummy;
-      BlockRows &Cop = inJ ? Tbuf    : dummy;
-      int64_t    cA  = inJ ? col0+WI : 0;
-      GatherGemm(one, Aop, cA, WJ,
-                 mid, r1,
-                 Cbuf, 0, WI,
-                 zero, Cop, 0);
-    }
-
-    // 7. U = B . Sinv       (I rows; gather Sinv from J owners)
-    BlockRows Ubuf;
-    if ( inI ) Ubuf.Resize(myNrows, WJ);
-    {
-      BlockRows &Aop = inI ? Bbuf : dummy;
-      BlockRows &Cop = inI ? Ubuf : dummy;
-      GatherGemm(one, Aop, 0, WJ,
-                 mid, r1,
-                 Arows, col0+WI, WJ,
-                 zero, Cop, 0);
-    }
-
-    // 8. X11 = A11inv + U . C  in place (I rows; gather C from J owners)
-    {
-      BlockRows &Aop = inI ? Ubuf  : dummy;
-      BlockRows &Cop = inI ? Arows : dummy;
-      int64_t    cC  = inI ? col0  : 0;
-      GatherGemm(one, Aop, 0, WJ,
-                 mid, r1,
-                 Cbuf, 0, WI,
-                 one, Cop, cC);
-    }
-
-    // 9. Off-diagonal signs, local
-    if ( inI ) NegateCopy(Ubuf, Arows, col0+WI, WJ);
-    if ( inJ ) NegateCopy(Tbuf, Arows, col0,    WI);
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // PUBLIC ENTRY.  Arows: my rows of the rank-major N x N matrix (fp64).
-  // On exit Arows holds my rows of the inverse, still fp64; the caller
-  // owns the single terminal rounding into its fp32 apply storage.
-  ///////////////////////////////////////////////////////////////////////////
-  void Invert(BlockRows &Arows)
-  {
-    GRID_ASSERT( Arows.rows == myNrows );
-    GRID_ASSERT( Arows.cols == N );
-
-    telNormB.resize(0);
-    telSratio.resize(0);
-    telLeafMaxInv = 0.0;
-
-    tMemset        = 0.0;
-    tDeposit       = 0.0;
-    tAllreduce     = 0.0;
-    tBarrier       = 0.0;
-    tRepack        = 0.0;
-    tGemm          = 0.0;
-    tLeaf          = 0.0;
-    tARmin         = 1.0e30;
-    tARmax         = 0.0;
-    tARall.clear();
-    bytesAllreduce = 0;
-    nAllreduce     = 0;
-    nGatherGemm    = 0;
-    nGather        = 0;
-    nGatherV       = 0;
-    nBcast         = 0;
-
-    SchurNode(0, P, 0, N, Arows);
-
-    RealD mx = telLeafMaxInv;
-    grid->GlobalMax(mx);
-    telLeafMaxInv = mx;
-  }
-
-  // All telemetry values are globally reduced or boss-local; safe to
-  // stream on every rank (Grid quiesces stdout to the boss unless
-  // --debug-stdout).  NOTE: tAllreduce INCLUDES wait/imbalance -- a rank
-  // arriving early books its wait here; the min/max spread across ranks
-  // separates true wire time (~min) from skew (max-min).
-  void ReportTelemetry(void)
-  {
-    for(uint64_t i=0; i<telNormB.size(); i++)
-    {
-      std::cout << GridLogPerformance
-                << "SchurNode " << i
-                << " ||B||_F "       << telNormB[i]
-                << " ||S||/||A22|| " << telSratio[i]
-                << std::endl;
-    }
-    std::cout << GridLogPerformance
-              << "Schur leaves max|Ainv| " << telLeafMaxInv
-              << std::endl;
-
-    RealD armax =  tAllreduce;
-    RealD armin = -tAllreduce;
-    grid->GlobalMax(armax);
-    grid->GlobalMax(armin);
-    armin = -armin;
-
-    std::cout << GridLogMessage << "Schur phases (boss rank, seconds):"
-              << "  memset "    << tMemset/1.0e6
-              << "  deposit "   << tDeposit/1.0e6
-              << "  allreduce " << tAllreduce/1.0e6
-              << "  barrier "   << tBarrier/1.0e6
-              << "  repack "    << tRepack/1.0e6
-              << "  gemm "      << tGemm/1.0e6
-              << "  leaf "      << tLeaf/1.0e6
-              << std::endl;
-    std::cout << GridLogMessage << "Schur comms:"
-              << "  [transports run: allreduce " << (nAllreduce - nGatherV - nBcast)
-              << " allgatherv " << nGatherV << " bcast " << nBcast << "]"
-              << ( barrierProbe ? " [barrier probe on]" : "" )
-              << "  GatherGemm calls " << nGatherGemm
-              << "  panel allreduces " << nAllreduce
-              << "  allreduce GB "     << bytesAllreduce/1024./1024./1024.
-              << "  allreduce s min/max over ranks " << armin/1.0e6
-              << " / " << armax/1.0e6
-              << std::endl;
-    std::cout << GridLogMessage << "Schur comms per-call (boss):"
-              << "  min " << (nAllreduce ? tARmin/1.0e3 : 0.0) << " ms"
-              << "  avg " << (nAllreduce ? tAllreduce/nAllreduce/1.0e3 : 0.0) << " ms"
-              << "  max " << tARmax/1.0e3 << " ms"
-              << "   effective " << (tAllreduce>0 ? bytesAllreduce/tAllreduce*1.0e6/1.0e9 : 0.0)
-              << " GB/s"
-              << std::endl;
-    //////////////////////////////////////////////////////////////////////
-    // Distribution, not just min/avg/max.  A barrier that merely REBOOKS
-    // skew shifts the median; a barrier that suppresses pathological
-    // collectives shortens the tail.  p50 vs p99 separates the two from a
-    // single run, which min/avg/max cannot.
-    //////////////////////////////////////////////////////////////////////
-    if ( tARall.size() ) {
-      std::vector<double> v = tARall;
-      std::sort(v.begin(),v.end());
-      size_t n = v.size();
-      auto pc = [&](double f){ size_t i=(size_t)(f*(n-1)); return v[i]/1.0e3; };
-      double med = pc(0.50);
-      // how much time is spent in calls that are gross outliers
-      double tail=0.0; size_t ntail=0;
-      for(size_t i=0;i<n;i++) if ( v[i] > 4.0*med*1.0e3 ) { tail+=v[i]; ntail++; }
-      std::cout << GridLogMessage << "Schur comms distribution (boss):"
-                << "  p50 "  << med
-                << "  p90 "  << pc(0.90)
-                << "  p99 "  << pc(0.99)
-                << " ms   calls > 4x median: " << ntail << "/" << n
-                << " carrying " << tail/1.0e6 << " s"
-                << std::endl;
-    }
-  }
-};
-
-NAMESPACE_END(Grid);

Grid/algorithms/multigrid/Smoothers.h

@@ -36,6 +36,8 @@ NAMESPACE_BEGIN(Grid);
 //                              smoother operator such as shifted PVdagM.
 //   ChebyshevInverter          one Chebyshev-corrected step with residual print.
 //   MirsSmoother               shifted-MdagM CG, HDCG arXiv:1402.2585.
+//   CGSmoother                 fixed-iteration CG on an already-shifted
+//                              Hermitian operator: the mrhs HDCG smoother.
 //   GCRReplaySmoother          replays a GCR's recorded step lengths a_k and
 //                              orthogonalisation coefficients b_kj with NO
 //                              inner products: one matvec per step, zero
@@ -191,6 +193,30 @@ public:
   }
 };
 
+//////////////////////////////////////////////////////////////////////////////
+// Fixed-iteration CG on an (already shifted) Hermitian operator, tolerance
+// zero so the work is fixed.  NOT a stationary preconditioner: the CG
+// coefficients depend on the input, so the map is nonlinear.  Use under a
+// flexible outer (fPcg); BlockCGrQ needs the Chebyshev smoother.
+//////////////////////////////////////////////////////////////////////////////
+template<class Field> class CGSmoother : public LinearFunction<Field>
+{
+public:
+  using LinearFunction<Field>::operator();
+  LinearOperatorBase<Field> &_Op;
+  int iters;
+  CGSmoother(int _iters, LinearOperatorBase<Field> &Op) : _Op(Op), iters(_iters)
+  {
+    std::cout << GridLogMessage << "CGSmoother order " << iters << std::endl;
+  }
+  void operator() (const Field &in, Field &out)
+  {
+    ConjugateGradient<Field> CG(0.0,iters,false);   // never converges: by design
+    out = Zero();
+    CG(_Op,in,out);
+  }
+};
+
 //////////////////////////////////////////////////////////////////////////////
 // Replay of a recorded GCR with a trivial preconditioner:
 //   p_0 = r_0 ;  x_{k+1} = x_k + a_k p_k ;  r_{k+1} = r_k - a_k A p_k ;

Grid/algorithms/multigrid/deprecated/GeneralCoarsenedMatrix.h

@@ -38,10 +38,10 @@ NAMESPACE_BEGIN(Grid);
 // Fine Object == (per site) type of fine field
 // nbasis      == number of deflation vectors
 template<class Fobj,class CComplex,int nbasis>
-class GeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
+class DeprecatedGeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
 public:
 
-  typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
+  typedef DeprecatedGeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
   typedef iVector<CComplex,nbasis >           siteVector;
   typedef iMatrix<CComplex,nbasis >           siteMatrix;
   typedef Lattice<iScalar<CComplex> >         CoarseComplexField;
@@ -103,7 +103,7 @@ public:
   void ProjectNearestNeighbour(RealD shift, GeneralCoarseOp &CopyMe)
   {
     int nfound=0;
-    std::cout << GridLogMessage <<"GeneralCoarsenedMatrix::ProjectNearestNeighbour "<< CopyMe._A[0].Grid()<<std::endl;
+    std::cout << GridLogMessage <<"DeprecatedGeneralCoarsenedMatrix::ProjectNearestNeighbour "<< CopyMe._A[0].Grid()<<std::endl;
     for(int p=0;p<geom.npoint;p++){
       for(int pp=0;pp<CopyMe.geom.npoint;pp++){
  	// Search for the same relative shift
@@ -120,7 +120,7 @@ public:
   }
   */
   
-  GeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridBase *FineGrid, GridCartesian * CoarseGrid,int _herm=1)
+  DeprecatedGeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridBase *FineGrid, GridCartesian * CoarseGrid,int _herm=1)
     : geom(_geom),
       _FineGrid(FineGrid),
       _CoarseGrid(CoarseGrid),

Grid/algorithms/multigrid/deprecated/GeneralCoarsenedMatrixMultiRHS.h

@@ -0,0 +1,758 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/GeneralCoarsenedMatrixMultiRHS.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+
+NAMESPACE_BEGIN(Grid);
+
+
+// Fine Object == (per site) type of fine field
+// nbasis      == number of deflation vectors
+template<class Fobj,class CComplex,int nbasis>
+class DeprecatedMultiGeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
+public:
+  typedef typename CComplex::scalar_object SComplex;
+  typedef DeprecatedGeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
+
+  typedef iVector<CComplex,nbasis >           siteVector;
+  typedef iMatrix<CComplex,nbasis >           siteMatrix;
+  typedef iVector<SComplex,nbasis >           calcVector;
+  typedef iMatrix<SComplex,nbasis >           calcMatrix;
+  typedef Lattice<iScalar<CComplex> >         CoarseComplexField;
+  typedef Lattice<siteVector>                 CoarseVector;
+  typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
+  typedef iMatrix<CComplex,nbasis >  Cobj;
+  typedef iVector<CComplex,nbasis >  Cvec;
+  typedef Lattice< CComplex >   CoarseScalar; // used for inner products on fine field
+  typedef Lattice<Fobj >        FineField;
+  typedef Lattice<CComplex >    FineComplexField;
+  typedef CoarseVector Field;
+
+  ////////////////////
+  // Data members
+  ////////////////////
+  GridCartesian *       _CoarseGridMulti; 
+  NonLocalStencilGeometry geom;
+  NonLocalStencilGeometry geom_srhs;
+  PaddedCell Cell;
+  GeneralLocalStencil Stencil;
+
+  deviceVector<calcVector> BLAS_B;
+  deviceVector<calcVector> BLAS_C;
+  std::vector<deviceVector<calcMatrix> > BLAS_A;
+
+  std::vector<deviceVector<ComplexD *> > BLAS_AP;
+  std::vector<deviceVector<ComplexD *> > BLAS_BP;
+  deviceVector<ComplexD *>               BLAS_CP;
+
+  ///////////////////////
+  // Interface
+  ///////////////////////
+  GridBase      * Grid(void)           { return _CoarseGridMulti; };   // this is all the linalg routines need to know
+  GridCartesian * CoarseGrid(void)     { return _CoarseGridMulti; };   // this is all the linalg routines need to know
+
+  //////////////////////////////////////////////////////////////////////////
+  // Accessors shared with DeprecatedGeneralCoarsenedMatrix. Grid() here is the
+  // D+1 multiRHS grid and this class never holds the D dimensional one, so
+  // ExtractMatrix writes into whatever grid the caller's lattice is on.
+  //////////////////////////////////////////////////////////////////////////
+  NonLocalStencilGeometry & Geometry(void)      { return geom_srhs; };
+  void ExtractMatrix(int p,CoarseMatrix &A)     { BLAStoGrid(A,BLAS_A[p]); };
+
+  // One stencil point in or out, matching MultiGeneralCoarsenedOperator, so a
+  // caller can read or write a point without knowing which of the two layouts
+  // (per-point here, site-major there) holds it.
+  void MatrixPointIn(int p,const deviceVector<calcMatrix> &src)
+  {
+    int64_t sites = BLAS_A[p].size();
+    GRID_ASSERT((int64_t)src.size() == sites);
+    calcMatrix       *dst = &BLAS_A[p][0];
+    const calcMatrix *sp  = &src[0];
+    accelerator_for(ss,sites,1,{ dst[ss] = sp[ss]; });
+  }
+  void MatrixPointOut(int p,deviceVector<calcMatrix> &dst)
+  {
+    int64_t sites = BLAS_A[p].size();
+    dst.resize(sites);
+    const calcMatrix *sp = &BLAS_A[p][0];
+    calcMatrix       *dp = &dst[0];
+    accelerator_for(ss,sites,1,{ dp[ss] = sp[ss]; });
+  }
+
+  // I/O on the operator matrices, via the BLAS layout array. The parameter is
+  // a vector over the geometry points; the body indexes A[p].
+  void SetMatrix (int p,std::vector<CoarseMatrix> & A)
+  {
+    GRID_ASSERT(A.size()==geom_srhs.npoint);
+    GridtoBLAS(A[p],BLAS_A[p]);
+  }
+  void GetMatrix (int p,std::vector<CoarseMatrix> & A)
+  {
+    GRID_ASSERT(A.size()==geom_srhs.npoint);
+    BLAStoGrid(A[p],BLAS_A[p]);
+  }
+  void CopyMatrix (GeneralCoarseOp &_Op)
+  {
+    for(int p=0;p<geom.npoint;p++){
+      auto Aup = _Op.Cell.Extract(_Op._A[p]);
+      //Unpadded
+      GridtoBLAS(Aup,BLAS_A[p]);
+    }
+  }
+  /*
+  void CheckMatrix (GeneralCoarseOp &_Op)
+  {
+    std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
+    for(int p=0;p<geom.npoint;p++){
+      //Unpadded
+      auto Aup = _Op.Cell.Extract(_Op._A[p]);
+      auto Ack = Aup;
+      BLAStoGrid(Ack,BLAS_A[p]);
+      std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
+      std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
+    }
+    std::cout <<"************* "<<std::endl;
+  }
+  */
+  
+  DeprecatedMultiGeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridCartesian *CoarseGridMulti) :
+    _CoarseGridMulti(CoarseGridMulti),
+    geom_srhs(_geom),
+    geom(_CoarseGridMulti,_geom.hops,_geom.skip+1),
+    Cell(geom.Depth(),_CoarseGridMulti),
+    Stencil(Cell.grids.back(),geom.shifts) // padded cell stencil
+  {
+    int32_t padded_sites   = Cell.grids.back()->lSites();
+    int32_t unpadded_sites = CoarseGridMulti->lSites();
+    
+    int32_t nrhs  = CoarseGridMulti->FullDimensions()[0];  // # RHS
+    int32_t orhs  = nrhs/CComplex::Nsimd();
+
+    padded_sites   = padded_sites/nrhs;
+    unpadded_sites = unpadded_sites/nrhs;
+    
+    /////////////////////////////////////////////////
+    // Device data vector storage
+    /////////////////////////////////////////////////
+    BLAS_A.resize(geom.npoint);
+    for(int p=0;p<geom.npoint;p++){
+      BLAS_A[p].resize (unpadded_sites); // no ghost zone, npoint elements
+    }
+    
+    BLAS_B.resize(nrhs *padded_sites);   // includes ghost zone
+    BLAS_C.resize(nrhs *unpadded_sites); // no ghost zone
+    BLAS_AP.resize(geom.npoint);
+    BLAS_BP.resize(geom.npoint);
+    for(int p=0;p<geom.npoint;p++){
+      BLAS_AP[p].resize(unpadded_sites);
+      BLAS_BP[p].resize(unpadded_sites);
+    }
+    BLAS_CP.resize(unpadded_sites);
+
+    /////////////////////////////////////////////////
+    // Pointers to data
+    /////////////////////////////////////////////////
+
+    // Site identity mapping for A
+    for(int p=0;p<geom.npoint;p++){
+      for(int ss=0;ss<unpadded_sites;ss++){
+	ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
+	acceleratorPut(BLAS_AP[p][ss],ptr);
+      }
+    }
+    // Site identity mapping for C
+    for(int ss=0;ss<unpadded_sites;ss++){
+      ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
+      acceleratorPut(BLAS_CP[ss],ptr);
+    }
+
+    // Neighbour table is more complicated
+    int32_t j=0; // Interior point counter (unpadded)
+    for(int32_t s=0;s<padded_sites;s++){ // 4 volume, padded
+      int ghost_zone=0;
+      for(int32_t point = 0 ; point < geom.npoint; point++){
+	int i=s*orhs*geom.npoint+point;
+	if( Stencil._entries[i]._wrap ) { // stencil is indexed by the oSite of the CoarseGridMulti, hence orhs factor
+	  ghost_zone=1; // If general stencil wrapped in any direction, wrap=1
+	}
+      }
+
+      if( ghost_zone==0) {
+	for(int32_t point = 0 ; point < geom.npoint; point++){
+	  int i=s*orhs*geom.npoint+point;
+ 	  int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite
+	  GRID_ASSERT(nbr<BLAS_B.size());
+	  ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
+	  acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
+	}
+	j++;
+      }
+    }
+    GRID_ASSERT(j==unpadded_sites);
+  }
+  template<class vobj> void GridtoBLAS(const Lattice<vobj> &from,deviceVector<typename vobj::scalar_object> &to)
+  {
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  GridBase *Fg = from.Grid();
+  GRID_ASSERT(!Fg->_isCheckerBoarded);
+  int nd = Fg->_ndimension;
+
+  to.resize(Fg->lSites());
+
+  Coordinate LocalLatt = Fg->LocalDimensions();
+  size_t nsite = 1;
+  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
+
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  // do the index calc on the GPU
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  Coordinate f_ostride = Fg->_ostride;
+  Coordinate f_istride = Fg->_istride;
+  Coordinate f_rdimensions = Fg->_rdimensions;
+
+  autoView(from_v,from,AcceleratorRead);
+  auto to_v = &to[0];
+
+  const int words=sizeof(vobj)/sizeof(vector_type);
+  accelerator_for(idx,nsite,1,{
+      
+      Coordinate from_coor, base;
+      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
+      for(int i=0;i<nd;i++){
+	from_coor[i] = base[i];
+      }
+      int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*(from_coor[d]%f_rdimensions[d]);
+      int from_lane = 0; for(int d=0;d<nd;d++) from_lane+=f_istride[d]*(from_coor[d]/f_rdimensions[d]);
+
+      const vector_type* from = (const vector_type *)&from_v[from_oidx];
+      scalar_type* to = (scalar_type *)&to_v[idx];
+      
+      scalar_type stmp;
+      for(int w=0;w<words;w++){
+	stmp = getlane(from[w], from_lane);
+	to[w] = stmp;
+      }
+    });
+  }    
+  template<class vobj> void BLAStoGrid(Lattice<vobj> &grid,deviceVector<typename vobj::scalar_object> &in)
+  {
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  GridBase *Tg = grid.Grid();
+  GRID_ASSERT(!Tg->_isCheckerBoarded);
+  int nd = Tg->_ndimension;
+  
+  GRID_ASSERT(in.size()==Tg->lSites());
+
+  Coordinate LocalLatt = Tg->LocalDimensions();
+  size_t nsite = 1;
+  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
+
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  // do the index calc on the GPU
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  Coordinate t_ostride = Tg->_ostride;
+  Coordinate t_istride = Tg->_istride;
+  Coordinate t_rdimensions = Tg->_rdimensions;
+
+  autoView(to_v,grid,AcceleratorWrite);
+  auto from_v = &in[0];
+
+  const int words=sizeof(vobj)/sizeof(vector_type);
+  accelerator_for(idx,nsite,1,{
+      
+      Coordinate to_coor, base;
+      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
+      for(int i=0;i<nd;i++){
+	to_coor[i] = base[i];
+      }
+      int to_oidx = 0; for(int d=0;d<nd;d++) to_oidx+=t_ostride[d]*(to_coor[d]%t_rdimensions[d]);
+      int to_lane = 0; for(int d=0;d<nd;d++) to_lane+=t_istride[d]*(to_coor[d]/t_rdimensions[d]);
+
+      vector_type* to = (vector_type *)&to_v[to_oidx];
+      scalar_type* from = (scalar_type *)&from_v[idx];
+      
+      scalar_type stmp;
+      for(int w=0;w<words;w++){
+	stmp=from[w];
+	putlane(to[w], stmp, to_lane);
+      }
+    });
+  }
+  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
+		       Aggregation<Fobj,CComplex,nbasis> & Subspace,
+		       GridBase *CoarseGrid)
+  {
+#if 0
+    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
+
+    GridBase *grid = Subspace.FineGrid;
+
+    /////////////////////////////////////////////////////////////
+    // Orthogonalise the subblocks over the basis
+    /////////////////////////////////////////////////////////////
+    CoarseScalar InnerProd(CoarseGrid); 
+    blockOrthogonalise(InnerProd,Subspace.subspace);
+
+    const int npoint = geom_srhs.npoint;
+
+    Coordinate clatt = CoarseGrid->GlobalDimensions();
+    int Nd = CoarseGrid->Nd();
+      /*
+       *     Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
+       *     Matrix index i is mapped to this shift via 
+       *               geom.shifts[i]
+       *
+       *     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block] 
+       *       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} > 
+       *       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
+       *       = M_{kl} A_ji^{b.b+l}
+       *
+       *     Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
+       *  
+       *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
+       *
+       *     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
+       */
+    Eigen::MatrixXcd Mkl    = Eigen::MatrixXcd::Zero(npoint,npoint);
+    Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
+    ComplexD ci(0.0,1.0);
+    for(int k=0;k<npoint;k++){ // Loop over momenta
+
+      for(int l=0;l<npoint;l++){ // Loop over nbr relative
+	ComplexD phase(0.0,0.0);
+	for(int mu=0;mu<Nd;mu++){
+	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
+	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
+	}
+	phase=exp(phase*ci);
+	Mkl(k,l) = phase;
+      }
+    }
+    invMkl = Mkl.inverse();
+
+    ///////////////////////////////////////////////////////////////////////
+    // Now compute the matrix elements of linop between the orthonormal
+    // set of vectors.
+    ///////////////////////////////////////////////////////////////////////
+    FineField phaV(grid); // Phased block basis vector
+    FineField MphaV(grid);// Matrix applied
+    std::vector<FineComplexField> phaF(npoint,grid);
+    std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
+    
+    CoarseVector coarseInner(CoarseGrid);
+    
+    typedef typename CComplex::scalar_type SComplex;
+    FineComplexField one(grid); one=SComplex(1.0);
+    FineComplexField zz(grid); zz = Zero();
+    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
+      /////////////////////////////////////////////////////
+      // Stick a phase on every block
+      /////////////////////////////////////////////////////
+      CoarseComplexField coor(CoarseGrid);
+      pha[p]=Zero();
+      for(int mu=0;mu<Nd;mu++){
+	LatticeCoordinate(coor,mu);
+	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
+	pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
+      }
+      pha[p]  =exp(pha[p]*ci);	
+
+      blockZAXPY(phaF[p],pha[p],one,zz);
+    }
+
+    // Could save on temporary storage here
+    std::vector<CoarseMatrix> _A;
+    _A.resize(geom_srhs.npoint,CoarseGrid);
+
+    std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
+    CoarseVector          FT(CoarseGrid);
+    for(int i=0;i<nbasis;i++){// Loop over basis vectors
+      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
+      for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
+
+	phaV = phaF[p]*Subspace.subspace[i];
+
+	/////////////////////////////////////////////////////////////////////
+	// Multiple phased subspace vector by matrix and project to subspace
+	// Remove local bulk phase to leave relative phases
+	/////////////////////////////////////////////////////////////////////
+	linop.Op(phaV,MphaV);
+
+	// Fixme, could use batched block projector here
+	blockProject(coarseInner,MphaV,Subspace.subspace);
+
+	coarseInner = conjugate(pha[p]) * coarseInner;
+
+	ComputeProj[p] = coarseInner;
+      }
+
+      // Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
+      for(int k=0;k<npoint;k++){
+
+	FT = Zero();
+	for(int l=0;l<npoint;l++){
+	  FT= FT+ invMkl(l,k)*ComputeProj[l];
+	}
+      
+	int osites=CoarseGrid->oSites();
+	autoView( A_v  , _A[k], AcceleratorWrite);
+	autoView( FT_v  , FT, AcceleratorRead);
+	accelerator_for(sss, osites, 1, {
+	    for(int j=0;j<nbasis;j++){
+	      A_v[sss](i,j) = FT_v[sss](j);
+	    }
+        });
+      }
+    }
+
+    // Only needed if nonhermitian
+    //    if ( ! hermitian ) {
+    //      std::cout << GridLogMessage<<"PopulateAdag  "<<std::endl;
+    //      PopulateAdag();
+    //    }
+    // Need to write something to populate Adag from A
+
+    for(int p=0;p<geom_srhs.npoint;p++){
+      GridtoBLAS(_A[p],BLAS_A[p]);
+    }
+    /*
+Grid : Message : 11698.730546 s : CoarsenOperator eigen  1334 us
+Grid : Message : 11698.730563 s : CoarsenOperator phase  34729 us
+Grid : Message : 11698.730565 s : CoarsenOperator phaseBZ 2423814 us
+Grid : Message : 11698.730566 s : CoarsenOperator mat    127890998 us
+Grid : Message : 11698.730567 s : CoarsenOperator proj   515840840 us
+Grid : Message : 11698.730568 s : CoarsenOperator inv    103948313 us
+Takes 600s to compute matrix elements, DOMINATED by the block project.
+Easy to speed up with the batched block project.
+Store npoint vectors, get npoint x Nbasis block projection, and 81 fold faster.
+
+// Block project below taks to 240s
+Grid : Message : 328.193418 s : CoarsenOperator phase      38338 us
+Grid : Message : 328.193434 s : CoarsenOperator phaseBZ  1711226 us
+Grid : Message : 328.193436 s : CoarsenOperator mat    122213270 us
+//Grid : Message : 328.193438 s : CoarsenOperator proj   1181154 us <-- this is mistimed
+//Grid : Message : 11698.730568 s : CoarsenOperator inv  103948313 us <-- Cut this ~10x if lucky by loop fusion
+     */
+#else
+    RealD tproj=0.0;
+    RealD tmat=0.0;
+    RealD tphase=0.0;
+    RealD tphaseBZ=0.0;
+    RealD tinv=0.0;
+
+    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
+
+    GridBase *grid = Subspace.FineGrid;
+
+    /////////////////////////////////////////////////////////////
+    // Orthogonalise the subblocks over the basis
+    /////////////////////////////////////////////////////////////
+    CoarseScalar InnerProd(CoarseGrid); 
+    blockOrthogonalise(InnerProd,Subspace.subspace);
+
+
+    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
+    Projector.Allocate(nbasis,grid,CoarseGrid);
+    Projector.ImportBasis(Subspace.subspace);
+    
+    const int npoint = geom_srhs.npoint;
+
+    Coordinate clatt = CoarseGrid->GlobalDimensions();
+    int Nd = CoarseGrid->Nd();
+      /*
+       *     Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
+       *     Matrix index i is mapped to this shift via 
+       *               geom.shifts[i]
+       *
+       *     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block] 
+       *       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} > 
+       *       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
+       *       = M_{kl} A_ji^{b.b+l}
+       *
+       *     Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
+       *  
+       *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
+       *
+       *     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
+       */
+    Eigen::MatrixXcd Mkl    = Eigen::MatrixXcd::Zero(npoint,npoint);
+    Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
+    ComplexD ci(0.0,1.0);
+    for(int k=0;k<npoint;k++){ // Loop over momenta
+
+      for(int l=0;l<npoint;l++){ // Loop over nbr relative
+	ComplexD phase(0.0,0.0);
+	for(int mu=0;mu<Nd;mu++){
+	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
+	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
+	}
+	phase=exp(phase*ci);
+	Mkl(k,l) = phase;
+      }
+    }
+    invMkl = Mkl.inverse();
+
+    ///////////////////////////////////////////////////////////////////////
+    // Now compute the matrix elements of linop between the orthonormal
+    // set of vectors.
+    ///////////////////////////////////////////////////////////////////////
+    FineField phaV(grid); // Phased block basis vector
+    FineField MphaV(grid);// Matrix applied
+    std::vector<FineComplexField> phaF(npoint,grid);
+    std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
+    
+    CoarseVector coarseInner(CoarseGrid);
+    
+    tphase=-usecond();
+    typedef typename CComplex::scalar_type SComplex;
+    FineComplexField one(grid); one=SComplex(1.0);
+    FineComplexField zz(grid); zz = Zero();
+    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
+      /////////////////////////////////////////////////////
+      // Stick a phase on every block
+      /////////////////////////////////////////////////////
+      CoarseComplexField coor(CoarseGrid);
+      pha[p]=Zero();
+      for(int mu=0;mu<Nd;mu++){
+	LatticeCoordinate(coor,mu);
+	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
+	pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
+      }
+      pha[p]  =exp(pha[p]*ci);	
+
+      blockZAXPY(phaF[p],pha[p],one,zz);
+    }
+    tphase+=usecond();
+
+    // Could save on temporary storage here
+    std::vector<CoarseMatrix> _A;
+    _A.resize(geom_srhs.npoint,CoarseGrid);
+
+    // Count use small chunks than npoint == 81 and save memory
+    int batch = 9;
+    std::vector<FineField>    _MphaV(batch,grid);
+    std::vector<CoarseVector> TmpProj(batch,CoarseGrid);
+
+    std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
+    CoarseVector          FT(CoarseGrid);
+    for(int i=0;i<nbasis;i++){// Loop over basis vectors
+      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
+
+      //      std::cout << GridLogMessage << " phasing the fine vector "<<std::endl;
+      // Fixme : do this in batches
+      for(int p=0;p<npoint;p+=batch){ // Loop over momenta in npoint
+
+	for(int b=0;b<MIN(batch,npoint-p);b++){
+	  tphaseBZ-=usecond();
+	  phaV = phaF[p+b]*Subspace.subspace[i];
+	  tphaseBZ+=usecond();
+
+	  /////////////////////////////////////////////////////////////////////
+	  // Multiple phased subspace vector by matrix and project to subspace
+	  // Remove local bulk phase to leave relative phases
+	  /////////////////////////////////////////////////////////////////////
+	  // Memory footprint was an issue
+	  tmat-=usecond();
+	  linop.Op(phaV,MphaV);
+	  _MphaV[b] = MphaV;
+	  tmat+=usecond();
+	}      
+
+	//	std::cout << GridLogMessage << " Calling block project "<<std::endl;
+	tproj-=usecond();
+	Projector.blockProject(_MphaV,TmpProj);
+	tproj+=usecond();
+	
+	//	std::cout << GridLogMessage << " conj phasing the coarse vectors "<<std::endl;
+	for(int b=0;b<MIN(batch,npoint-p);b++){
+	  ComputeProj[p+b] = conjugate(pha[p+b])*TmpProj[b];
+	}
+      }
+
+      // Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
+      
+      // std::cout << GridLogMessage << " Starting FT inv "<<std::endl;
+      tinv-=usecond();
+      for(int k=0;k<npoint;k++){
+	FT = Zero();
+	// 81 kernel calls as many ComputeProj vectors
+	// Could fuse with a vector of views, but ugly
+	// Could unroll the expression and run fewer kernels -- much more attractive
+	// Could also do non blocking.
+#if 0	
+	for(int l=0;l<npoint;l++){
+	  FT= FT+ invMkl(l,k)*ComputeProj[l];
+	}
+#else
+	const int radix = 9;
+	int ll;
+	for(ll=0;ll+radix-1<npoint;ll+=radix){
+	  // When ll = npoint-radix, ll+radix-1 = npoint-1, and we do it all.
+	  FT = FT 
+	    + invMkl(ll+0,k)*ComputeProj[ll+0]
+	    + invMkl(ll+1,k)*ComputeProj[ll+1]
+	    + invMkl(ll+2,k)*ComputeProj[ll+2]
+	    + invMkl(ll+3,k)*ComputeProj[ll+3]
+	    + invMkl(ll+4,k)*ComputeProj[ll+4]
+	    + invMkl(ll+5,k)*ComputeProj[ll+5]
+	    + invMkl(ll+6,k)*ComputeProj[ll+6]
+	    + invMkl(ll+7,k)*ComputeProj[ll+7]
+	    + invMkl(ll+8,k)*ComputeProj[ll+8];
+	}
+	for(int l=ll;l<npoint;l++){
+	  FT= FT+ invMkl(l,k)*ComputeProj[l];
+	}
+#endif
+      
+	// 1 kernel call -- must be cheaper
+	int osites=CoarseGrid->oSites();
+	autoView( A_v  , _A[k], AcceleratorWrite);
+	autoView( FT_v  , FT, AcceleratorRead);
+	accelerator_for(sss, osites, 1, {
+	    for(int j=0;j<nbasis;j++){
+	      A_v[sss](i,j) = FT_v[sss](j);
+	    }
+        });
+      }
+      tinv+=usecond();
+    }
+
+    // Only needed if nonhermitian
+    //    if ( ! hermitian ) {
+    //      std::cout << GridLogMessage<<"PopulateAdag  "<<std::endl;
+    //      PopulateAdag();
+    //    }
+    // Need to write something to populate Adag from A
+    //    std::cout << GridLogMessage << " Calling GridtoBLAS "<<std::endl;
+    for(int p=0;p<geom_srhs.npoint;p++){
+      GridtoBLAS(_A[p],BLAS_A[p]);
+    }
+    std::cout << GridLogMessage<<"CoarsenOperator phase  "<<tphase<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator mat    "<<tmat <<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator proj   "<<tproj<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator inv    "<<tinv<<" us"<<std::endl;
+#endif
+  }
+  void Mdag(const CoarseVector &in, CoarseVector &out)
+  {
+    this->M(in,out);
+  }
+  void M (const CoarseVector &in, CoarseVector &out)
+  {
+    //    std::cout << GridLogMessage << "New Mrhs coarse"<<std::endl;
+    conformable(CoarseGrid(),in.Grid());
+    conformable(in.Grid(),out.Grid());
+    out.Checkerboard() = in.Checkerboard();
+
+    RealD t_tot;
+    RealD t_exch;
+    RealD t_GtoB;
+    RealD t_BtoG;
+    RealD t_mult;
+
+    t_tot=-usecond();
+    CoarseVector tin=in;
+    t_exch=-usecond();
+    CoarseVector pin = Cell.ExchangePeriodic(tin); //padded input
+    t_exch+=usecond();
+
+    CoarseVector pout(pin.Grid());
+
+    int npoint = geom.npoint;
+    typedef calcMatrix* Aview;
+    typedef LatticeView<Cvec> Vview;
+      
+    const int Nsimd = CComplex::Nsimd();
+
+    int64_t nrhs  =pin.Grid()->GlobalDimensions()[0];
+    GRID_ASSERT(nrhs>=1);
+
+    RealD flops,bytes;
+    int64_t osites=in.Grid()->oSites(); // unpadded
+    int64_t unpadded_vol = CoarseGrid()->lSites()/nrhs;
+    
+    flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
+    bytes = 1.0*osites*sizeof(siteMatrix)*npoint/pin.Grid()->GlobalDimensions()[0]
+          + 2.0*osites*sizeof(siteVector)*npoint;
+    
+
+    t_GtoB=-usecond();
+    GridtoBLAS(pin,BLAS_B);
+    t_GtoB+=usecond();
+
+    GridBLAS BLAS;
+
+    t_mult=-usecond();
+    for(int p=0;p<geom.npoint;p++){
+      RealD c = 1.0;
+      if (p==0) c = 0.0;
+      ComplexD beta(c);
+
+      BLAS.gemmBatched(nbasis,nrhs,nbasis,
+		       ComplexD(1.0),
+		       BLAS_AP[p], 
+		       BLAS_BP[p], 
+		       ComplexD(c), 
+		       BLAS_CP);
+    }
+    BLAS.synchronise();
+    t_mult+=usecond();
+
+    t_BtoG=-usecond();
+    BLAStoGrid(out,BLAS_C);
+    t_BtoG+=usecond();
+    t_tot+=usecond();
+    /*
+    std::cout << GridLogMessage << "New Mrhs coarse DONE "<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult exch "<<t_exch<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult mult "<<t_mult<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult GtoB  "<<t_GtoB<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult BtoG  "<<t_BtoG<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult tot  "<<t_tot<<" us"<<std::endl;
+    */
+    //    std::cout << GridLogMessage<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse Kernel flops "<< flops<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse Kernel flop/s "<< flops/t_mult<<" mflop/s"<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse Kernel bytes/s "<< bytes/t_mult/1000<<" GB/s"<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse overall flops/s "<< flops/t_tot<<" mflop/s"<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse total bytes   "<< bytes/1e6<<" MB"<<std::endl;
+  };
+  virtual  void Mdiag    (const Field &in, Field &out){ GRID_ASSERT(0);};
+  virtual  void Mdir     (const Field &in, Field &out,int dir, int disp){assert(0);};
+  virtual  void MdirAll  (const Field &in, std::vector<Field> &out){assert(0);};
+};
+  
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/deprecated/TwoLevelADEF2.h

@@ -0,0 +1,136 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./Grid/algorithms/multigrid/deprecated/TwoLevelADEF2.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#pragma once
+
+// DEPRECATED with the other classes in deprecated/: the single-RHS ADEF-2 on an
+// Aggregation (ProjectToSubspace/PromoteFromSubspace) and a D-dimensional
+// coarse operator.  The mrhs solver in AdefMrhs.h covers one right-hand
+// side through its LinearFunction interface, on MultiGeneralCoarsenedOperator.
+
+NAMESPACE_BEGIN(Grid);
+
+template<class Field, class CoarseField, class Aggregation>
+class DeprecatedTwoLevelADEF2 : public TwoLevelCG<Field>
+{
+ public:
+  ///////////////////////////////////////////////////////////////////////////////////
+  // Need something that knows how to get from Coarse to fine and back again
+  //  void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
+  //  void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
+  ///////////////////////////////////////////////////////////////////////////////////
+  GridBase *coarsegrid;
+  Aggregation &_Aggregates;                    
+  LinearFunction<CoarseField> &_CoarseSolver;
+  LinearFunction<CoarseField> &_CoarseSolverPrecise;
+  ///////////////////////////////////////////////////////////////////////////////////
+  
+  // more most opertor functions
+  DeprecatedTwoLevelADEF2(RealD tol,
+		Integer maxit,
+		LinearOperatorBase<Field>    &FineLinop,
+		LinearFunction<Field>        &Smoother,
+		LinearFunction<CoarseField>  &CoarseSolver,
+		LinearFunction<CoarseField>  &CoarseSolverPrecise,
+		Aggregation &Aggregates
+		) :
+      TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,Aggregates.FineGrid),
+      _CoarseSolver(CoarseSolver),
+      _CoarseSolverPrecise(CoarseSolverPrecise),
+      _Aggregates(Aggregates)
+  {
+    coarsegrid = Aggregates.CoarseGrid;
+  };
+
+  virtual void PcgM1(Field & in, Field & out)
+  {
+    GRID_TRACE("MultiGridPreconditioner ");
+    // [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
+
+    Field tmp(this->grid);
+    Field Min(this->grid);
+    CoarseField PleftProj(this->coarsegrid);
+    CoarseField PleftMss_proj(this->coarsegrid);
+
+    this->SmoothTimer.Start();
+    this->_Smoother(in,Min);
+    this->SmoothTimer.Stop();
+    this->SmoothCalls++;
+
+    this->MatrixTimer.Start();
+    this->_FineLinop.HermOp(Min,out);
+    this->MatrixTimer.Stop();
+    this->MatrixCalls++;
+    axpy(tmp,-1.0,out,in);          // tmp  = in - A Min
+
+    this->ProjectTimer.Start();
+    this->_Aggregates.ProjectToSubspace(PleftProj,tmp);
+    this->ProjectTimer.Stop();
+    this->ProjectCalls++;
+    this->CoarseTimer.Start();
+    this->_CoarseSolver(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
+    this->CoarseTimer.Stop();
+    this->CoarseCalls++;
+    this->PromoteTimer.Start();
+    this->_Aggregates.PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
+    this->PromoteTimer.Stop();
+    this->PromoteCalls++;
+
+    axpy(out,1.0,Min,tmp); // Min+tmp
+  }
+
+  virtual void Vstart(Field & x,const Field & src)
+  {
+    std::cout << GridLogMessage<<"HDCG: fPcg Vstart "<<std::endl;
+    ///////////////////////////////////
+    // Choose x_0 such that 
+    // x_0 = guess +  (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
+    //                               = [1 - Ass_inv A] Guess + Assinv src
+    //                               = P^T guess + Assinv src 
+    //                               = Vstart  [Tang notation]
+    // This gives:
+    // W^T (src - A x_0) = src_s - A guess_s - r_s
+    //                   = src_s - (A guess)_s - src_s  + (A guess)_s 
+    //                   = 0 
+    ///////////////////////////////////
+    Field r(this->grid);
+    Field mmp(this->grid);
+    CoarseField PleftProj(this->coarsegrid);
+    CoarseField PleftMss_proj(this->coarsegrid);
+
+    std::cout << GridLogMessage<<"HDCG: fPcg Vstart projecting "<<std::endl;
+    this->_Aggregates.ProjectToSubspace(PleftProj,src);     
+    std::cout << GridLogMessage<<"HDCG: fPcg Vstart coarse solve "<<std::endl;
+    this->_CoarseSolverPrecise(PleftProj,PleftMss_proj); // Ass^{-1} r_s
+    std::cout << GridLogMessage<<"HDCG: fPcg Vstart promote "<<std::endl;
+    this->_Aggregates.PromoteFromSubspace(PleftMss_proj,x);  
+
+  }
+
+};
+
+NAMESPACE_END(Grid);

Grid/allocator/MemoryManager.cc

@@ -366,7 +366,7 @@ void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries
   GRID_ASSERT(omp_in_parallel()==0);
 #endif 
 
-  if (ncache == 0) return ptr;
+  if (ncache == 0) { if(freed) *freed = bytes; return ptr; }   // uncached: the incoming block is the one freed
 
   void * ret = NULL;
   int v = -1;

Grid/communicator/Communicator_base.h

@@ -133,6 +133,7 @@ public:
 
   template<class obj> void GlobalSumP2P(obj &o)
   {
+    GRID_TRACE("GlobalSumP2P");
     std::vector<obj> column;
     obj accum = o;
     int source,dest;

Grid/communicator/Communicator_mpi3.cc

@@ -191,6 +191,7 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const
   // Set up from the new split communicator
   //////////////////////////////////////////////////////////////////////////////////////////////////////
   InitFromMPICommunicator(processors,comm_split);
+  _shm_processors = shm_processors;
 
   //////////////////////////////////////////////////////////////////////////////////////////////////////
   // Take the right SHM buffers

Grid/communicator/RingAllReduce.h

@@ -201,4 +201,54 @@ void CartesianRingAllGather(CartesianCommunicator *comm, T *buf, uint64_t chunk,
   if ( cur != buf ) acceleratorCopyDeviceToDevice((void *)cur, (void *)buf, blk*sizeof(T));
 }
 
+/////////////////////////////////////////////////////////////////////////////
+// Cartesian ALL-TO-ALL along one processor dimension, point-to-point only.
+//
+//   CartesianRingAllToAll(comm, sbuf, rbuf, chunk, dim)
+//     sbuf, rbuf each hold P_dim*chunk elements of T.  Block index is the
+//     process COORDINATE along dim (not the MPI rank -- ShiftedRanks resolves
+//     the OptimalCommunicator relabelling): sbuf[c*chunk] is my data destined
+//     for the rank at coordinate c along dim; on exit rbuf[c*chunk] is the
+//     block that rank sent me.  All other coordinates equal mine.
+//
+// Direct-pairwise, NOT ring-forwarding: P_dim-1 symmetric exchanges, step k
+// with the +/-k neighbours (ShiftedRanks == MPI_Cart_shift, arbitrary shift),
+// each moving one chunk with no store-and-forward.  Per-rank traffic
+// (P_dim-1)*chunk -- a factor P below the AllGather's replicated P*chunk, which
+// is the point for the pencil-FFT transpose (partition the orthogonal coords
+// rather than replicate the whole transform line on every rank).
+//
+// On a switched fabric a distance-k send is one logical hop, so direct-pairwise
+// moves the minimum; ring-forwarding would only win on a true physical ring.
+// Every step is one SendToRecvFrom of one chunk: no collective, no size cliff.
+// Send/recv-rank binding follows Cshift_mpi.h: ShiftedRanks(dim,k,xmit,recv)
+// sends to coord-k, receives from coord+k.  Steps: P_dim-1.  Exact.
+/////////////////////////////////////////////////////////////////////////////
+template<class T>
+void CartesianRingAllToAll(CartesianCommunicator *comm,
+                           T *sbuf, T *rbuf, uint64_t chunk, int dim)
+{
+  int Nd = comm->_ndimension;
+  GRID_ASSERT( dim >= 0 && dim < Nd );
+  int P  = comm->_processors[dim];
+  int me = comm->_processor_coor[dim];
+  if ( chunk==0 ) return;
+  GRID_ASSERT( (chunk*sizeof(T))%4 == 0 );      // SendToRecvFrom counts int32 words
+  uint64_t bytes = chunk*sizeof(T);
+
+  // my own block never goes on the wire
+  acceleratorCopyDeviceToDevice((void *)&sbuf[(uint64_t)me*chunk],
+                                (void *)&rbuf[(uint64_t)me*chunk], bytes);
+  if ( P==1 ) return;
+
+  for(int k=1;k<P;k++){
+    int xmit_to_rank, recv_from_rank;
+    comm->ShiftedRanks(dim, k, xmit_to_rank, recv_from_rank); // xmit=coord-k, recv=coord+k
+    uint64_t sidx = (uint64_t)((me - k + P) % P);   // block destined for the coord-k rank
+    uint64_t ridx = (uint64_t)((me + k) % P);       // block arriving from the coord+k rank
+    comm->SendToRecvFrom((void *)&sbuf[sidx*chunk], xmit_to_rank,
+                         (void *)&rbuf[ridx*chunk], recv_from_rank, bytes);
+  }
+}
+
 NAMESPACE_END(Grid);

Grid/lattice/Lattice_comparison.h

@@ -175,5 +175,68 @@ template<class lobj,class robj>
 inline Lattice<vPredicate<robj> > operator != (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(vne<lobj,robj>(),lhs,rhs);
 }
+
+//////////////////////////////////////////////////////////////////////////
+// Real-part relational comparison for COMPLEX lattices.
+//
+// Complex has no ordering, so operator<,>,<=,>= are (deliberately) undefined for
+// complex operands -- Comparison() in Lattice_comparison_utils.h is IfNotComplex --
+// and where() cannot be driven by a complex lattice directly.  It is however often
+// useful to threshold on the REAL PART of a complex (scalar/singlet) field, e.g. a
+// momentum-magnitude mask phat^2 > pc^2.  These free functions compare the real part
+// to a real threshold and return the matching IntegerPredicate, so the result feeds
+// where() exactly like the built-in relationals.
+//
+// Written with the per-lane getlane/putlane accessors (as in FFT.h, PaddedCell.h),
+// not extract/merge buffers.  One wrinkle: the predicate type IntegerPredicate<CComplex>
+// is vInteger, whose Nsimd (>= the widest real type's) exceeds the complex operand's
+// Nsimd by s = Npred/Nsimd.  where() only reads the ii=0 representative of each group,
+// at physical lane lane*s (the "s-fold skip" -- cf. extract()'s getlane(i*s)), so
+// filling the rest of the group is not functionally required; we replicate the value
+// across all s lanes anyway because some Grid code asserts the s replicas are equal
+// (and it mirrors what merge() does).  When vInteger is reworked to carry the operand
+// Nsimd (scope later) s becomes 1 and the inner loop drops out.
+//////////////////////////////////////////////////////////////////////////
+template<class scalar> class sRealLt { public:
+  accelerator_inline Integer operator()(const scalar &a, RealD b) const { return a.real() <  b ? 1 : 0; } };
+template<class scalar> class sRealLe { public:
+  accelerator_inline Integer operator()(const scalar &a, RealD b) const { return a.real() <= b ? 1 : 0; } };
+template<class scalar> class sRealGt { public:
+  accelerator_inline Integer operator()(const scalar &a, RealD b) const { return a.real() >  b ? 1 : 0; } };
+template<class scalar> class sRealGe { public:
+  accelerator_inline Integer operator()(const scalar &a, RealD b) const { return a.real() >= b ? 1 : 0; } };
+
+template<class sfunctor,class CComplex>
+inline Lattice<vPredicate<CComplex> > RealPartComparison(sfunctor op,const Lattice<CComplex> &lhs, RealD thr)
+{
+  Lattice<vPredicate<CComplex> > ret(lhs.Grid());
+  autoView( lv, lhs, AcceleratorRead);
+  autoView( rv, ret, AcceleratorWrite);
+  typedef typename CComplex::vector_type vsimd;
+  const int Nsimd = vsimd::Nsimd();
+  const int s     = IntegerPredicate<CComplex>::Nsimd() / Nsimd;   // lane-replication factor (see note)
+  accelerator_for(ss, lhs.Grid()->oSites(), Nsimd, {
+    vsimd v = TensorRemove(lv[ss]);                 // strip iScalar nest to the bare complex SIMD word
+#ifdef GRID_SIMT
+    { int lane = acceleratorSIMTlane(Nsimd);        // GPU: this thread == this operand lane
+#else
+    for(int lane=0;lane<Nsimd;lane++){              // CPU: walk the packed operand lanes
+#endif
+      Integer p = op(v.getlane(lane), thr);
+      for(int ii=0;ii<s;ii++) rv[ss]._internal.putlane(p, lane*s+ii);   // replicate across the group
+#ifdef GRID_SIMT
+    }
+#else
+    }
+#endif
+  });
+  return ret;
+}
+
+template<class CComplex> inline Lattice<vPredicate<CComplex> > RealPartLessThan    (const Lattice<CComplex> &a, RealD b){ return RealPartComparison(sRealLt<typename CComplex::vector_type::scalar_type>(),a,b); }
+template<class CComplex> inline Lattice<vPredicate<CComplex> > RealPartLessEqual    (const Lattice<CComplex> &a, RealD b){ return RealPartComparison(sRealLe<typename CComplex::vector_type::scalar_type>(),a,b); }
+template<class CComplex> inline Lattice<vPredicate<CComplex> > RealPartGreaterThan  (const Lattice<CComplex> &a, RealD b){ return RealPartComparison(sRealGt<typename CComplex::vector_type::scalar_type>(),a,b); }
+template<class CComplex> inline Lattice<vPredicate<CComplex> > RealPartGreaterEqual (const Lattice<CComplex> &a, RealD b){ return RealPartComparison(sRealGe<typename CComplex::vector_type::scalar_type>(),a,b); }
+
 NAMESPACE_END(Grid);
 #endif

Grid/lattice/Lattice_reduction.h

@@ -898,7 +898,8 @@ template<int B,class vobj>
 void axpyMultiChunk(Lattice<vobj> &z,const ComplexD *b,
 		    const std::vector<const Lattice<vobj>*> &x,int m,
 		    int do_norm,
-		    decltype(innerProduct(vobj(),vobj())) *inner_tmp_v)
+                    decltype(innerProduct(vobj(),vobj())) *inner_tmp_v,
+                    const Lattice<vobj> *from=nullptr)
 {
   typedef decltype(z.View(AcceleratorRead)) View;
   GRID_ASSERT(m>=1 && m<=B);
@@ -917,13 +918,29 @@ void axpyMultiChunk(Lattice<vobj> &z,const ComplexD *b,
   }
   for(int j=m;j<B;j++){ pack.set(j,h_v[0]); pack.b[j] = ComplexD(0.0); }
 
-  autoView(z_v,z,AcceleratorWrite);
+  if ( from == nullptr ) {
-  accelerator_for(ss,sites,nsimd,{
+    autoView(z_v,z,AcceleratorWrite);
-      auto acc = coalescedRead(z_v[ss]);
+    accelerator_for(ss,sites,nsimd,{
-      for(int j=0;j<B;j++) if ( j<m ) acc = acc + pack.b[j]*coalescedRead(pack.v(j)[ss]);
+        auto acc = coalescedRead(z_v[ss]);
-      coalescedWrite(z_v[ss],acc);
+        for(int j=0;j<B;j++) if ( j<m ) acc = acc + pack.b[j]*coalescedRead(pack.v(j)[ss]);
-      if ( do_norm ) coalescedWrite(inner_tmp_v[ss],innerProduct(acc,acc));
+        coalescedWrite(z_v[ss],acc);
-  });
+        if ( do_norm ) coalescedWrite(inner_tmp_v[ss],innerProduct(acc,acc));
+    });
+  } else {
+    // z = from + sum_j b[j] x[j].  The accumulator is seeded from another field,
+    // so z is written without being read and the caller needs no copy into it.
+    conformable(*from,z);
+    GRID_ASSERT(from!=&z);
+    const Lattice<vobj> &y = *from;   // named: autoView is a macro, *p.View() misparses
+    autoView(y_v,y,AcceleratorRead);
+    autoView(z_v,z,AcceleratorWriteDiscard);
+    accelerator_for(ss,sites,nsimd,{
+        auto acc = coalescedRead(y_v[ss]);
+        for(int j=0;j<B;j++) if ( j<m ) acc = acc + pack.b[j]*coalescedRead(pack.v(j)[ss]);
+        coalescedWrite(z_v[ss],acc);
+        if ( do_norm ) coalescedWrite(inner_tmp_v[ss],innerProduct(acc,acc));
+    });
+  }
   for(int j=0;j<m;j++) h_v[j].ViewClose();
 }
 
@@ -931,7 +948,8 @@ void axpyMultiChunk(Lattice<vobj> &z,const ComplexD *b,
 // (last) pass.  Chunks of 16 for windows longer than 16.
 template<class vobj>
 RealD axpyMultiNormImpl(Lattice<vobj> &z,const std::vector<ComplexD> &b,
-			const std::vector<const Lattice<vobj>*> &x,int do_norm)
+                        const std::vector<const Lattice<vobj>*> &x,int do_norm,
+                        const Lattice<vobj> *from=nullptr)
 {
   typedef decltype(innerProduct(vobj(),vobj())) inner_t;
   int m = x.size();
@@ -943,6 +961,7 @@ RealD axpyMultiNormImpl(Lattice<vobj> &z,const std::vector<ComplexD> &b,
   inner_t *inner_tmp_v = &inner_tmp[0];
 
   if ( m==0 ) {
+    if ( from ) z = *from;
     if ( do_norm ) return norm2(z);
     return 0.0;
   }
@@ -951,10 +970,12 @@ RealD axpyMultiNormImpl(Lattice<vobj> &z,const std::vector<ComplexD> &b,
     int last = (j0+mm>=m);
     std::vector<const Lattice<vobj>*> sub(x.begin()+j0,x.begin()+j0+mm);
     int dn = do_norm && last;
-    if      ( mm<=2 ) axpyMultiChunk<2> (z,&b[j0],sub,mm,dn,inner_tmp_v);
+    // Only the first chunk is seeded from `from`; later chunks accumulate into z.
-    else if ( mm<=4 ) axpyMultiChunk<4> (z,&b[j0],sub,mm,dn,inner_tmp_v);
+    const Lattice<vobj> *f = (j0==0) ? from : nullptr;
-    else if ( mm<=8 ) axpyMultiChunk<8> (z,&b[j0],sub,mm,dn,inner_tmp_v);
+    if      ( mm<=2 ) axpyMultiChunk<2> (z,&b[j0],sub,mm,dn,inner_tmp_v,f);
-    else              axpyMultiChunk<16>(z,&b[j0],sub,mm,dn,inner_tmp_v);
+    else if ( mm<=4 ) axpyMultiChunk<4> (z,&b[j0],sub,mm,dn,inner_tmp_v,f);
+    else if ( mm<=8 ) axpyMultiChunk<8> (z,&b[j0],sub,mm,dn,inner_tmp_v,f);
+    else              axpyMultiChunk<16>(z,&b[j0],sub,mm,dn,inner_tmp_v,f);
   }
 
   RealD nrm = 0.0;
@@ -969,6 +990,15 @@ void axpyMulti(Lattice<vobj> &z,const std::vector<ComplexD> &b,const std::vector
 {
   axpyMultiNormImpl(z,b,x,0);
 }
+// z = y + sum_j b[j] x[j].  Same single pass as axpyMulti, but the accumulator
+// starts from y, so a caller whose z would otherwise have to be primed with a
+// copy of y does not make that copy.
+template<class vobj>
+void axpyMultiFrom(Lattice<vobj> &z,const Lattice<vobj> &y,
+                   const std::vector<ComplexD> &b,const std::vector<const Lattice<vobj>*> &x)
+{
+  axpyMultiNormImpl(z,b,x,0,&y);
+}
 template<class vobj>
 RealD axpyMultiNorm(Lattice<vobj> &z,const std::vector<ComplexD> &b,const std::vector<const Lattice<vobj>*> &x)
 {

Grid/lattice/Lattice_transfer.h

@@ -210,6 +210,14 @@ accelerator_inline void convertType(vComplexD2 & out, const vComplexF & in) {
   precisionChange(out,in);
 }
 
+// Precision change in the lex (unvectorised) chart: one lane, so a plain
+// scalar conversion.  Reached by the block operations on an fp32 coarse
+// space, whose inner products accumulate in double and convert back.
+accelerator_inline void convertType(sComplexF & out, const sComplexD & in) { out.v = ComplexF(in.v); }
+accelerator_inline void convertType(sComplexD & out, const sComplexF & in) { out.v = ComplexD(in.v); }
+accelerator_inline void convertType(sRealF    & out, const sRealD    & in) { out.v = RealF(in.v); }
+accelerator_inline void convertType(sRealD    & out, const sRealF    & in) { out.v = RealD(in.v); }
+
 template<typename T1,typename T2>
 accelerator_inline void convertType(iScalar<T1> & out, const iScalar<T2> & in) {
   convertType(out._internal,in._internal);
@@ -792,6 +800,39 @@ void localCopyRegion(const Lattice<vobj> &From,Lattice<vobj> & To,Coordinate Fro
   autoView(from_v,From,AcceleratorRead);
   autoView(to_v,To,AcceleratorWrite);
 
+  if constexpr ( vobj::Nsimd() == 1 ) {
+
+    // Unvectorised: there is one lane, so the lane indices are identically zero
+    // and the internal word index is free to carry the thread.  Threading over
+    // (site,word) with the word fastest makes consecutive threads read and write
+    // consecutive elements; one thread per site instead leaves them `words`
+    // apart, which is what made this uncoalesced.  The site count and the word
+    // count are folded into one range so the large product lands on the only
+    // block dimension that can hold it.
+    accelerator_for(ss,nsite*words,1,{
+
+        int      w   = ss % words;
+        uint64_t idx = ss / words;
+
+        Coordinate from_coor, to_coor, base;
+        Lexicographic::CoorFromIndex(base,idx,RegionSize);
+        for(int i=0;i<nd;i++){
+          from_coor[i] = base[i] + FromLowerLeft[i];
+          to_coor[i] = base[i] + ToLowerLeft[i];
+        }
+        // rdimensions == ldimensions when unvectorised and the region is local,
+        // so the oSite sums need no modulo and there is no lane term.
+        int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*from_coor[d];
+        int to_oidx   = 0; for(int d=0;d<nd;d++) to_oidx  +=t_ostride[d]*to_coor[d];
+
+        const vector_type* from = (const vector_type *)&from_v[from_oidx];
+        vector_type* to = (vector_type *)&to_v[to_oidx];
+
+        to[w] = from[w];
+    });
+
+  } else {
+
   accelerator_for(idx,nsite,1,{
 
       Coordinate from_coor, to_coor, base;
@@ -807,13 +848,15 @@ void localCopyRegion(const Lattice<vobj> &From,Lattice<vobj> & To,Coordinate Fro
 
       const vector_type* from = (const vector_type *)&from_v[from_oidx];
       vector_type* to = (vector_type *)&to_v[to_oidx];
-      
+
       scalar_type stmp;
       for(int w=0;w<words;w++){
 	stmp = getlane(from[w], from_lane);
 	putlane(to[w], stmp, to_lane);
       }
   });
+
+  }
 }
 
 template<class vobj>
@@ -1507,174 +1550,277 @@ void precisionChange(Lattice<VobjOut> &out, const Lattice<VobjIn> &in){
 ////////////////////////////////////////////////////////////////////////////////
 // Communicate between grids
 ////////////////////////////////////////////////////////////////////////////////
-///////////////////////////////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////////////////
-// SIMPLE CASE:
+// Grid_split / Grid_unsplit
-///////////////////////////////////////////////////////////////////////////////////////////////////////////
 //
-// Mesh of nodes (2x2) ; subdivide to  1x1 subdivisions
+// The split grid divides the full grid's processor layout Pf into partitions of layout
+// Ps; ratio R = Pf/Ps per dimension, nvector = prod R partitions. A rank at full processor
+// coordinate c belongs to partition q = c/Ps at split coordinate s = c%Ps, and partition q
+// holds vector index lex(q,R) (GridSplitVectorIndex). Local volumes satisfy
+// Ls = Lf * R per dimension.
 //
-// Lex ord:   
+// Every full-grid rank's local block of field v lies wholly inside one split-grid rank's
-//          N0 va0 vb0 vc0 vd0       N1 va1 vb1 vc1 vd1  
+// local volume, in the sub-box at offset o = c%R (units of Lf), on the rank of partition
-//          N2 va2 vb2 vc2 vd2       N3 va3 vb3 vc3 vd3 
+// q(v) at split coordinate c/R. The redistribution is therefore nvector whole-block
+// point-to-point exchanges, nothing finer, and no data moves twice. Step k (a coordinate in
+// the R box) pairs every rank with one sender and one receiver:
 //
-// Ratio = full[dim] / split[dim]
+//   send  field q = (c%R + k)%R      to   full coordinate q*Ps + c/R
+//   recv  sub-box o = (c/Ps - k)%R   from full coordinate (c%Ps)*R + o
 //
-// For each dimension do an all to all; get Nvec -> Nvec / ratio
+// Within a step the send map is a permutation of the ranks and the receive map is its
-//                                          Ldim -> Ldim * ratio
+// inverse, so each step is one symmetric SendToRecvFrom per rank; a rank paired with
-//                                          LocalVol -> LocalVol * ratio
+// itself copies locally. Grid_unsplit runs the same steps with the roles exchanged.
-// full AllToAll(0)
-//          N0 va0 vb0 va1 vb1       N1 vc0 vd0 vc1 vd1   
-//          N2 va2 vb2 va3 vb3       N3 vc2 vd2 vc3 vd3 
 //
-// REARRANGE
+// Data stays on the device. A block on the full grid is the local volume in lexicographic
-//          N0 va01 vb01      N1 vc01 vd01
+// order; the split grid's local volume is staged box-major (sub-box index, then
-//          N2 va23 vb23      N3 vc23 vd23
+// lexicographic within the sub-box), so every exchanged block is contiguous. Packing and
-//
+// unpacking are accelerator kernels. Buffers are deviceVectors: the staged split volume
-// full AllToAll(1)           // Not what is wanted. FIXME
+// plus one block. Without ACCELERATOR_AWARE_MPI each block is staged through host memory.
-//          N0 va01 va23      N1 vc01 vc23 
+/////////////////////////////////////////////////////////////////////////////////////////////
-//          N2 vb01 vb23      N3 vd01 vd23
+
-// 
+// Scalar sites of in, written to buf in box-major order: sub-boxes of extent bdims taken
-// REARRANGE
+// lexicographically, sites lexicographically within each. bdims = local dimensions gives
-//          N0 va0123      N1 vc0123
+// plain lexicographic order.
-//          N2 vb0123      N3 vd0123
+template<class vobj>
-//
+void GridSplitUnvectorise(const Lattice<vobj> &in,typename vobj::scalar_object *buf,const Coordinate &bdims)
-// Must also rearrange data to get into the NEW lex order of grid at each stage. Some kind of "insert/extract".
+{
-// NB: Easiest to programme if keep in lex order.
+  GridBase *grid = in.Grid();
-/*
+  int nd = grid->_ndimension;
- *  Let chunk = (fvol*nvec)/sP be size of a chunk.         ( Divide lexico vol * nvec into fP/sP = M chunks )
+  Coordinate rdims = grid->_rdimensions;
- *  
+  Coordinate simd  = grid->_simd_layout;
- *  2nd A2A (over sP nodes; subdivide the fP into sP chunks of M)
+  Coordinate nbox(nd);
- * 
+  int64_t bvol = 1;
- *     node 0     1st chunk of node 0M..(1M-1); 2nd chunk of node 0M..(1M-1)..   data chunk x M x sP = fL / sP * M * sP = fL * M growth
+  for(int d=0;d<nd;d++){
- *     node 1     1st chunk of node 1M..(2M-1); 2nd chunk of node 1M..(2M-1)..
+    GRID_ASSERT( (grid->_ldimensions[d] % bdims[d]) == 0 );
- *     node 2     1st chunk of node 2M..(3M-1); 2nd chunk of node 2M..(3M-1)..
+    nbox[d] = grid->_ldimensions[d] / bdims[d];
- *     node 3     1st chunk of node 3M..(3M-1); 2nd chunk of node 2M..(3M-1)..
+    bvol   *= bdims[d];
- *  etc...
+  }
- */
+  const int Nsimd = grid->Nsimd();
+  autoView( in_v , in, AcceleratorRead);
+  accelerator_for(ss, grid->oSites(), 1, {
+    Coordinate ocoor(nd);
+    Coordinate icoor(nd);
+    Coordinate bcoor(nd);
+    Coordinate wcoor(nd);
+    Lexicographic::CoorFromIndex(ocoor,ss,rdims);
+    for(int lane=0;lane<Nsimd;lane++){
+      Lexicographic::CoorFromIndex(icoor,lane,simd);
+      for(int d=0;d<nd;d++){
+        int l = ocoor[d] + rdims[d]*icoor[d];
+        bcoor[d] = l / bdims[d];
+        wcoor[d] = l % bdims[d];
+      }
+      int64_t b;
+      int64_t w;
+      Lexicographic::IndexFromCoor(bcoor,b,nbox);
+      Lexicographic::IndexFromCoor(wcoor,w,bdims);
+      buf[b*bvol+w] = extractLane(lane,in_v[ss]);
+    }
+  });
+}
+
+// Inverse of GridSplitUnvectorise: every site of out is written
+template<class vobj>
+void GridSplitVectorise(const typename vobj::scalar_object *buf,Lattice<vobj> &out,const Coordinate &bdims)
+{
+  GridBase *grid = out.Grid();
+  int nd = grid->_ndimension;
+  Coordinate rdims = grid->_rdimensions;
+  Coordinate simd  = grid->_simd_layout;
+  Coordinate nbox(nd);
+  int64_t bvol = 1;
+  for(int d=0;d<nd;d++){
+    GRID_ASSERT( (grid->_ldimensions[d] % bdims[d]) == 0 );
+    nbox[d] = grid->_ldimensions[d] / bdims[d];
+    bvol   *= bdims[d];
+  }
+  const int Nsimd = grid->Nsimd();
+  autoView( out_v , out, AcceleratorWriteDiscard);
+  accelerator_for(ss, grid->oSites(), 1, {
+    Coordinate ocoor(nd);
+    Coordinate icoor(nd);
+    Coordinate bcoor(nd);
+    Coordinate wcoor(nd);
+    Lexicographic::CoorFromIndex(ocoor,ss,rdims);
+    for(int lane=0;lane<Nsimd;lane++){
+      Lexicographic::CoorFromIndex(icoor,lane,simd);
+      for(int d=0;d<nd;d++){
+        int l = ocoor[d] + rdims[d]*icoor[d];
+        bcoor[d] = l / bdims[d];
+        wcoor[d] = l % bdims[d];
+      }
+      int64_t b;
+      int64_t w;
+      Lexicographic::IndexFromCoor(bcoor,b,nbox);
+      Lexicographic::IndexFromCoor(wcoor,w,bdims);
+      insertLane(lane,out_v[ss],buf[b*bvol+w]);
+    }
+  });
+}
+
+// Step schedule shared by Grid_split and Grid_unsplit; see the description above
+class GridSplitSchedule
+{
+public:
+  int        nd;
+  int        nvector;
+  Coordinate ratio;      // R
+  Coordinate fldims;     // full grid local dimensions = sub-box extent on the split grid
+  GridBase  *full_grid;
+
+  // Step k: this rank sends field Field(k) to rank FieldRank(k) and receives sub-box
+  // Box(k) from rank BoxRank(k) (Grid_split); Grid_unsplit reverses both
+  std::vector<int> Field;
+  std::vector<int> FieldRank;
+  std::vector<int> Box;
+  std::vector<int> BoxRank;
+
+  GridSplitSchedule(GridBase *_full_grid,GridBase *split_grid) : full_grid(_full_grid)
+  {
+    nd = full_grid->_ndimension;
+    GRID_ASSERT(split_grid->_ndimension == nd);
+    ratio.resize(nd);
+    fldims = full_grid->_ldimensions;
+    Coordinate sprocs = split_grid->_processors;
+    Coordinate fcoor  = full_grid->_processor_coor;
+    nvector = 1;
+    for(int d=0;d<nd;d++){
+      GRID_ASSERT(full_grid->_gdimensions[d] == split_grid->_gdimensions[d]);
+      GRID_ASSERT(full_grid->_fdimensions[d] == split_grid->_fdimensions[d]);
+      GRID_ASSERT( (full_grid->_processors[d] % sprocs[d]) == 0 );
+      ratio[d] = full_grid->_processors[d] / sprocs[d];
+      GRID_ASSERT(split_grid->_ldimensions[d] == fldims[d]*ratio[d]);
+      nvector *= ratio[d];
+    }
+    GRID_ASSERT(nvector*split_grid->_Nprocessors == full_grid->_Nprocessors);
+
+    Field.resize(nvector);
+    FieldRank.resize(nvector);
+    Box.resize(nvector);
+    BoxRank.resize(nvector);
+    for(int k=0;k<nvector;k++){
+      Coordinate kc(nd);
+      Coordinate q(nd);
+      Coordinate dest(nd);
+      Coordinate o(nd);
+      Coordinate from(nd);
+      Lexicographic::CoorFromIndex(kc,k,ratio);
+      for(int d=0;d<nd;d++){
+        q[d]    = (fcoor[d] % ratio[d] + kc[d]) % ratio[d];
+        dest[d] = q[d]*sprocs[d] + fcoor[d] / ratio[d];
+        o[d]    = (fcoor[d] / sprocs[d] - kc[d] + ratio[d]) % ratio[d];
+        from[d] = (fcoor[d] % sprocs[d])*ratio[d] + o[d];
+      }
+      Lexicographic::IndexFromCoor(q,Field[k],ratio);
+      Lexicographic::IndexFromCoor(o,Box[k],ratio);
+      FieldRank[k] = full_grid->RankFromProcessorCoor(dest);
+      BoxRank[k]   = full_grid->RankFromProcessorCoor(from);
+    }
+  }
+
+  // One step's exchange; returns the bytes that left this rank
+  uint64_t Exchange(void *xmit,int dest,void *recv,int from,uint64_t bytes)
+  {
+    int me = full_grid->ThisRank();
+    // The send map is a permutation and the receive map its inverse: self pairs with self
+    GRID_ASSERT( (dest == me) == (from == me) );
+    if ( dest == me ) {
+      acceleratorCopyDeviceToDevice(xmit,recv,bytes);
+      return 0;
+    }
+    GRID_ASSERT( (bytes % sizeof(int32_t)) == 0 );
+    GRID_ASSERT( bytes/sizeof(int32_t) < (1ULL<<31) );   // SendToRecvFrom counts int32 words
+#ifdef ACCELERATOR_AWARE_MPI
+    full_grid->SendToRecvFrom(xmit,dest,recv,from,bytes);
+#else
+    std::vector<char> hxmit(bytes);
+    std::vector<char> hrecv(bytes);
+    acceleratorCopyFromDevice(xmit,&hxmit[0],bytes);
+    full_grid->SendToRecvFrom(&hxmit[0],dest,&hrecv[0],from,bytes);
+    acceleratorCopyToDevice(&hrecv[0],recv,bytes);
+#endif
+    return bytes;
+  }
+};
+
+// Common body: full[v] may alias one another (the single field overload)
 template<class Vobj>
-void Grid_split(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
+void Grid_split(const std::vector<const Lattice<Vobj> *> &full,Lattice<Vobj> &split)
 {
   typedef typename Vobj::scalar_object Sobj;
 
-  int full_vecs   = full.size();
+  GRID_ASSERT(full.size()>=1);
-
+  GridBase *full_grid  = full[0]->Grid();
-  GRID_ASSERT(full_vecs>=1);
-
-  GridBase * full_grid = full[0].Grid();
   GridBase *split_grid = split.Grid();
 
-  int       ndim  = full_grid->_ndimension;
+  int cb = full[0]->Checkerboard();
-  int  full_nproc = full_grid->_Nprocessors;
-  int split_nproc =split_grid->_Nprocessors;
-
-  ////////////////////////////////
-  // Checkerboard management
-  ////////////////////////////////
-  int cb = full[0].Checkerboard();
   split.Checkerboard() = cb;
-
+  for(int n=0;n<full.size();n++){
-  //////////////////////////////
+    GRID_ASSERT(full[n]->Grid() == full_grid);
-  // Checks
+    GRID_ASSERT(full[n]->Checkerboard() == cb);
-  //////////////////////////////
-  GRID_ASSERT(full_grid->_ndimension==split_grid->_ndimension);
-  for(int n=0;n<full_vecs;n++){
-    GRID_ASSERT(full[n].Checkerboard() == cb);
-    for(int d=0;d<ndim;d++){
-      GRID_ASSERT(full[n].Grid()->_gdimensions[d]==split.Grid()->_gdimensions[d]);
-      GRID_ASSERT(full[n].Grid()->_fdimensions[d]==split.Grid()->_fdimensions[d]);
-    }
   }
 
-  int   nvector   =full_nproc/split_nproc; 
+  double t_start = usecond();
-  GRID_ASSERT(nvector*split_nproc==full_nproc);
+  GridSplitSchedule sched(full_grid,split_grid);
-  GRID_ASSERT(nvector == full_vecs);
+  GRID_ASSERT(sched.nvector == full.size());
 
-  Coordinate ratio(ndim);
+  uint64_t fvol  = full_grid->lSites();
-  for(int d=0;d<ndim;d++){
+  uint64_t bytes = fvol*sizeof(Sobj);
-    ratio[d] = full_grid->_processors[d]/ split_grid->_processors[d];
+  deviceVector<Sobj> staged(split_grid->lSites());
+  deviceVector<Sobj> block(fvol);
+  Sobj *staged_p = &staged[0];
+  Sobj *block_p  = &block[0];
+  double t_alloc = usecond() - t_start;
+
+  double t_pack = 0;
+  double t_comm = 0;
+  uint64_t offrank = 0;
+  for(int k=0;k<sched.nvector;k++){
+    double t0 = usecond();
+    GridSplitUnvectorise(*full[sched.Field[k]],block_p,full_grid->_ldimensions);
+    double t1 = usecond();
+    offrank += sched.Exchange((void *)block_p,sched.FieldRank[k],
+                              (void *)&staged_p[sched.Box[k]*fvol],sched.BoxRank[k],bytes);
+    double t2 = usecond();
+    t_pack += t1-t0;
+    t_comm += t2-t1;
   }
 
-  uint64_t lsites = full_grid->lSites();
+  double t0 = usecond();
-  uint64_t     sz = lsites * nvector;
+  GridSplitVectorise(staged_p,split,sched.fldims);
-  std::vector<Sobj> tmpdata(sz);
+  double t_unpack = usecond() - t0;
-  std::vector<Sobj> alldata(sz);
+  double t_total  = usecond() - t_start;
-  std::vector<Sobj> scalardata(lsites); 
 
-  for(int v=0;v<nvector;v++){
+  std::cout << GridLogPerformance << "Grid_split: " << sched.nvector << " x " << bytes/1.0e6
-    unvectorizeToLexOrdArray(scalardata,full[v]);    
+            << " MB/rank total " << t_total/1.0e6 << " s : alloc " << t_alloc/1.0e6
-    thread_for(site,lsites,{
+            << " pack " << t_pack/1.0e6
-      alldata[v*lsites+site] = scalardata[site];
+            << " exchange " << t_comm/1.0e6 << " (" << offrank/1.0e6 << " MB off rank, "
-    });
+            << offrank/(t_comm+1.0e-9)/1.0e3 << " GB/s)"
+            << " unpack " << t_unpack/1.0e6 << std::endl;
+  // Staging buffers are still live here, so this is the high-water point of the call
+  if ( GridLogPerformance.isActive() ) {
+    HostMemoryReport(full_grid,GridLogPerformance,"Grid_split");
   }
-
-  int nvec = nvector; // Counts down to 1 as we collapse dims
-  Coordinate ldims = full_grid->_ldimensions;
-
-  for(int d=ndim-1;d>=0;d--){
-
-    if ( ratio[d] != 1 ) {
-
-      full_grid ->AllToAll(d,alldata,tmpdata);
-      if ( split_grid->_processors[d] > 1 ) {
-	alldata=tmpdata;
-	split_grid->AllToAll(d,alldata,tmpdata);
-      }
-
-      auto rdims = ldims; 
-      auto     M = ratio[d];
-      auto rsites= lsites*M;// increases rsites by M
-      nvec      /= M;       // Reduce nvec by subdivision factor
-      rdims[d]  *= M;       // increase local dim by same factor
-
-      int sP =   split_grid->_processors[d];
-      int fP =    full_grid->_processors[d];
-
-      int fvol   = lsites;
-      
-      int chunk  = (nvec*fvol)/sP;          GRID_ASSERT(chunk*sP == nvec*fvol);
-
-      // Loop over reordered data post A2A
-      thread_for(c, chunk, {
-	Coordinate coor(ndim);
-	for(int m=0;m<M;m++){
-	  for(int s=0;s<sP;s++){
-	    
-	    // addressing; use lexico
-	    int lex_r;
-	    uint64_t lex_c        = c+chunk*m+chunk*M*s;
-	    uint64_t lex_fvol_vec = c+chunk*s;
-	    uint64_t lex_fvol     = lex_fvol_vec%fvol;
-	    uint64_t lex_vec      = lex_fvol_vec/fvol;
-
-	    // which node sets an adder to the coordinate
-	    Lexicographic::CoorFromIndex(coor, lex_fvol, ldims);	  
-	    coor[d] += m*ldims[d];
-	    Lexicographic::IndexFromCoor(coor, lex_r, rdims);	  
-	    lex_r += lex_vec * rsites;
-
-	    // LexicoFind coordinate & vector number within split lattice
-	    alldata[lex_r] = tmpdata[lex_c];
-
-	  }
-	}
-      });
-      ldims[d]*= ratio[d];
-      lsites  *= ratio[d];
-
-    }
-  }
-  vectorizeFromLexOrdArray(alldata,split);    
 }
 
+template<class Vobj>
+void Grid_split(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
+{
+  std::vector<const Lattice<Vobj> *> full_p(full.size());
+  for(int n=0;n<full.size();n++){
+    full_p[n] = &full[n];
+  }
+  Grid_split(full_p,split);
+}
+
+// The same field to every partition
 template<class Vobj>
 void Grid_split(Lattice<Vobj> &full,Lattice<Vobj>   & split)
 {
   int nvector = full.Grid()->_Nprocessors / split.Grid()->_Nprocessors;
-  std::vector<Lattice<Vobj> > full_v(nvector,full.Grid());
+  std::vector<const Lattice<Vobj> *> full_p(nvector,&full);
-  for(int n=0;n<nvector;n++){
+  Grid_split(full_p,split);
-    full_v[n] = full;
-  }
-  Grid_split(full_v,split);
 }
 
 template<class Vobj>
@@ -1682,119 +1828,57 @@ void Grid_unsplit(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
 {
   typedef typename Vobj::scalar_object Sobj;
 
-  int full_vecs   = full.size();
+  GRID_ASSERT(full.size()>=1);
-
+  GridBase *full_grid  = full[0].Grid();
-  GRID_ASSERT(full_vecs>=1);
-
-  GridBase * full_grid = full[0].Grid();
   GridBase *split_grid = split.Grid();
 
-  int       ndim  = full_grid->_ndimension;
-  int  full_nproc = full_grid->_Nprocessors;
-  int split_nproc =split_grid->_Nprocessors;
-
-  ////////////////////////////////
-  // Checkerboard management
-  ////////////////////////////////
   int cb = full[0].Checkerboard();
   split.Checkerboard() = cb;
-
+  for(int n=0;n<full.size();n++){
-  //////////////////////////////
+    GRID_ASSERT(full[n].Grid() == full_grid);
-  // Checks
-  //////////////////////////////
-  GRID_ASSERT(full_grid->_ndimension==split_grid->_ndimension);
-  for(int n=0;n<full_vecs;n++){
     GRID_ASSERT(full[n].Checkerboard() == cb);
-    for(int d=0;d<ndim;d++){
-      GRID_ASSERT(full[n].Grid()->_gdimensions[d]==split.Grid()->_gdimensions[d]);
-      GRID_ASSERT(full[n].Grid()->_fdimensions[d]==split.Grid()->_fdimensions[d]);
-    }
   }
 
-  int   nvector   =full_nproc/split_nproc; 
+  double t_start = usecond();
-  GRID_ASSERT(nvector*split_nproc==full_nproc);
+  GridSplitSchedule sched(full_grid,split_grid);
-  GRID_ASSERT(nvector == full_vecs);
+  GRID_ASSERT(sched.nvector == full.size());
 
-  Coordinate ratio(ndim);
+  uint64_t fvol  = full_grid->lSites();
-  for(int d=0;d<ndim;d++){
+  uint64_t bytes = fvol*sizeof(Sobj);
-    ratio[d] = full_grid->_processors[d]/ split_grid->_processors[d];
+  deviceVector<Sobj> staged(split_grid->lSites());
+  deviceVector<Sobj> block(fvol);
+  Sobj *staged_p = &staged[0];
+  Sobj *block_p  = &block[0];
+  double t_alloc = usecond() - t_start;
+
+  double t0 = usecond();
+  GridSplitUnvectorise(split,staged_p,sched.fldims);
+  double t_pack = usecond() - t0;
+
+  double t_comm   = 0;
+  double t_unpack = 0;
+  uint64_t offrank = 0;
+  for(int k=0;k<sched.nvector;k++){
+    double t1 = usecond();
+    offrank += sched.Exchange((void *)&staged_p[sched.Box[k]*fvol],sched.BoxRank[k],
+                              (void *)block_p,sched.FieldRank[k],bytes);
+    double t2 = usecond();
+    GridSplitVectorise(block_p,full[sched.Field[k]],full_grid->_ldimensions);
+    double t3 = usecond();
+    t_comm   += t2-t1;
+    t_unpack += t3-t2;
   }
+  double t_total = usecond() - t_start;
 
-  uint64_t lsites = full_grid->lSites();
+  std::cout << GridLogPerformance << "Grid_unsplit: " << sched.nvector << " x " << bytes/1.0e6
-  uint64_t     sz = lsites * nvector;
+            << " MB/rank total " << t_total/1.0e6 << " s : alloc " << t_alloc/1.0e6
-  std::vector<Sobj> tmpdata(sz);
+            << " pack " << t_pack/1.0e6
-  std::vector<Sobj> alldata(sz);
+            << " exchange " << t_comm/1.0e6 << " (" << offrank/1.0e6 << " MB off rank, "
-  std::vector<Sobj> scalardata(lsites); 
+            << offrank/(t_comm+1.0e-9)/1.0e3 << " GB/s)"
-
+            << " unpack " << t_unpack/1.0e6 << std::endl;
-  unvectorizeToLexOrdArray(alldata,split);    
+  // Staging buffers are still live here, so this is the high-water point of the call
-
+  if ( GridLogPerformance.isActive() ) {
-  /////////////////////////////////////////////////////////////////
+    HostMemoryReport(full_grid,GridLogPerformance,"Grid_unsplit");
-  // Start from split grid and work towards full grid
-  /////////////////////////////////////////////////////////////////
-
-  int nvec = 1;
-  uint64_t rsites        = split_grid->lSites();
-  Coordinate rdims = split_grid->_ldimensions;
-
-  for(int d=0;d<ndim;d++){
-
-    if ( ratio[d] != 1 ) {
-
-      auto     M = ratio[d];
-
-      int sP =   split_grid->_processors[d];
-      int fP =    full_grid->_processors[d];
-      
-      auto ldims = rdims;  ldims[d]  /= M;  // Decrease local dims by same factor
-      auto lsites= rsites/M;                // Decreases rsites by M
-      
-      int fvol   = lsites;
-      int chunk  = (nvec*fvol)/sP;          GRID_ASSERT(chunk*sP == nvec*fvol);
-	
-      {
-	// Loop over reordered data post A2A
-	thread_for(c, chunk,{
-	  Coordinate coor(ndim);
-	  for(int m=0;m<M;m++){
-	    for(int s=0;s<sP;s++){
-
-	      // addressing; use lexico
-	      int lex_r;
-	      uint64_t lex_c = c+chunk*m+chunk*M*s;
-	      uint64_t lex_fvol_vec = c+chunk*s;
-	      uint64_t lex_fvol     = lex_fvol_vec%fvol;
-	      uint64_t lex_vec      = lex_fvol_vec/fvol;
-	      
-	      // which node sets an adder to the coordinate
-	      Lexicographic::CoorFromIndex(coor, lex_fvol, ldims);	  
-	      coor[d] += m*ldims[d];
-	      Lexicographic::IndexFromCoor(coor, lex_r, rdims);	  
-	      lex_r += lex_vec * rsites;
-	      
-	      // LexicoFind coordinate & vector number within split lattice
-	      tmpdata[lex_c] = alldata[lex_r];
-	    }
-	  }
-        });
-      }
-
-      if ( split_grid->_processors[d] > 1 ) {
-	split_grid->AllToAll(d,tmpdata,alldata);
-	tmpdata=alldata;
-      }
-      full_grid ->AllToAll(d,tmpdata,alldata);
-      rdims[d]/= M;
-      rsites  /= M;
-      nvec    *= M;       // Increase nvec by subdivision factor
-    }
-  }
-
-  lsites = full_grid->lSites();
-  for(int v=0;v<nvector;v++){
-    thread_for(site, lsites,{
-      scalardata[site] = alldata[v*lsites+site];
-    });
-    vectorizeFromLexOrdArray(scalardata,full[v]);    
   }
 }
 

Grid/lattice/PaddedCell.h

@@ -94,7 +94,7 @@ template<class vobj> inline void ScatterSlice(const deviceVector<vobj> &buf,
   // FIXME -- can put internal indices into thread loop
   auto buf_p = & buf[0];
   autoView(lat_v, lat, AcceleratorWrite);
-  accelerator_for(ss, face_ovol/simd[dim],Nsimd,{
+  accelerator_forNB(ss, face_ovol/simd[dim],Nsimd,{
 
     // scalar layout won't coalesce
 #ifdef GRID_SIMT
@@ -180,7 +180,7 @@ template<class vobj> inline void GatherSlice(deviceVector<vobj> &buf,
   //for cross platform
   //For CPU perhaps just run a loop over Nsimd
   auto buf_p = & buf[0];
-  accelerator_for(ss, face_ovol/simd[dim],Nsimd,{
+  accelerator_forNB(ss, face_ovol/simd[dim],Nsimd,{
 
     // scalar layout won't coalesce
 #ifdef GRID_SIMT
@@ -303,10 +303,17 @@ public:
   template<class vobj>
   inline Lattice<vobj> Exchange(const Lattice<vobj> &in, const CshiftImplBase<vobj> &cshift = CshiftImplDefault<vobj>()) const
   {
-    GridBase *old_grid = in.Grid();
+    Coordinate processors=unpadded_grid->_processors;
-    int dims = old_grid->Nd();
+    int dims = in.Grid()->Nd();
-    Lattice<vobj> tmp = in;
+    // An undecomposed dimension is not padded, so Expand on it would only copy
-    for(int d=0;d<dims;d++){
+    // its input onto the same grid: those are skipped.  The first decomposed
+    // dimension expands from `in` itself, so the chain needs no seed copy.
+    int first=-1;
+    for(int d=0;d<dims;d++) if ( processors[d] > 1 ) { first=d; break; }
+    if ( first < 0 ) return in;  // nothing decomposed: the padded grid IS the input grid
+    Lattice<vobj> tmp = Expand(first,in,cshift);
+    for(int d=first+1;d<dims;d++){
+      if ( processors[d] == 1 ) continue;
       tmp = Expand(d,tmp,cshift); // rvalue && assignment
     }
     return tmp;
@@ -314,10 +321,18 @@ public:
   template<class vobj>
   inline Lattice<vobj> ExchangePeriodic(const Lattice<vobj> &in) const
   {
-    GridBase *old_grid = in.Grid();
+    Coordinate processors=unpadded_grid->_processors;
-    int dims = old_grid->Nd();
+    int dims = in.Grid()->Nd();
-    Lattice<vobj> tmp = in;
+    // As Exchange: undecomposed dimensions are pure copies and are skipped, and
-    for(int d=0;d<dims;d++){
+    // the first decomposed dimension expands from `in` so there is no seed copy.
+    // On the D+1 mrhs coarse grid (nrhs,1,x,y,z,t) this removes three full
+    // coarse-field copies per operator application.
+    int first=-1;
+    for(int d=0;d<dims;d++) if ( processors[d] > 1 ) { first=d; break; }
+    if ( first < 0 ) return in;  // nothing decomposed: the padded grid IS the input grid
+    Lattice<vobj> tmp = ExpandPeriodic(first,in);
+    for(int d=first+1;d<dims;d++){
+      if ( processors[d] == 1 ) continue;
       tmp = ExpandPeriodic(d,tmp); // rvalue && assignment
     }
     return tmp;
@@ -513,12 +528,18 @@ public:
     RealD t_tot=-usecond();
     int plane=0;
     for ( int d=0;d < depth ; d ++ ) {
-      int tag = d*1024 + dimension*2+0;
-
       t=usecond();
       GatherSlice(send_buf,from,d,dimension,plane*buffer_size); plane++;
       t_gather+=usecond()-t;
-
+    }
+    for ( int d=0;d < depth ; d ++ ) {
+      t=usecond();
+      GatherSlice(send_buf,from,ld-depth+d,dimension,plane*buffer_size); plane++;
+      t_gather+= usecond() - t;
+    }
+    accelerator_barrier();
+    for ( int d=0;d < depth ; d ++ ) {
+      int tag = d*1024 + dimension*2+0;
       t=usecond();
 
       if(d==0) fwd_trace = traceStart("PaddedCellFwdMPI");
@@ -533,14 +554,9 @@ public:
 				(void *)&hrecv_buf[d*buffer_size], recv_from_rank, bytes, tag);
 #endif
       t_comms+=usecond()-t;
-     }
+    }
     for ( int d=0;d < depth ; d ++ ) {
       int tag = d*1024 + dimension*2+1;
-
-      t=usecond();
-      GatherSlice(send_buf,from,ld-depth+d,dimension,plane*buffer_size); plane++;
-      t_gather+= usecond() - t;
-
       t=usecond();
       if (d==0) bwd_trace = traceStart("PaddedCellBwdMPI");
 #ifdef ACCELERATOR_AWARE_MPI
@@ -603,9 +619,11 @@ public:
     for ( int d=0;d < depth ; d ++ ) {
       ScatterSlice(recv_buf,to,d,dimension,plane*buffer_size); plane++;
     }
+    accelerator_barrier();
     t_scatter+= usecond() - t;
     t_tot+=usecond();
 
+
     std::cout << GridLogPerformance << "PaddedCell::Expand new timings: gather :" << t_gather/1000  << "ms"<<std::endl;
     std::cout << GridLogPerformance << "PaddedCell::Expand new timings: scatter:" << t_scatter/1000   << "ms"<<std::endl;
     std::cout << GridLogPerformance << "PaddedCell::Expand new timings: copy   :" << t_copy/1000      << "ms"<<std::endl;

Grid/parallelIO/BinaryIO.h

@@ -266,9 +266,10 @@ class BinaryIO {
   // GRID_BINARYIO_NOAGGREGATE falls back to plain lexicographic I/O.
   static int DefaultControl(void)
   {
-    static int ctrl = getenv("GRID_BINARYIO_NOAGGREGATE")
+    //    static int ctrl = getenv("GRID_BINARYIO_NOAGGREGATE")
-                    ? BINARYIO_LEXICOGRAPHIC
+    //                    ? BINARYIO_LEXICOGRAPHIC
-                    : BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE;
+    //                    : BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE;
+    static int ctrl = BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE;
     return ctrl;
   }
 

Grid/perfmon/HostMemory.h

@@ -0,0 +1,227 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./lib/perfmon/HostMemory.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+#include <sys/resource.h>
+#include <unistd.h>
+#include <cctype>
+#include <fstream>
+#ifdef __APPLE__
+#include <mach/mach.h>
+#endif
+
+NAMESPACE_BEGIN(Grid);
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Host memory as the OOM killer sees it, plus the device allocator footprint.
+// Targets are Apple or Linux; on Apple only the RSS and device fields are filled.
+//
+//  RSS      : this process, now and its high-water mark (getrusage).
+//  Node     : MemAvailable and MemTotal from /proc/meminfo (Linux).
+//  Cgroup   : the nearest enclosing memory cgroup with a finite limit (the one the batch
+//             system enforces), its usage now, its high-water mark and the limit (Linux,
+//             v2 or v1 hierarchy). Cgroup usage includes page cache, tmpfs and kernel memory
+//             charged to the job, none of which appear in any process RSS.
+//  Device   : MemoryManager Lattice footprint on the device and its free-block cache. Device
+//             allocations outside the MemoryManager (comms buffers) are not included.
+//
+// All values in GB; -1 where the source does not exist on this platform or kernel.
+/////////////////////////////////////////////////////////////////////////////////////////////
+struct HostMemoryStatus
+{
+  RealD RSS           = -1;
+  RealD RSSPeak       = -1;
+  RealD NodeAvailable = -1;
+  RealD NodeTotal     = -1;
+  RealD CgroupCurrent = -1;
+  RealD CgroupPeak    = -1;
+  RealD CgroupLimit   = -1;
+  RealD DeviceLattice = -1;
+  RealD DeviceCache   = -1;
+};
+
+#ifndef __APPLE__ // Linux
+
+// One number from a file; -1 if the file is absent or holds no number (e.g. "max")
+inline RealD HostMemoryReadBytes(const std::string &file)
+{
+  std::ifstream f(file);
+  if ( !f.good() ) {
+    return -1;
+  }
+  std::string word;
+  f >> word;
+  if ( word.empty() || !isdigit(word[0]) ) {
+    return -1;
+  }
+  return std::stod(word);
+}
+
+// Value of a "Key: N kB" line in /proc/meminfo, in bytes; -1 if absent
+inline RealD HostMemoryMeminfo(const std::string &key)
+{
+  std::ifstream f("/proc/meminfo");
+  std::string name;
+  RealD value;
+  std::string unit;
+  while ( f >> name >> value >> unit ) {
+    if ( name == key + ":" ) {
+      return value*1024.0;
+    }
+  }
+  return -1;
+}
+
+// Fill the cgroup fields from the nearest enclosing memory cgroup with a finite limit, or
+// from this process's own cgroup (limit left at -1) if no ancestor has one
+inline void HostMemoryCgroup(HostMemoryStatus &s)
+{
+  std::ifstream f("/proc/self/cgroup");
+  std::string line;
+  std::string root;
+  std::string path;
+  std::string current;
+  std::string peak;
+  std::string limit;
+  while ( std::getline(f,line) ) {
+    size_t c1 = line.find(':');
+    size_t c2 = line.find(':',c1+1);
+    if ( c1 == std::string::npos || c2 == std::string::npos ) {
+      continue;
+    }
+    std::string controllers = line.substr(c1+1,c2-c1-1);
+    // v2 unified hierarchy: "0::/path"
+    if ( controllers.empty() && root.empty() ) {
+      root    = "/sys/fs/cgroup";
+      path    = line.substr(c2+1);
+      current = "memory.current";
+      peak    = "memory.peak";
+      limit   = "memory.max";
+    }
+    // v1 memory controller: "N:memory:/path", takes precedence
+    if ( (","+controllers+",").find(",memory,") != std::string::npos ) {
+      root    = "/sys/fs/cgroup/memory";
+      path    = line.substr(c2+1);
+      current = "memory.usage_in_bytes";
+      peak    = "memory.max_usage_in_bytes";
+      limit   = "memory.limit_in_bytes";
+      break;
+    }
+  }
+  if ( root.empty() ) {
+    return;
+  }
+
+  // v1 reports "no limit" as a number near 2^63
+  const RealD unlimited = 1.0e18;
+  std::string dir = path;
+  while ( true ) {
+    std::string base = root + dir + "/";
+    RealD lim = HostMemoryReadBytes(base+limit);
+    if ( lim > 0 && lim < unlimited ) {
+      s.CgroupCurrent = HostMemoryReadBytes(base+current)/1.0e9;
+      s.CgroupPeak    = HostMemoryReadBytes(base+peak)/1.0e9;
+      s.CgroupLimit   = lim/1.0e9;
+      return;
+    }
+    if ( dir.empty() || dir == "/" ) {
+      break;
+    }
+    dir = dir.substr(0,dir.rfind('/'));
+  }
+  std::string base = root + path + "/";
+  s.CgroupCurrent = HostMemoryReadBytes(base+current)/1.0e9;
+  s.CgroupPeak    = HostMemoryReadBytes(base+peak)/1.0e9;
+}
+
+#endif
+
+// This rank only; no communication
+inline HostMemoryStatus HostMemoryQuery(void)
+{
+  HostMemoryStatus s;
+
+  struct rusage ru;
+  getrusage(RUSAGE_SELF,&ru);
+#ifdef __APPLE__
+  s.RSSPeak = ru.ru_maxrss/1.0e9;                // bytes on macOS
+  mach_task_basic_info_data_t info;
+  mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT;
+  if ( task_info(mach_task_self(),MACH_TASK_BASIC_INFO,(task_info_t)&info,&count) == KERN_SUCCESS ) {
+    s.RSS = info.resident_size/1.0e9;
+  }
+#else // Linux
+  s.RSSPeak = ru.ru_maxrss*1024.0/1.0e9;         // kilobytes on Linux
+  std::ifstream statm("/proc/self/statm");
+  long pages    = 0;
+  long resident = 0;
+  if ( statm >> pages >> resident ) {
+    s.RSS = resident*(RealD)sysconf(_SC_PAGESIZE)/1.0e9;
+  }
+  RealD avail = HostMemoryMeminfo("MemAvailable");
+  RealD total = HostMemoryMeminfo("MemTotal");
+  if ( avail >= 0 ) {
+    s.NodeAvailable = avail/1.0e9;
+  }
+  if ( total >= 0 ) {
+    s.NodeTotal = total/1.0e9;
+  }
+  HostMemoryCgroup(s);
+#endif
+
+  s.DeviceLattice = MemoryManager::DeviceBytes/1.0e9;
+  s.DeviceCache   = MemoryManager::DeviceCacheBytes()/1.0e9;
+  return s;
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Collective over grid. One line on log: maxima over ranks, except node MemAvailable which
+// is the minimum over ranks (the tightest node).
+/////////////////////////////////////////////////////////////////////////////////////////////
+inline void HostMemoryReport(GridBase *grid,GridLogger &log,const std::string &label)
+{
+  HostMemoryStatus s = HostMemoryQuery();
+  RealD minus_available = -s.NodeAvailable;
+  grid->GlobalMax(s.RSS);
+  grid->GlobalMax(s.RSSPeak);
+  grid->GlobalMax(minus_available);
+  grid->GlobalMax(s.NodeTotal);
+  grid->GlobalMax(s.CgroupCurrent);
+  grid->GlobalMax(s.CgroupPeak);
+  grid->GlobalMax(s.CgroupLimit);
+  grid->GlobalMax(s.DeviceLattice);
+  grid->GlobalMax(s.DeviceCache);
+  std::cout << log << "HOSTMEM " << label
+            << " : rank RSS " << s.RSS << " peak " << s.RSSPeak
+            << " ; node available " << -minus_available << " of " << s.NodeTotal
+            << " ; cgroup " << s.CgroupCurrent << " peak " << s.CgroupPeak << " limit " << s.CgroupLimit
+            << " ; device lattice " << s.DeviceLattice << " cache " << s.DeviceCache
+            << " GB" << std::endl;
+}
+
+NAMESPACE_END(Grid);

Grid/qcd/action/ActionParams.h

@@ -53,6 +53,7 @@ struct WilsonImplParams {
   AcceleratorVector<Real,Nd> twist_n_2pi_L;
   AcceleratorVector<Complex,Nd> boundary_phases;
   WilsonImplParams()  {
+    overlapCommsCompute=true;
     dirichlet.resize(0);
     partialDirichlet=0;
     boundary_phases.resize(Nd, 1.0);

Grid/qcd/action/fermion/CayleyFermion5D.h

@@ -81,6 +81,15 @@ public:
     mass_minus=_mass_minus;
     SetCoefficientsInternal(_zolo_hi,_gamma,_b,_c);  // Reset coeffs
   } ;
+
+  // Everything SetMass needs, plus the boost, then rebuild the coefficient arrays in clone
+  void CloneCoefficientsInto(CayleyFermion5D<Impl> &clone)
+  {
+    clone.mass_plus  = mass_plus;
+    clone.mass_minus = mass_minus;
+    clone.qmu        = qmu;
+    clone.SetCoefficientsInternal(_zolo_hi,_gamma,_b,_c);
+  }
   void  P(const FermionField &psi, FermionField &chi);
   void  Pdag(const FermionField &psi, FermionField &chi);
   

Grid/qcd/action/fermion/CompactWilsonCloverFermion.h

@@ -189,6 +189,71 @@ public:
 
   void ImportGauge(const GaugeField& _Umu) override;
 
+  // Install already computed clover fields and inverses, and derive the checkerboarded parts
+  void ImportCloverTerms(const CloverDiagonalField& D,
+                         const CloverTriangleField& T,
+                         const CloverDiagonalField& Dinv,
+                         const CloverTriangleField& Tinv)
+  {
+    Diagonal    = D;
+    Triangle    = T;
+    DiagonalInv = Dinv;
+    TriangleInv = Tinv;
+    pickCheckerboard(Even, DiagonalEven,    Diagonal);
+    pickCheckerboard(Even, TriangleEven,    Triangle);
+    pickCheckerboard(Odd,  DiagonalOdd,     Diagonal);
+    pickCheckerboard(Odd,  TriangleOdd,     Triangle);
+    pickCheckerboard(Even, DiagonalInvEven, DiagonalInv);
+    pickCheckerboard(Even, TriangleInvEven, TriangleInv);
+    pickCheckerboard(Odd,  DiagonalInvOdd,  DiagonalInv);
+    pickCheckerboard(Odd,  TriangleInvOdd,  TriangleInv);
+  }
+
+  // Exact type only: derived operators must not inherit this
+  SplitOperator<FermionField> *SplitClone(const Coordinate& mpi_split) override
+  {
+    if ( typeid(*this) != typeid(CompactWilsonCloverFermion<Impl,CloverHelpers>) ) {
+      return nullptr;
+    }
+    if ( Impl::isGparity ) {
+      return nullptr;
+    }
+    SplitOperator<FermionField> *split = this->MakeSplitGrids(mpi_split);
+
+    // Placeholder links; the doubled field and clover fields are overwritten below.
+    // Boundary masks are rebuilt by the constructor from global coordinates.
+    GaugeField Uplaceholder(split->GaugeGrid);
+    Uplaceholder = Zero();
+
+    CompactWilsonCloverFermion<Impl,CloverHelpers> *clone =
+      new CompactWilsonCloverFermion<Impl,CloverHelpers>(Uplaceholder,
+                                                         *split->GaugeGrid,
+                                                         *split->GaugeRBGrid,
+                                                         this->mass,
+                                                         csw_r,
+                                                         csw_t,
+                                                         cF,
+                                                         this->anisotropyCoeff,
+                                                         this->Params);
+    // The stored coefficients are post-scaled; the constructor would scale them again
+    clone->csw_r = csw_r;
+    clone->csw_t = csw_t;
+
+    CloverDiagonalField sD(split->GaugeGrid);
+    CloverTriangleField sT(split->GaugeGrid);
+    CloverDiagonalField sDinv(split->GaugeGrid);
+    CloverTriangleField sTinv(split->GaugeGrid);
+    Grid_split(Diagonal,sD);
+    Grid_split(Triangle,sT);
+    Grid_split(DiagonalInv,sDinv);
+    Grid_split(TriangleInv,sTinv);
+    clone->ImportCloverTerms(sD,sT,sDinv,sTinv);
+
+    this->SplitDoubledGaugeInto(*clone);
+    split->Matrix = clone;
+    return split;
+  }
+
   /////////////////////////////////////////////
   // Helpers
   /////////////////////////////////////////////

Grid/qcd/action/fermion/ContinuedFractionFermion5D.h

@@ -65,8 +65,8 @@ public:
     std::cout << "Free Propagator for PartialFraction"<<std::endl;
     FermionField in_k(in.Grid());
     FermionField prop_k(in.Grid());
-    
+
-    FFT theFFT((GridCartesian *) in.Grid());
+    PlannedFFT<typename FermionField::vector_object> &theFFT = this->ThePlannedFFT(in.Grid());
 
     //phase for boundary condition
     ComplexField coor(in.Grid());

Grid/qcd/action/fermion/DWFSlow.h

@@ -109,10 +109,69 @@ public:
   ////////////////////////
   // Derivative interface
   ////////////////////////
-  // Interface calls an internal routine
+  // Force on every link of every s-slice, in the form WilsonFermion5D uses:
-  void DhopDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag)  { GRID_ASSERT(0);};
+  //   F5[mu](x,s) = tr_spin[ Btilde(x,s) A(x,s)^dag ]
-  void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){ GRID_ASSERT(0);};
+  //   Btilde(x,s) = -1/2 U_mu(x,s) (1 -+ gamma_mu) B(x+mu,s), lower sign for dag
-  void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){ GRID_ASSERT(0);};
+  // Dhop5 carries no gauge links and contributes nothing.
+  void DhopDeriv5D(std::vector<GaugeLinkField> &F5,const FermionField &A,const FermionField &B,int dag)
+  {
+    RealD sgn= 1.0;
+    if (dag ) sgn=-1.0;
+
+    Gamma::Algebra Gmu [] = {
+			 Gamma::Algebra::GammaX,
+			 Gamma::Algebra::GammaY,
+			 Gamma::Algebra::GammaZ,
+			 Gamma::Algebra::GammaT
+    };
+
+    FermionField tmp(B.Grid());
+    FermionField Btilde(B.Grid());
+    for(int mu=0;mu<Nd;mu++){
+      tmp    = Umu[mu] * Cshift(B,mu+1,1);
+      Btilde = tmp - Gamma(Gmu[mu])*tmp*sgn;
+      Btilde = -0.5*Btilde;
+      F5[mu] = TraceIndex<SpinIndex>(outerProduct(Btilde,A));
+    }
+  }
+
+  // Every slice holds the same 4D gauge field, so the 4D force is the sum over slices
+  void DhopDeriv(GaugeField &mat,const FermionField &A,const FermionField &B,int dag)
+  {
+    std::vector<GaugeLinkField> F5(Nd,_grid);
+    DhopDeriv5D(F5,A,B,dag);
+
+    GaugeLinkField slice(_grid4);
+    GaugeLinkField sum(_grid4);
+    for(int mu=0;mu<Nd;mu++){
+      sum = Zero();
+      for(int s=0;s<Ls;s++){
+        ExtractSlice(slice,F5[mu],s,0);
+        sum = sum + slice;
+      }
+      PokeIndex<LorentzIndex>(mat,sum,mu);
+    }
+  }
+
+  // The checkerboard here is 5D, so a field on one checkerboard touches links at both 4D
+  // parities: the even-odd forces are returned on the full 4D gauge grid.
+  int CheckerboardedForce(void) { return 0; };
+
+  // A and B on opposite 5D checkerboards, placed in zeroed full fields: the full-lattice
+  // derivative then picks up exactly the links Meooe uses between them.
+  void DhopDerivCB(GaugeField &mat,const FermionField &A,const FermionField &B,int dag)
+  {
+    GRID_ASSERT(A.Checkerboard() != B.Checkerboard());
+    FermionField Afull(_grid);
+    FermionField Bfull(_grid);
+    Afull = Zero();
+    Bfull = Zero();
+    setCheckerboard(Afull,A);
+    setCheckerboard(Bfull,B);
+    DhopDeriv(mat,Afull,Bfull,dag);
+  }
+  void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){ DhopDerivCB(mat,U,V,dag);};
+  void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){ DhopDerivCB(mat,U,V,dag);};
 
   ///////////////////////////////////////////////////////////////
   // non-hermitian hopping term; half cb or both
@@ -200,6 +259,7 @@ public:
     _grid(&Fgrid),
     _cbgrid(&Hgrid),
     _grid4(_Umu.Grid()),
+    _cbgrid4(nullptr),  // the checkerboard is 5D; there is no 4D red-black grid
     Umu(Nd,&Fgrid),
     UmuEven(Nd,&Hgrid),
     UmuOdd(Nd,&Hgrid),
@@ -221,10 +281,6 @@ public:
       MassField =scalar(-mass);
       one       =scalar(1.0);
       MassField =where(coor==Integer(Ls-1),MassField,one);
-      for(int mu=0;mu<Nd;mu++){
-	pickCheckerboard(Even,UmuEven[mu],Umu[mu]);
-	pickCheckerboard(Odd ,UmuOdd[mu],Umu[mu]);
-      }
       pickCheckerboard(Even,MassFieldEven,MassField);
       pickCheckerboard(Odd ,MassFieldOdd,MassField);
       
@@ -239,6 +295,8 @@ public:
       for(int s=0;s<this->Ls;s++){
 	InsertSlice(U4,Umu[mu],s,0);
       }
+      pickCheckerboard(Even,UmuEven[mu],Umu[mu]);
+      pickCheckerboard(Odd ,UmuOdd[mu],Umu[mu]);
     }
   }
 

Grid/qcd/action/fermion/DomainWallFermion.h

@@ -45,7 +45,7 @@ public:
 	FermionField in_k(in.Grid());
 	FermionField prop_k(in.Grid());
 
-	FFT theFFT((GridCartesian *) in.Grid());
+	PlannedFFT<typename FermionField::vector_object> &theFFT = this->ThePlannedFFT(in.Grid());
 
 	//phase for boundary condition
 	ComplexField coor(in.Grid());
@@ -132,6 +132,35 @@ public:
     Approx::zolotarev_free(zdata);
   }
 
+  // Exact type only: derived operators must not inherit this
+  virtual SplitOperator<FermionField> *SplitClone(const Coordinate &mpi_split)
+  {
+    if ( typeid(*this) != typeid(DomainWallFermion<Impl>) ) {
+      return nullptr;
+    }
+    if ( this->Dirichlet || Impl::LsVectorised || Impl::isGparity ) {
+      return nullptr;
+    }
+    SplitOperator<FermionField> *split = this->MakeSplitGrids(mpi_split);
+
+    // Placeholder links; the doubled field is overwritten below
+    GaugeField Uplaceholder(split->GaugeGrid);
+    Uplaceholder = Zero();
+
+    DomainWallFermion<Impl> *clone = new DomainWallFermion<Impl>(Uplaceholder,
+                                                                 *split->FermionGrid,
+                                                                 *split->FermionRBGrid,
+                                                                 *split->GaugeGrid,
+                                                                 *split->GaugeRBGrid,
+                                                                 this->mass_plus,
+                                                                 this->M5,
+                                                                 this->Params);
+    this->CloneCoefficientsInto(*clone);
+    this->SplitDoubledGaugeInto(*clone);
+    split->Matrix = clone;
+    return split;
+  }
+
 };
 
 NAMESPACE_END(Grid);

Grid/qcd/action/fermion/FermionOperator.h

@@ -45,7 +45,7 @@ public:
   INHERIT_IMPL_TYPES(Impl);
 
   FermionOperator(const ImplParams &p= ImplParams()) : Impl(p) {};
-  virtual ~FermionOperator(void) = default;
+  virtual ~FermionOperator(void) { if ( _thePlannedFFT ) delete _thePlannedFFT; }
 
   virtual FermionField &tmp(void) = 0;
 
@@ -59,6 +59,38 @@ public:
   virtual GridBase *GaugeGrid(void)           =0;
   virtual GridBase *GaugeRedBlackGrid(void)   =0;
 
+  ////////////////////////////////////////////////////////////////
+  // Split-communicator copies of this operator's gauge and fermion
+  // grids, returned in a new bundle with no Matrix or Linop yet.
+  // Partitions have MPI layout mpi_split. Collective.
+  ////////////////////////////////////////////////////////////////
+  SplitOperator<FermionField> *MakeSplitGrids(const Coordinate &mpi_split)
+  {
+    GridCartesian *U = dynamic_cast<GridCartesian *>(GaugeGrid());
+    GRID_ASSERT(U != nullptr);
+    GRID_ASSERT(mpi_split.size() == Nd);
+
+    double t0 = usecond();
+
+    SplitOperator<FermionField> *split = new SplitOperator<FermionField>();
+
+    split->GaugeGrid   = new GridCartesian(U->FullDimensions(),U->_simd_layout,mpi_split,*U);
+    split->GaugeRBGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(split->GaugeGrid);
+    split->Partition   = GridSplitVectorIndex(U,split->GaugeGrid);
+    split->Partitions  = U->ProcessorCount()/split->GaugeGrid->ProcessorCount();
+
+    if ( FermionGrid()->Nd() == Nd+1 ) {
+      int Ls = FermionGrid()->_fdimensions[0];
+      split->FermionGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,split->GaugeGrid);
+      split->FermionRBGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,split->GaugeGrid);
+    } else {
+      split->FermionGrid   = split->GaugeGrid;
+      split->FermionRBGrid = split->GaugeRBGrid;
+    }
+    std::cout << GridLogPerformance << "MakeSplitGrids: " << (usecond()-t0)/1.0e6 << " s" << std::endl;
+    return split;
+  }
+
   // override multiply
   virtual void  M    (const FermionField &in, FermionField &out)=0;
   virtual void  Mdag (const FermionField &in, FermionField &out)=0;
@@ -88,6 +120,10 @@ public:
   virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag)=0;
   virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag)=0;
 
+  // 1 if MoeDeriv/MeoDeriv return a force on a single 4D checkerboard (4D red-black),
+  // 0 if they return it on the full 4D gauge grid (a checkerboard that includes s)
+  virtual int CheckerboardedForce(void) { return 1; };
+
   virtual void  Mdiag  (const FermionField &in, FermionField &out) { Mooee(in,out);};   // Same as Mooee applied to both CB's
   virtual void  Mdir   (const FermionField &in, FermionField &out,int dir,int disp)=0;   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
   virtual void  MdirAll(const FermionField &in, std::vector<FermionField> &out)=0;   // case by case Wilson, Clover, Cayley, ContFrac, PartFrac
@@ -95,9 +131,28 @@ public:
 
   virtual void  MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m,std::vector<double> twist) { GRID_ASSERT(0);};
 
-  virtual void  FreePropagator(const FermionField &in,FermionField &out,RealD mass,std::vector<Complex> boundary,std::vector<double> twist) 
+protected:
+  // Cached planned FFT for the ACTIVE field grid -- a general utility (FreePropagator
+  // today; smoother, smearings and other users anticipated).  Frontier FFTW plan
+  // create+destroy is ~22 ms/call (measured, Test_fft_prop PLANCOST) -- ~4x the
+  // transform itself and ~80% of an unplanned call -- so a per-call `FFT theFFT(grid)`
+  // dominates.  We cache BOTH the FFT and the grid it was built on; when called with a
+  // different grid (e.g. the 5D propagator vs the 4D physical propagator) we rebuild.
+  // Lazily built, owned, deleted in the dtor.  Raw pointer, no smart pointers.
+  PlannedFFT<typename FermionField::vector_object> *_thePlannedFFT{nullptr};
+  GridBase                                         *_thePlannedFFTGrid{nullptr};
+  PlannedFFT<typename FermionField::vector_object> & ThePlannedFFT(GridBase *grid) {
+    if ( _thePlannedFFT == nullptr || _thePlannedFFTGrid != grid ) {
+      if ( _thePlannedFFT != nullptr ) delete _thePlannedFFT;   // active grid changed: rebuild
+      _thePlannedFFT     = new PlannedFFT<typename FermionField::vector_object>((GridCartesian *)grid);
+      _thePlannedFFTGrid = grid;
+    }
+    return *_thePlannedFFT;
+  }
+public:
+  virtual void  FreePropagator(const FermionField &in,FermionField &out,RealD mass,std::vector<Complex> boundary,std::vector<double> twist)
       {
-	FFT theFFT((GridCartesian *) in.Grid());
+	PlannedFFT<typename FermionField::vector_object> &theFFT = this->ThePlannedFFT(in.Grid());
 
 	typedef typename Simd::scalar_type Scalar;
 

Grid/qcd/action/fermion/MobiusFermion.h

@@ -70,6 +70,37 @@ public:
  
   }
 
+  // Exact type only: derived operators (e.g. ScaledShamirFermion) must not inherit this
+  virtual SplitOperator<FermionField> *SplitClone(const Coordinate &mpi_split)
+  {
+    if ( typeid(*this) != typeid(MobiusFermion<Impl>) ) {
+      return nullptr;
+    }
+    if ( this->Dirichlet || Impl::LsVectorised || Impl::isGparity ) {
+      return nullptr;
+    }
+    SplitOperator<FermionField> *split = this->MakeSplitGrids(mpi_split);
+
+    // Placeholder links; the doubled field is overwritten below
+    GaugeField Uplaceholder(split->GaugeGrid);
+    Uplaceholder = Zero();
+
+    MobiusFermion<Impl> *clone = new MobiusFermion<Impl>(Uplaceholder,
+                                                         *split->FermionGrid,
+                                                         *split->FermionRBGrid,
+                                                         *split->GaugeGrid,
+                                                         *split->GaugeRBGrid,
+                                                         this->mass_plus,
+                                                         this->M5,
+                                                         this->_b,
+                                                         this->_c,
+                                                         this->Params);
+    this->CloneCoefficientsInto(*clone);
+    this->SplitDoubledGaugeInto(*clone);
+    split->Matrix = clone;
+    return split;
+  }
+
 };
 
 NAMESPACE_END(Grid);

Grid/qcd/action/fermion/PartialFractionFermion5D.h

@@ -95,8 +95,8 @@ public:
     std::cout << "Free Propagator for PartialFraction"<<std::endl;
     FermionField in_k(in.Grid());
     FermionField prop_k(in.Grid());
-    
+
-    FFT theFFT((GridCartesian *) in.Grid());
+    PlannedFFT<typename FermionField::vector_object> &theFFT = this->ThePlannedFFT(in.Grid());
 
     //phase for boundary condition
     ComplexField coor(in.Grid());

Grid/qcd/action/fermion/WilsonCloverFermion.h

@@ -91,6 +91,60 @@ public:
   // Derivative parts unpreconditioned pseudofermions
   void MDeriv(GaugeField &force, const FermionField &X, const FermionField &Y, int dag);
 
+  // Install already computed clover term and inverse, and derive the checkerboarded parts
+  void ImportCloverTerms(const CloverField &C, const CloverField &Cinv)
+  {
+    CloverTerm    = C;
+    CloverTermInv = Cinv;
+    pickCheckerboard(Even, CloverTermEven, CloverTerm);
+    pickCheckerboard(Odd,  CloverTermOdd,  CloverTerm);
+    pickCheckerboard(Even, CloverTermDagEven, adj(CloverTerm));
+    pickCheckerboard(Odd,  CloverTermDagOdd,  adj(CloverTerm));
+    pickCheckerboard(Even, CloverTermInvEven, CloverTermInv);
+    pickCheckerboard(Odd,  CloverTermInvOdd,  CloverTermInv);
+    pickCheckerboard(Even, CloverTermInvDagEven, adj(CloverTermInv));
+    pickCheckerboard(Odd,  CloverTermInvDagOdd,  adj(CloverTermInv));
+  }
+
+  // Exact type only: derived operators must not inherit this
+  virtual SplitOperator<FermionField> *SplitClone(const Coordinate &mpi_split)
+  {
+    if ( typeid(*this) != typeid(WilsonCloverFermion<Impl,CloverHelpers>) ) {
+      return nullptr;
+    }
+    if ( Impl::isGparity ) {
+      return nullptr;
+    }
+    SplitOperator<FermionField> *split = this->MakeSplitGrids(mpi_split);
+
+    // Placeholder links; the doubled field and clover terms are overwritten below
+    GaugeField Uplaceholder(split->GaugeGrid);
+    Uplaceholder = Zero();
+
+    WilsonCloverFermion<Impl,CloverHelpers> *clone =
+      new WilsonCloverFermion<Impl,CloverHelpers>(Uplaceholder,
+                                                  *split->GaugeGrid,
+                                                  *split->GaugeRBGrid,
+                                                  this->mass,
+                                                  csw_r,
+                                                  csw_t,
+                                                  this->anisotropyCoeff,
+                                                  this->Params);
+    // The stored coefficients are post-scaled; the constructor would scale them again
+    clone->csw_r = csw_r;
+    clone->csw_t = csw_t;
+
+    CloverField sC(split->GaugeGrid);
+    CloverField sCinv(split->GaugeGrid);
+    Grid_split(CloverTerm,sC);
+    Grid_split(CloverTermInv,sCinv);
+    clone->ImportCloverTerms(sC,sCinv);
+
+    this->SplitDoubledGaugeInto(*clone);
+    split->Matrix = clone;
+    return split;
+  }
+
 public:
   // here fixing the 4 dimensions, make it more general?
 

Grid/qcd/action/fermion/WilsonFermion.h

@@ -147,6 +147,48 @@ public:
   // DoubleStore impl dependent
   void ImportGauge(const GaugeField &_Umu);
 
+  // Install an already doubled gauge field (phases, twists, anisotropy and -1/2 applied)
+  void ImportDoubledGauge(const DoubledGaugeField &U)
+  {
+    Umu = U;
+    pickCheckerboard(Even,UmuEven,Umu);
+    pickCheckerboard(Odd ,UmuOdd ,Umu);
+  }
+
+  // Redistribute this operator's doubled gauge field onto clone's split grid. Collective.
+  void SplitDoubledGaugeInto(WilsonFermion<Impl> &clone)
+  {
+    DoubledGaugeField sUmu(clone.GaugeGrid());
+    Grid_split(Umu,sUmu);
+    clone.ImportDoubledGauge(sUmu);
+  }
+
+  // Exact type only: derived operators (e.g. WilsonTMFermion) must not inherit this
+  virtual SplitOperator<FermionField> *SplitClone(const Coordinate &mpi_split)
+  {
+    if ( typeid(*this) != typeid(WilsonFermion<Impl>) ) {
+      return nullptr;
+    }
+    if ( Impl::isGparity ) {
+      return nullptr;
+    }
+    SplitOperator<FermionField> *split = this->MakeSplitGrids(mpi_split);
+
+    // Placeholder links; the doubled field is overwritten below
+    GaugeField Uplaceholder(split->GaugeGrid);
+    Uplaceholder = Zero();
+
+    WilsonFermion<Impl> *clone = new WilsonFermion<Impl>(Uplaceholder,
+                                                         *split->GaugeGrid,
+                                                         *split->GaugeRBGrid,
+                                                         mass,
+                                                         this->Params,
+                                                         anisotropyCoeff);
+    SplitDoubledGaugeInto(*clone);
+    split->Matrix = clone;
+    return split;
+  }
+
   ///////////////////////////////////////////////////////////////
   // Data members require to support the functionality
   ///////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/WilsonFermion5D.h

@@ -181,6 +181,27 @@ public:
   // DoubleStore
   void ImportGauge(const GaugeField &_Umu);
     
+  // Install an already doubled gauge field (phases, twists and -1/2 applied)
+  void ImportDoubledGauge(const DoubledGaugeField &U)
+  {
+    Umu = U;
+    pickCheckerboard(Even,UmuEven,Umu);
+    pickCheckerboard(Odd ,UmuOdd ,Umu);
+  }
+
+  // Redistribute this operator's doubled gauge field onto clone's split grid. Collective.
+  void SplitDoubledGaugeInto(WilsonFermion5D<Impl> &clone)
+  {
+    double t0 = usecond();
+    DoubledGaugeField sUmu(clone.GaugeGrid());
+    Grid_split(Umu,sUmu);
+    double t1 = usecond();
+    clone.ImportDoubledGauge(sUmu);
+    double t2 = usecond();
+    std::cout << GridLogPerformance << "SplitDoubledGaugeInto: Grid_split " << (t1-t0)/1.0e6
+              << " s import " << (t2-t1)/1.0e6 << " s" << std::endl;
+  }
+
   ///////////////////////////////////////////////////////////////
   // Data members require to support the functionality
   ///////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/ZMobiusFermion.h

@@ -67,6 +67,44 @@ public:
     this->SetCoefficientsInternal(1.0,zgamma,b,c);
   }
 
+  // Exact type only: derived operators must not inherit this
+  virtual SplitOperator<FermionField> *SplitClone(const Coordinate &mpi_split)
+  {
+    if ( typeid(*this) != typeid(ZMobiusFermion<Impl>) ) {
+      return nullptr;
+    }
+    if ( this->Dirichlet || Impl::LsVectorised || Impl::isGparity ) {
+      return nullptr;
+    }
+    SplitOperator<FermionField> *split = this->MakeSplitGrids(mpi_split);
+
+    // Placeholder links; the doubled field is overwritten below
+    GaugeField Uplaceholder(split->GaugeGrid);
+    Uplaceholder = Zero();
+
+    // Constructor argument only; CloneCoefficientsInto restores _gamma at full precision
+    std::vector<ComplexD> gamma(this->Ls);
+    for ( int s=0; s<this->Ls; s++ ) {
+      gamma[s] = this->_gamma[s];
+    }
+
+    ZMobiusFermion<Impl> *clone = new ZMobiusFermion<Impl>(Uplaceholder,
+                                                           *split->FermionGrid,
+                                                           *split->FermionRBGrid,
+                                                           *split->GaugeGrid,
+                                                           *split->GaugeRBGrid,
+                                                           this->mass_plus,
+                                                           this->M5,
+                                                           gamma,
+                                                           this->_b,
+                                                           this->_c,
+                                                           this->Params);
+    this->CloneCoefficientsInto(*clone);
+    this->SplitDoubledGaugeInto(*clone);
+    split->Matrix = clone;
+    return split;
+  }
+
 };
 
 NAMESPACE_END(Grid);

Grid/qcd/action/pseudofermion/EvenOddSchurDifferentiable.h

@@ -64,6 +64,20 @@ public:
     GRID_ASSERT(U.Checkerboard()==Odd);
     GRID_ASSERT(V.Checkerboard()==U.Checkerboard());
 
+    // Forces returned on the full 4D grid (checkerboard including s): same sequence, summed
+    if ( !this->_Mat.CheckerboardedForce() ) {
+      GaugeField ForceO(Force.Grid());
+      GaugeField ForceE(Force.Grid());
+      this->_Mat.Meooe   (V,tmp1);
+      this->_Mat.MooeeInv(tmp1,tmp2);
+      this->_Mat.MoeDeriv(ForceO,U,tmp2,DaggerNo);
+      this->_Mat.MeooeDag   (U,tmp1);
+      this->_Mat.MooeeInvDag(tmp1,tmp2);
+      this->_Mat.MeoDeriv(ForceE,tmp2,V,DaggerNo);
+      Force = -(ForceO+ForceE);
+      return;
+    }
+
     // NOTE Guido: WE DO NOT WANT TO USE THE ucbgrid GRID FOR THE FORCE
     // it is not conformable with the HMC force field
     // Case: Ls vectorised fields
@@ -109,6 +123,20 @@ public:
     GRID_ASSERT(V.Checkerboard()==Odd);
     GRID_ASSERT(V.Checkerboard()==V.Checkerboard());
 
+    // Forces returned on the full 4D grid (checkerboard including s): same sequence, summed
+    if ( !this->_Mat.CheckerboardedForce() ) {
+      GaugeField ForceO(Force.Grid());
+      GaugeField ForceE(Force.Grid());
+      this->_Mat.MeooeDag   (V,tmp1);
+      this->_Mat.MooeeInvDag(tmp1,tmp2);
+      this->_Mat.MoeDeriv(ForceO,U,tmp2,DaggerYes);
+      this->_Mat.Meooe   (U,tmp1);
+      this->_Mat.MooeeInv(tmp1,tmp2);
+      this->_Mat.MeoDeriv(ForceE,tmp2,V,DaggerYes);
+      Force = -(ForceO+ForceE);
+      return;
+    }
+
     // NOTE Guido: WE DO NOT WANT TO USE THE ucbgrid GRID FOR THE FORCE
     // it is not conformable with the HMC force field
     // INHERIT FROM THE Force field instead

Grid/qcd/smearing/GaugeConfiguration.h

@@ -125,6 +125,16 @@ public:
     return SmearedSet[Level];
   }
 
+  /*! @brief Level n of the smearing chain: 0 is the thin links, n is n smearing steps */
+  const GaugeField &get_level(int n) const
+  {
+    GRID_ASSERT(n >= 0 && n <= (int)smearingLevels);
+    if ( n == 0 ) {
+      return *ThinLinks;
+    }
+    return SmearedSet[n-1];
+  }
+
   //====================================================================
   void set_iLambda(GaugeLinkField& iLambda, GaugeLinkField& e_iQ,
                    const GaugeLinkField& iQ, const GaugeLinkField& Sigmap,
@@ -277,6 +287,45 @@ public:
     SigmaTilde=Gimpl::projectForce(SigmaTilde); // Ta
       
   }
+
+  //====================================================================
+  // Force on the thin links from an action that depends on every level of the chain.
+  // LevelForce[n], n = 0..smearingLevels, is the force on level n in the same
+  // "U times dS/dU" form smeared_force takes (zero fields allowed). The chain rule is
+  // applied from the top level down, adding each level's own force on the way:
+  //   Sigma_L = U_L^dag F_L
+  //   Sigma_n = AnalyticSmearedForce(Sigma_{n+1}, U_n) + U_n^dag F_n
+  //   ThinForce = Ta( U_0 Sigma_0 )
+  // With force on the top level only this is smeared_force.
+  // SmearedConfigurationMasked would need its own: its chain rule is level dependent.
+  virtual void smeared_force_levels(const std::vector<GaugeField> &LevelForce, GaugeField &ThinForce)
+  {
+    GRID_ASSERT(LevelForce.size() == smearingLevels+1);
+
+    GridBase *grid = LevelForce[0].Grid();
+    GaugeField Sigma(grid);
+    GaugeLinkField tmp_mu(grid);
+
+    // Sigma = U_L^dag F_L
+    for (int mu = 0; mu < Nd; mu++) {
+      tmp_mu = adj(peekLorentz(get_level(smearingLevels), mu)) * peekLorentz(LevelForce[smearingLevels], mu);
+      pokeLorentz(Sigma, tmp_mu, mu);
+    }
+
+    for (int n = (int)smearingLevels - 1; n >= 0; n--) {
+      Sigma = AnalyticSmearedForce(Sigma, get_level(n));
+      for (int mu = 0; mu < Nd; mu++) {
+        tmp_mu = peekLorentz(Sigma, mu) + adj(peekLorentz(get_level(n), mu)) * peekLorentz(LevelForce[n], mu);
+        pokeLorentz(Sigma, tmp_mu, mu);
+      }
+    }
+
+    for (int mu = 0; mu < Nd; mu++) {
+      tmp_mu = peekLorentz(*ThinLinks, mu) * peekLorentz(Sigma, mu);
+      pokeLorentz(ThinForce, tmp_mu, mu);
+    }
+    ThinForce = Gimpl::projectForce(ThinForce); // Ta
+  }
   //====================================================================
 
   virtual GaugeField& get_SmearedU() { return SmearedSet[smearingLevels - 1]; }

Grid/qcd/smearing/WilsonFlow.h

@@ -52,7 +52,7 @@ protected:
 public:
 
 //Define the action used to evolve the plaquettes
-//(Lüscher: https://arxiv.org/pdf/1006.4518 eq. 1.4)
+//(Luscher: https://arxiv.org/pdf/1006.4518 eq. 1.4)
 //V'(t) = -g^2 * ( d/dVt S[Vt](g) ) * Vt
 //      = -g^2 * ( d/dVt (1/g^2 * sum_p Re tr{ 1 - Vt(p) } ) ) * Vt
 //      = - d/dVt ( sum_p ( Nc - Re tr Vt(p) ) * Vt
@@ -157,7 +157,7 @@ public:
 ////////////////////////////////////////////////////////////////////////////////
 
 //Compute t^2 <E(t)> for time from the plaquette form
-//(Lüscher: https://arxiv.org/pdf/1006.4518 eq. 3.1)
+//(Luscher: https://arxiv.org/pdf/1006.4518 eq. 3.1)
 //E(t) = 2 * sum_p Retr{ 1 - Vt(p) } =
 //     = 2 * sum_p ( Nc - Retr Vt(p) ) =
 //     = 2 * Nc * sum_p ( 1 - Retr Vt(p)/Nc )

Grid/simd/Grid_vector_types.h

@@ -1002,10 +1002,10 @@ accelerator_inline void precisionChange(vRealD    *out,const vRealF    *in,int n
     Optimization::PrecisionChange::StoD(in[m].v,out[n].v,out[n+1].v);
     // Bug in gcc 10.0.1 and gcc 10.1 using fixed-size SVE ACLE data types  CAS-159553-Y1K4C6
     // function call results in compile-time error:
-    // In function ‘void Grid::precisionChange(Grid::vRealD*, Grid::vRealF*, int)’:
+    // In function 'void Grid::precisionChange(Grid::vRealD*, Grid::vRealF*, int)':
     // .../Grid_vector_types.h:961:56: error:
-    // cannot bind non-const lvalue reference of type ‘vecd&’ {aka ‘svfloat64_t&’}
+    // cannot bind non-const lvalue reference of type 'vecd&' {aka 'svfloat64_t&'}
-    // to an rvalue of type ‘vecd’ {aka ‘svfloat64_t’}
+    // to an rvalue of type 'vecd' {aka 'svfloat64_t'}
     // 961 |     Optimization::PrecisionChange::StoD(in[m].v,out[n].v,out[n+1].v);
     //  |                                                 ~~~~~~~^
   }

Grid/stencil/Stencil.h

@@ -82,7 +82,7 @@ void DslashLogPartial(void);
 void DslashLogDirichlet(void);
 
 struct StencilEntry {
-#ifdef GRID_CUDA
+#if defined(GRID_CUDA) || defined(GRID_HIP)
   uint64_t _byte_offset;       // 8 bytes
   uint32_t _offset;            // 4 bytes
 #else
@@ -453,9 +453,19 @@ public:
 	double *dbuf =(double *) packet.recv_buf;
 	float  *fbuf =(float  *) packet.compressed_recv_buf;
 
+	// The lane index is a GPU coalescing optimisation: lane = threadIdx.y
+	// puts adjacent threads on adjacent words.  On a CPU build there are no
+	// SIMT lanes and acceleratorSIMTlane() is identically zero, so the plain
+	// form would convert only 1 word of every nsimd; the #else covers all
+	// lanes explicitly.
 	accelerator_forNB(ss,outer,nsimd,{
+#ifdef GRID_SIMT
 	  int lane = acceleratorSIMTlane(nsimd);
 	  dbuf[ss*nsimd+lane] = fbuf[ss*nsimd+lane]; //conversion
+#else
+	  for(int lane=0;lane<nsimd;lane++)
+	    dbuf[ss*nsimd+lane] = fbuf[ss*nsimd+lane]; //conversion
+#endif
 	});
 
       } else if ( sizeof(word)==4){
@@ -467,8 +477,13 @@ public:
 	uint16_t *hbuf =(uint16_t *) packet.compressed_recv_buf;
 
 	accelerator_forNB(ss,outer,nsimd,{
+#ifdef GRID_SIMT
 	  int lane = acceleratorSIMTlane(nsimd);
 	  fbuf[ss*nsimd+lane] = ((uint32_t)hbuf[ss*nsimd+lane])<<16; //copy back and pad each word with zeroes
+#else
+	  for(int lane=0;lane<nsimd;lane++)
+	    fbuf[ss*nsimd+lane] = ((uint32_t)hbuf[ss*nsimd+lane])<<16; //copy back and pad each word with zeroes
+#endif
 	});
 
       } else {
@@ -515,9 +530,15 @@ public:
 	double *dbuf =(double *) packet.send_buf;
 	float  *fbuf =(float  *) packet.compressed_send_buf;
 
+	// CPU lane coverage as in DecompressPacket.
 	accelerator_forNB(ss,outer,nsimd,{
+#ifdef GRID_SIMT
 	  int lane = acceleratorSIMTlane(nsimd);
 	  fbuf[ss*nsimd+lane] = dbuf[ss*nsimd+lane]; // convert fp64 to fp32
+#else
+	  for(int lane=0;lane<nsimd;lane++)
+	    fbuf[ss*nsimd+lane] = dbuf[ss*nsimd+lane]; // convert fp64 to fp32
+#endif
 	});
 
       } else if ( sizeof(word)==4){
@@ -526,8 +547,13 @@ public:
 	uint16_t *hbuf =(uint16_t *) packet.compressed_send_buf;
 	
 	accelerator_forNB(ss,outer,nsimd,{
+#ifdef GRID_SIMT
 	  int lane = acceleratorSIMTlane(nsimd);
 	  hbuf[ss*nsimd+lane] = fbuf[ss*nsimd+lane]>>16; // convert as in Bagel/BFM ; bfloat16 ; s7e8 Intel patent
+#else
+	  for(int lane=0;lane<nsimd;lane++)
+	    hbuf[ss*nsimd+lane] = fbuf[ss*nsimd+lane]>>16; // convert as in Bagel/BFM ; bfloat16 ; s7e8 Intel patent
+#endif
 	});
 
       } else {

Grid/tensors/Tensor_traits.h

@@ -110,6 +110,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef RealD DoublePrecision;
     typedef RealD DoublePrecision2;
+    typedef RealF SinglePrecision;
   };
   template<> struct GridTypeMapper<RealD> : public GridTypeMapper_Base {
     typedef RealD scalar_type;
@@ -124,6 +125,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef RealD DoublePrecision;
     typedef RealD DoublePrecision2;
+    typedef RealF SinglePrecision;
   };
   template<> struct GridTypeMapper<ComplexF> : public GridTypeMapper_Base {
     typedef ComplexF scalar_type;
@@ -138,6 +140,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
+    typedef ComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<ComplexD> : public GridTypeMapper_Base {
     typedef ComplexD scalar_type;
@@ -152,6 +155,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
+    typedef ComplexF SinglePrecision;
   };
 
 #if defined(GRID_CUDA) || defined(GRID_HIP)  
@@ -168,6 +172,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef scalar_typeD DoublePrecision;
     typedef scalar_typeD DoublePrecision2;
+    typedef std::complex<float> SinglePrecision;
   };
   template<> struct GridTypeMapper<std::complex<double> > : public GridTypeMapper_Base {
     typedef std::complex<double> scalar_type;
@@ -182,6 +187,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef scalar_typeD DoublePrecision;
     typedef scalar_typeD DoublePrecision2;
+    typedef std::complex<float> SinglePrecision;
   };
 #endif
 
@@ -198,6 +204,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
+    typedef void SinglePrecision;
   };
 
   template<> struct GridTypeMapper<vRealF> : public GridTypeMapper_Base {
@@ -213,6 +220,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD2 DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vRealD> : public GridTypeMapper_Base {
     typedef RealD  scalar_type;
@@ -227,6 +235,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vRealD2> : public GridTypeMapper_Base {
     typedef RealD  scalar_type;
@@ -241,6 +250,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD2 DoublePrecision;
     typedef vRealD2 DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vRealH> : public GridTypeMapper_Base {
     // Fixme this is incomplete until Grid supports fp16 or bfp16 arithmetic types
@@ -256,6 +266,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexH> : public GridTypeMapper_Base {
     // Fixme this is incomplete until Grid supports fp16 or bfp16 arithmetic types
@@ -271,6 +282,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexF> : public GridTypeMapper_Base {
     typedef ComplexF  scalar_type;
@@ -285,6 +297,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD2 DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexD> : public GridTypeMapper_Base {
     typedef ComplexD  scalar_type;
@@ -299,6 +312,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexD2> : public GridTypeMapper_Base {
     typedef ComplexD  scalar_type;
@@ -313,6 +327,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD2 DoublePrecision;
     typedef vComplexD2 DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vInteger> : public GridTypeMapper_Base {
     typedef  Integer scalar_type;
@@ -327,6 +342,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
+    typedef void SinglePrecision;
   };
   template<> struct GridTypeMapper<sRealF> : public GridTypeMapper_Base {
     typedef RealF  scalar_type;
@@ -341,6 +357,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sRealD DoublePrecision;
     typedef sRealD DoublePrecision2;
+    typedef sRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<sRealD> : public GridTypeMapper_Base {
     typedef RealD  scalar_type;
@@ -355,6 +372,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sRealD DoublePrecision;
     typedef sRealD DoublePrecision2;
+    typedef sRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<sComplexF> : public GridTypeMapper_Base {
     typedef ComplexF  scalar_type;
@@ -369,6 +387,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sComplexD DoublePrecision;
     typedef sComplexD DoublePrecision2;
+    typedef sComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<sComplexD> : public GridTypeMapper_Base {
     typedef ComplexD  scalar_type;
@@ -383,6 +402,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sComplexD DoublePrecision;
     typedef sComplexD DoublePrecision2;
+    typedef sComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<sInteger> : public GridTypeMapper_Base {
     typedef Integer  scalar_type;
@@ -397,6 +417,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
+    typedef void SinglePrecision;
   };
 
 #define GridTypeMapper_RepeatedTypes \
@@ -417,6 +438,7 @@ NAMESPACE_BEGIN(Grid);
     using Integerified    = iScalar<typename BaseTraits::Integerified>;
     using DoublePrecision = iScalar<typename BaseTraits::DoublePrecision>;
     using DoublePrecision2= iScalar<typename BaseTraits::DoublePrecision2>;
+    using SinglePrecision = iScalar<typename BaseTraits::SinglePrecision>;
     static constexpr int Rank = BaseTraits::Rank + 1;
     static constexpr std::size_t count = BaseTraits::count;
     static constexpr int Dimension(int dim) {
@@ -433,6 +455,7 @@ NAMESPACE_BEGIN(Grid);
     using Integerified    = iVector<typename BaseTraits::Integerified,    N>;
     using DoublePrecision = iVector<typename BaseTraits::DoublePrecision, N>;
     using DoublePrecision2= iVector<typename BaseTraits::DoublePrecision2, N>;
+    using SinglePrecision = iVector<typename BaseTraits::SinglePrecision, N>;
     static constexpr int Rank = BaseTraits::Rank + 1;
     static constexpr std::size_t count = BaseTraits::count * N;
     static constexpr int Dimension(int dim) {
@@ -449,6 +472,7 @@ NAMESPACE_BEGIN(Grid);
     using Integerified    = iMatrix<typename BaseTraits::Integerified,    N>;
     using DoublePrecision = iMatrix<typename BaseTraits::DoublePrecision, N>;
     using DoublePrecision2= iMatrix<typename BaseTraits::DoublePrecision2, N>;
+    using SinglePrecision = iMatrix<typename BaseTraits::SinglePrecision, N>;
     static constexpr int Rank = BaseTraits::Rank + 2;
     static constexpr std::size_t count = BaseTraits::count * N * N;
     static constexpr int Dimension(int dim) {

Grid/util/Init.cc

@@ -95,6 +95,8 @@ NAMESPACE_BEGIN(Grid);
 //////////////////////////////////////////////////////
 static Coordinate Grid_default_latt;
 static Coordinate Grid_default_mpi;
+static Coordinate Grid_default_batched_solver_split;
+static bool       Grid_default_batched_solver_split_node = false;
 
 int GridThread::_threads =1;
 int GridThread::_hyperthreads=1;
@@ -108,6 +110,8 @@ char *GridHostname(void)
 }
 const Coordinate &GridDefaultLatt(void)     {return Grid_default_latt;};
 const Coordinate &GridDefaultMpi(void)      {return Grid_default_mpi;};
+const Coordinate &GridDefaultBatchedSolverSplit(void)     {return Grid_default_batched_solver_split;};
+bool              GridDefaultBatchedSolverSplitNode(void) {return Grid_default_batched_solver_split_node;};
 const Coordinate GridDefaultSimd(int dims,int nsimd)
 {
   Coordinate layout(dims);
@@ -562,6 +566,7 @@ void Grid_init(int *argc,char ***argv)
     std::cout<<GridLogMessage<<"Performance:"<<std::endl;
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"  --comms-overlap    : Overlap comms with compute "<<std::endl;    
+    std::cout<<GridLogMessage<<"  --batched-solver-split a.b.c.d|node : batched solvers run independent solves on partitions of this MPI layout (node = one per node)"<<std::endl;
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"  --dslash-generic: Wilson kernel for generic Nc"<<std::endl;    
     std::cout<<GridLogMessage<<"  --dslash-unroll : Wilson kernel for Nc=3"<<std::endl;    
@@ -613,6 +618,16 @@ void Grid_init(int *argc,char ***argv)
 		  Grid_default_latt,
 		  Grid_default_mpi);
 
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--batched-solver-split") ){
+    std::string arg = GridCmdOptionPayload(*argv,*argv+*argc,"--batched-solver-split");
+    if ( arg == "node" ) {
+      Grid_default_batched_solver_split_node = true;
+    } else {
+      GridCmdOptionIntVector(arg,Grid_default_batched_solver_split);
+      GRID_ASSERT(Grid_default_batched_solver_split.size() == Grid_default_mpi.size());
+    }
+  }
+
   if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
     std::cout<<GridLogMessage<<"Grid Default Decomposition patterns\n";
     std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;

Grid/util/Init.h

@@ -49,6 +49,11 @@ void Grid_unquiesce_nodes(void);
 const Coordinate  GridDefaultSimd(int dims,int nsimd);
 const Coordinate &GridDefaultLatt(void);
 const Coordinate &GridDefaultMpi(void);
+// --batched-solver-split a.b.c.d : MPI layout of each partition for batched solves.
+// --batched-solver-split node    : one partition per node.
+// Not given: empty Coordinate and false, meaning no split.
+const Coordinate &GridDefaultBatchedSolverSplit(void);
+bool              GridDefaultBatchedSolverSplitNode(void);
 const int        &GridThreads(void)  ;
 void              GridSetThreads(int t) ;
 void GridLogTimestamp(int);

MPI_benchmark/halo_mpi.cc

@@ -104,6 +104,119 @@ inline double usecond(void) {
   gettimeofday(&tv,NULL);
   return 1.0e6*tv.tv_sec + 1.0*tv.tv_usec;
 }
+/**************************************************************
+ * Point to point cost, per Cartesian direction.
+ *
+ * PaddedCell::Face_exchange issues one MPI_Sendrecv per dimension and waits,
+ * so the quantity that bounds a halo round is the sendrecv time for that
+ * direction, not a one way ping-pong.  Sweeping the size gives the latency
+ * intercept and the bandwidth slope separately.
+ *
+ * Each direction is labelled on- or off-node, which is what decides whether a
+ * halo round rides the intra-node fabric or the network, and therefore which
+ * dimension ordering costs least.  Times are reduced across ranks: the max is
+ * the one that bounds a collective halo round.
+ **************************************************************
+ */
+void PingPong(std::vector<int> cart_geom,bool use_device,int ncall)
+{
+  int Nd=cart_geom.size();
+  std::vector<int> periodic(Nd,1);
+  std::vector<int> coor(Nd);
+  int rank;
+
+  MPI_Comm communicator;
+  MPI_Cart_create(WorldComm,Nd,&cart_geom[0],&periodic[0],0,&communicator);
+  MPI_Comm_rank(communicator,&rank);
+  MPI_Cart_coords(communicator,rank,Nd,&coor[0]);
+
+  // An identifier every rank on a node agrees on, so a neighbour can be
+  // classified by comparing it.
+  int node_id = WorldRank;
+  MPI_Bcast(&node_id,1,MPI_INT,0,WorldShmComm);
+
+  size_t max_bytes = 2*1024*1024;
+  void *xmit, *recv;
+  if ( use_device ) {
+    xmit = acceleratorAllocDevice(max_bytes);
+    recv = acceleratorAllocDevice(max_bytes);
+  } else {
+    xmit = malloc(max_bytes);
+    recv = malloc(max_bytes);
+  }
+
+  if ( !WorldRank ) {
+    printf("= dim  dir       bytes      us(max)      us(min)        MB/s   off-node ranks\n");
+    fflush(stdout);
+  }
+
+  for(int d=0;d<Nd;d++){
+
+    // An undecomposed dimension is a local wrap in PaddedCell and carries no
+    // message at all; a shift here would only time a rank talking to itself.
+    if ( cart_geom[d] == 1 ) continue;
+
+    for(int sign=-1;sign<=1;sign+=2){
+
+      int from,to;
+      MPI_Cart_shift(communicator,d,sign,&from,&to);
+
+      // Fetch the identifier of the neighbour we SEND to, which means sending
+      // ours the other way round the shift.
+      int peer_node=node_id;
+      MPI_Sendrecv(&node_id, 1,MPI_INT,from,rank,
+		   &peer_node,1,MPI_INT,to,  to,
+		   communicator,MPI_STATUS_IGNORE);
+      int offnode = (peer_node != node_id) ? 1 : 0;
+      int offnode_count=0;
+      MPI_Reduce(&offnode,&offnode_count,1,MPI_INT,MPI_SUM,0,communicator);
+
+      for(size_t bytes=8; bytes<=max_bytes; bytes*=8){
+
+	// Latency needs the repeats; bandwidth does not, and large messages
+	// would otherwise dominate the run time.
+	int nc = (bytes<=32768) ? ncall : (ncall/20 > 10 ? ncall/20 : 10);
+
+	// Untimed warm-up: the first exchange on a fresh buffer pays connection
+	// setup and registration, and would otherwise land in the first row.
+	MPI_Sendrecv(xmit,bytes,MPI_CHAR,to,rank,
+		     recv,bytes,MPI_CHAR,from,from,
+		     communicator,MPI_STATUS_IGNORE);
+
+	MPI_Barrier(communicator);
+	double t0=usecond();
+	for(int i=0;i<nc;i++){
+	  MPI_Sendrecv(xmit,bytes,MPI_CHAR,to,rank,
+		       recv,bytes,MPI_CHAR,from,from,
+		       communicator,MPI_STATUS_IGNORE);
+	}
+	double t1=usecond();
+
+	double us = (t1-t0)/(double)nc;
+	double us_max, us_min;
+	MPI_Reduce(&us,&us_max,1,MPI_DOUBLE,MPI_MAX,0,communicator);
+	MPI_Reduce(&us,&us_min,1,MPI_DOUBLE,MPI_MIN,0,communicator);
+
+	if ( !WorldRank ) {
+	  printf("= %3d  %s  %10zu  %11.2f  %11.2f  %10.1f   %d\n",
+		 d, sign>0?"+":"-", bytes, us_max, us_min,
+		 (double)bytes/us_max, offnode_count);
+	  fflush(stdout);
+	}
+      }
+    }
+  }
+
+  if ( use_device ) {
+    acceleratorFreeDevice(xmit);
+    acceleratorFreeDevice(recv);
+  } else {
+    free(xmit);
+    free(recv);
+  }
+  MPI_Comm_free(&communicator);
+}
+
 /**************************************************************
  * Main benchmark routine
  **************************************************************
@@ -303,6 +416,20 @@ int main(int argc, char **argv)
   }
 
   
+  if( !WorldRank ) {
+    printf("=========================================================\n");
+    printf("= Point to point cost per direction, HOST memory         \n");
+    printf("=========================================================\n");fflush(stdout);
+  }
+  PingPong(mpi,false,1000);
+
+  if( !WorldRank ) {
+    printf("=========================================================\n");
+    printf("= Point to point cost per direction, DEVICE memory       \n");
+    printf("=========================================================\n");fflush(stdout);
+  }
+  PingPong(mpi,true,1000);
+
   if( !WorldRank ) {
     printf("=========================================================\n");
     printf("= Benchmarking HOST memory MPI performance               \n");

MPI_benchmark/simple_reproducer_noncache.cc

@@ -0,0 +1,214 @@
+// Minimal reproducer for the CXI uncached-registration defect.
+//
+// Hypothesis: with the libfabric MR cache disabled, a device pointer at a non-zero offset
+// into a hipMalloc'd allocation is registered as the whole allocation and the offset is
+// discarded, so a send transmits from the base and a receive lands at the base.
+//
+// Each test therefore prints, for both the data and its landing place, what a correct
+// implementation must give, what the hypothesis predicts instead, and what was observed.
+//
+//   hipcc -O2 -o simple_reproducer_noncache simple_reproducer_noncache.cc \
+//         -I$MPICH_DIR/include -L$MPICH_DIR/lib -lmpi -L$MPICH_DIR/gtl/lib -lmpi_gtl_hsa
+//
+//   MPICH_GPU_SUPPORT_ENABLED=1 FI_MR_CACHE_MAX_COUNT=0 \
+//     srun -N2 -n2 --ntasks-per-node=1 ./simple_reproducer_noncache
+
+#include <mpi.h>
+#include <hip/hip_runtime.h>
+#include <cstdio>
+#include <cstdint>
+#include <cstdlib>
+#include <vector>
+
+// The message must be large enough to go rendezvous: an eager message is staged through a host
+// bounce buffer with hipMemcpy, which honours the offset, and never registers the user pointer.
+static const int NWORDS = 262144; // 2 MB buffers
+static int       NMSG   = 16384;  // 128 KB messages          (argv[1], in words)
+static int       OFF    = 16384;  // offending offset in words (argv[2]); clearest at >= NMSG,
+                                  // which keeps the intended and predicted windows disjoint
+
+static int rank, size, peer;
+static uint64_t *A, *B;
+static std::vector<uint64_t> h(NWORDS);
+
+#define HIP(cmd) do { hipError_t e=(cmd); if ( e != hipSuccess ) {			\
+      printf("hip error %s at line %d\n",hipGetErrorString(e),__LINE__);		\
+      MPI_Abort(MPI_COMM_WORLD,1); } } while(0)
+
+static uint64_t word(uint64_t tag,int r,int n)
+{
+  return (tag<<60) | ((uint64_t)r<<32) | (uint64_t)n;
+}
+
+static const char *verdict(int observed,int correct,int predicted)
+{
+  if ( observed == correct   ) return "CORRECT";
+  if ( observed == predicted ) return "WRONG, as predicted";
+  return "WRONG, and not as predicted";
+}
+
+static void refresh(void)
+{
+  for(int n=0;n<NWORDS;n++) h[n] = word(0xa,rank,n);
+  HIP(hipMemcpy(A,h.data(),NWORDS*sizeof(uint64_t),hipMemcpyHostToDevice));
+  for(int n=0;n<NWORDS;n++) h[n] = word(0xb,rank,n);
+  HIP(hipMemcpy(B,h.data(),NWORDS*sizeof(uint64_t),hipMemcpyHostToDevice));
+  MPI_Barrier(MPI_COMM_WORLD);
+}
+
+static void exchange(int sendoff,int recvoff)
+{
+  MPI_Request req[2];
+  MPI_Irecv(B+recvoff,NMSG*sizeof(uint64_t),MPI_BYTE,peer,0,MPI_COMM_WORLD,&req[0]);
+  MPI_Isend(A+sendoff,NMSG*sizeof(uint64_t),MPI_BYTE,peer,0,MPI_COMM_WORLD,&req[1]);
+  MPI_Waitall(2,req,MPI_STATUSES_IGNORE);
+}
+
+// Report the state of one NMSG-word window of B: how much of it was overwritten, and by what.
+static void window(const char *label,int at)
+{
+  int mod=0, m0=-1;
+  for(int i=0;i<NMSG && at+i<NWORDS;i++) {
+    if ( h[at+i] != word(0xb,rank,at+i) ) { if ( m0 < 0 ) m0 = at+i; mod++; }
+  }
+  if ( mod == 0 ) {
+    printf("rank %d    %-9s B[%d..%d] : untouched\n",rank,label,at,at+NMSG-1);
+    return;
+  }
+  uint64_t w = h[m0];
+  printf("rank %d    %-9s B[%d..%d] : %d of %d words overwritten, B[%d] holds A[%d] from rank %d\n",
+	 rank,label,at,at+NMSG-1,mod,NMSG,m0,(int)(w & 0xffffffff),(int)((w>>32) & 0xfffffff));
+}
+
+// Name a word: still the receive pattern, or a word of the neighbour's send buffer.
+static const char *describe(uint64_t w,int idx)
+{
+  static char s[64];
+  if      ( w == word(0xb,rank,idx) ) snprintf(s,sizeof(s),"untouched");
+  else if ( (w>>60) == 0xa )          snprintf(s,sizeof(s),"A[%d] of rank %d",
+					       (int)(w & 0xffffffff),(int)((w>>32) & 0xfffffff));
+  else                                snprintf(s,sizeof(s),"unrecognised");
+  return s;
+}
+
+static void report(const char *name,int sendoff,int recvoff)
+{
+  HIP(hipMemcpy(h.data(),B,NWORDS*sizeof(uint64_t),hipMemcpyDeviceToHost));
+
+  int first=-1, last=-1;
+  for(int n=0;n<NWORDS;n++) {
+    if ( h[n] != word(0xb,rank,n) ) { if ( first < 0 ) first = n; last = n; }
+  }
+
+  printf("rank %d  %s : send &A[%d] -> recv &B[%d]\n",rank,name,sendoff,recvoff);
+  if ( first < 0 ) { printf("rank %d    nothing was written\n",rank); return; }
+
+  uint64_t w   = h[first];
+  int      src = (int)(w & 0xffffffff);
+  int      sr  = (int)((w>>32) & 0xfffffff);
+
+  // The hypothesis predicts both offsets are discarded: the bytes at A[0], landing at B[0].
+  printf("rank %d    data      : want A[%d]  predict A[0]  got A[%d] from rank %d   %s\n",
+	 rank,sendoff,src,sr,verdict(src,sendoff,0));
+  printf("rank %d    place     : want B[%d]  predict B[0]  got B[%d]                %s\n",
+	 rank,recvoff,first,verdict(first,recvoff,0));
+
+  if ( src == sendoff && first == recvoff ) return;
+
+  // Both candidate landing sites in our receive buffer, and both candidate source words in the
+  // neighbour's send buffer, which the fill pattern fixes by construction.
+  printf("rank %d    Boff %d\n",rank,recvoff);
+  printf("rank %d      B[0]      = 0x%016llx  %s\n",
+	 rank,(unsigned long long)h[0],describe(h[0],0));
+  if ( recvoff != 0 )
+    printf("rank %d      B[%d]\t= 0x%016llx  %s\n",
+	   rank,recvoff,(unsigned long long)h[recvoff],describe(h[recvoff],recvoff));
+  printf("rank %d    Neighbour rank %d send buffer holds\n",rank,peer);
+  printf("rank %d      A[0]\t= 0x%016llx\n",
+	 rank,(unsigned long long)word(0xa,peer,0));
+  if ( sendoff != 0 )
+    printf("rank %d      A[%d]\t= 0x%016llx\n",
+	   rank,sendoff,(unsigned long long)word(0xa,peer,sendoff));
+
+  window("intended",recvoff);
+  if ( recvoff != 0 ) window("predicted",0);
+  if ( first < recvoff || last >= recvoff+NMSG ) {
+    if ( first != 0 || last != NMSG-1 ) {
+      printf("rank %d    stray     : modified words span B[%d..%d], outside both windows\n",
+	     rank,first,last);
+    }
+  }
+
+  int bad=0;
+  for(int i=0;i<NMSG && first+i<NWORDS;i++) {
+    if ( h[first+i] != word(0xa,peer,src+i) ) bad++;
+  }
+  if ( bad ) printf("rank %d    body      : %d of %d words are not a contiguous run from A[%d]\n",
+		    rank,bad,NMSG,src);
+}
+
+// Let each rank print in turn: flush, then a barrier so rank r is on the page before r+1 starts.
+static void ordered(void (*print)(void))
+{
+  for(int r=0;r<size;r++) {
+    if ( r == rank ) { print(); fflush(stdout); }
+    MPI_Barrier(MPI_COMM_WORLD);
+  }
+}
+
+static const char *test_name;
+static int         test_sendoff, test_recvoff;
+static void        print_report(void) { report(test_name,test_sendoff,test_recvoff); }
+static void        print_setup (void)
+{
+  printf("rank %d  A %p  B %p  message %d words (%d bytes)  offset %d words (%d bytes)\n",
+	 rank,(void *)A,(void *)B,NMSG,(int)(NMSG*sizeof(uint64_t)),OFF,(int)(OFF*sizeof(uint64_t)));
+}
+
+int main(int argc,char **argv)
+{
+  MPI_Init(&argc,&argv);
+  MPI_Comm_rank(MPI_COMM_WORLD,&rank);
+  MPI_Comm_size(MPI_COMM_WORLD,&size);
+  if ( size != 2 ) {
+    if ( rank == 0 ) printf("run with two ranks, one per node\n");
+    MPI_Finalize();
+    return 1;
+  }
+  peer = 1-rank;
+
+  if ( argc > 1 ) NMSG = atoi(argv[1]);
+  if ( argc > 2 ) OFF  = atoi(argv[2]);
+  if ( NMSG < 1 || OFF < 0 || OFF+NMSG > NWORDS ) {
+    if ( rank == 0 ) printf("message %d and offset %d words do not fit in %d\n",NMSG,OFF,NWORDS);
+    MPI_Finalize();
+    return 1;
+  }
+
+  int ndev;
+  HIP(hipGetDeviceCount(&ndev));
+  if ( ndev < 1 ) {
+    printf("rank %d sees no GPU\n",rank);
+    MPI_Abort(MPI_COMM_WORLD,1);
+  }
+  HIP(hipSetDevice(rank%ndev));
+  HIP(hipMalloc(&A,NWORDS*sizeof(uint64_t)));
+  HIP(hipMalloc(&B,NWORDS*sizeof(uint64_t)));
+
+  ordered(print_setup);
+
+  const int  offsets[4][2] = { {0,0}, {OFF,0}, {0,OFF}, {OFF,OFF} };
+  const char *names[4]     = { "test 1","test 2","test 3","test 4" };
+
+  for(int t=0;t<4;t++) {
+    refresh();
+    exchange(offsets[t][0],offsets[t][1]);
+    test_name = names[t]; test_sendoff = offsets[t][0]; test_recvoff = offsets[t][1];
+    ordered(print_report);
+  }
+
+  HIP(hipFree(A));
+  HIP(hipFree(B));
+  MPI_Finalize();
+  return 0;
+}

benchmarks/Benchmark_usqcd.cc

@@ -276,14 +276,31 @@ public:
 
     GridBLAS blas;
 
+    // The batched GEMMs of a coarse multigrid level are memory bound, not
+    // flop bound: each call streams the matrices once and does O(1) flops per
+    // element.  So report the effective bandwidth beside the flop rate --
+    // measured against the device's peak it says whether a shape is running
+    // well, which the flop rate alone does not.  beta=1, so C is read as well
+    // as written.
+    auto Report = [&](int M,int N,int K,int BATCH)
+    {
+      double gf    = blas.benchmark<CComplex>(M,N,K,BATCH);
+      double flops = 8.0*M*N*K*BATCH;
+      double bytes = 1.0*sizeof(CComplex)*(1.0*M*K + 1.0*K*N + 2.0*M*N)*BATCH;
+      double gbs   = gf*bytes/flops;
+      fprintf(FP,"%d, %d, %d, %d, %f, %f\n", M, N, K, BATCH, gf, gbs);
+      std::cout<<GridLogMessage<<std::setprecision(3)
+	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<gf<<"\t\t"<<gbs<<std::endl;
+    };
+
     int fpbits = sizeof(CComplex)*4;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << "= batched GEMM fp"<<fpbits<<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"Gflop/s / rank (coarse mrhs)"<<std::endl;
+    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (coarse mrhs)"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
-    fprintf(FP,"GEMM\n\n M, N, K, BATCH, GF/s per rank fp%d\n",fpbits);
+    fprintf(FP,"GEMM\n\n M, N, K, BATCH, GF/s per rank, GB/s per rank fp%d\n",fpbits);
 
     for(int b=0;b<3;b++){
     for(int r=0;r<3;r++){
@@ -291,15 +308,10 @@ public:
       int N=rhs[r];
       int K=basis[b];
       int BATCH=vol;
-      double p=blas.benchmark<CComplex>(M,N,K,BATCH);
+      Report(M,N,K,BATCH);
-
-      fprintf(FP,"%d, %d, %d, %d, %f\n", M, N, K, BATCH, p);
-      
-      std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<p<<std::endl;
     }}
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"Gflop/s / rank (block project)"<<std::endl;
+    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (block project)"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
     for(int b=0;b<3;b++){
     for(int r=0;r<3;r++){
@@ -307,14 +319,10 @@ public:
       int N=rhs[r];
       int K=blk;
       int BATCH=vol;
-      double p=blas.benchmark<CComplex>(M,N,K,BATCH);
+      Report(M,N,K,BATCH);
-
-      fprintf(FP,"%d, %d, %d, %d, %f\n", M, N, K, BATCH, p);
-      std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<p<<std::endl;
     }}
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"Gflop/s / rank (block promote)"<<std::endl;
+    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (block promote)"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
     for(int b=0;b<3;b++){
     for(int r=0;r<3;r++){
@@ -322,12 +330,54 @@ public:
       int N=blk;
       int K=basis[b];
       int BATCH=vol;
-      double p=blas.benchmark<CComplex>(M,N,K,BATCH);
+      Report(M,N,K,BATCH);
-
-      fprintf(FP,"%d, %d, %d, %d, %f\n", M, N, K, BATCH, p);
-      std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<p<<std::endl;
     }}
+
+    ///////////////////////////////////////////////////////////////////////////
+    // The coarse stencil apply as the multigrid actually issues it, swept in
+    // the batch count.  One call per stencil point, M = K = nbasis, N = Nrhs,
+    // batch = the LOCAL coarse volume: at 48^3x96 over 288 GCDs with a
+    // {2,2,3,3} block that is 1024 sites, which is small, and the trace shows
+    // the single-precision kernel taking as long as the double one.  The
+    // sweep spans that point and the alternative organisation in which the
+    // stencil legs ride in the batch dimension instead of being separate
+    // calls: 9x and 33x the sites, with the partial results summed after.
+    // Nrhs 1 is the latency-bound case that a single-right-hand-side solve
+    // pays throughout.
+    ///////////////////////////////////////////////////////////////////////////
+    {
+      int  nb[]    = { 60, 64 };
+      int  nrhs_[] = { 1, 12, 24, 32 };
+      // 1024 is the local coarse volume at the production point; the rest are
+      // that times the number of stencil legs folded into the batch.  3 is
+      // the common divisor of the 33-point (PVdagM) and 81-point (HDCG)
+      // stencils, so x3 is the grouping that serves both and leaves only a
+      // three-way linear combination at the end.
+      int  batches[] = { 256, 1024, 3072, 4096, 9216, 33792 };
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (coarse stencil, production shapes)"<<std::endl;
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      for(int b=0;b<2;b++){
+      for(int r=0;r<4;r++){
+      for(int v=0;v<6;v++){
+	Report(nb[b],nrhs_[r],nb[b],batches[v]);
+      }}}
+
+      // The same work with the stencil sum folded along the CONTRACTION
+      // instead of the batch: one GEMM per site with K = npoint*nbasis.  Both
+      // precisions reach their best rates at large K (the block-project
+      // shape, K=256), which this pushes further, at the price of gathering
+      // the neighbour vectors.  33 points is the next-to-nearest stencil,
+      // 81 the full box.
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (stencil folded along K)"<<std::endl;
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      for(int b=0;b<2;b++){
+      for(int r=0;r<4;r++){
+	Report(nb[b],nrhs_[r],33*nb[b],1024);
+	Report(nb[b],nrhs_[r],81*nb[b],1024);
+      }}
+    }
     fprintf(FP,"\n\n\n");
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   };

configure.ac

@@ -192,6 +192,18 @@ case ${ac_TRACING} in
 	AC_DEFINE([GRID_TRACING_NONE],[1],[no tracing]);;
 esac
 
+############### distributed dense inverse precision (multigrid coarse-coarse bottom)
+AC_ARG_ENABLE([dense-inverse-precision],
+    [AS_HELP_STRING([--enable-dense-inverse-precision=double|single],[precision of the distributed dense coarse-coarse inversion (default double); independent of the coarse-space precision])],
+    [ac_DENSE_INVERSE=${enable_dense_inverse_precision}], [ac_DENSE_INVERSE=double])
+
+case ${ac_DENSE_INVERSE} in
+    single)
+        AC_DEFINE([GRID_DENSE_INVERSE_SINGLE],[1],[fp32 distributed dense inversion]);;
+    *)
+        AC_DEFINE([GRID_DENSE_INVERSE_DOUBLE],[1],[fp64 distributed dense inversion]);;
+esac
+
 ############### fermions
 AC_ARG_ENABLE([fermion-instantiations],
      [AS_HELP_STRING([--enable-fermion-instantiations=yes|no],[enable fermion instantiations])],

examples/Example_hdcg_multigrid.cc

@@ -0,0 +1,172 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_hdcg_multigrid.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// The two-level mrhs HDCG driver on the library objects
+// (Grid/algorithms/multigrid/HDCGMultiGrid.h): the Schur-preconditioned
+// M^dag M of a Mobius/Shamir fermion on one checkerboard, coarsened on V2.
+//
+//   ./Example_hdcg_multigrid --grid 48.48.48.96 --mpi ... --hdcg-params params.xml
+//
+// With no --hdcg-params the built-in defaults run (hot-start gauge field
+// unless Config is set).  A missing file gets a template written next to
+// it and the program exits: the template documents every parameter.
+//
+// Compile-time: -DNBASIS=8 cuts the basis down for laptop runs;
+// -DCOARSE_SINGLE builds the fp32 coarse space, which examples/Makefile.am
+// builds as its own binary, Example_hdcg_multigrid_fp32coarse.
+//
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/HDCGMultiGrid.h>
+
+using namespace std;
+using namespace Grid;
+
+#ifndef NBASIS
+#define NBASIS 62
+#endif
+
+#ifdef COARSE_SINGLE
+typedef sTComplexF CoarseScalar_t;
+#else
+typedef sTComplexD CoarseScalar_t;
+#endif
+
+struct HDCGDriverParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(HDCGDriverParams,
+                                  int,         Ls,
+                                  RealD,       Mass,
+                                  RealD,       M5,
+                                  RealD,       MobiusB,
+                                  RealD,       MobiusC,
+                                  std::string, Config,          // empty: hot start
+                                  int,         Checkerboard,    // 0 Even, 1 Odd: the Schur operator's checkerboard
+                                  int,         Nrhs,
+                                  int,         SolveSingleRHS,  // also run Nrhs=1 through the same objects
+                                  HDCGMultiGridParams, MultiGrid);
+  HDCGDriverParams()
+    : Ls(24), Mass(0.00078), M5(1.8), MobiusB(1.5), MobiusC(0.5),
+      Config(""), Checkerboard(0), Nrhs(12), SolveSingleRHS(1) {};
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  HDCGDriverParams P;
+  {
+    std::string pfile("");
+    if( GridCmdOptionExists(argv,argv+argc,"--hdcg-params") )
+      pfile = GridCmdOptionPayload(argv,argv+argc,"--hdcg-params");
+    if ( pfile.length() ) {
+      bool good; { std::ifstream f(pfile); good = f.good(); }
+      if ( !good ) {
+        if ( GlobalSharedMemory::WorldRank == 0 ) {
+          XmlWriter WR(pfile+".templ");
+          write(WR, "HDCGDriver", P);
+          std::cout << GridLogMessage << pfile << " does not exist; template written to "
+                    << pfile << ".templ" << std::endl;
+        }
+        Grid_finalize(); return 0;
+      }
+      XmlReader RD(pfile);
+      read(RD, "HDCGDriver", P);
+    }
+    CheckValidity(P.MultiGrid);
+    std::cout << GridLogMessage << "HDCGDriver parameters ("
+              << (pfile.length() ? pfile : std::string("defaults")) << "):" << std::endl;
+    std::cout << P << std::endl;
+  }
+  const int cb = P.Checkerboard;
+
+  Coordinate latt = GridDefaultLatt();
+  Coordinate mpi  = GridDefaultMpi();
+  Coordinate fsimd= GridDefaultSimd(Nd,vComplex::Nsimd());
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt,fsimd,mpi);
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(P.Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(P.Ls,UGrid);
+
+  GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
+  GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
+
+  LatticeGaugeField Umu(UGrid);
+  if ( P.Config.length() ) {
+    std::cout << GridLogMessage << "Reading gauge field " << P.Config << std::endl;
+    FieldMetaData header;
+    NerscIO::readConfiguration(Umu,header,P.Config);
+  } else {
+    std::cout << GridLogMessage << "Hot start gauge field" << std::endl;
+    SU<Nc>::HotConfiguration(RNG4,Umu);
+  }
+
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,P.Mass,P.M5,P.MobiusB,P.MobiusC);
+  SchurDiagMooeeOperator<MobiusFermionD,LatticeFermionD> HermOpEO(Ddwf);
+  HermOpAdaptor<LatticeFermionD> FineOp(HermOpEO);     // Op = HermOp, for the coarsening
+
+  //////////////////////////////////////////////////////////////////////
+  // Grids -> coarsening.  Scope order is lifetime order.  The fp32 fine
+  // grids and fermion operator serve the fp32 fine level of the
+  // preconditioner (MultiGrid.Setup.FinePrecision, a run-time choice).
+  //////////////////////////////////////////////////////////////////////
+  typedef HDCGCoarsening<vSpinColourVector,CoarseScalar_t,NBASIS> Coarsening_t;
+  std::cout << GridLogMessage << "Coarse sector precision (compiled): "
+            << (sizeof(typename GridTypeMapper<CoarseScalar_t>::scalar_type)==sizeof(ComplexF) ? "fp32" : "fp64")
+            << ", nbasis " << NBASIS << std::endl;
+
+  MGCoarseGrids CGrids(FGrid, P.MultiGrid.Setup);
+  MGFineGridsF  FGridsF(FGrid);
+
+  LatticeGaugeFieldF UmuF(FGridsF.UGridF);
+  precisionChange(UmuF,Umu);
+  MobiusFermionF DdwfF(UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,P.Mass,P.M5,P.MobiusB,P.MobiusC);
+
+  Coarsening_t Coarsening(CGrids, FGridsF, FrbGrid, cb, P.MultiGrid);
+
+  Coarsening.GetSubspace(RNG5, FineOp);
+  HDCGRefineSubspace(Coarsening, Ddwf, DdwfF, FineOp, P.Nrhs);    // no-op unless Refine is HDCG
+  Coarsening.Coarsen(FineOp);
+  Coarsening.CertifyCoarsening(FineOp);
+  Coarsening.CoarseLanczos(P.Nrhs);
+
+  //////////////////////////////////////////////////////////////////////
+  // Solves: mrhs then (optionally) single RHS through the SAME objects.
+  //////////////////////////////////////////////////////////////////////
+  auto RunSolve = [&](int nr)
+  {
+    HDCGSolver<MobiusFermionD,MobiusFermionF,Coarsening_t> Solver(Ddwf,DdwfF,Coarsening,P.MultiGrid,nr);
+    std::vector<LatticeFermionD> src(nr,FrbGrid), sol(nr,FrbGrid);
+    for(int r=0;r<nr;r++){ src[r].Checkerboard()=cb; sol[r].Checkerboard()=cb; gaussian(RNG5,src[r]); sol[r]=Zero(); }
+    Solver.Solve(src,sol);
+    if ( nr == 1 ) {            // the same solve through the LinearFunction interface
+      LatticeFermionD x(FrbGrid); x.Checkerboard()=cb; x=Zero();
+      Solver.Solve(src[0],x);
+    }
+  };
+
+  RunSolve(P.Nrhs);
+  if ( P.SolveSingleRHS && P.Nrhs != 1 ) RunSolve(1);
+
+  Grid_finalize();
+}

examples/Example_mdagm.cc

@@ -32,50 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 using namespace std;
 using namespace Grid;
 
-// Routes Op/AdjOp -> HermOp so that CoarsenOperator and CreateSubspace
-// both see the HPD operator M†M rather than bare M.
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> &wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
-  void HermOp (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
-  void AdjOp  (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
-  void OpDiag (const Field &in, Field &out)                  { GRID_ASSERT(0); }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) { GRID_ASSERT(0); }
-  void OpDirAll(const Field &in, std::vector<Field> &out)    { GRID_ASSERT(0); }
-  void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
-    wrapped.HermOp(in, out);
-    ComplexD dot = innerProduct(in, out);
-    n1 = real(dot);
-    n2 = norm2(out);
-  }
-};
 
-// Fixed-iteration CG smoother: runs exactly `iters` steps of CG on the
-// shifted operator.  tolerance=0 so CG never exits early.
-template<class Field>
-class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator &_SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator)
-    : _SmootherOperator(SmootherOperator), iters(_iters)
-  {
-    std::cout << GridLogMessage << " CGSmoother order " << iters << std::endl;
-  }
-  void operator()(const Field &in, Field &out)
-  {
-    ConjugateGradient<Field> CG(0.0, iters, false);
-    out = Zero();
-    CG(_SmootherOperator, in, out);
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -130,7 +87,7 @@ int main (int argc, char ** argv)
   HermOpAdaptor<LatticeFermionD> HermFineOp(MdagMOp);
 
   // ── Coarse geometry ────────────────────────────────────────────────────
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
 
@@ -214,7 +171,7 @@ int main (int argc, char ** argv)
   LatticeFermionD src(FGrid);    random(RNG5, src);
   LatticeFermionD result(FGrid); result = Zero();
 
-  TwoLevelADEF2<LatticeFermionD, CoarseVector, Subspace>
+  DeprecatedTwoLevelADEF2<LatticeFermionD, CoarseVector, Subspace>
     HDCG(1.0e-8, 1000,
          HermFineOp,
          Smoother,

examples/Example_mdagm_cg.cc

@@ -31,7 +31,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 //
 // Operator hierarchy:
 //   Fine:   M†M, acted on via HermOpAdaptor so Op = HermOp = M†M
-//   Coarse: 33-point GeneralCoarsenedMatrix (NextToNearest, 2-hop M†M)
+//   Coarse: 33-point DeprecatedGeneralCoarsenedMatrix (NextToNearest, 2-hop M†M)
 //
 // Setup:
 //   1. Chebyshev filter (T_600 x T_2500) on M†M to build near-null subspace {ψᵢ}
@@ -57,46 +57,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 using namespace std;
 using namespace Grid;
 
-// Wraps any LinearOperatorBase so that Op = AdjOp = HermOp.
-// Required when coarsening an HPD operator whose Op != HermOp
-// (e.g. MdagMLinearOperator where Op=M, HermOp=M†M).
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> &wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {}
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
-  void HermOp (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
-  void AdjOp  (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
-  void OpDiag (const Field &in, Field &out)                    { GRID_ASSERT(0); }
-  void OpDir  (const Field &in, Field &out, int dir, int disp) { GRID_ASSERT(0); }
-  void OpDirAll(const Field &in, std::vector<Field> &out)      { GRID_ASSERT(0); }
-  void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
-    wrapped.HermOp(in, out);
-    ComplexD dot = innerProduct(in, out);
-    n1 = real(dot); n2 = norm2(out);
-  }
-};
 
-// Fixed-iteration CG as a smoother (LinearFunction).
-// Used as the IR-shifted smoother: solves (M†M + lo*I) x = b approximately.
-template<class Field>
-class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  LinearOperatorBase<Field> &_op;
-  int iters;
-  CGSmoother(int _iters, LinearOperatorBase<Field> &op) : _op(op), iters(_iters) {
-    std::cout << GridLogMessage << "CGSmoother order " << iters << std::endl;
-  }
-  void operator()(const Field &in, Field &out) {
-    ConjugateGradient<Field> CG(0.0, iters, false);
-    out = Zero();
-    CG(_op, in, out);
-  }
-};
 
 // Two-level V-cycle preconditioner (LinearFunction).
 template<class Fobj, class CComplex, int nbasis>
@@ -237,7 +198,7 @@ int main(int argc, char **argv)
   ///////////////////////////////////////////////////////////
   // Coarse operator: 33-point stencil for 2-hop M†M
   ///////////////////////////////////////////////////////////
-  typedef GeneralCoarsenedMatrix<vSpinColourVector, vTComplex, nbasis> LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector, vTComplex, nbasis> LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector CoarseVector;
 
   NextToNearestStencilGeometry5D geom(Coarse5d);

examples/Example_pvdagm.cc

@@ -364,7 +364,7 @@ void runMG(
 ) {
 
   // typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
-  // typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+  // typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
   // typedef LittleDiracOperator::CoarseVector CoarseVector;
   ParseEnvironment();
 
@@ -620,7 +620,7 @@ int main (int argc, char ** argv)
 
   // assert(nbasis <= Nevecs);     // need to have enough evecs
 
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector CoarseVector;
 
   NextToNearestStencilGeometry5D geom(Coarse5d);

examples/Example_pvdagm_3level.cc

@@ -190,8 +190,8 @@ public:
   }
 };
 
-// Lüscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
+// Luscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
-// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi† src.
+// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi^dag src.
 template<class Field>
 class LuscherGuesser : public LinearFunction<Field> {
   const std::vector<Field> &psi;
@@ -342,7 +342,7 @@ void runMG(
   TrivialPrecon<LatticeFermionD>   simple_fine;
 
   //////////////////////////////////////////////////////////////////////
-  // Level 0→1: coarsen PVdagM, build LinOpCoarse
+  // Level 0->1: coarsen PVdagM, build LinOpCoarse
   //////////////////////////////////////////////////////////////////////
   LittleDiracOperator LittleDiracOpPV(geom, FGrid, Coarse5d);
   LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesPD);
@@ -366,7 +366,7 @@ void runMG(
   //////////////////////////////////////////////////////////////////////
   // psi_coarse: coarse projections of pre-GS fine null vectors.
   // These are the Level 1 near-null vectors, promoted from Level 0.
-  // Used as the aggregation basis for Level 1→2 coarsening.
+  // Used as the aggregation basis for Level 1->2 coarsening.
   //////////////////////////////////////////////////////////////////////
   std::vector<CoarseVector> psi_coarse(nbasis, Coarse5d);
   for (int k = 0; k < nbasis; k++)
@@ -396,22 +396,22 @@ void runMG(
     RealD normC      = C.norm();
     RealD normCmCdag = (C - C.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse null matrix ||C||            = " << normC << std::endl;
-    std::cout << GridLogMessage << "Coarse null matrix ||C - C†||/||C|| = " << normCmCdag/normC << std::endl;
+    std::cout << GridLogMessage << "Coarse null matrix ||C - C^dag||/||C|| = " << normCmCdag/normC << std::endl;
     std::cout << GridLogMessage << "Galerkin check ||C||/||W||          = " << normC/normW << std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2: set up aggregation using psi_coarse as subspace.
+  // Level 1->2: set up aggregation using psi_coarse as subspace.
   // Block factor 2,2,3,2 (removes odd local sublattice in z given MPI
-  // geometry 3×6×4×4 where z-local at Level 1 is 6).
+  // geometry 3x6x4x4 where z-local at Level 1 is 6).
   // psi_coarse are assigned directly; CoarsenOperator performs
   // block-GS orthogonalisation before building LinOpCoarseCoarse.
   //////////////////////////////////////////////////////////////////////
   // innerProduct(CoarseSiteObj, CoarseSiteObj) returns iScalar<vTComplex>, so CComplex
-  // for the L1→L2 level must be iScalar<vTComplex>, not vTComplex.
+  // for the L1->L2 level must be iScalar<vTComplex>, not vTComplex.
   typedef typename CoarseVector::vector_object                            CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
   typedef typename LittleDiracOperatorL2::CoarseVector                   CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,NB>                       SubspaceL2;
   typedef MGPreconditioner<CoarseSiteObj,vTTComplex,NB>                  L1to2MG;
@@ -429,12 +429,12 @@ void runMG(
   TrivialPrecon<CoarseCoarseVector> simpleCC;
 
   //////////////////////////////////////////////////////////////////////
-  // Lüscher deflation guesser for L3PGCR.
+  // Luscher deflation guesser for L3PGCR.
   // Step 1: project psi_coarse[k] (promoted fine null vectors) to
-  //         CoarseCoarseVector space — these cover the zero-momentum
+  //         CoarseCoarseVector space -- these cover the zero-momentum
   //         component of the near-null space of LinOpCC.
   // Step 2: breed Nextra additional null vectors directly on LinOpCC
-  //         using GCR with random sources — these pick up near-null
+  //         using GCR with random sources -- these pick up near-null
   //         modes at all spatial frequencies not spanned by step 1.
   // Step 3: build C_{st} = <psi_cc[s]|LinOpCC|psi_cc[t]> over the
   //         full augmented basis and invert directly via Eigen LU.
@@ -476,7 +476,7 @@ void runMG(
     RealD normCcc      = Ccc.norm();
     RealD normCccmCdag = (Ccc - Ccc.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc||              = " << normCcc << std::endl;
-    std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc-Ccc†||/||Ccc|| = " << normCccmCdag/normCcc << std::endl;
+    std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc-Ccc^dag||/||Ccc|| = " << normCccmCdag/normCcc << std::endl;
   }
   Eigen::MatrixXcd Ccc_inv = Ccc.inverse();
   LuscherGuesser<CoarseCoarseVector> CCDeflGuesser(psi_cc, Ccc_inv);
@@ -489,7 +489,7 @@ void runMG(
   L3PGCR.Name("CCouter");
 
   //////////////////////////////////////////////////////////////////////
-  // Coarse-level GCR smoother for Level 1→2 V-cycle.
+  // Coarse-level GCR smoother for Level 1->2 V-cycle.
   // Mirrors fine-grid SmootherGCR: shifted operator + fixed step count.
   // coarse_smoother_shift and coarse_smoother_nstep are the tuning knobs.
   //////////////////////////////////////////////////////////////////////
@@ -505,7 +505,7 @@ void runMG(
   CoarseSmootherGCR.Name("Csmoother");
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2 V-cycle preconditioner.
+  // Level 1->2 V-cycle preconditioner.
   //////////////////////////////////////////////////////////////////////
   L1to2MG L1to2Precon(AggregatesL2,
                        LinOpCoarse,
@@ -513,7 +513,7 @@ void runMG(
                        CoarseSmootherGCR,  // post-smoother: 12 GCR steps
                        LinOpCC,
                        L3PGCR,
-                       CCDeflGuesser);     // Lüscher guesser: psi_cc C^{-1} psi_cc†
+                       CCDeflGuesser);     // Luscher guesser: psi_cc C^{-1} psi_cc^dag
 
   //////////////////////////////////////////////////////////////////////
   // Standalone Level 1 two-level solve test.
@@ -548,7 +548,7 @@ void runMG(
   f_src = one;
 
   // Pre-smoother: none (TrivialPrecon); post-smoother: shifted PGCR.
-  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1→2 V-cycle).
+  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1->2 V-cycle).
   TwoLevelMG ThreeLevelPrecon(AggregatesPD,
                                PVdagM,
                                simple_fine,
@@ -592,7 +592,7 @@ int main (int argc, char ** argv)
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
-  // Level 1 coarse grid: block 2^4 from fine (48×48×48×96 → 24×24×24×48, Ls=1)
+  // Level 1 coarse grid: block 2^4 from fine (48x48x48x96 -> 24x24x24x48, Ls=1)
   Coordinate clatt = lat_size;
   for (int d = 0; d < 4; d++) clatt[d] /= 2;
   std::cout << GridLogMessage << "Level 1 coarse lattice: " << clatt << std::endl;
@@ -600,13 +600,13 @@ int main (int argc, char ** argv)
   GridCartesian *Coarse4d  = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridCartesian *Coarse5d  = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
 
-  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24×24×24×48 → 12×12×8×16, Ls=1).
+  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24x24x24x48 -> 12x12x8x16, Ls=1).
   // MPI geometry 3.6.4.4 (288 ranks): fine local {16,8,12,24}.
   // Level 1 local {8,4,6,12}; Level 2 local {4,2,2,4}.
-  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) → must use 3.
+  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) -> must use 3.
   // t blocked by 3: t-Level1-local=12; 12/3=4 divisible by Nsimd=4 (gen-simd-width=64).
-  //   t-block=2 gives t2-local=6, 6 mod 4 ≠ 0, fails Grid SIMD assertion. ✓
+  //   t-block=2 gives t2-local=6, 6 mod 4 != 0, fails Grid SIMD assertion.
-  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4). ✓
+  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4).
   Coordinate clatt2 = clatt;
   clatt2[0] /= 2;
   clatt2[1] /= 2;
@@ -635,7 +635,7 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
   typedef MGPreconditioner<vSpinColourVector,vTComplex,nbasis>        TwoLevelMG;

examples/Example_pvdagm_3level_DenseCoarseMatrix.cc

@@ -0,0 +1,990 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_pvdagm_3level_DenseCoarseMatrix.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// PVdagM three level multigrid on MultiGeneralCoarsenedOperator.
+//
+// STAGES ONE AND TWO: grids, types, subspace, and the L1 and L2 coarsenings.
+// The dense bottom and the solves are not here yet.
+//
+// Differences from Example_pvdagm_mrhs_3level_DenseCoarseMatrix.cc:
+//
+//  * The coarse space is UNVECTORISED (sComplexD). The fine space stays
+//    vectorised. MultiRHSBlockProject carries the mixed layout.
+//
+//  * One operator, not two. The deprecated path needed
+//    DeprecatedGeneralCoarsenedMatrix to coarsen and
+//    DeprecatedMultiGeneralCoarsenedMatrix to apply, bridged by CopyMatrix.
+//    This operator does both, and single versus multiRHS is SetGrid on the
+//    same object with the matrix elements built once.
+//
+//  * Nrhs is unconstrained. The deprecated path required nrhs % vComplex::Nsimd() == 0 because
+//    its multiRHS grid carried the SIMD in the rhs direction.
+//
+//  * CoarsenOperator takes the subspace vectors, not an Aggregation. It block
+//    orthonormalises them IN PLACE -- the vectors are far too large to copy
+//    defensively -- so rawNull is taken first and the RAW vectors are what
+//    define the L2 null space. Do not insert an Orthogonalise() anywhere:
+//    projecting a block-orthonormal vector onto its own block-orthonormalised
+//    aggregation gives e_k, and the near null content is silently gone. The
+//    ||<psi|psi> - I||_F guard below is what catches that.
+//
+// Env: LATT LS MASS NBASIS(compile time) NRHS BLOCK BLOCK2 COARSEN_BATCH
+//      HOT_START CONFIG SUBSPACE_FILE DEPRECATED_CHECK MRHS_COARSEN
+//
+
+#include <Grid/Grid.h>
+#include <algorithm>              // std::sort, for the runtime-environment dump in ParseEnvironment
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
+#include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
+
+#include <memory>
+
+using namespace std;
+using namespace Grid;
+
+// Compile time so it can be cut down for laptop runs: -DNBASIS=8
+#ifndef NBASIS
+#define NBASIS 60
+#endif
+
+RealD mass      = 0.00078;
+int   Nrhs      = 12;
+int   Ls        = 24;
+int   CoarsenBatch = 9;
+std::vector<int> lat_size({48,48,48,96});
+
+// Solver tuning.  PRINCIPLE (PB, 2026-08-24): the defaults ARE the current
+// optimum, so an unset environment reproduces the best banked result; they
+// are updated as and when a better point is found, and every change is
+// dated here.  Environment variables of the same names override for sweeps.
+//
+// Current optimum: 2026-08-24, slurm-5335492 F4, 48^3x96 Ls=24 on 288 GCDs,
+//   17.2 s/RHS at Nrhs=4, 32.2 s at Nrhs=1 (exact-halo FINAL ~1e-8 pending
+//   the exact-outer rerun).  Smoother mmax == order (full GCR history);
+//   PB's mmax=1 trial gave 72 vs ~60 outer iterations and was slower.
+RealD FineSmootherShift    = 0.1;
+int   FineSmootherOrder    = 6;
+int   FineSmootherMmax     = 6;
+RealD CoarseSmootherShift  = 0.1;
+int   PowerIterations      = 0;     // >0: power-iterate the smoother operators before the solves (spectral edge)
+// Smoother implementation per level (Smoothers.h):
+//   gcr    : the adaptive PGCR (default, as always)
+//   replay : run the PGCR with a coefficient recorder for the first
+//            PolyRecordIters outer steps, then switch to GCRReplaySmoother
+//            (same polynomial, no inner products) -- 1402.2585 p.13 revisited
+//   cheb   : ChebyshevNonHermitianSmoother, 1/x on [ChebLo,ChebHi], order
+//            = the GCR step count of that level
+std::string FineSmootherMode   = "gcr";
+std::string CoarseSmootherMode = "gcr";
+int   PolyRecordIters      = 4;
+int   PolyRecordStart      = 0;     // outer step at which recording begins (0: from the first step)
+std::string PolyRecordSelect = "last"; // which recorded call to replay: last|first|mean (mean is the bad one)
+int   PolyRefresh          = 0;     // >0: every PolyRefresh outer steps, one adaptive step re-records the polynomial (HDCG: every 10)
+int   PolyVerbose          = 0;     // 1: fixed-polynomial smoothers print |r_m|/|r_0| per call
+RealD FineChebLo   = 3.0,  FineChebHi   = 137.0;   // from the harvested polynomial and PowerIterations edge
+RealD CoarseChebLo = 8.0,  CoarseChebHi = 45.0;
+int   CoarseSmootherNstep  = 2;
+int   CoarseSmootherMmax   = 2;
+RealD CoarseSolverTol      = 0.05;
+int   CoarseSolverOrder    = 200;
+int   CoarseSolverMmax     = 16;
+RealD OuterTol             = 1.0e-8;
+int   OuterMmax            = 4;
+int   OuterNstep           = 8;
+
+// "It's legal to get the same answer faster, not to get a less correct
+// answer."  (PB, 2026-08-24)
+//
+// Halo-precision POLICY: reduced-precision (fp32 wire)
+// halos belong in the PRECONDITIONER -- the smoother, the V-cycle's own
+// residuals, and the coarsening -- and NEVER in the outer Krylov.  The
+// outer operator's applications define what "converged" means; making them
+// sloppy turns the stopping criterion into a statement about the wrong
+// operator (measured: solver stops at computed 9.8e-9 while the true
+// residual is 3.4e-8).  Exactness costs one exact fine matvec per outer
+// iteration, a few percent of the solve.
+//
+// Stencil::SloppyComms is a free runtime setter (Stencil.h:303), so the
+// policy is implemented by SCOPED toggling: SetFineSloppy(1) on entering
+// the preconditioner / coarsening, SetFineSloppy(0) on leaving.  The
+// operators default to EXACT.  FineSloppyComms therefore now means
+// "sloppy inside the preconditioner"; =0 makes everything exact.
+int   FineSloppyComms      = 1;
+// SmootherCoeffLog=1 : print the GCR step lengths a_k and orthogonalisation
+// coefficients b_kj of BOTH smoothers every call -- the harvest for a fixed
+// polynomial smoother (stable coefficients => stationary p(A), no reductions).
+int   SmootherCoeffLog     = 0;
+std::function<void(int)> SetFineSloppy = [](int){};
+
+void ParseEnvironment(void)
+{
+  if(getenv("MASS"))          mass        = atof(getenv("MASS"));
+  if(getenv("NRHS"))          Nrhs        = atoi(getenv("NRHS"));
+  if(getenv("LS"))            Ls          = atoi(getenv("LS"));
+  if(getenv("COARSEN_BATCH")) CoarsenBatch= atoi(getenv("COARSEN_BATCH"));
+  if(getenv("FineSmootherShift"))  FineSmootherShift  = atof(getenv("FineSmootherShift"));
+  if(getenv("FineSmootherOrder"))  FineSmootherOrder  = atoi(getenv("FineSmootherOrder"));
+  if(getenv("FineSmootherMmax"))   FineSmootherMmax   = atoi(getenv("FineSmootherMmax"));
+  if(getenv("CoarseSmootherShift"))CoarseSmootherShift= atof(getenv("CoarseSmootherShift"));
+  if(getenv("PowerIterations"))    PowerIterations    = atoi(getenv("PowerIterations"));
+  if(getenv("FineSmootherMode"))   FineSmootherMode   = getenv("FineSmootherMode");
+  if(getenv("CoarseSmootherMode")) CoarseSmootherMode = getenv("CoarseSmootherMode");
+  if(getenv("PolyRecordIters"))    PolyRecordIters    = atoi(getenv("PolyRecordIters"));
+  if(getenv("PolyRecordStart"))    PolyRecordStart    = atoi(getenv("PolyRecordStart"));
+  if(getenv("PolyRecordSelect"))   PolyRecordSelect   = getenv("PolyRecordSelect");
+  if(getenv("PolyRefresh"))        PolyRefresh        = atoi(getenv("PolyRefresh"));
+  if(getenv("PolyVerbose"))        PolyVerbose        = atoi(getenv("PolyVerbose"));
+  if(getenv("FineChebLo"))         FineChebLo         = atof(getenv("FineChebLo"));
+  if(getenv("FineChebHi"))         FineChebHi         = atof(getenv("FineChebHi"));
+  if(getenv("CoarseChebLo"))       CoarseChebLo       = atof(getenv("CoarseChebLo"));
+  if(getenv("CoarseChebHi"))       CoarseChebHi       = atof(getenv("CoarseChebHi"));
+  if(getenv("CoarseSmootherNstep"))CoarseSmootherNstep= atoi(getenv("CoarseSmootherNstep"));
+  if(getenv("CoarseSmootherMmax")) CoarseSmootherMmax = atoi(getenv("CoarseSmootherMmax"));
+  if(getenv("CoarseSolverTol"))    CoarseSolverTol    = atof(getenv("CoarseSolverTol"));
+  if(getenv("CoarseSolverOrder"))  CoarseSolverOrder  = atoi(getenv("CoarseSolverOrder"));
+  if(getenv("CoarseSolverMmax"))   CoarseSolverMmax   = atoi(getenv("CoarseSolverMmax"));
+  if(getenv("OuterTol"))           OuterTol           = atof(getenv("OuterTol"));
+  if(getenv("OuterMmax"))          OuterMmax          = atoi(getenv("OuterMmax"));
+  if(getenv("FineSloppyComms"))    FineSloppyComms    = atoi(getenv("FineSloppyComms"));
+  if(getenv("SmootherCoeffLog"))   SmootherCoeffLog   = atoi(getenv("SmootherCoeffLog"));
+  if(getenv("OuterNstep"))         OuterNstep         = atoi(getenv("OuterNstep"));
+  if(getenv("LATT")){
+    Coordinate l;
+    GridCmdOptionIntVector(std::string(getenv("LATT")),l);
+    GRID_ASSERT(l.size()==4);
+    for(int d=0;d<4;d++) lat_size[d]=l[d];
+  }
+
+  std::cout << GridLogMessage << "PARAM: LATT               "
+	    << lat_size[0]<<"."<<lat_size[1]<<"."<<lat_size[2]<<"."<<lat_size[3] << std::endl;
+  std::cout << GridLogMessage << "PARAM: LS                 " << Ls           << std::endl;
+  std::cout << GridLogMessage << "PARAM: MASS               " << mass         << std::endl;
+  std::cout << GridLogMessage << "PARAM: NBASIS             " << NBASIS       << std::endl;
+  std::cout << GridLogMessage << "PARAM: NRHS               " << Nrhs         << std::endl;
+  std::cout << GridLogMessage << "PARAM: COARSEN_BATCH      " << CoarsenBatch << std::endl;
+  // EVERY knob this programme parsed, so a log identifies its own run (2026-08-28:
+  // four jobs in the queue differing in FineSloppyComms/NRHS/mmax and none of it
+  // printed).  Interim until the Serializable parameter struct replaces all of this.
+  auto P = [](const char *n, auto v){ std::cout << GridLogMessage << "PARAM: " << std::left << std::setw(20) << n << std::right << " " << v << std::endl; };
+  P("FineSmootherShift",FineSmootherShift);   P("FineSmootherOrder",FineSmootherOrder);   P("FineSmootherMmax",FineSmootherMmax);
+  P("FineSmootherMode",FineSmootherMode);     P("FineChebLo",FineChebLo);                 P("FineChebHi",FineChebHi);
+  P("FineSloppyComms",FineSloppyComms);
+  P("CoarseSmootherShift",CoarseSmootherShift); P("CoarseSmootherNstep",CoarseSmootherNstep); P("CoarseSmootherMmax",CoarseSmootherMmax);
+  P("CoarseSmootherMode",CoarseSmootherMode); P("CoarseChebLo",CoarseChebLo);             P("CoarseChebHi",CoarseChebHi);
+  P("CoarseSolverTol",CoarseSolverTol);       P("CoarseSolverOrder",CoarseSolverOrder);   P("CoarseSolverMmax",CoarseSolverMmax);
+  P("OuterTol",OuterTol);                     P("OuterMmax",OuterMmax);                   P("OuterNstep",OuterNstep);
+  P("PowerIterations",PowerIterations);       P("PolyRecordIters",PolyRecordIters);       P("PolyRecordStart",PolyRecordStart);
+  P("PolyRecordSelect",PolyRecordSelect);     P("PolyRefresh",PolyRefresh);               P("PolyVerbose",PolyVerbose);
+  P("SmootherCoeffLog",SmootherCoeffLog);
+  // library-side knobs, read straight from the environment as the library will
+  const char *envs[] = {"BLOCK","BLOCK2","SUBSPACE_FILE","CONFIG","HOT_START","MRHS_COARSEN","DEPRECATED_CHECK",
+                        "DENSE_CC","DENSE_SCHUR","DENSE_SCHUR2D","DENSE_DEVICE_SUM","DENSE_SPLITK","DENSE_NB",
+                        "SCHUR2D_LEAF_SPAN","SCHUR2D_LEAF_LU","SCHUR2D_PROBE","SUMMA_HANDSHAKE","DENSE_APPLY_PROFILE","SLAB_FILE",
+                        "SOLVE_SRHS"};
+  for(auto e : envs) P(e, getenv(e) ? std::string(getenv(e)) : std::string("(unset)"));
+
+  ////////////////////////////////////////////////////////////////////////////////////////
+  // The COMPLETE fabric/runtime environment, scanned from environ rather than a curated
+  // list.  2026-08-30: we could not tell from a failing log whether FI_CXI_ATS was set,
+  // because it was not in the list -- and a module or the submitting shell can set
+  // anything.  A log must describe its own run without reference to the job script.
+  ////////////////////////////////////////////////////////////////////////////////////////
+  {
+    extern char **environ;
+    const char *prefixes[] = {"FI_","OFI_","MPICH_","MPIR_","PMI_","CXI_","HSA_","HIP_","ROCR_",
+                              "ROCM_","AMD_","GPU_","NUMA_","OMP_","GOMP_","GRID_","CRAY_","LIBFABRIC"};
+    std::vector<std::string> hits;
+    for(char **e = environ; e && *e; e++){
+      std::string s(*e);
+      for(auto p : prefixes){
+        if( s.compare(0,strlen(p),p)==0 ){
+          if(s.size()>200) s = s.substr(0,197)+"...";     // paths can be enormous
+          hits.push_back(s);
+          break;
+        }
+      }
+    }
+    std::sort(hits.begin(),hits.end());
+    std::cout << GridLogMessage << "PARAM: ---- runtime environment: "<<hits.size()<<" variables set ----"<<std::endl;
+    for(auto &s : hits) std::cout << GridLogMessage << "PARAM: ENV " << s << std::endl;
+    // Variables whose ABSENCE is as informative as their value (defaults bite: an unset
+    // FI_MR_CACHE_MONITOR means libfabric's defective memhooks, see systems/WorkArounds.txt).
+    const char *notable[] = {"FI_MR_CACHE_MONITOR","FI_MR_CACHE_MAX_COUNT","FI_MR_ROCR_CACHE_MONITOR_ENABLED",
+                             "FI_CXI_ATS","FI_CXI_RDZV_THRESHOLD","FI_CXI_DISABLE_HMEM_DEV_REGISTER",
+                             "FI_HMEM_ROCR_USE_DMABUF","MPICH_GPU_SUPPORT_ENABLED","MPICH_OFI_NIC_POLICY",
+                             "MPICH_SMP_SINGLE_COPY_MODE","OMP_NUM_THREADS","GRID_ALLOC_NCACHE_LARGE"};
+    for(auto n : notable) if(!getenv(n))
+      std::cout << GridLogMessage << "PARAM: ENV " << n << " (UNSET - provider/runtime default applies)" << std::endl;
+  }
+}
+
+template <class Field>
+void saveSubspace(std::vector<Field> &subspace, std::string const fname){
+#ifdef HAVE_LIME
+  Grid::emptyUserRecord record;
+  Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
+  SW.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++) SW.writeScidacFieldRecord(subspace[k], record);
+  SW.close();
+#endif
+}
+template <class Field>
+void loadSubspace(std::vector<Field> &subspace, std::string const fname){
+#ifdef HAVE_LIME
+  Grid::emptyUserRecord record;
+  Grid::ScidacReader SR;
+  SR.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++) SR.readScidacFieldRecord(subspace[k], record);
+  SR.close();
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////
+// A = PV^dag M (non-Hermitian)
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class Field>
+class PVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat; Matrix &_PV;
+public:
+  PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV) {};
+  void OpDiag (const Field &in, Field &out) { assert(0); }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) { assert(0); }
+  void OpDirAll  (const Field &in, std::vector<Field> &out){ assert(0); };
+  void Op     (const Field &in, Field &out){ Field tmp(in.Grid()); _Mat.M(in,tmp); _PV.Mdag(tmp,out); }
+  void AdjOp  (const Field &in, Field &out){ Field tmp(in.Grid()); _PV.M(in,tmp); _Mat.Mdag(tmp,out); }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ HermOp(in,out); ComplexD d=innerProduct(in,out); n1=real(d); n2=norm2(out); }
+  void HermOp(const Field &in, Field &out){ Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
+};
+
+//////////////////////////////////////////////////////////////////////
+// ||<v|v> - I||_F over a set of coarse vectors. Small means the raw near null
+// content survived the projection; see GramGuard for where a leak lands.
+//////////////////////////////////////////////////////////////////////
+template<class CoarseField>
+RealD GramDefect(std::vector<CoarseField> &v)
+{
+  RealD s2=0.0;
+  for(int i=0;i<(int)v.size();i++){
+    for(int j=0;j<(int)v.size();j++){
+      ComplexD sij=TensorRemove(innerProduct(v[i],v[j]));
+      ComplexD d=sij-(i==j?ComplexD(1.0):ComplexD(0.0));
+      s2+=real(d)*real(d)+imag(d)*imag(d);
+    }
+  }
+  return std::sqrt(s2);
+}
+
+// On a leak every image collapses to the block unit e_k, the Gram becomes
+// N*I, and the defect lands at (N-1)*sqrt(nbasis) -- orders above the ~0.2
+// of a content preserving projection. Trip well below that so a mis-set
+// threshold costs a log line rather than the run.
+template<class CoarseField>
+void GramGuard(const std::string &name,std::vector<CoarseField> &v,GridBase *grid)
+{
+  RealD defect = GramDefect(v);
+  RealD N      = (RealD)grid->gSites();
+  RealD leak   = (N-1.0)*std::sqrt((RealD)v.size());
+  RealD trip   = std::sqrt(N);
+  std::cout << GridLogMessage << "GUARD: ||<"<<name<<"|"<<name<<"> - I||_F = " << defect
+	    << "   (e_k leak would be " << leak << ", trip at " << trip << ")" << std::endl;
+  GRID_ASSERT( defect < trip );
+}
+
+
+//////////////////////////////////////////////////////////////////////
+// Shifted variants for the smoothers
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class Field>
+class ShiftedPVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat; Matrix &_PV;
+public:
+  RealD shift;
+  ShiftedPVdagMLinearOperator(RealD _shift,Matrix &Mat,Matrix &PV): shift(_shift),_Mat(Mat),_PV(PV){};
+  void OpDiag (const Field &in, Field &out) { assert(0); }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) { assert(0); }
+  void OpDirAll  (const Field &in, std::vector<Field> &out){ assert(0); };
+  void Op     (const Field &in, Field &out){ Field tmp(in.Grid()); _Mat.M(in,tmp); _PV.Mdag(tmp,out); out = out + shift*in; }
+  void AdjOp  (const Field &in, Field &out){ Field tmp(in.Grid()); _PV.M(tmp,out); _Mat.Mdag(in,tmp); out = out + shift*in; }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ assert(0); }
+  void HermOp(const Field &in, Field &out){ Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
+};
+
+template<class Field>
+class ShiftedLinearOperator : public LinearOperatorBase<Field> {
+  LinearOperatorBase<Field> &_Op; RealD shift;
+public:
+  ShiftedLinearOperator(RealD _shift, LinearOperatorBase<Field> &Op) : _Op(Op), shift(_shift) {}
+  void OpDiag  (const Field &in, Field &out) { assert(0); }
+  void OpDir   (const Field &in, Field &out,int dir,int disp) { assert(0); }
+  void OpDirAll (const Field &in, std::vector<Field> &out) { assert(0); }
+  void Op      (const Field &in, Field &out) { _Op.Op(in,out);    out = out + shift*in; }
+  void AdjOp   (const Field &in, Field &out) { _Op.AdjOp(in,out); out = out + shift*in; }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ assert(0); }
+  void HermOp  (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
+};
+
+//////////////////////////////////////////////////////////////////////
+// Power iteration on a (non-Hermitian) operator: the spectral edge the
+// smoother polynomial must not exceed.  Reports
+//   step 0 : |A v|/|v| on a RANDOM unit v -- a one-sample lower bound on
+//            sigma_max(A).  If this and the converged value agree, the
+//            operator is near-normal and the spectral picture (R_m(lambda)
+//            on the spectrum) is trustworthy; if not, the field of values
+//            sets the safe interval and the spectrum understates it.
+//   step k : |A v_k|/|v_k| -> |lambda_max| as v_k -> the dominant
+//            eigenvector; the complex Rayleigh quotient <v,Av> gives its
+//            phase (real => on the axis).  A non-converging oscillation
+//            means a complex-conjugate pair of equal modulus at the top.
+// Uses Op(), not HermOp(): this is the operator the smoother sees.
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+void PowerIteration(const std::string &name, LinearOperatorBase<Field> &Op, GridBase *grid, int iters)
+{
+  GRID_TRACE("PowerIteration");
+  GridParallelRNG RNG(grid); RNG.SeedFixedIntegers(std::vector<int>({7,11,13,17}));
+  Field v(grid), Av(grid);
+  gaussian(RNG,v);
+  RealD nv = std::sqrt(norm2(v)); v = v*(1.0/nv);
+  RealD ratio=0.0, ratio0=0.0; ComplexD rq(0.0);
+  for(int i=0;i<iters;i++){
+    Op.Op(v,Av);
+    RealD nAv = std::sqrt(norm2(Av));
+    rq = innerProduct(v,Av);                 // v is unit
+    ratio = nAv;
+    if ( i==0 ) {
+      ratio0 = ratio;
+      std::cout << GridLogMessage << "PowerIteration " << name << " step 0 (random v): |Av|/|v| = " << ratio
+                << "   [lower bound on sigma_max]" << std::endl;
+    }
+    if ( (i%10==0) || (i==iters-1) )
+      std::cout << GridLogMessage << "PowerIteration " << name << " step " << i << " |Av|/|v| = " << ratio
+                << "  Rayleigh = (" << real(rq) << "," << imag(rq) << ")" << std::endl;
+    v = Av*(1.0/nAv);
+  }
+  std::cout << GridLogMessage << "PowerIteration " << name << " SUMMARY: |lambda_max| ~ " << ratio
+            << "  Rayleigh (" << real(rq) << "," << imag(rq) << ")"
+            << "  phase " << std::atan2(imag(rq),real(rq)) << " rad"
+            << "  step-0 ratio / converged = " << ratio0/ratio
+            << (ratio0/ratio > 1.2 ? "   ** non-normal: sigma_max well above |lambda_max| **" : "   (near-normal)")
+            << std::endl;
+}
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+  ParseEnvironment();
+
+  RealD M5=1.8, b=1.5, c=0.5;
+  const int nbasis=NBASIS;
+  const int nrhs=Nrhs;
+  const int batch=CoarsenBatch;
+
+  Coordinate mpi = GridDefaultMpi();
+  Coordinate fsimd= GridDefaultSimd(Nd,vComplex::Nsimd());
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(lat_size,fsimd,mpi);
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  // Level 1 blocking (default 2^4)
+  Coordinate clatt = lat_size;
+  Coordinate Block({2,2,3,3});         // L1->L2 blocking; banked optimum 2026-08-24 (env BLOCK overrides)
+  if ( getenv("BLOCK") ){ GridCmdOptionIntVector(std::string(getenv("BLOCK")),Block); GRID_ASSERT(Block.size()==4); }
+  for(int d=0;d<4;d++){ GRID_ASSERT(lat_size[d]%Block[d]==0); clatt[d]=lat_size[d]/Block[d]; }
+  std::cout << GridLogMessage << "Block  " << Block  << "  coarse lattice " << clatt << std::endl;
+
+  //////////////////////////////////////////////////////////////////////
+  // The coarse space is unvectorised. The 5D coarse grid is built here
+  // rather than through SpaceTimeGrid so the SIMD layout is ours.
+  //////////////////////////////////////////////////////////////////////
+  Coordinate c5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  Coordinate c5simd({1,1,1,1,1});
+  Coordinate c5mpi ({1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  GridCartesian *Coarse5d = new GridCartesian(c5latt,c5simd,c5mpi);
+
+  // 6D coarse multiRHS grid: rhs is dim 0, undistributed and unvectorised.
+  // No divisibility constraint on nrhs, unlike the deprecated operators.
+  Coordinate cmlatt({nrhs,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  Coordinate cmsimd({1,1,1,1,1,1});
+  Coordinate cmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  GridCartesian *CoarseMrhs = new GridCartesian(cmlatt,cmsimd,cmmpi);
+
+  // 6D coarse grid at the coarsening batch, used only while CoarsenOperator
+  // runs. The matrix elements survive the change back to nrhs.
+  Coordinate cblatt({batch,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  GridCartesian *CoarseBatch = new GridCartesian(cblatt,cmsimd,cmmpi);
+
+  // 6D fine grid carrying the coarsening batch: fine SIMD layout preserved
+  Coordinate fmlatt({batch,Ls,lat_size[0],lat_size[1],lat_size[2],lat_size[3]});
+  Coordinate fmsimd({1,1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
+  Coordinate fmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  GridCartesian *FineMrhs = new GridCartesian(fmlatt,fmsimd,fmmpi);
+
+  std::cout << GridLogMessage << "Nsimd  fine " << FGrid->Nsimd()
+	    << "   coarse " << Coarse5d->Nsimd() << std::endl;
+
+  GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
+  GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
+
+  //////////////////////////////////////////////////////////////////////
+  // Gauge field
+  //////////////////////////////////////////////////////////////////////
+  LatticeGaugeField Umu(UGrid);
+  if ( getenv("HOT_START") ) {
+    std::cout << GridLogMessage << "Hot start gauge field" << std::endl;
+    SU<Nc>::HotConfiguration(RNG4,Umu);
+  } else {
+    std::string file("/ccs/home/poare/ckpoint_lat.1000");
+    if ( getenv("CONFIG") ) file = std::string(getenv("CONFIG"));
+    std::cout << GridLogMessage << "Reading gauge field " << file << std::endl;
+    FieldMetaData header;
+    NerscIO::readConfiguration(Umu,header,file);
+  }
+
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
+  MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
+
+  // PVdagM and ShiftedPVdagM are thin wrappers over these same two objects,
+  // so this one callback controls every fine halo in the program.  Default
+  // EXACT; the preconditioner and the coarsening turn sloppiness on for
+  // their own scope only (policy note at FineSloppyComms).
+  SetFineSloppy = [&Ddwf,&Dpv](int sloppy){
+    Ddwf.SloppyComms(sloppy);
+    Dpv .SloppyComms(sloppy);
+  };
+  SetFineSloppy(0);
+  std::cout << GridLogMessage << "Fine halo policy: preconditioner+coarsening "
+            << (FineSloppyComms ? "SLOPPY (fp32 wire)" : "exact")
+            << ", outer Krylov EXACT" << std::endl;
+
+  typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
+  typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
+  PVdagM_t PVdagM(Ddwf,Dpv);
+
+  //////////////////////////////////////////////////////////////////////
+  // Level 1 types: unvectorised coarse scalar
+  //////////////////////////////////////////////////////////////////////
+  typedef sTComplexD                                                          CComplexS;
+  typedef MultiGeneralCoarsenedOperator<vSpinColourVector,CComplexS,nbasis> CoarseOperator;
+  typedef CoarseOperator::CoarseVector                                        CoarseVector;
+  typedef Aggregation<vSpinColourVector,CComplexS,nbasis>                     Subspace;
+
+  NextToNearestStencilGeometry5D geom(Coarse5d);
+
+  //////////////////////////////////////////////////////////////////////
+  // Subspace: load RAW (no Orthogonalise!), or generate.
+  //
+  // The Aggregation is scaffolding for CreateSubspaceGCR only. That runs
+  // entirely on the fine grid and ends in GlobalOrthonormalise, which is a
+  // whole-lattice Gram-Schmidt, so the coarse grid it holds is never
+  // dereferenced and may be the unvectorised one.
+  //////////////////////////////////////////////////////////////////////
+  std::string subspace_file = "subspace_nb" + std::to_string(nbasis) + ".scidac";
+  if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
+  uint64_t file_exists=0;
+  if ( UGrid->IsBoss() ){ std::ifstream f(subspace_file); file_exists=f.good()?1:0; }
+  UGrid->GlobalSum(file_exists);
+
+  const int cb=0;
+  Subspace AggregatesGCR(Coarse5d,FGrid,cb);
+  if ( file_exists ){
+    std::cout << GridLogMessage << "*** Loading subspace from disk (kept RAW) ***" << std::endl;
+    loadSubspace(AggregatesGCR.subspace, subspace_file);
+  } else {
+    std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
+    AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
+    saveSubspace(AggregatesGCR.subspace, subspace_file);
+  }
+
+  // RAW copy BEFORE CoarsenOperator block-orthonormalises in place.
+  std::vector<LatticeFermionD> rawNull(nbasis,FGrid);
+  for(int k=0;k<nbasis;k++) rawNull[k]=AggregatesGCR.subspace[k];
+
+  //////////////////////////////////////////////////////////////////////
+  // L1 coarsening. The fine operator is single RHS, so it is promoted to
+  // the 6D batch grid; a natively multiRHS fine operator would substitute
+  // here with no other change.
+  //////////////////////////////////////////////////////////////////////
+  CoarseOperator CoarseOpPV(geom,Coarse5d);
+  CoarseOpPV.SetGrid(CoarseBatch);
+
+  std::cout << GridLogMessage << "*** L1 CoarsenOperator, batch "<<batch<<" ***" << std::endl;
+  SetFineSloppy(FineSloppyComms);   // coarsening builds the PRECONDITIONER
+  if ( getenv("MRHS_COARSEN") ) {
+    // Promote the single RHS operator and pack the batch: costs an
+    // ExtractSlice/InsertSlice pair per rhs. Here for the A/B only; this is
+    // the path a natively multiRHS fine operator would take.
+    MrhsPromotedOperator<LatticeFermionD> MrhsPVdagM(PVdagM,FGrid,batch);
+    CoarseOpPV.CoarsenOperator(MrhsPVdagM,FineMrhs,AggregatesGCR.subspace,Coarse5d);
+  } else {
+    // PVdagM is single RHS: apply it directly, batch on the coarse side.
+    CoarseOpPV.CoarsenOperator(PVdagM,AggregatesGCR.subspace,Coarse5d,batch);
+  }
+  SetFineSloppy(0);
+
+  // (moved here 2026-08-28: it needs AggregatesGCR.subspace, which is freed right after
+  //  the projector imports it below -- the 60 fine vectors are 20 GB of host memory per
+  //  rank and 60 LRU-eligible device fields competing with the solver's working set)
+  //////////////////////////////////////////////////////////////////////
+  // Optional cross check of the coarse matrix elements against the
+  // deprecated path, which needs a vectorised coarse space. Block Gram-Schmidt
+  // is idempotent, so the deprecated path may re-orthonormalise the same vectors in place
+  // without a second copy of the subspace.
+  //////////////////////////////////////////////////////////////////////
+  if ( getenv("DEPRECATED_CHECK") ) {
+
+    typedef DeprecatedGeneralCoarsenedMatrix     <vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+    typedef DeprecatedMultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
+    typedef Aggregation                <vSpinColourVector,vTComplex,nbasis> SubspaceV;
+
+    Coordinate v5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    Coordinate v5simd({1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
+    GridCartesian *Coarse5dV = new GridCartesian(v5latt,v5simd,c5mpi);
+
+    int nrhs_dep = vComplex::Nsimd();
+    Coordinate vmlatt({nrhs_dep,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    Coordinate vmsimd({vComplex::Nsimd(),1,1,1,1,1});
+    GridCartesian *CoarseMrhsV = new GridCartesian(vmlatt,vmsimd,cmmpi);
+
+    NextToNearestStencilGeometry5D geomV(Coarse5dV);
+
+    SubspaceV AggV(Coarse5dV,FGrid,cb);
+    for(int k=0;k<nbasis;k++) AggV.subspace[k]=AggregatesGCR.subspace[k];
+
+    LittleDiracOperator LittleDiracOpPV(geomV,FGrid,Coarse5dV);
+    std::cout << GridLogMessage << "*** deprecated CoarsenOperator (cross check) ***" << std::endl;
+    SetFineSloppy(FineSloppyComms);
+    LittleDiracOpPV.CoarsenOperator(PVdagM,AggV);
+    SetFineSloppy(0);
+
+    MrhsLittleDiracOperator mrhsDep(geomV,CoarseMrhsV);
+    mrhsDep.CopyMatrix(LittleDiracOpPV);
+
+    // BLAS_A is written by GridtoBLAS in lSite order and both sides carry
+    // the same scalar_object, so the two are directly comparable.
+    typedef MrhsLittleDiracOperator::calcMatrix calcMatrix;
+    int npoint = geom.npoint;
+    RealD num=0.0, den=0.0;
+    // The operator holds ONE site-major matrix buffer; MatrixPointOut hands back
+    // stencil point p in the same lSite order as the deprecated per-point
+    // array, and sizes a2.
+    deviceVector<calcMatrix> a2;
+    for(int p=0;p<npoint;p++){
+      int64_t sites = mrhsDep.BLAS_A[p].size();
+      CoarseOpPV.MatrixPointOut(p,a2);
+      GRID_ASSERT(sites == (int64_t)a2.size());
+      std::vector<calcMatrix> h1(sites),h2(sites);
+      acceleratorCopyFromDevice(&mrhsDep.BLAS_A[p][0],&h1[0],sites*sizeof(calcMatrix));
+      acceleratorCopyFromDevice(&a2[0],              &h2[0],sites*sizeof(calcMatrix));
+      ComplexD *w1=(ComplexD *)&h1[0];
+      ComplexD *w2=(ComplexD *)&h2[0];
+      int64_t words = sites*sizeof(calcMatrix)/sizeof(ComplexD);
+      for(int64_t i=0;i<words;i++){
+        ComplexD d=w1[i]-w2[i];
+        num += real(d)*real(d)+imag(d)*imag(d);
+        den += real(w1[i])*real(w1[i])+imag(w1[i])*imag(w1[i]);
+      }
+    }
+    std::cout << GridLogMessage << "DEPRECATED_CHECK: |A_dep|^2 = " << den << std::endl;
+    std::cout << GridLogMessage << "DEPRECATED_CHECK: |A_dep - A|^2 / |A_dep|^2 = " << num/den << std::endl;
+    GRID_ASSERT( den > 0.0 );
+    GRID_ASSERT( num/den < 1.0e-18 );
+  }
+
+  // Stay on the batch grid: the L2 coarsening drives this operator at the
+  // batch. It is switched to the solve Nrhs once L2 is built.
+
+  //////////////////////////////////////////////////////////////////////
+  // psi_coarse = P^dag (RAW fine null) -> Galerkin images that carry the
+  // near null content, and are free: A_c (P psi) = P A psi.
+  //////////////////////////////////////////////////////////////////////
+  MultiRHSBlockProject<LatticeFermionD> MrhsProjector;
+  MrhsProjector.Allocate(nbasis,FGrid,Coarse5d);
+  MrhsProjector.ImportBasis(AggregatesGCR.subspace);   // block orthonormal basis
+  // The basis now lives in the projector's BLAS_V (device); nothing after this
+  // point reads AggregatesGCR.subspace (DEPRECATED_CHECK moved above).  Free it: 60 fine
+  // fields = 20 GB host per rank, and 60 LRU-eligible device copies.
+  AggregatesGCR.subspace.clear(); AggregatesGCR.subspace.shrink_to_fit();
+
+  std::vector<CoarseVector> psi_coarse(nbasis,Coarse5d);
+  MrhsProjector.blockProject(rawNull,psi_coarse);      // RAW vectors in
+  rawNull.clear(); rawNull.shrink_to_fit();
+
+  GramGuard("psi_coarse",psi_coarse,Coarse5d);
+
+  //////////////////////////////////////////////////////////////////////
+  // STAGE TWO: L2 -> L3.
+  //
+  // The fine operator here is the L1 operator, which is natively multiRHS, so the
+  // multiRHS driver applies with no promotion adapter: its D+1 grid IS the
+  // batch grid the L1 operator is currently set to.
+  //////////////////////////////////////////////////////////////////////
+  Coordinate cclatt = clatt;
+  Coordinate Block2({4,4,2,4});        // L2->L3 blocking; banked optimum 2026-08-24 (env BLOCK2 overrides)
+  if ( getenv("BLOCK2") ){ GridCmdOptionIntVector(std::string(getenv("BLOCK2")),Block2); GRID_ASSERT(Block2.size()==4); }
+  for(int d=0;d<4;d++){ GRID_ASSERT(clatt[d]%Block2[d]==0); cclatt[d]=clatt[d]/Block2[d]; }
+  std::cout << GridLogMessage << "Block2 " << Block2 << "  coarse-coarse lattice " << cclatt << std::endl;
+
+  Coordinate cc5latt({1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+  GridCartesian *CoarseCoarse5d = new GridCartesian(cc5latt,c5simd,c5mpi);
+
+  Coordinate ccmlatt({nrhs,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+  GridCartesian *CoarseCoarseMrhs = new GridCartesian(ccmlatt,cmsimd,cmmpi);
+
+  Coordinate ccblatt({batch,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+  GridCartesian *CoarseCoarseBatch = new GridCartesian(ccblatt,cmsimd,cmmpi);
+
+  // Coarsening deepens the tensor nest by one iScalar
+  typedef CoarseVector::vector_object                                         CoarseSiteObj;
+  typedef iScalar<CComplexS>                                                  CComplexS2;
+  typedef MultiGeneralCoarsenedOperator<CoarseSiteObj,CComplexS2,nbasis>    CoarseCoarseOperator;
+  typedef CoarseCoarseOperator::CoarseVector                                  CoarseCoarseVector;
+
+  NextToNearestStencilGeometry5D geom2(CoarseCoarse5d);
+
+  // RAW copy of the coarse null vectors, for the same reason as rawNull:
+  // the L2 CoarsenOperator block-orthonormalises its subspace in place, and
+  // the L3 basis must be defined by the vectors that still carry content.
+  std::vector<CoarseVector> rawPsi(nbasis,Coarse5d);
+  for(int k=0;k<nbasis;k++) rawPsi[k]=psi_coarse[k];
+
+  CoarseCoarseOperator CoarseOpL2(geom2,CoarseCoarse5d);
+  CoarseOpL2.SetGrid(CoarseCoarseBatch);
+
+  NonHermitianLinearOperator<CoarseOperator,CoarseVector> LinOpCoarse(CoarseOpPV);
+
+  std::cout << GridLogMessage << "*** L2 CoarsenOperator, batch "<<batch<<" ***" << std::endl;
+  CoarseOpL2.CoarsenOperator(LinOpCoarse,CoarseBatch,psi_coarse,CoarseCoarse5d);
+
+  //////////////////////////////////////////////////////////////////////
+  // Both operators to the solve Nrhs. The matrix elements are Nrhs
+  // independent and survive the change.
+  //////////////////////////////////////////////////////////////////////
+  CoarseOpPV.SetGrid(CoarseMrhs);
+  CoarseOpL2.SetGrid(CoarseCoarseMrhs);
+  std::cout << GridLogMessage << "L1 operator at Nrhs " << CoarseOpPV.Nrhs()
+	    << ",  L2 operator at Nrhs " << CoarseOpL2.Nrhs() << std::endl;
+
+  //////////////////////////////////////////////////////////////////////
+  // psi_cc from the RAW coarse null vectors, and the same guard
+  //////////////////////////////////////////////////////////////////////
+  MultiRHSBlockProject<CoarseVector> MrhsProjectorL2;
+  MrhsProjectorL2.Allocate(nbasis,Coarse5d,CoarseCoarse5d);
+  MrhsProjectorL2.ImportBasis(psi_coarse);            // block orthonormal basis
+
+  {
+    std::vector<CoarseCoarseVector> psi_cc(nbasis,CoarseCoarse5d);
+    MrhsProjectorL2.blockProject(rawPsi,psi_cc);      // RAW vectors in
+
+    GramGuard("psi_cc",psi_cc,CoarseCoarse5d);
+  }
+  rawPsi.clear(); rawPsi.shrink_to_fit();
+
+  //////////////////////////////////////////////////////////////////////
+  // Both coarse operators apply on their solve grids
+  //////////////////////////////////////////////////////////////////////
+  {
+    GridParallelRNG cRNG(Coarse5d); cRNG.SeedFixedIntegers({3,4,5,6});
+    CoarseVector cin(CoarseMrhs), cout_(CoarseMrhs);
+    random(cRNG,cin);
+    CoarseOpPV.M(cin,cout_);
+    std::cout << GridLogMessage << "L1 apply |in|^2 = " << norm2(cin)
+	      << "   |M in|^2 = " << norm2(cout_) << std::endl;
+    GRID_ASSERT( norm2(cout_) > 0.0 );
+
+    GridParallelRNG ccRNG(CoarseCoarse5d); ccRNG.SeedFixedIntegers({7,8,9,10});
+    CoarseCoarseVector ccin(CoarseCoarseMrhs), ccout(CoarseCoarseMrhs);
+    random(ccRNG,ccin);
+    CoarseOpL2.M(ccin,ccout);
+    std::cout << GridLogMessage << "L2 apply |in|^2 = " << norm2(ccin)
+	      << "   |M in|^2 = " << norm2(ccout) << std::endl;
+    GRID_ASSERT( norm2(ccout) > 0.0 );
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // STAGE THREE (part one): the dense bottom on L2.
+  //
+  // DenseCoarseMatrix is bilingual: it takes the elements through
+  // Geometry()/ExtractMatrix(), so the L2 operator serves directly. It does
+  // detect that a multiRHS op cannot apply on the D dimensional grid and
+  // skips its own certificate and VERIFY, so the equivalent check is done
+  // here instead, driving the L2 operator at Nrhs 1 through a slice.
+  //////////////////////////////////////////////////////////////////////
+  typedef DenseCoarseMatrix<CComplexS2,nbasis> DenseCC_t;
+  std::unique_ptr<DenseCC_t> DenseCC;
+
+  if ( getenv("DENSE_CC")==nullptr || atoi(getenv("DENSE_CC")) ) {
+
+    std::cout << GridLogMessage << "*** L3 dense bottom: import from the L2 operator ***" << std::endl;
+    DenseCC.reset(new DenseCC_t(CoarseCoarse5d));
+    DenseCC->Import(CoarseOpL2);
+
+    ////////////////////////////////////////////////////////////////////
+    // ||A Ainv x - x|| / ||x||, the check Import could not run itself
+    ////////////////////////////////////////////////////////////////////
+    Coordinate cc1latt({1,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+    GridCartesian *CoarseCoarseOne = new GridCartesian(cc1latt,cmsimd,cmmpi);
+
+    CoarseOpL2.SetGrid(CoarseCoarseOne);
+
+    CoarseCoarseVector x(CoarseCoarse5d),y(CoarseCoarse5d),z(CoarseCoarse5d);
+    GridParallelRNG dRNG(CoarseCoarse5d); dRNG.SeedFixedIntegers({11,12,13,14});
+    random(dRNG,x);
+
+    (*DenseCC)(x,y);                    // y = Ainv x
+
+    CoarseCoarseVector y1(CoarseCoarseOne),z1(CoarseCoarseOne);
+    InsertSliceFast(y,y1,0,0);
+    CoarseOpL2.M(y1,z1);                // z = A y
+    ExtractSliceFast(z,z1,0,0);
+
+    z = z - x;
+    RealD rel = std::sqrt(norm2(z)/norm2(x));
+    std::cout << GridLogMessage << "L3 dense: ||A Ainv x - x||/||x|| = " << rel << std::endl;
+    GRID_ASSERT( rel < 1.0e-2 );
+
+    CoarseOpL2.SetGrid(CoarseCoarseMrhs);
+    delete CoarseCoarseOne;
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // STAGE THREE (part two): the solves.
+  //
+  // Both operators are driven from the SAME objects at whatever Nrhs is
+  // asked for -- the matrix elements were built once and survive SetGrid --
+  // so single RHS and multiRHS are the same code path with a different grid.
+  //////////////////////////////////////////////////////////////////////
+  GRID_ASSERT(DenseCC != nullptr);   // the PGCR bottom is not ported yet
+
+  typedef PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD> FineSmoother_t;
+
+  ShiftedPVdagM_t ShiftedPVdagM(FineSmootherShift,Ddwf,Dpv);
+  TrivialPrecon<LatticeFermionD> simple_fine;
+  TrivialPrecon<CoarseVector>    simpleC;
+
+  auto RunSolve = [&](int nr)
+  {
+    std::cout << GridLogMessage << "**********************************************" << std::endl;
+    std::cout << GridLogMessage << " THREE-level solve, Nrhs = " << nr << std::endl;
+    // Device-memory budget BEFORE the solve (2026-08-28: NRHS=6 died in hipMalloc at the
+    // first fine-smoother history allocation -- reported as an asynchronous "memory
+    // access fault" unless AMD_SERIALIZE_KERNEL/COPY made it a clean OOM).  The outer
+    // mRHS GCR holds src, sol, r, Az and OuterMmax x (p,q) fine fields PER RHS; the fine
+    // smoother adds FineSmootherMmax x (p,q) once.  The MemoryManager LRU cap
+    // (--device-mem) must be BELOW what is physically left after the non-LRU allocations
+    // (comms buffers, dense slab, stencil buffers), or the device fills before anything
+    // is evicted.  Print the estimate, the LRU state, and the device's own free count.
+    {
+      uint64_t fieldBytes = (uint64_t)FGrid->lSites()*sizeof(typename LatticeFermionD::scalar_object);
+      double outerGB = (double)nr*(4 + 2*OuterMmax)*fieldBytes/1.0e9;
+      double smthGB  = (double)(2*FineSmootherMmax + 4)*fieldBytes/1.0e9;
+      std::cout << GridLogMessage << "Device budget: fine field " << fieldBytes/1.0e6 << " MB; outer GCR history "
+                << nr << " x (4 + 2 x " << OuterMmax << ") fields = " << outerGB << " GB; fine smoother history + temps ~ "
+                << smthGB << " GB; MemoryManager device LRU " << MemoryManager::DeviceCacheBytes()/1.0e9 << " GB now, cap "
+                << MemoryManager::DeviceMaxBytes/1.0e9 << " GB" << std::endl;
+      // Empty the device LRU: every setup-era Lattice copy (coarse null vectors,
+      // coarsening temporaries) goes back to host, so the solve's working set
+      // starts from a clean device and the cap applies to it alone.
+      MemoryManager::EvictAll();
+      // ...and release the allocation caches' held blocks (setup-era deviceVector
+      // scratch that is "free" to the caller but not to hipMalloc).
+      MemoryManager::DropCache();
+      MemoryManager::PrintBytes();
+      acceleratorMem();
+    }
+    std::cout << GridLogMessage << "**********************************************" << std::endl;
+
+    Coordinate cml({nr,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    Coordinate ccml({nr,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+    GridCartesian *CMrhs  = new GridCartesian(cml, cmsimd,cmmpi);
+    GridCartesian *CCMrhs = new GridCartesian(ccml,cmsimd,cmmpi);
+
+    CoarseOpPV.SetGrid(CMrhs);
+    CoarseOpL2.SetGrid(CCMrhs);
+
+    NonHermitianLinearOperator<CoarseOperator,CoarseVector>            LinOpC (CoarseOpPV);
+    NonHermitianLinearOperator<CoarseCoarseOperator,CoarseCoarseVector> LinOpCC(CoarseOpL2);
+
+    MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector> ccSolve(*DenseCC,nr);
+
+    ShiftedLinearOperator<CoarseVector> ShiftedC(CoarseSmootherShift, LinOpC);
+    if ( PowerIterations > 0 ) {
+      // Spectral edges the smoother polynomials must respect (see
+      // scripts/gcr_polynomial.py: |R_m|>1 beyond the edge = amplification).
+      PowerIteration<CoarseVector>   ("CoarseSmootherOp(shift="+std::to_string(CoarseSmootherShift)+")", ShiftedC, CMrhs, PowerIterations);
+      PowerIteration<CoarseVector>   ("CoarseOp(unshifted)", LinOpC, CMrhs, PowerIterations);
+      PowerIteration<LatticeFermionD>("FineSmootherOp(shift="+std::to_string(FineSmootherShift)+")", ShiftedPVdagM, Ddwf.FermionGrid(), PowerIterations);
+    }
+    PrecGeneralisedConjugateResidualNonHermitian<CoarseVector>
+      CoarseSmootherGCR(0.01,1,ShiftedC,simpleC,CoarseSmootherMmax,CoarseSmootherNstep);
+    CoarseSmootherGCR.Level(2); CoarseSmootherGCR.Name("Csmoother"); CoarseSmootherGCR.SetZeroGuess(1);
+
+    SwitchableSmoother<CoarseVector> CoarseSmootherSlot(CoarseSmootherGCR,"Csmoother GCR");
+    MrhsCoarseThreeLevelPrec<CoarseVector,CoarseCoarseVector>
+      L2to3Precon(LinOpC, CoarseSmootherSlot, MrhsProjectorL2, ccSolve,
+                  Coarse5d, CoarseCoarse5d, CCMrhs, nr);
+
+    PrecGeneralisedConjugateResidualNonHermitian<CoarseVector>
+      L2PGCR(CoarseSolverTol, CoarseSolverOrder/16, LinOpC, L2to3Precon, CoarseSolverMmax, 16);
+    L2PGCR.Level(2); L2PGCR.Name("Couter"); L2PGCR.SetZeroGuess(1);
+
+    FineSmoother_t SmootherGCR(0.0,1,ShiftedPVdagM,simple_fine,FineSmootherMmax,FineSmootherOrder);
+    SmootherGCR.Level(1); SmootherGCR.Name("Fsmoother"); SmootherGCR.SetZeroGuess(1);
+    SmootherGCR.LogCoefficients(SmootherCoeffLog);
+    CoarseSmootherGCR.LogCoefficients(SmootherCoeffLog);
+
+    SwitchableSmoother<LatticeFermionD> FineSmootherSlot(SmootherGCR,"Fsmoother GCR");
+    MrhsTwoLevelMG<LatticeFermionD,CoarseVector,SwitchableSmoother<LatticeFermionD> >
+      ThreeLevelPrecon(PVdagM, FineSmootherSlot, MrhsProjector, L2PGCR, Coarse5d, CMrhs);
+    ThreeLevelPrecon.SetSloppy   = SetFineSloppy;
+    ThreeLevelPrecon.SloppyComms = FineSloppyComms;
+
+    MrhsPGCRNonHermitian<LatticeFermionD>
+      L1PGCR(OuterTol,1000,PVdagM,ThreeLevelPrecon,OuterMmax,OuterNstep);
+    L1PGCR.Level(1); L1PGCR.Name("Fouter"); L1PGCR.SetZeroGuess(1);
+
+    //////////////////////////////////////////////////////////////////////
+    // Smoother modes.  Objects live for the duration of this RunSolve.
+    //////////////////////////////////////////////////////////////////////
+    std::unique_ptr<ChebyshevNonHermitianSmoother<LatticeFermionD> > FineCheb;
+    std::unique_ptr<ChebyshevNonHermitianSmoother<CoarseVector> >    CoarseCheb;
+    std::unique_ptr<GCRReplaySmoother<LatticeFermionD> > FineReplay;
+    std::unique_ptr<GCRReplaySmoother<CoarseVector> >    CoarseReplay;
+    GCRCoefficients recF, recC;
+    {
+      GCRCoefficients::Select sel = GCRCoefficients::Last;
+      if ( PolyRecordSelect=="first" ) sel = GCRCoefficients::First;
+      if ( PolyRecordSelect=="mean" )  sel = GCRCoefficients::Mean;
+      recF.select = sel; recC.select = sel;
+    }
+    if ( FineSmootherMode == "cheb" ) {
+      FineCheb.reset(new ChebyshevNonHermitianSmoother<LatticeFermionD>(FineChebLo,FineChebHi,FineSmootherOrder,ShiftedPVdagM));
+      FineCheb->Verbose = PolyVerbose; FineCheb->name = "Fsmoother";
+      FineSmootherSlot.Set(*FineCheb,"Fsmoother Chebyshev");
+    }
+    if ( CoarseSmootherMode == "cheb" ) {
+      CoarseCheb.reset(new ChebyshevNonHermitianSmoother<CoarseVector>(CoarseChebLo,CoarseChebHi,CoarseSmootherNstep,ShiftedC));
+      CoarseCheb->Verbose = PolyVerbose; CoarseCheb->name = "Csmoother";
+      CoarseSmootherSlot.Set(*CoarseCheb,"Csmoother Chebyshev");
+    }
+    // Recording window [PolyRecordStart, PolyRecordStart+PolyRecordIters).
+    // M3 (2026-08-26): the GCR polynomial changes fast over the first outer
+    // steps (per-call |r|/|r0| 0.0034 -> 0.017 over steps 1-4) and the MEAN
+    // of those is a poor smoother (replay 0.033-0.049 per call); record a
+    // settled window instead.
+    if ( PolyRecordStart == 0 ) {
+      if ( FineSmootherMode   == "replay" ) SmootherGCR.SetCoefficientRecorder(&recF);
+      if ( CoarseSmootherMode == "replay" ) CoarseSmootherGCR.SetCoefficientRecorder(&recC);
+    }
+    // Record -> replay, with optional periodic re-recording ("re-record, not
+    // fade away": HDCG refreshed its polynomial every 10 steps, tracking the
+    // evolving spectral content of the residual).  Schedule on outer steps:
+    //   [PolyRecordStart, +PolyRecordIters)          adaptive GCR, recording
+    //   then replay of the selected recorded call;
+    //   if PolyRefresh>0: every PolyRefresh steps, ONE adaptive recording
+    //   step, then replay of that call.
+    auto BuildReplays = [&](void){
+      if ( FineSmootherMode == "replay" ) {
+        SmootherGCR.SetCoefficientRecorder(nullptr);
+        recF.Flush(); recF.Report("Fsmoother");
+        FineReplay.reset(new GCRReplaySmoother<LatticeFermionD>(ShiftedPVdagM,recF));
+        FineReplay->Verbose = PolyVerbose; FineReplay->name = "Fsmoother";
+        FineSmootherSlot.Set(*FineReplay,"Fsmoother replay");
+        SmootherGCR.ReleaseHistory();
+      }
+      if ( CoarseSmootherMode == "replay" ) {
+        CoarseSmootherGCR.SetCoefficientRecorder(nullptr);
+        recC.Flush(); recC.Report("Csmoother");
+        CoarseReplay.reset(new GCRReplaySmoother<CoarseVector>(ShiftedC,recC));
+        CoarseReplay->Verbose = PolyVerbose; CoarseReplay->name = "Csmoother";
+        CoarseSmootherSlot.Set(*CoarseReplay,"Csmoother replay");
+        CoarseSmootherGCR.ReleaseHistory();
+      }
+    };
+    auto StartRecording = [&](int step){
+      if ( FineSmootherMode   == "replay" ) { { auto sel=recF.select; recF = GCRCoefficients(); recF.select=sel; } SmootherGCR.SetCoefficientRecorder(&recF);       FineSmootherSlot.Set(SmootherGCR,"Fsmoother GCR (recording)"); }
+      if ( CoarseSmootherMode == "replay" ) { { auto sel=recC.select; recC = GCRCoefficients(); recC.select=sel; } CoarseSmootherGCR.SetCoefficientRecorder(&recC); CoarseSmootherSlot.Set(CoarseSmootherGCR,"Csmoother GCR (recording)"); }
+      std::cout << GridLogMessage << "Smoother coefficient recording starts at outer step " << step << std::endl;
+    };
+    int switchStep = PolyRecordStart + PolyRecordIters;
+    L1PGCR.OnStep = [&](int step){
+      if ( FineSmootherMode != "replay" && CoarseSmootherMode != "replay" ) return;
+      if ( step == PolyRecordStart && PolyRecordStart > 0 ) StartRecording(step);
+      if ( step == switchStep ) { BuildReplays(); return; }
+      if ( PolyRefresh > 0 && step > switchStep ) {
+        int since = step - switchStep;
+        if ( since % PolyRefresh == 0 )     { StartRecording(step); }       // one adaptive, recorded step
+        if ( since % PolyRefresh == 1 )     { BuildReplays(); }             // then replay it
+      }
+    };
+    std::cout << GridLogMessage << "Smoother modes: fine " << FineSmootherMode << "  coarse " << CoarseSmootherMode
+              << (FineSmootherMode=="replay"||CoarseSmootherMode=="replay" ? "  (record outer steps "+std::to_string(PolyRecordStart)+".."+std::to_string(PolyRecordStart+PolyRecordIters)+")" : "")
+              << std::endl;
+
+    std::vector<LatticeFermionD> src(nr,FGrid), sol(nr,FGrid);
+    for(int r=0;r<nr;r++){ gaussian(RNG5,src[r]); sol[r]=Zero(); }
+
+    GridStopWatch w; w.Start();
+    L1PGCR(src,sol);
+    w.Stop();
+    std::cout << GridLogMessage << "3-level solve Nrhs "<<nr<<" total " << w.Elapsed()
+              << "  (per RHS: " << w.useconds()/1.0e6/nr << " s)" << std::endl;
+
+    // The outer operator is exact by policy; assert the state rather than
+    // trust it -- a preconditioner that failed to restore would surface here.
+    SetFineSloppy(0);
+    { LatticeFermionD Ax(FGrid); RealD worst=0.0;
+      for(int r=0;r<nr;r++){ PVdagM.Op(sol[r],Ax); Ax=Ax-src[r];
+        RealD rn=std::sqrt(norm2(Ax)/norm2(src[r]));
+        std::cout << GridLogMessage << "FINAL Nrhs "<<nr<<": rhs["<<r<<"] true residual = " << rn << std::endl;
+        worst=std::max(worst,rn); }
+      std::cout << GridLogMessage << "FINAL Nrhs "<<nr<<": worst-case residual = " << worst
+                << "   (exact-halo verification)" << std::endl;
+    }
+
+
+    // The operators borrow these grids and build a PaddedCell on them, so
+    // they must let go before the grids are destroyed.
+    CoarseOpPV.ReleaseGrid();
+    CoarseOpL2.ReleaseGrid();
+    delete CMrhs; delete CCMrhs;
+  };
+
+  RunSolve(nrhs);
+  if ( getenv("SOLVE_SRHS")==nullptr || atoi(getenv("SOLVE_SRHS")) ) RunSolve(1);
+
+  std::cout << GridLogMessage << "*** stage three complete: solves done ***" << std::endl;
+
+  Grid_finalize();
+}

examples/Example_pvdagm_3level_FourierSmoother.cc

@@ -2,7 +2,7 @@
 
     Grid physics library, www.github.com/paboyle/Grid
 
-    Source file: ./examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc
+    Source file: ./examples/Example_pvdagm_3level_FourierSmoother.cc
 
     Copyright (C) 2026
 
@@ -27,7 +27,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 /*  END LEGAL */
 
 //
-// PVdagM three level multigrid on the V2 coarse operator.
+// PVdagM three level multigrid on MultiGeneralCoarsenedOperator.
 //
 // STAGES ONE AND TWO: grids, types, subspace, and the L1 and L2 coarsenings.
 // The dense bottom and the solves are not here yet.
@@ -37,12 +37,13 @@ Author: Peter Boyle <pboyle@bnl.gov>
 //  * The coarse space is UNVECTORISED (sComplexD). The fine space stays
 //    vectorised. MultiRHSBlockProject carries the mixed layout.
 //
-//  * One operator, not two. V1 needed GeneralCoarsenedMatrix to coarsen and
+//  * One operator, not two. The deprecated path needed
-//    MultiGeneralCoarsenedMatrix to apply, bridged by CopyMatrix. V2 does
+//    DeprecatedGeneralCoarsenedMatrix to coarsen and
-//    both, and single versus multiRHS is SetGrid on the same object with the
+//    DeprecatedMultiGeneralCoarsenedMatrix to apply, bridged by CopyMatrix.
-//    matrix elements built once.
+//    This operator does both, and single versus multiRHS is SetGrid on the
+//    same object with the matrix elements built once.
 //
-//  * Nrhs is unconstrained. V1 required nrhs % vComplex::Nsimd() == 0 because
+//  * Nrhs is unconstrained. The deprecated path required nrhs % vComplex::Nsimd() == 0 because
 //    its multiRHS grid carried the SIMD in the rhs direction.
 //
 //  * CoarsenOperator takes the subspace vectors, not an Aggregation. It block
@@ -54,14 +55,16 @@ Author: Peter Boyle <pboyle@bnl.gov>
 //    ||<psi|psi> - I||_F guard below is what catches that.
 //
 // Env: LATT LS MASS NBASIS(compile time) NRHS BLOCK BLOCK2 COARSEN_BATCH
-//      HOT_START CONFIG SUBSPACE_FILE V1_CHECK MRHS_COARSEN
+//      HOT_START CONFIG SUBSPACE_FILE DEPRECATED_CHECK MRHS_COARSEN
 //
 
 #include <Grid/Grid.h>
+#include <Grid/qcd/utils/FreeMobius5D_claude.h>
 #include <algorithm>              // std::sort, for the runtime-environment dump in ParseEnvironment
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 #include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
 
 #include <memory>
@@ -110,6 +113,16 @@ int   PolyRefresh          = 0;     // >0: every PolyRefresh outer steps, one ad
 int   PolyVerbose          = 0;     // 1: fixed-polynomial smoothers print |r_m|/|r_0| per call
 RealD FineChebLo   = 3.0,  FineChebHi   = 137.0;   // from the harvested polynomial and PowerIterations edge
 RealD CoarseChebLo = 8.0,  CoarseChebHi = 45.0;
+// Landau frame + Fourier fine smoother (FineSmootherMode "fourier"):
+// the configuration is rotated to the Landau frame at load; the smoother is the
+// free Mobius PVdagM inverse, F(1)^-dag then F(m_s), by 4D FFT (FreeMobius5D).
+// FourierSmootherMass caps the free IR gain (structural at large Ls: the free
+// operator's own e^{-alpha Ls} cap vanishes, so m_s must supply it).
+RealD GaugeFixAlpha       = 0.05;   // stable window 0.02-0.05 (Grid FA normalisation)
+RealD GaugeFixTol         = 1.0e-5; // smoothness saturates here (tol scan 2026-09-01)
+RealD FourierSmootherMass = 0.1;
+RealD SmootherGainCap     = 0.0;    // >0: cap free-block singular values (IR gain limiter); 0 = off
+int   RotateSubspace      = 1;      // rotate a LOADED subspace into the frame (legacy files)
 int   CoarseSmootherNstep  = 2;
 int   CoarseSmootherMmax   = 2;
 RealD CoarseSolverTol      = 0.05;
@@ -161,6 +174,11 @@ void ParseEnvironment(void)
   if(getenv("PolyRecordSelect"))   PolyRecordSelect   = getenv("PolyRecordSelect");
   if(getenv("PolyRefresh"))        PolyRefresh        = atoi(getenv("PolyRefresh"));
   if(getenv("PolyVerbose"))        PolyVerbose        = atoi(getenv("PolyVerbose"));
+  if(getenv("GaugeFixAlpha"))       GaugeFixAlpha       = atof(getenv("GaugeFixAlpha"));
+  if(getenv("GaugeFixTol"))         GaugeFixTol         = atof(getenv("GaugeFixTol"));
+  if(getenv("FourierSmootherMass")) FourierSmootherMass = atof(getenv("FourierSmootherMass"));
+  if(getenv("SmootherGainCap"))     SmootherGainCap     = atof(getenv("SmootherGainCap"));
+  if(getenv("RotateSubspace"))      RotateSubspace      = atoi(getenv("RotateSubspace"));
   if(getenv("FineChebLo"))         FineChebLo         = atof(getenv("FineChebLo"));
   if(getenv("FineChebHi"))         FineChebHi         = atof(getenv("FineChebHi"));
   if(getenv("CoarseChebLo"))       CoarseChebLo       = atof(getenv("CoarseChebLo"));
@@ -203,8 +221,10 @@ void ParseEnvironment(void)
   P("PowerIterations",PowerIterations);       P("PolyRecordIters",PolyRecordIters);       P("PolyRecordStart",PolyRecordStart);
   P("PolyRecordSelect",PolyRecordSelect);     P("PolyRefresh",PolyRefresh);               P("PolyVerbose",PolyVerbose);
   P("SmootherCoeffLog",SmootherCoeffLog);
+  P("GaugeFixAlpha",GaugeFixAlpha);           P("GaugeFixTol",GaugeFixTol);
+  P("FourierSmootherMass",FourierSmootherMass); P("SmootherGainCap",SmootherGainCap); P("RotateSubspace",RotateSubspace);
   // library-side knobs, read straight from the environment as the library will
-  const char *envs[] = {"BLOCK","BLOCK2","SUBSPACE_FILE","CONFIG","HOT_START","MRHS_COARSEN","V1_CHECK",
+  const char *envs[] = {"BLOCK","BLOCK2","SUBSPACE_FILE","CONFIG","HOT_START","MRHS_COARSEN","DEPRECATED_CHECK",
                         "DENSE_CC","DENSE_SCHUR","DENSE_SCHUR2D","DENSE_DEVICE_SUM","DENSE_SPLITK","DENSE_NB",
                         "SCHUR2D_LEAF_SPAN","SCHUR2D_LEAF_LU","SCHUR2D_PROBE","SUMMA_HANDSHAKE","DENSE_APPLY_PROFILE","SLAB_FILE",
                         "SOLVE_SRHS"};
@@ -245,6 +265,88 @@ void ParseEnvironment(void)
   }
 }
 
+// Apply the 4D gauge transformation g to every s slice of a 5D fermion.
+void RotateBasis5D(LatticeFermionD &ferm5,const LatticeColourMatrixD &g)
+{
+  GridBase *FGrid = ferm5.Grid();
+  GridBase *UGrid = g.Grid();
+  int Ls5 = FGrid->_fdimensions[0];
+  LatticeFermionD ferm4((GridCartesian *)UGrid);
+  for(int s=0;s<Ls5;s++){
+    ExtractSlice(ferm4,ferm5,s,0);
+    ferm4 = g*ferm4;
+    InsertSlice(ferm4,ferm5,s,0);
+  }
+}
+
+// Fourier fine smoother: the free Mobius PVdagM inverse in the Landau frame,
+//   (Dpv^dag Dm)^-1_free = F(m_s) . F(1)^-dag .
+// F(1)^-dag has the same phase/FFT structure as F(1) with every momentum block
+// conjugate-transposed; DaggerBlocks() post-processes a built F in place.
+// Cap the singular values of every per-momentum free block inverse at Gmax.
+// The smoother is a hyperparameter, not a physical action, so S(p) is unconstrained
+// by Nielsen-Ninomiya: this bounds the free-IR gain (kills the amplifying band that
+// destabilises a stationary smoother) while leaving the UV -- where the block gain is
+// already < Gmax -- untouched.  One scalar, self-targeting to the IR pole.  The
+// interacting IR is Q's job (the nbasis subspace); the cap only stops the smoother
+// harming it.  <=0 disables.
+void GainCap(FreeMobius5DInverse<WilsonImplD> &F, double Gmax)
+{
+  if ( Gmax <= 0.0 ) return;
+  int n5 = 4*F.Ls;
+  size_t nslot = F.Minv_dev.size()/((size_t)n5*n5);
+  for(size_t slot=0;slot<nslot;slot++){
+    size_t base = slot*(size_t)n5*n5;
+    Eigen::MatrixXcd B(n5,n5);
+    for(int r=0;r<n5;r++) for(int cc=0;cc<n5;cc++){
+      Grid::Complex z = F.Minv_dev[base+(size_t)r*n5+cc];
+      B(r,cc) = std::complex<double>(z.real(),z.imag());
+    }
+    Eigen::JacobiSVD<Eigen::MatrixXcd> svd(B,Eigen::ComputeFullU|Eigen::ComputeFullV);
+    Eigen::VectorXd s = svd.singularValues();
+    for(int i=0;i<s.size();i++) if(s(i)>Gmax) s(i)=Gmax;
+    Eigen::MatrixXcd Bc = svd.matrixU()*s.asDiagonal()*svd.matrixV().adjoint();
+    for(int r=0;r<n5;r++) for(int cc=0;cc<n5;cc++){
+      std::complex<double> z = Bc(r,cc);
+      F.Minv_dev[base+(size_t)r*n5+cc] = Grid::Complex(z.real(),z.imag());
+    }
+  }
+}
+
+void DaggerBlocks(FreeMobius5DInverse<WilsonImplD> &F)
+{
+  int n5 = 4*F.Ls;
+  for(int i=0;i<(int)F.Minv.size();i++) F.Minv[i] = F.Minv[i].adjoint().eval();
+  size_t nslot = F.Minv_dev.size()/((size_t)n5*n5);
+  for(size_t slot=0;slot<nslot;slot++){
+    size_t base = slot*(size_t)n5*n5;
+    for(int r=0;r<n5;r++){
+      for(int cc=r+1;cc<n5;cc++){
+        Grid::Complex a = F.Minv_dev[base+(size_t)r*n5+cc];
+        F.Minv_dev[base+(size_t)r*n5+cc] = conjugate(F.Minv_dev[base+(size_t)cc*n5+r]);
+        F.Minv_dev[base+(size_t)cc*n5+r] = conjugate(a);
+      }
+      F.Minv_dev[base+(size_t)r*n5+r] = conjugate(F.Minv_dev[base+(size_t)r*n5+r]);
+    }
+  }
+}
+
+class FourierPVdagMSmoother : public LinearFunction<LatticeFermionD> {
+public:
+  using LinearFunction<LatticeFermionD>::operator();
+  FreeMobius5DInverse<WilsonImplD> &Fm;     // F(m_s)
+  FreeMobius5DInverse<WilsonImplD> &F1dag;  // F(1) with daggered blocks
+  FourierPVdagMSmoother(FreeMobius5DInverse<WilsonImplD> &_Fm,
+                        FreeMobius5DInverse<WilsonImplD> &_F1dag) : Fm(_Fm), F1dag(_F1dag) {}
+  void operator()(const LatticeFermionD &in,LatticeFermionD &out){
+    // Outer operator A = PV^dag M (PVdagMLinearOperator::Op = _Mat.M then _PV.Mdag).
+    // A^-1 = M^-1 (PV^dag)^-1 = F(m) . F(1)^-dag:  apply F(1)^-dag first, then F(m).
+    LatticeFermionD tmp(in.Grid());
+    F1dag(in,tmp);
+    Fm(tmp,out);
+  }
+};
+
 template <class Field>
 void saveSubspace(std::vector<Field> &subspace, std::string const fname){
 #ifdef HAVE_LIME
@@ -396,220 +498,6 @@ void PowerIteration(const std::string &name, LinearOperatorBase<Field> &Op, Grid
             << std::endl;
 }
 
-//////////////////////////////////////////////////////////////////////
-// Dense L3 solve on the packed D+1 coarse-coarse field
-//////////////////////////////////////////////////////////////////////
-template<class DenseType, class CoarseCoarseField>
-class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
-public:
-  DenseType &_Dense;
-  int _nrhs;
-  MrhsDenseCCSolve(DenseType &D, int nrhs) : _Dense(D), _nrhs(nrhs) {}
-  using LinearFunction<CoarseCoarseField>::operator();
-  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
-    _Dense.ApplyBatch6D(in, out, _nrhs);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  std::function<void(int)> OnStep;      // called with the outer step count after every step (smoother switching)
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("MrhsPGCR::vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),Az(nrhs,grid);   // Az: restart residual scratch only
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    // p[0]=Prec(r), q[0]=A p[0], produced directly in the history slots (no copies)
-    Preconditioner(r,p[0]); vOp(p[0],q[0]); qq[0]=vnorm2(q[0]); cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      if ( OnStep ) OnStep(steps);
-      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      // New direction straight into its history slot: p=Prec(r), q=A p.
-      Preconditioner(r,p[peri_kp]);
-      vOp(p[peri_kp],q[peri_kp]);
-      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      MemoryManager::Snapshot(name+" orthog begin step "+std::to_string(steps));
-      {
-	GRID_TRACE("MrhsPGCR orthog");
-        // Classical Gram-Schmidt: all coefficients against the UN-updated new q
-        // (independent, batchable), then apply.  Complex coefficient: the
-        // operator is non-Hermitian, real(<q_j,Aq>) alone left q's non-orthogonal.
-        // Batched per rhs (one fused kernel + one reduction each), the shared
-        // coefficient summed over rhs on the host, ONE GlobalSumVector.
-        std::vector<ComplexD> bcoef(northog,ComplexD(0.0)), part;
-        for(int rr=0;rr<nrhs;rr++){
-          std::vector<const Field*> qwin(northog);
-          for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0); qwin[back]=&q[peri_back][rr]; }
-          rankInnerProductMulti(part,qwin,q[peri_kp][rr]);
-          for(int back=0;back<northog;back++) bcoef[back]+=part[back];
-        }
-        if(northog) grid->GlobalSumVector(&bcoef[0],northog);
-        for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; bcoef[back]=-bcoef[back]/qq[peri_back]; }
-        for(int rr=0;rr<nrhs;rr++){
-          std::vector<const Field*> qwin(northog), pwin(northog);
-          for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; qwin[back]=&q[peri_back][rr]; pwin[back]=&p[peri_back][rr]; }
-          axpyMulti(p[peri_kp][rr],bcoef,pwin);
-          axpyMulti(q[peri_kp][rr],bcoef,qwin);
-        }
-      }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-      MemoryManager::Snapshot(name+" orthog+vnorm2 end   step "+std::to_string(steps));
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle on the D+1 coarse field
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;
-  LinearFunction<CoarseField>              &_CoarseSmoother;
-  MultiRHSBlockProject<CoarseField>        &_Projector;
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve;
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-    out = in;
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict, through the mixed blockProject: D+1 coarse in, D+1 cc out
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-    _Projector.blockProject(vec1,CCsrc);
-
-    //    CCsol=Zero(); // Is this necessary?
-    _CoarseCoarseSolve(CCsrc,CCsol);
-
-    _Projector.blockPromote(vec1,CCsol);
-    add(out,out,vec1);
-
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-    //    vec2=Zero(); // Zero guess
-    _CoarseSmoother(vec1,vec2);
-    add(out,out,vec2);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    // The whole V-cycle is preconditioner: its fine residuals and the
-    // smoother run with sloppy halos; the caller (the outer Krylov) gets
-    // the exact operator back on exit.
-    GRID_TRACE("MGVcycle");
-    SetFineSloppy(FineSloppyComms);
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid();
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-    { GRID_TRACE("MGFineResidual");
-      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    }
-    // fine vector -> D+1 coarse, via the mixed blockProject
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    { GRID_TRACE("MGProject");
-      _Projector.blockProject(vec1,CsrcMrhs);
-    }
-    CsolMrhs=Zero();
-    { GRID_TRACE("MGCoarseSolve");
-      _CoarseSolve(CsrcMrhs,CsolMrhs);
-    }
-    { GRID_TRACE("MGPromote");
-      _Projector.blockPromote(vec1,CsolMrhs);
-      for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    }
-    { GRID_TRACE("MGFineResidual2");
-      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    }
-    { GRID_TRACE("MGPostSmooth");
-      for(int r=0;r<nrhs;r++){
-	//	vec2[r]=Zero();
-	_PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]);
-      }
-    }
-    SetFineSloppy(0);
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -646,7 +534,7 @@ int main (int argc, char ** argv)
   GridCartesian *Coarse5d = new GridCartesian(c5latt,c5simd,c5mpi);
 
   // 6D coarse multiRHS grid: rhs is dim 0, undistributed and unvectorised.
-  // No divisibility constraint on nrhs, unlike V1.
+  // No divisibility constraint on nrhs, unlike the deprecated operators.
   Coordinate cmlatt({nrhs,1,clatt[0],clatt[1],clatt[2],clatt[3]});
   Coordinate cmsimd({1,1,1,1,1,1});
   Coordinate cmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
@@ -684,6 +572,27 @@ int main (int argc, char ** argv)
     NerscIO::readConfiguration(Umu,header,file);
   }
 
+  //////////////////////////////////////////////////////////////////////
+  // Landau frame.  Fix a copy, project Omega back onto the group, rotate
+  // Umu in place: everything downstream runs in the smooth frame.  The
+  // plaquette is gauge invariant (checked); Omega is retained to rotate a
+  // subspace that was generated in the original frame.
+  //////////////////////////////////////////////////////////////////////
+  LatticeColourMatrixD Omega(UGrid);
+  {
+    LatticeGaugeField Ufix(UGrid); Ufix = Umu;
+    FourierAcceleratedGaugeFixer<PeriodicGimplR>::SteepestDescentGaugeFix(
+        Ufix,Omega,GaugeFixAlpha,10000,GaugeFixTol,GaugeFixTol,true,-1,false);
+    Omega = ProjectOnGroup(Omega);
+    RealD plaq0 = WilsonLoops<PeriodicGimplR>::avgPlaquette(Umu);
+    SU<Nc>::GaugeTransform<PeriodicGimplR>(Umu,Omega);
+    RealD plaq1 = WilsonLoops<PeriodicGimplR>::avgPlaquette(Umu);
+    std::cout << GridLogMessage << "Landau frame: plaquette " << plaq0 << " -> " << plaq1
+              << " (invariant); 1-linkTrace/Nc = "
+              << 1.0-WilsonLoops<PeriodicGimplR>::linkTrace(Umu) << std::endl;
+    GRID_ASSERT(fabs(plaq1-plaq0) < 1.0e-10);
+  }
+
   MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
   MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
 
@@ -708,7 +617,7 @@ int main (int argc, char ** argv)
   // Level 1 types: unvectorised coarse scalar
   //////////////////////////////////////////////////////////////////////
   typedef sTComplexD                                                          CComplexS;
-  typedef MultiGeneralCoarsenedOperatorV2<vSpinColourVector,CComplexS,nbasis> CoarseOperator;
+  typedef MultiGeneralCoarsenedOperator<vSpinColourVector,CComplexS,nbasis> CoarseOperator;
   typedef CoarseOperator::CoarseVector                                        CoarseVector;
   typedef Aggregation<vSpinColourVector,CComplexS,nbasis>                     Subspace;
 
@@ -733,6 +642,12 @@ int main (int argc, char ** argv)
   if ( file_exists ){
     std::cout << GridLogMessage << "*** Loading subspace from disk (kept RAW) ***" << std::endl;
     loadSubspace(AggregatesGCR.subspace, subspace_file);
+    if ( RotateSubspace ) {
+      // Legacy file, generated in the original frame: the near-null space is
+      // gauge covariant, so rotating by Omega gives the exact frame basis.
+      std::cout << GridLogMessage << "*** Rotating loaded subspace into the Landau frame ***" << std::endl;
+      for(int k=0;k<nbasis;k++) RotateBasis5D(AggregatesGCR.subspace[k],Omega);
+    }
   } else {
     std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
     AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
@@ -769,23 +684,23 @@ int main (int argc, char ** argv)
   //  the projector imports it below -- the 60 fine vectors are 20 GB of host memory per
   //  rank and 60 LRU-eligible device fields competing with the solver's working set)
   //////////////////////////////////////////////////////////////////////
-  // Optional cross check of the coarse matrix elements against the V1
+  // Optional cross check of the coarse matrix elements against the
-  // path, which needs a vectorised coarse space. Block Gram-Schmidt is
+  // deprecated path, which needs a vectorised coarse space. Block Gram-Schmidt
-  // idempotent, so V1 may re-orthonormalise the same vectors in place
+  // is idempotent, so the deprecated path may re-orthonormalise the same vectors in place
   // without a second copy of the subspace.
   //////////////////////////////////////////////////////////////////////
-  if ( getenv("V1_CHECK") ) {
+  if ( getenv("DEPRECATED_CHECK") ) {
 
-    typedef GeneralCoarsenedMatrix     <vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+    typedef DeprecatedGeneralCoarsenedMatrix     <vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
-    typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
+    typedef DeprecatedMultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
     typedef Aggregation                <vSpinColourVector,vTComplex,nbasis> SubspaceV;
 
     Coordinate v5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
     Coordinate v5simd({1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
     GridCartesian *Coarse5dV = new GridCartesian(v5latt,v5simd,c5mpi);
 
-    int nrhs_v1 = vComplex::Nsimd();
+    int nrhs_dep = vComplex::Nsimd();
-    Coordinate vmlatt({nrhs_v1,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    Coordinate vmlatt({nrhs_dep,1,clatt[0],clatt[1],clatt[2],clatt[3]});
     Coordinate vmsimd({vComplex::Nsimd(),1,1,1,1,1});
     GridCartesian *CoarseMrhsV = new GridCartesian(vmlatt,vmsimd,cmmpi);
 
@@ -795,25 +710,30 @@ int main (int argc, char ** argv)
     for(int k=0;k<nbasis;k++) AggV.subspace[k]=AggregatesGCR.subspace[k];
 
     LittleDiracOperator LittleDiracOpPV(geomV,FGrid,Coarse5dV);
-    std::cout << GridLogMessage << "*** V1 CoarsenOperator (cross check) ***" << std::endl;
+    std::cout << GridLogMessage << "*** deprecated CoarsenOperator (cross check) ***" << std::endl;
     SetFineSloppy(FineSloppyComms);
     LittleDiracOpPV.CoarsenOperator(PVdagM,AggV);
     SetFineSloppy(0);
 
-    MrhsLittleDiracOperator mrhsV1(geomV,CoarseMrhsV);
+    MrhsLittleDiracOperator mrhsDep(geomV,CoarseMrhsV);
-    mrhsV1.CopyMatrix(LittleDiracOpPV);
+    mrhsDep.CopyMatrix(LittleDiracOpPV);
 
     // BLAS_A is written by GridtoBLAS in lSite order and both sides carry
     // the same scalar_object, so the two are directly comparable.
     typedef MrhsLittleDiracOperator::calcMatrix calcMatrix;
     int npoint = geom.npoint;
     RealD num=0.0, den=0.0;
+    // The operator holds ONE site-major matrix buffer; MatrixPointOut hands back
+    // stencil point p in the same lSite order as the deprecated per-point
+    // array, and sizes a2.
+    deviceVector<calcMatrix> a2;
     for(int p=0;p<npoint;p++){
-      int64_t sites = mrhsV1.BLAS_A[p].size();
+      int64_t sites = mrhsDep.BLAS_A[p].size();
-      GRID_ASSERT(sites == (int64_t)CoarseOpPV.BLAS_A[p].size());
+      CoarseOpPV.MatrixPointOut(p,a2);
+      GRID_ASSERT(sites == (int64_t)a2.size());
       std::vector<calcMatrix> h1(sites),h2(sites);
-      acceleratorCopyFromDevice(&mrhsV1.BLAS_A[p][0],    &h1[0],sites*sizeof(calcMatrix));
+      acceleratorCopyFromDevice(&mrhsDep.BLAS_A[p][0],&h1[0],sites*sizeof(calcMatrix));
-      acceleratorCopyFromDevice(&CoarseOpPV.BLAS_A[p][0],&h2[0],sites*sizeof(calcMatrix));
+      acceleratorCopyFromDevice(&a2[0],              &h2[0],sites*sizeof(calcMatrix));
       ComplexD *w1=(ComplexD *)&h1[0];
       ComplexD *w2=(ComplexD *)&h2[0];
       int64_t words = sites*sizeof(calcMatrix)/sizeof(ComplexD);
@@ -823,8 +743,8 @@ int main (int argc, char ** argv)
         den += real(w1[i])*real(w1[i])+imag(w1[i])*imag(w1[i]);
       }
     }
-    std::cout << GridLogMessage << "V1_CHECK: |A_V1|^2 = " << den << std::endl;
+    std::cout << GridLogMessage << "DEPRECATED_CHECK: |A_dep|^2 = " << den << std::endl;
-    std::cout << GridLogMessage << "V1_CHECK: |A_V1 - A_V2|^2 / |A_V1|^2 = " << num/den << std::endl;
+    std::cout << GridLogMessage << "DEPRECATED_CHECK: |A_dep - A|^2 / |A_dep|^2 = " << num/den << std::endl;
     GRID_ASSERT( den > 0.0 );
     GRID_ASSERT( num/den < 1.0e-18 );
   }
@@ -840,7 +760,7 @@ int main (int argc, char ** argv)
   MrhsProjector.Allocate(nbasis,FGrid,Coarse5d);
   MrhsProjector.ImportBasis(AggregatesGCR.subspace);   // block orthonormal basis
   // The basis now lives in the projector's BLAS_V (device); nothing after this
-  // point reads AggregatesGCR.subspace (V1_CHECK moved above).  Free it: 60 fine
+  // point reads AggregatesGCR.subspace (DEPRECATED_CHECK moved above).  Free it: 60 fine
   // fields = 20 GB host per rank, and 60 LRU-eligible device copies.
   AggregatesGCR.subspace.clear(); AggregatesGCR.subspace.shrink_to_fit();
 
@@ -853,7 +773,7 @@ int main (int argc, char ** argv)
   //////////////////////////////////////////////////////////////////////
   // STAGE TWO: L2 -> L3.
   //
-  // The fine operator here is V2 at L1, which is natively multiRHS, so the
+  // The fine operator here is the L1 operator, which is natively multiRHS, so the
   // multiRHS driver applies with no promotion adapter: its D+1 grid IS the
   // batch grid the L1 operator is currently set to.
   //////////////////////////////////////////////////////////////////////
@@ -875,7 +795,7 @@ int main (int argc, char ** argv)
   // Coarsening deepens the tensor nest by one iScalar
   typedef CoarseVector::vector_object                                         CoarseSiteObj;
   typedef iScalar<CComplexS>                                                  CComplexS2;
-  typedef MultiGeneralCoarsenedOperatorV2<CoarseSiteObj,CComplexS2,nbasis>    CoarseCoarseOperator;
+  typedef MultiGeneralCoarsenedOperator<CoarseSiteObj,CComplexS2,nbasis>    CoarseCoarseOperator;
   typedef CoarseCoarseOperator::CoarseVector                                  CoarseCoarseVector;
 
   NextToNearestStencilGeometry5D geom2(CoarseCoarse5d);
@@ -943,7 +863,7 @@ int main (int argc, char ** argv)
   // STAGE THREE (part one): the dense bottom on L2.
   //
   // DenseCoarseMatrix is bilingual: it takes the elements through
-  // Geometry()/ExtractMatrix(), so the V2 operator serves directly. It does
+  // Geometry()/ExtractMatrix(), so the L2 operator serves directly. It does
   // detect that a multiRHS op cannot apply on the D dimensional grid and
   // skips its own certificate and VERIFY, so the equivalent check is done
   // here instead, driving the L2 operator at Nrhs 1 through a slice.
@@ -953,7 +873,7 @@ int main (int argc, char ** argv)
 
   if ( getenv("DENSE_CC")==nullptr || atoi(getenv("DENSE_CC")) ) {
 
-    std::cout << GridLogMessage << "*** L3 dense bottom: import from the V2 L2 operator ***" << std::endl;
+    std::cout << GridLogMessage << "*** L3 dense bottom: import from the L2 operator ***" << std::endl;
     DenseCC.reset(new DenseCC_t(CoarseCoarse5d));
     DenseCC->Import(CoarseOpL2);
 
@@ -1003,7 +923,7 @@ int main (int argc, char ** argv)
   auto RunSolve = [&](int nr)
   {
     std::cout << GridLogMessage << "**********************************************" << std::endl;
-    std::cout << GridLogMessage << " V2 THREE-level solve, Nrhs = " << nr << std::endl;
+    std::cout << GridLogMessage << " THREE-level solve, Nrhs = " << nr << std::endl;
     // Device-memory budget BEFORE the solve (2026-08-28: NRHS=6 died in hipMalloc at the
     // first fine-smoother history allocation -- reported as an asynchronous "memory
     // access fault" unless AMD_SERIALIZE_KERNEL/COPY made it a clean OOM).  The outer
@@ -1074,6 +994,8 @@ int main (int argc, char ** argv)
     SwitchableSmoother<LatticeFermionD> FineSmootherSlot(SmootherGCR,"Fsmoother GCR");
     MrhsTwoLevelMG<LatticeFermionD,CoarseVector,SwitchableSmoother<LatticeFermionD> >
       ThreeLevelPrecon(PVdagM, FineSmootherSlot, MrhsProjector, L2PGCR, Coarse5d, CMrhs);
+    ThreeLevelPrecon.SetSloppy   = SetFineSloppy;
+    ThreeLevelPrecon.SloppyComms = FineSloppyComms;
 
     MrhsPGCRNonHermitian<LatticeFermionD>
       L1PGCR(OuterTol,1000,PVdagM,ThreeLevelPrecon,OuterMmax,OuterNstep);
@@ -1093,6 +1015,34 @@ int main (int argc, char ** argv)
       if ( PolyRecordSelect=="mean" )  sel = GCRCoefficients::Mean;
       recF.select = sel; recC.select = sel;
     }
+    std::unique_ptr<FreeMobius5DInverse<WilsonImplD> > Fms;
+    std::unique_ptr<FreeMobius5DInverse<WilsonImplD> > F1dag;
+    std::unique_ptr<FourierPVdagMSmoother>             FineFourier;
+    std::unique_ptr<FineSmoother_t>                    FineFourierGCR;   // fgcr: Fourier as the preconditioner of a low-order fine GCR ("2-deep" smoother)
+    if ( FineSmootherMode == "fourier" || FineSmootherMode == "fgcr" ) {
+      // Free PVdagM inverse in the Landau frame (config already rotated).
+      // Block build is (4Ls)^2 per local momentum in memory; Ls=24 at large
+      // local volume is heavy -- fine on the laptop harness.
+      std::vector<Complex> pbc = {1,1,1,1};
+      Fms.reset  (new FreeMobius5DInverse<WilsonImplD>(FGrid,Ls,M5,b,c,FourierSmootherMass,pbc));
+      F1dag.reset(new FreeMobius5DInverse<WilsonImplD>(FGrid,Ls,M5,b,c,1.0,pbc));
+      DaggerBlocks(*F1dag);
+      GainCap(*Fms,  SmootherGainCap);
+      GainCap(*F1dag,SmootherGainCap);
+      FineFourier.reset(new FourierPVdagMSmoother(*Fms,*F1dag));
+      if ( FineSmootherMode == "fourier" ) {
+        FineSmootherSlot.Set(*FineFourier,"Fsmoother Fourier");
+      } else {
+        // fgcr: identical to the "gcr" fine smoother but with the Fourier free-inverse
+        // as the preconditioner instead of simple_fine.  A low-order (FineSmootherOrder)
+        // GCR on ShiftedPVdagM, Fourier-preconditioned -- the outer GCR absorbs the
+        // non-normal transient that makes raw Fourier amplify (cf. (iii): S0 alone rho=13
+        // but S0-as-preconditioner converged FGMRES in 181).  Prints its residuals.
+        FineFourierGCR.reset(new FineSmoother_t(0.0,1,ShiftedPVdagM,*FineFourier,FineSmootherMmax,FineSmootherOrder));
+        FineFourierGCR->Level(1); FineFourierGCR->Name("Fsmoother-FGCR"); FineFourierGCR->SetZeroGuess(1);
+        FineSmootherSlot.Set(*FineFourierGCR,"Fsmoother FourierGCR");
+      }
+    }
     if ( FineSmootherMode == "cheb" ) {
       FineCheb.reset(new ChebyshevNonHermitianSmoother<LatticeFermionD>(FineChebLo,FineChebHi,FineSmootherOrder,ShiftedPVdagM));
       FineCheb->Verbose = PolyVerbose; FineCheb->name = "Fsmoother";
@@ -1163,7 +1113,7 @@ int main (int argc, char ** argv)
     GridStopWatch w; w.Start();
     L1PGCR(src,sol);
     w.Stop();
-    std::cout << GridLogMessage << "V2 3-level solve Nrhs "<<nr<<" total " << w.Elapsed()
+    std::cout << GridLogMessage << "3-level solve Nrhs "<<nr<<" total " << w.Elapsed()
               << "  (per RHS: " << w.useconds()/1.0e6/nr << " s)" << std::endl;
 
     // The outer operator is exact by policy; assert the state rather than

examples/Example_pvdagm_3level_dense.cc

@@ -212,8 +212,8 @@ public:
   }
 };
 
-// Lüscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
+// Luscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
-// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi† src.
+// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi^dag src.
 template<class Field>
 class LuscherGuesser : public LinearFunction<Field> {
   const std::vector<Field> &psi;
@@ -704,7 +704,7 @@ void runMG(
   TrivialPrecon<LatticeFermionD>   simple_fine;
 
   //////////////////////////////////////////////////////////////////////
-  // Level 0→1: coarsen PVdagM, build LinOpCoarse
+  // Level 0->1: coarsen PVdagM, build LinOpCoarse
   //////////////////////////////////////////////////////////////////////
   LittleDiracOperator LittleDiracOpPV(geom, FGrid, Coarse5d);
   LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesPD);
@@ -736,7 +736,7 @@ void runMG(
   // Optional sigma-ordering of psi_coarse (SVD_REORDER set): replace the crude
   // first-NB_CC slice with the NB_CC most-null directions of span(psi_coarse)
   // under LinOpCoarse.  Nullness measure = singular values (eig of the
-  // Gram-whitened Psi†A†APsi), NOT the numerical range Q†AQ which
+  // Gram-whitened Psi^dag A^dag APsi), NOT the numerical range Q^dag AQ which
   // non-normality contaminates.  The printed sigma spectrum shows where the
   // truncation cliff sits.  Unset => raw first-30 (crude GS-ordered slice).
   //////////////////////////////////////////////////////////////////////
@@ -787,7 +787,7 @@ void runMG(
   }
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2: SUPERCOARSE aggregation using psi_coarse as subspace.
+  // Level 1->2: SUPERCOARSE aggregation using psi_coarse as subspace.
   // Maximal block {8,4,3,6}: CC = [3,6,8,8] = the dense-invertible floor.
   //
   // UNBLOCKING TRUNCATION NB_CC = 30 (first-30 slice of psi_coarse):
@@ -811,7 +811,7 @@ void runMG(
 
   typedef typename CoarseVector::vector_object                            CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB_CC>         LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB_CC>         LittleDiracOperatorL2;
   typedef typename LittleDiracOperatorL2::CoarseVector                   CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,NB_CC>                    SubspaceL2;
   typedef MGPreconditioner<CoarseSiteObj,vTTComplex,NB_CC>               L1to2MG;
@@ -830,7 +830,7 @@ void runMG(
 
   //////////////////////////////////////////////////////////////////////
   // CC solve: DENSE (exact, non-iterative) by default; DENSE_CC=0 gives
-  // the previous iterative L3PGCR + Lüscher-guesser path for A/B.
+  // the previous iterative L3PGCR + Luscher-guesser path for A/B.
   //////////////////////////////////////////////////////////////////////
   int use_dense = 1;
   if (getenv("DENSE_CC")) use_dense = atoi(getenv("DENSE_CC"));
@@ -859,7 +859,7 @@ void runMG(
     ccGuess = &simpleCC;      // exact solve ignores/overwrites any guess
   } else {
     ////////////////////////////////////////////////////////////////////
-    // Lüscher deflation guesser (arXiv:0706.2298 A.3) for the iterative CC
+    // Luscher deflation guesser (arXiv:0706.2298 A.3) for the iterative CC
     // solve, as in the earlier supercoarse configuration.
     ////////////////////////////////////////////////////////////////////
     psi_cc.resize(nbasis, CoarseCoarse5d);
@@ -900,7 +900,7 @@ void runMG(
   }
 
   //////////////////////////////////////////////////////////////////////
-  // Coarse-level GCR smoother for Level 1→2 V-cycle.
+  // Coarse-level GCR smoother for Level 1->2 V-cycle.
   //////////////////////////////////////////////////////////////////////
   RealD coarse_smoother_shift = 0.1;
   int   coarse_smoother_nstep = 2;
@@ -914,7 +914,7 @@ void runMG(
   CoarseSmootherGCR.SetZeroGuess(1);  // post-smoother slot: caller zeroes vec2 (NOT L2MGsolver: it takes the Luscher guess)
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2 V-cycle preconditioner.
+  // Level 1->2 V-cycle preconditioner.
   //////////////////////////////////////////////////////////////////////
   L1to2MG L1to2Precon(AggregatesL2,
                        LinOpCoarse,
@@ -998,7 +998,7 @@ int main (int argc, char ** argv)
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
-  // Level 1 coarse grid: block 2^4 from fine (48×48×48×96 → 24×24×24×48, Ls=1)
+  // Level 1 coarse grid: block 2^4 from fine (48x48x48x96 -> 24x24x24x48, Ls=1)
   Coordinate clatt = lat_size;
   //  Coordinate Block1({2,2,2,2});
   //  Coordinate Block2({8,4,3,6});
@@ -1047,7 +1047,7 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
   typedef MGPreconditioner<vSpinColourVector,vTComplex,nbasis>        TwoLevelMG;

examples/Example_pvdagm_3level_madj.cc

@@ -323,7 +323,7 @@ void runMG(
   TrivialPrecon<LatticeFermionD>   simple_fine;
 
   //////////////////////////////////////////////////////////////////////
-  // Level 0→1: coarsen PVdagM, build LinOpCoarse
+  // Level 0->1: coarsen PVdagM, build LinOpCoarse
   //////////////////////////////////////////////////////////////////////
   LittleDiracOperator LittleDiracOpPV(geom, FGrid, Coarse5d);
   LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesPD);
@@ -348,7 +348,7 @@ void runMG(
   //////////////////////////////////////////////////////////////////////
   // psi_coarse: coarse projections of pre-GS fine null vectors.
   // These are the Level 1 near-null vectors, promoted from Level 0.
-  // Used as the aggregation basis for Level 1→2 coarsening.
+  // Used as the aggregation basis for Level 1->2 coarsening.
   //////////////////////////////////////////////////////////////////////
   std::vector<CoarseVector> psi_coarse(nbasis, Coarse5d);
   for (int k = 0; k < nbasis; k++)
@@ -378,22 +378,22 @@ void runMG(
     RealD normC      = C.norm();
     RealD normCmCdag = (C - C.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse null matrix ||C||            = " << normC << std::endl;
-    std::cout << GridLogMessage << "Coarse null matrix ||C - C†||/||C|| = " << normCmCdag/normC << std::endl;
+    std::cout << GridLogMessage << "Coarse null matrix ||C - C^dag||/||C|| = " << normCmCdag/normC << std::endl;
     std::cout << GridLogMessage << "Galerkin check ||C||/||W||          = " << normC/normW << std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2: set up aggregation using psi_coarse as subspace.
+  // Level 1->2: set up aggregation using psi_coarse as subspace.
   // Block factor 2,2,3,2 (removes odd local sublattice in z given MPI
-  // geometry 3×6×4×4 where z-local at Level 1 is 6).
+  // geometry 3x6x4x4 where z-local at Level 1 is 6).
   // psi_coarse are assigned directly; CoarsenOperator performs
   // block-GS orthogonalisation before building LinOpCoarseCoarse.
   //////////////////////////////////////////////////////////////////////
   // innerProduct(CoarseSiteObj, CoarseSiteObj) returns iScalar<vTComplex>, so CComplex
-  // for the L1→L2 level must be iScalar<vTComplex>, not vTComplex.
+  // for the L1->L2 level must be iScalar<vTComplex>, not vTComplex.
   typedef typename CoarseVector::vector_object                            CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
   typedef typename LittleDiracOperatorL2::CoarseVector                   CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,NB>                       SubspaceL2;
   typedef MGPreconditioner<CoarseSiteObj,vTTComplex,NB>                  L1to2MG;
@@ -412,14 +412,14 @@ void runMG(
   // Level 2 solver: plain GCR, no further coarsening
   //////////////////////////////////////////////////////////////////////
   TrivialPrecon<CoarseCoarseVector> simpleCC;
-  // L3PGCR is an inner solver inside the L1→2 V-cycle; does not need to converge
+  // L3PGCR is an inner solver inside the L1->2 V-cycle; does not need to converge
   // to fine-grid precision. Loose tolerance (3e-2) and large restart (64) to allow
   // the Krylov space to span enough of the near-null spectrum of LinOpCC per cycle.
   PrecGeneralisedConjugateResidualNonHermitian<CoarseCoarseVector> L3PGCR(1.0e-4,5,LinOpCC,simpleCC,64,64);
   L3PGCR.Level(3);
 
   //////////////////////////////////////////////////////////////////////
-  // Coarse-level GCR smoother for Level 1→2 V-cycle.
+  // Coarse-level GCR smoother for Level 1->2 V-cycle.
   // Mirrors fine-grid SmootherGCR: shifted operator + fixed step count.
   // coarse_smoother_shift and coarse_smoother_nstep are the tuning knobs.
   //////////////////////////////////////////////////////////////////////
@@ -433,7 +433,7 @@ void runMG(
   CoarseSmootherGCR.Level(2);
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2 V-cycle preconditioner.
+  // Level 1->2 V-cycle preconditioner.
   //////////////////////////////////////////////////////////////////////
   L1to2MG L1to2Precon(AggregatesL2,
 		      LinOpCoarse,
@@ -474,7 +474,7 @@ void runMG(
   f_src = one;
 
   // Pre-smoother: none (TrivialPrecon); post-smoother: shifted PGCR.
-  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1→2 V-cycle).
+  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1->2 V-cycle).
   TwoLevelMG ThreeLevelPrecon(AggregatesPD,
 			      PVdagM,
 			      simple_fine,
@@ -518,7 +518,7 @@ int main (int argc, char ** argv)
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
-  // Level 1 coarse grid: block 2^4 from fine (48×48×48×96 → 24×24×24×48, Ls=1)
+  // Level 1 coarse grid: block 2^4 from fine (48x48x48x96 -> 24x24x24x48, Ls=1)
   Coordinate clatt = lat_size;
   for (int d = 0; d < 4; d++) clatt[d] /= 2;
   std::cout << GridLogMessage << "Level 1 coarse lattice: " << clatt << std::endl;
@@ -526,13 +526,13 @@ int main (int argc, char ** argv)
   GridCartesian *Coarse4d  = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridCartesian *Coarse5d  = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
 
-  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24×24×24×48 → 12×12×8×16, Ls=1).
+  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24x24x24x48 -> 12x12x8x16, Ls=1).
   // MPI geometry 3.6.4.4 (288 ranks): fine local {16,8,12,24}.
   // Level 1 local {8,4,6,12}; Level 2 local {4,2,2,4}.
-  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) → must use 3.
+  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) -> must use 3.
   // t blocked by 3: t-Level1-local=12; 12/3=4 divisible by Nsimd=4 (gen-simd-width=64).
-  //   t-block=2 gives t2-local=6, 6 mod 4 ≠ 0, fails Grid SIMD assertion. ✓
+  //   t-block=2 gives t2-local=6, 6 mod 4 != 0, fails Grid SIMD assertion.
-  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4). ✓
+  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4).
   Coordinate clatt2 = clatt;
   clatt2[0] /= 2;
   clatt2[1] /= 2;
@@ -566,7 +566,7 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
   typedef MGPreconditioner<vSpinColourVector,vTComplex,nbasis>        TwoLevelMG;

examples/Example_pvdagm_4level.cc

@@ -190,8 +190,8 @@ public:
   }
 };
 
-// Lüscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
+// Luscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
-// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi† src.
+// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi^dag src.
 template<class Field>
 class LuscherGuesser : public LinearFunction<Field> {
   const std::vector<Field> &psi;
@@ -343,7 +343,7 @@ void runMG(
   TrivialPrecon<LatticeFermionD>   simple_fine;
 
   //////////////////////////////////////////////////////////////////////
-  // Level 0→1: coarsen PVdagM, build LinOpCoarse
+  // Level 0->1: coarsen PVdagM, build LinOpCoarse
   //////////////////////////////////////////////////////////////////////
   LittleDiracOperator LittleDiracOpPV(geom, FGrid, Coarse5d);
   LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesPD);
@@ -367,7 +367,7 @@ void runMG(
   //////////////////////////////////////////////////////////////////////
   // psi_coarse: coarse projections of pre-GS fine null vectors.
   // These are the Level 1 near-null vectors, promoted from Level 0.
-  // Used as the aggregation basis for Level 1→2 coarsening.
+  // Used as the aggregation basis for Level 1->2 coarsening.
   //////////////////////////////////////////////////////////////////////
   std::vector<CoarseVector> psi_coarse(nbasis, Coarse5d);
   for (int k = 0; k < nbasis; k++)
@@ -397,22 +397,22 @@ void runMG(
     RealD normC      = C.norm();
     RealD normCmCdag = (C - C.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse null matrix ||C||            = " << normC << std::endl;
-    std::cout << GridLogMessage << "Coarse null matrix ||C - C†||/||C|| = " << normCmCdag/normC << std::endl;
+    std::cout << GridLogMessage << "Coarse null matrix ||C - C^dag||/||C|| = " << normCmCdag/normC << std::endl;
     std::cout << GridLogMessage << "Galerkin check ||C||/||W||          = " << normC/normW << std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2: set up aggregation using psi_coarse as subspace.
+  // Level 1->2: set up aggregation using psi_coarse as subspace.
   // Block factor 2,2,3,2 (removes odd local sublattice in z given MPI
-  // geometry 3×6×4×4 where z-local at Level 1 is 6).
+  // geometry 3x6x4x4 where z-local at Level 1 is 6).
   // psi_coarse are assigned directly; CoarsenOperator performs
   // block-GS orthogonalisation before building LinOpCoarseCoarse.
   //////////////////////////////////////////////////////////////////////
   // innerProduct(CoarseSiteObj, CoarseSiteObj) returns iScalar<vTComplex>, so CComplex
-  // for the L1→L2 level must be iScalar<vTComplex>, not vTComplex.
+  // for the L1->L2 level must be iScalar<vTComplex>, not vTComplex.
   typedef typename CoarseVector::vector_object                            CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
   typedef typename LittleDiracOperatorL2::CoarseVector                   CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,NB>                       SubspaceL2;
   typedef MGPreconditioner<CoarseSiteObj,vTTComplex,NB>                  L1to2MG;
@@ -430,12 +430,12 @@ void runMG(
   TrivialPrecon<CoarseCoarseVector> simpleCC;
 
   //////////////////////////////////////////////////////////////////////
-  // Lüscher deflation guesser for L3PGCR.
+  // Luscher deflation guesser for L3PGCR.
   // Step 1: project psi_coarse[k] (promoted fine null vectors) to
-  //         CoarseCoarseVector space — these cover the zero-momentum
+  //         CoarseCoarseVector space -- these cover the zero-momentum
   //         component of the near-null space of LinOpCC.
   // Step 2: breed Nextra additional null vectors directly on LinOpCC
-  //         using GCR with random sources — these pick up near-null
+  //         using GCR with random sources -- these pick up near-null
   //         modes at all spatial frequencies not spanned by step 1.
   // Step 3: build C_{st} = <psi_cc[s]|LinOpCC|psi_cc[t]> over the
   //         full augmented basis and invert directly via Eigen LU.
@@ -477,13 +477,13 @@ void runMG(
     RealD normCcc      = Ccc.norm();
     RealD normCccmCdag = (Ccc - Ccc.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc||              = " << normCcc << std::endl;
-    std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc-Ccc†||/||Ccc|| = " << normCccmCdag/normCcc << std::endl;
+    std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc-Ccc^dag||/||Ccc|| = " << normCccmCdag/normCcc << std::endl;
   }
   Eigen::MatrixXcd Ccc_inv = Ccc.inverse();
   LuscherGuesser<CoarseCoarseVector> CCDeflGuesser(psi_cc, Ccc_inv);
 
   //////////////////////////////////////////////////////////////////////
-  // Level 2→3: coarsen LinOpCC using the RAW promoted psi_cc as aggregation
+  // Level 2->3: coarsen LinOpCC using the RAW promoted psi_cc as aggregation
   // to build the Level 4 (coarse-coarse-coarse) operator.
   //   psi_cc[0..nbasis-1] are the coarse-coarse near-null vectors, projected
   //   from the RAW psi_coarse (themselves projected from the RAW fine null
@@ -492,11 +492,11 @@ void runMG(
   //   so assign COPIES of psi_cc and keep psi_cc itself raw.
   //
   // Tensor depth deepens once more: innerProduct(CoarseCoarseSiteObj,...) returns
-  // iScalar<vTTComplex>, so CComplex for the L2→L3 level is iScalar<iScalar<vTComplex>>.
+  // iScalar<vTTComplex>, so CComplex for the L2->L3 level is iScalar<iScalar<vTComplex>>.
   //////////////////////////////////////////////////////////////////////
   typedef typename CoarseCoarseVector::vector_object                     CoarseCoarseSiteObj;
   typedef iScalar<vTTComplex>                                            vTTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseCoarseSiteObj,vTTTComplex,NB>     LittleDiracOperatorL3;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseCoarseSiteObj,vTTTComplex,NB>     LittleDiracOperatorL3;
   typedef typename LittleDiracOperatorL3::CoarseVector                   CoarseCoarseCoarseVector;
   typedef Aggregation<CoarseCoarseSiteObj,vTTTComplex,NB>               SubspaceL3;
   typedef MGPreconditioner<CoarseCoarseSiteObj,vTTTComplex,NB>          L2to3MG;
@@ -531,7 +531,7 @@ void runMG(
   L4PGCR.Name("CCCouter");
 
   //////////////////////////////////////////////////////////////////////
-  // Level 2→3 V-cycle: depth-2 SHIFTED smoother on LinOpCC + L4 bottom.
+  // Level 2->3 V-cycle: depth-2 SHIFTED smoother on LinOpCC + L4 bottom.
   // The shift slides the coarse-coarse field of values off the origin so a
   // 2-step smoother has something to bite on a non-normal operator (IRS idea).
   //////////////////////////////////////////////////////////////////////
@@ -556,7 +556,7 @@ void runMG(
                       simpleCCC);                // trivial guesser at the bottom
 
   //////////////////////////////////////////////////////////////////////
-  // Level 3 (coarse-coarse) solve: GCR preconditioned by the L2→L3 V-cycle.
+  // Level 3 (coarse-coarse) solve: GCR preconditioned by the L2->L3 V-cycle.
   // Replaces the plain L3PGCR of the 3-level build -- the coarse-coarse level
   // is now smoothed shallowly and recursed rather than solved deeply.
   //////////////////////////////////////////////////////////////////////
@@ -565,7 +565,7 @@ void runMG(
   L3MGsolver.Name("CCouter");
 
   //////////////////////////////////////////////////////////////////////
-  // Coarse-level GCR smoother for Level 1→2 V-cycle.
+  // Coarse-level GCR smoother for Level 1->2 V-cycle.
   // Mirrors fine-grid SmootherGCR: shifted operator + fixed step count.
   // coarse_smoother_shift and coarse_smoother_nstep are the tuning knobs.
   //////////////////////////////////////////////////////////////////////
@@ -581,15 +581,15 @@ void runMG(
   CoarseSmootherGCR.Name("Csmoother");
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2 V-cycle preconditioner.
+  // Level 1->2 V-cycle preconditioner.
   //////////////////////////////////////////////////////////////////////
   L1to2MG L1to2Precon(AggregatesL2,
                        LinOpCoarse,
                        simpleC,            // no pre-smoother (matches fine-grid setup)
                        CoarseSmootherGCR,  // post-smoother: depth-2 shifted GCR
                        LinOpCC,
-                       L3MGsolver,         // coarse-coarse solve is now the L2→L3 V-cycle
+                       L3MGsolver,         // coarse-coarse solve is now the L2->L3 V-cycle
-                       CCDeflGuesser);     // Lüscher guesser: psi_cc C^{-1} psi_cc†
+                       CCDeflGuesser);     // Luscher guesser: psi_cc C^{-1} psi_cc^dag
 
   //////////////////////////////////////////////////////////////////////
   // Standalone Level 1 two-level solve test.
@@ -624,7 +624,7 @@ void runMG(
   f_src = one;
 
   // Pre-smoother: none (TrivialPrecon); post-smoother: shifted PGCR.
-  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1→2 V-cycle).
+  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1->2 V-cycle).
   TwoLevelMG ThreeLevelPrecon(AggregatesPD,
                                PVdagM,
                                simple_fine,
@@ -668,7 +668,7 @@ int main (int argc, char ** argv)
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
-  // Level 1 coarse grid: block 2^4 from fine (48×48×48×96 → 24×24×24×48, Ls=1)
+  // Level 1 coarse grid: block 2^4 from fine (48x48x48x96 -> 24x24x24x48, Ls=1)
   Coordinate clatt = lat_size;
   for (int d = 0; d < 4; d++) clatt[d] /= 2;
   std::cout << GridLogMessage << "Level 1 coarse lattice: " << clatt << std::endl;
@@ -676,13 +676,13 @@ int main (int argc, char ** argv)
   GridCartesian *Coarse4d  = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridCartesian *Coarse5d  = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
 
-  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24×24×24×48 → 12×12×8×16, Ls=1).
+  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24x24x24x48 -> 12x12x8x16, Ls=1).
   // MPI geometry 3.6.4.4 (288 ranks): fine local {16,8,12,24}.
   // Level 1 local {8,4,6,12}; Level 2 local {4,2,2,4}.
-  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) → must use 3.
+  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) -> must use 3.
   // t blocked by 3: t-Level1-local=12; 12/3=4 divisible by Nsimd=4 (gen-simd-width=64).
-  //   t-block=2 gives t2-local=6, 6 mod 4 ≠ 0, fails Grid SIMD assertion. ✓
+  //   t-block=2 gives t2-local=6, 6 mod 4 != 0, fails Grid SIMD assertion.
-  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4). ✓
+  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4).
   Coordinate clatt2 = clatt;
   clatt2[0] /= 2;
   clatt2[1] /= 2;
@@ -728,7 +728,7 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
   typedef MGPreconditioner<vSpinColourVector,vTComplex,nbasis>        TwoLevelMG;

examples/Example_pvdagm_5level.cc

@@ -190,8 +190,8 @@ public:
   }
 };
 
-// Lüscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
+// Luscher deflated guesser (arXiv:0706.2298 Sec A.3) for a non-Hermitian solve.
-// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi† src.
+// C_{st} = <psi[s] | LinOp | psi[t]>;  guess = sum_s c_s psi[s]  where c = C^{-1} psi^dag src.
 template<class Field>
 class LuscherGuesser : public LinearFunction<Field> {
   const std::vector<Field> &psi;
@@ -344,7 +344,7 @@ void runMG(
   TrivialPrecon<LatticeFermionD>   simple_fine;
 
   //////////////////////////////////////////////////////////////////////
-  // Level 0→1: coarsen PVdagM, build LinOpCoarse
+  // Level 0->1: coarsen PVdagM, build LinOpCoarse
   //////////////////////////////////////////////////////////////////////
   LittleDiracOperator LittleDiracOpPV(geom, FGrid, Coarse5d);
   LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesPD);
@@ -368,7 +368,7 @@ void runMG(
   //////////////////////////////////////////////////////////////////////
   // psi_coarse: coarse projections of pre-GS fine null vectors.
   // These are the Level 1 near-null vectors, promoted from Level 0.
-  // Used as the aggregation basis for Level 1→2 coarsening.
+  // Used as the aggregation basis for Level 1->2 coarsening.
   //////////////////////////////////////////////////////////////////////
   std::vector<CoarseVector> psi_coarse(nbasis, Coarse5d);
   for (int k = 0; k < nbasis; k++)
@@ -398,22 +398,22 @@ void runMG(
     RealD normC      = C.norm();
     RealD normCmCdag = (C - C.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse null matrix ||C||            = " << normC << std::endl;
-    std::cout << GridLogMessage << "Coarse null matrix ||C - C†||/||C|| = " << normCmCdag/normC << std::endl;
+    std::cout << GridLogMessage << "Coarse null matrix ||C - C^dag||/||C|| = " << normCmCdag/normC << std::endl;
     std::cout << GridLogMessage << "Galerkin check ||C||/||W||          = " << normC/normW << std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2: set up aggregation using psi_coarse as subspace.
+  // Level 1->2: set up aggregation using psi_coarse as subspace.
   // Block factor 2,2,3,2 (removes odd local sublattice in z given MPI
-  // geometry 3×6×4×4 where z-local at Level 1 is 6).
+  // geometry 3x6x4x4 where z-local at Level 1 is 6).
   // psi_coarse are assigned directly; CoarsenOperator performs
   // block-GS orthogonalisation before building LinOpCoarseCoarse.
   //////////////////////////////////////////////////////////////////////
   // innerProduct(CoarseSiteObj, CoarseSiteObj) returns iScalar<vTComplex>, so CComplex
-  // for the L1→L2 level must be iScalar<vTComplex>, not vTComplex.
+  // for the L1->L2 level must be iScalar<vTComplex>, not vTComplex.
   typedef typename CoarseVector::vector_object                            CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,NB>            LittleDiracOperatorL2;
   typedef typename LittleDiracOperatorL2::CoarseVector                   CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,NB>                       SubspaceL2;
   typedef MGPreconditioner<CoarseSiteObj,vTTComplex,NB>                  L1to2MG;
@@ -431,12 +431,12 @@ void runMG(
   TrivialPrecon<CoarseCoarseVector> simpleCC;
 
   //////////////////////////////////////////////////////////////////////
-  // Lüscher deflation guesser for L3PGCR.
+  // Luscher deflation guesser for L3PGCR.
   // Step 1: project psi_coarse[k] (promoted fine null vectors) to
-  //         CoarseCoarseVector space — these cover the zero-momentum
+  //         CoarseCoarseVector space -- these cover the zero-momentum
   //         component of the near-null space of LinOpCC.
   // Step 2: breed Nextra additional null vectors directly on LinOpCC
-  //         using GCR with random sources — these pick up near-null
+  //         using GCR with random sources -- these pick up near-null
   //         modes at all spatial frequencies not spanned by step 1.
   // Step 3: build C_{st} = <psi_cc[s]|LinOpCC|psi_cc[t]> over the
   //         full augmented basis and invert directly via Eigen LU.
@@ -478,13 +478,13 @@ void runMG(
     RealD normCcc      = Ccc.norm();
     RealD normCccmCdag = (Ccc - Ccc.adjoint()).norm();
     std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc||              = " << normCcc << std::endl;
-    std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc-Ccc†||/||Ccc|| = " << normCccmCdag/normCcc << std::endl;
+    std::cout << GridLogMessage << "Coarse-coarse deflation matrix ||Ccc-Ccc^dag||/||Ccc|| = " << normCccmCdag/normCcc << std::endl;
   }
   Eigen::MatrixXcd Ccc_inv = Ccc.inverse();
   LuscherGuesser<CoarseCoarseVector> CCDeflGuesser(psi_cc, Ccc_inv);
 
   //////////////////////////////////////////////////////////////////////
-  // Level 2→3: coarsen LinOpCC using the RAW promoted psi_cc as aggregation
+  // Level 2->3: coarsen LinOpCC using the RAW promoted psi_cc as aggregation
   // to build the Level 4 (coarse-coarse-coarse) operator.
   //   psi_cc[0..nbasis-1] are the coarse-coarse near-null vectors, projected
   //   from the RAW psi_coarse (themselves projected from the RAW fine null
@@ -493,11 +493,11 @@ void runMG(
   //   so assign COPIES of psi_cc and keep psi_cc itself raw.
   //
   // Tensor depth deepens once more: innerProduct(CoarseCoarseSiteObj,...) returns
-  // iScalar<vTTComplex>, so CComplex for the L2→L3 level is iScalar<iScalar<vTComplex>>.
+  // iScalar<vTTComplex>, so CComplex for the L2->L3 level is iScalar<iScalar<vTComplex>>.
   //////////////////////////////////////////////////////////////////////
   typedef typename CoarseCoarseVector::vector_object                     CoarseCoarseSiteObj;
   typedef iScalar<vTTComplex>                                            vTTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseCoarseSiteObj,vTTTComplex,NB>     LittleDiracOperatorL3;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseCoarseSiteObj,vTTTComplex,NB>     LittleDiracOperatorL3;
   typedef typename LittleDiracOperatorL3::CoarseVector                   CoarseCoarseCoarseVector;
   typedef Aggregation<CoarseCoarseSiteObj,vTTTComplex,NB>               SubspaceL3;
   typedef MGPreconditioner<CoarseCoarseSiteObj,vTTTComplex,NB>          L2to3MG;
@@ -514,7 +514,7 @@ void runMG(
   TrivialPrecon<CoarseCoarseCoarseVector> simpleCCC;
 
   //////////////////////////////////////////////////////////////////////
-  // Level 3→4: coarsen LinOpCCC to build the Level 5 operator, using a
+  // Level 3->4: coarsen LinOpCCC to build the Level 5 operator, using a
   // TRUNCATED basis of only the first NB5 (< nbasis) raw promoted null vectors.
   //   psi_ccc[k] = raw psi_cc projected through the (block-GS'd) L3 aggregation
   //   -- the pre-block-GS chain continued one level deeper.  We keep only the
@@ -526,7 +526,7 @@ void runMG(
   //   NB: a positive result is conservative (sigma-ordering can only help); a
   //   negative one is inconclusive until the sigma-ordered NB5 is tried.
   //
-  // Tensor depth deepens once more: CComplex for the L3→L4 level is
+  // Tensor depth deepens once more: CComplex for the L3->L4 level is
   // iScalar<vTTTComplex>.  NB5 (the coarse dimension) is independent of the
   // depth -- it just makes the coarsest site vector NB5-dimensional.
   //////////////////////////////////////////////////////////////////////
@@ -542,10 +542,10 @@ void runMG(
   // Optional sigma-ordering of psi_ccc (SVD_REORDER set): replace the crude
   // first-NB5 slice with the NB5 genuinely-most-null directions of span(psi_ccc)
   // under LinOpCCC.  For a NON-NORMAL operator the nullness measure is the
-  // singular value of A restricted to the span -- eig of Q†A†AQ -- NOT the
+  // singular value of A restricted to the span -- eig of Q^dag A^dag AQ -- NOT the
-  // numerical range Q†AQ (which non-normality contaminates).  Robust route:
+  // numerical range Q^dag AQ (which non-normality contaminates).  Robust route:
   // whiten by the Gram (drop near-dependent directions), Hermitian-eig the
-  // whitened A†A, rotate.  The printed singular spectrum IS the SVD study: where
+  // whitened A^dag A, rotate.  The printed singular spectrum IS the SVD study: where
   // it falls off tells you the natural NB5, and the same numbers illuminate why
   // the earlier singular-subspace deflation re-entered.  Safe here because we
   // ORDER vectors that then feed a Galerkin projection, not REMOVE a subspace.
@@ -603,7 +603,7 @@ void runMG(
 
   typedef typename CoarseCoarseCoarseVector::vector_object              CoarseCoarseCoarseSiteObj;
   typedef iScalar<vTTTComplex>                                          vTTTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseCoarseCoarseSiteObj,vTTTTComplex,NB5> LittleDiracOperatorL4;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseCoarseCoarseSiteObj,vTTTTComplex,NB5> LittleDiracOperatorL4;
   typedef typename LittleDiracOperatorL4::CoarseVector                  CoarseCoarseCoarseCoarseVector;
   typedef Aggregation<CoarseCoarseCoarseSiteObj,vTTTTComplex,NB5>       SubspaceL4;
   typedef MGPreconditioner<CoarseCoarseCoarseSiteObj,vTTTTComplex,NB5>  L3to4MG;
@@ -635,7 +635,7 @@ void runMG(
   L5PGCR.Name("CCCCouter");
 
   //////////////////////////////////////////////////////////////////////
-  // Level 3→4 V-cycle: depth-2 SHIFTED smoother on LinOpCCC + Level 5 bottom.
+  // Level 3->4 V-cycle: depth-2 SHIFTED smoother on LinOpCCC + Level 5 bottom.
   // Level 4 is no longer the bottom -- it is smoothed shallowly and recursed to
   // Level 5, mirroring how Level 3 recurses to Level 4.
   //////////////////////////////////////////////////////////////////////
@@ -660,14 +660,14 @@ void runMG(
                       simpleCCCC);                   // trivial guesser at the bottom
 
   //////////////////////////////////////////////////////////////////////
-  // Level 4 (coarse-coarse-coarse) solve: GCR preconditioned by the L3→L4 V-cycle.
+  // Level 4 (coarse-coarse-coarse) solve: GCR preconditioned by the L3->L4 V-cycle.
   //////////////////////////////////////////////////////////////////////
   PrecGeneralisedConjugateResidualNonHermitian<CoarseCoarseCoarseVector> L4MGsolver(1.0e-1,200,LinOpCCC,L3to4Precon,16,16);
   L4MGsolver.Level(4);
   L4MGsolver.Name("CCCouter");
 
   //////////////////////////////////////////////////////////////////////
-  // Level 2→3 V-cycle: depth-2 SHIFTED smoother on LinOpCC + Level 4 solve.
+  // Level 2->3 V-cycle: depth-2 SHIFTED smoother on LinOpCC + Level 4 solve.
   // The shift slides the coarse-coarse field of values off the origin so a
   // 2-step smoother has something to bite on a non-normal operator (IRS idea).
   //////////////////////////////////////////////////////////////////////
@@ -688,11 +688,11 @@ void runMG(
                       simpleCC,                  // no pre-smoother
                       CoarseCoarseSmootherGCR,   // post-smoother: depth-2 shifted GCR
                       LinOpCCC,
-                      L4MGsolver,                // coarse solve is now the L3→L4 V-cycle
+                      L4MGsolver,                // coarse solve is now the L3->L4 V-cycle
                       simpleCCC);                // trivial guesser
 
   //////////////////////////////////////////////////////////////////////
-  // Level 3 (coarse-coarse) solve: GCR preconditioned by the L2→L3 V-cycle.
+  // Level 3 (coarse-coarse) solve: GCR preconditioned by the L2->L3 V-cycle.
   // Replaces the plain L3PGCR of the 3-level build -- the coarse-coarse level
   // is now smoothed shallowly and recursed rather than solved deeply.
   //////////////////////////////////////////////////////////////////////
@@ -701,7 +701,7 @@ void runMG(
   L3MGsolver.Name("CCouter");
 
   //////////////////////////////////////////////////////////////////////
-  // Coarse-level GCR smoother for Level 1→2 V-cycle.
+  // Coarse-level GCR smoother for Level 1->2 V-cycle.
   // Mirrors fine-grid SmootherGCR: shifted operator + fixed step count.
   // coarse_smoother_shift and coarse_smoother_nstep are the tuning knobs.
   //////////////////////////////////////////////////////////////////////
@@ -717,15 +717,15 @@ void runMG(
   CoarseSmootherGCR.Name("Csmoother");
 
   //////////////////////////////////////////////////////////////////////
-  // Level 1→2 V-cycle preconditioner.
+  // Level 1->2 V-cycle preconditioner.
   //////////////////////////////////////////////////////////////////////
   L1to2MG L1to2Precon(AggregatesL2,
                        LinOpCoarse,
                        simpleC,            // no pre-smoother (matches fine-grid setup)
                        CoarseSmootherGCR,  // post-smoother: depth-2 shifted GCR
                        LinOpCC,
-                       L3MGsolver,         // coarse-coarse solve is now the L2→L3 V-cycle
+                       L3MGsolver,         // coarse-coarse solve is now the L2->L3 V-cycle
-                       CCDeflGuesser);     // Lüscher guesser: psi_cc C^{-1} psi_cc†
+                       CCDeflGuesser);     // Luscher guesser: psi_cc C^{-1} psi_cc^dag
 
   //////////////////////////////////////////////////////////////////////
   // Standalone Level 1 two-level solve test.
@@ -760,7 +760,7 @@ void runMG(
   f_src = one;
 
   // Pre-smoother: none (TrivialPrecon); post-smoother: shifted PGCR.
-  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1→2 V-cycle).
+  // Coarse solver: L2MGsolver (PGCR preconditioned by Level 1->2 V-cycle).
   TwoLevelMG ThreeLevelPrecon(AggregatesPD,
                                PVdagM,
                                simple_fine,
@@ -804,7 +804,7 @@ int main (int argc, char ** argv)
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
   GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
-  // Level 1 coarse grid: block 2^4 from fine (48×48×48×96 → 24×24×24×48, Ls=1)
+  // Level 1 coarse grid: block 2^4 from fine (48x48x48x96 -> 24x24x24x48, Ls=1)
   Coordinate clatt = lat_size;
   for (int d = 0; d < 4; d++) clatt[d] /= 2;
   std::cout << GridLogMessage << "Level 1 coarse lattice: " << clatt << std::endl;
@@ -812,13 +812,13 @@ int main (int argc, char ** argv)
   GridCartesian *Coarse4d  = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridCartesian *Coarse5d  = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
 
-  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24×24×24×48 → 12×12×8×16, Ls=1).
+  // Level 2 coarse-coarse grid: block 2,2,3,3 from Level 1 (24x24x24x48 -> 12x12x8x16, Ls=1).
   // MPI geometry 3.6.4.4 (288 ranks): fine local {16,8,12,24}.
   // Level 1 local {8,4,6,12}; Level 2 local {4,2,2,4}.
-  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) → must use 3.
+  // z blocked by 3: z-Level1-local=6; 6/3=2 (even), 6/2=3 (odd) -> must use 3.
   // t blocked by 3: t-Level1-local=12; 12/3=4 divisible by Nsimd=4 (gen-simd-width=64).
-  //   t-block=2 gives t2-local=6, 6 mod 4 ≠ 0, fails Grid SIMD assertion. ✓
+  //   t-block=2 gives t2-local=6, 6 mod 4 != 0, fails Grid SIMD assertion.
-  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4). ✓
+  // With {4,2,2,4}: Nsimd=4 goes into x or t (both =4).
   Coordinate clatt2 = clatt;
   clatt2[0] /= 2;
   clatt2[1] /= 2;
@@ -880,7 +880,7 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
   typedef MGPreconditioner<vSpinColourVector,vTComplex,nbasis>        TwoLevelMG;

examples/Example_pvdagm_census.cc

@@ -49,7 +49,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 //   sigma_min << min|lambda|                       : non-normal near origin
 //   lambda_min(H) < 0                              : half-plane condition violated
 //
-// Requires the dagger code path in GeneralCoarsenedMatrix:
+// Requires the dagger code path in DeprecatedGeneralCoarsenedMatrix:
 //   _Adag allocated, PopulateAdag active, _Adag exchanged, hermitian=0.
 //
 // Env vars:
@@ -308,7 +308,7 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
 

examples/Example_pvdagm_coherence.cc

@@ -0,0 +1,464 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_pvdagm_coherence.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+
+// Local-coherence census of the COARSE operator's low singular vectors.
+// (Fig 4 of arXiv:2409.03904 transplanted one level down.)
+//
+// Inputs: the raw fine subspace cache AND the coarse right-singular
+// vectors saved by Example_pvdagm_census (CENSUS_EVEC_FILE).
+//
+// Measurements:
+//  CHECK 0  u/v alignment per mode: u_k = A v_k/||A v_k||, align=|<u,v>|.
+//           Low-sector non-normality meter AND the "2-for-1 coupon" test
+//           (u adds deflation span iff align is small). Also re-measures
+//           sigma_k = ||A v_k|| as an evec-file integrity check.
+//  CHECK 1  Band overlap: |projection of v_k onto span(psi_c)|^2 where
+//           psi_c = coarse images of the 60 RAW fine null vectors.
+//           Detached-vs-submerged Ritz band question.
+//  CHECK A  Self-coherence: level-2 blocks (BLOCK2) with basis = evecs
+//           0..K-1 (block-orthonormalised); completeness of held-out
+//           evecs K..NEV-1. Local coherence of the coarse op's own tail.
+//  CHECK B  Capture by the available basis: same blocks, basis = psi_c;
+//           completeness of every evec. What a 3-level would have had.
+//  GS census (inside every block orthonormalisation): per-vector global
+//           remainder fraction and MIN-over-blocks remainder fraction.
+//           Detects finite-precision rank loss (near-parallel vectors
+//           within a block -> normalised noise). Values ~0 flag fiction
+//           in the "orthonormal" basis.
+//  Idempotency assertion ||(PP+)^2 v - PP+ v||/||PP+ v|| ~ eps on the
+//           first probe of each check: catches non-orthonormal-basis and
+//           rank-loss corruption in the checks themselves.
+//
+// Level-2 machinery uses the free block primitives directly on
+// std::vector<CoarseVector>, so basis rank K is a RUNTIME parameter
+// (no Aggregation template instantiation at level 2).
+//
+// Env:
+//   MASS, SUBSPACE_FILE          as elsewhere (cache MUST exist)
+//   CENSUS_EVEC_FILE             REQUIRED: evecs from the census run
+//   NEV                          number of evecs to load (default 96)
+//   NBASIS2                      K, basis size for CHECK A (default 48)
+//   BLOCK                        fine->coarse blocking; MUST match the
+//                                census run that made the evecs (4.4.4.4)
+//   BLOCK2                       coarse->coarsecoarse blocking (2.2.2.2)
+
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+
+using namespace std;
+using namespace Grid;
+
+RealD mass = 0.00078;
+int   Nev     = 96;
+int   Nbasis2 = 48;
+
+void ParseEnvironment(void)
+{
+  if(getenv("MASS"))    mass    = atof(getenv("MASS"));
+  if(getenv("NEV"))     Nev     = atoi(getenv("NEV"));
+  if(getenv("NBASIS2")) Nbasis2 = atoi(getenv("NBASIS2"));
+  std::cout << GridLogMessage << "PARAM: MASS    " << mass    << std::endl;
+  std::cout << GridLogMessage << "PARAM: NEV     " << Nev     << std::endl;
+  std::cout << GridLogMessage << "PARAM: NBASIS2 " << Nbasis2 << std::endl;
+}
+
+template <class Field>
+void loadFields(std::vector<Field> &v, std::string const fname){
+#ifdef HAVE_LIME
+  std::cout << Grid::GridLogMessage << "Loading " << v.size() << " fields from: " << fname << std::endl;
+  Grid::emptyUserRecord record;
+  Grid::ScidacReader SR;
+  SR.open(fname);
+  for (int k = 0; k < (int)v.size(); k++)
+    SR.readScidacFieldRecord(v[k], record);
+  SR.close();
+#endif
+}
+
+template<class Matrix,class Field>
+class PVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat;
+  Matrix &_PV;
+public:
+  PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV) {};
+  void OpDiag (const Field &in, Field &out) {    assert(0);  }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) {    assert(0);  }
+  void OpDirAll  (const Field &in, std::vector<Field> &out){    assert(0);  };
+  void Op     (const Field &in, Field &out){
+    Field tmp(in.Grid());
+    _Mat.M(in,tmp);
+    _PV.Mdag(tmp,out);
+  }
+  void AdjOp     (const Field &in, Field &out){
+    Field tmp(in.Grid());
+    _PV.M(in,tmp);
+    _Mat.Mdag(tmp,out);
+  }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+    HermOp(in,out);
+    ComplexD dot = innerProduct(in,out);
+    n1=real(dot); n2=norm2(out);
+  }
+  void HermOp(const Field &in, Field &out){
+    Field tmp(in.Grid());
+    Op(in,tmp);
+    AdjOp(tmp,out);
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// Block Gram-Schmidt with remainder census.
+// Mirrors blockOrthonormalize (one GS pass + normalise) but reports,
+// per vector: the GLOBAL remainder fraction ||v_k^perp||^2/||v_k||^2
+// and the MIN over blocks of the same ratio. A second silent pass
+// tightens orthonormality for the completeness measurements.
+//////////////////////////////////////////////////////////////////////
+template<class Field, class CField>
+void blockGSCensus(std::vector<Field> &basis, GridBase *ccGrid, const std::string tag)
+{
+  int n = basis.size();
+  CField ip(ccGrid);
+  CField n0(ccGrid), n1(ccGrid);
+
+  std::cout << GridLogMessage << "=== block GS census [" << tag << "] : " << n
+            << " vectors, blocks -> " << ccGrid->GlobalDimensions() << std::endl;
+
+  for(int v=0; v<n; v++){
+    RealD before = norm2(basis[v]);
+    blockInnerProduct(n0, basis[v], basis[v]);
+    for(int u=0; u<v; u++){
+      blockInnerProductD(ip, basis[u], basis[v]);
+      ip = -ip;
+      blockZAXPY(basis[v], ip, basis[u], basis[v]);
+    }
+    RealD after = norm2(basis[v]);
+    blockInnerProduct(n1, basis[v], basis[v]);
+
+    // min over blocks of remainder fraction n1/n0
+    RealD locmin = 1.0e60;
+    {
+      typedef typename CField::scalar_object sobj;
+      for(int64_t s=0; s<ccGrid->lSites(); s++){
+        Coordinate lcoor(ccGrid->_ndimension);
+        ccGrid->LocalIndexToLocalCoor(s,lcoor);
+        sobj s0, s1;
+        peekLocalSite(s0, n0, lcoor);
+        peekLocalSite(s1, n1, lcoor);
+        RealD r0 = real(TensorRemove(s0));
+        RealD r1 = real(TensorRemove(s1));
+        if ( r0 > 0.0 ) {
+          RealD frac = r1/r0;
+          if (frac < locmin) locmin = frac;
+        }
+      }
+      RealD neg = -locmin;
+      ccGrid->GlobalMax(neg);
+      locmin = -neg;
+    }
+
+    std::cout << GridLogMessage << "GS[" << tag << "] vec " << v
+              << "  global remainder " << after/before
+              << "  min-block remainder " << locmin
+              << ( locmin < 1.0e-10 ? "   <== RANK LOSS FLAG" : "" ) << std::endl;
+
+    blockNormalise(ip, basis[v]);
+  }
+  // Second (silent) pass for numerical hygiene of downstream projections
+  for(int v=0; v<n; v++){
+    for(int u=0; u<v; u++){
+      blockInnerProductD(ip, basis[u], basis[v]);
+      ip = -ip;
+      blockZAXPY(basis[v], ip, basis[u], basis[v]);
+    }
+    blockNormalise(ip, basis[v]);
+  }
+}
+
+//////////////////////////////////////////////////////////////////////
+// P P^dag v for a block-orthonormal basis, via the block primitives.
+// Runtime basis size; no Aggregation template needed.
+//////////////////////////////////////////////////////////////////////
+template<class Field, class CField>
+void blockPPdag(const std::vector<Field> &q, GridBase *ccGrid, const Field &v, Field &w)
+{
+  CField ip(ccGrid);
+  w = Zero();
+  for(int j=0; j<(int)q.size(); j++){
+    blockInnerProduct(ip, q[j], v);
+    blockZAXPY(w, ip, q[j], w);
+  }
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+  ParseEnvironment();
+
+  const int Ls=24;
+  RealD M5=1.8, b=1.5, c=0.5;
+  const int nbasis = 60;
+
+  std::vector<int> lat_size {48, 48, 48, 96};
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(lat_size, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  // Fine->coarse blocking: MUST match the census run that made the evecs.
+  Coordinate clatt = lat_size;
+  Coordinate Block({4,4,4,4});
+  if ( getenv("BLOCK") ) {
+    GridCmdOptionIntVector(std::string(getenv("BLOCK")),Block);
+    GRID_ASSERT(Block.size()==4);
+  }
+  for(int d=0;d<clatt.size();d++){
+    GRID_ASSERT(lat_size[d] % Block[d] == 0);
+    clatt[d] = lat_size[d]/Block[d];
+  }
+  std::cout << GridLogMessage << "Block " << Block << "  coarse lattice " << clatt << std::endl;
+
+  GridCartesian *Coarse4d =  SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian *Coarse5d =  SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
+
+  // Coarse->coarsecoarse blocking for the coherence checks
+  Coordinate cclatt = clatt;
+  Coordinate Block2({2,2,2,2});
+  if ( getenv("BLOCK2") ) {
+    GridCmdOptionIntVector(std::string(getenv("BLOCK2")),Block2);
+    GRID_ASSERT(Block2.size()==4);
+  }
+  for(int d=0;d<cclatt.size();d++){
+    GRID_ASSERT(clatt[d] % Block2[d] == 0);
+    cclatt[d] = clatt[d]/Block2[d];
+  }
+  std::cout << GridLogMessage << "Block2 " << Block2 << "  coarsecoarse lattice " << cclatt << std::endl;
+
+  GridCartesian *CC4d =  SpaceTimeGrid::makeFourDimGrid(cclatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian *CC5d =  SpaceTimeGrid::makeFiveDimGrid(1,CC4d);
+
+  GridParallelRNG RNG5(FGrid);  RNG5.SeedFixedIntegers({5,6,7,8});
+
+  LatticeGaugeField Umu(UGrid);
+  std::cout << GridLogMessage << "Reading gauge field" << std::endl;
+  FieldMetaData header;
+  std::string file("/ccs/home/poare/ckpoint_lat.1000");
+  NerscIO::readConfiguration(Umu,header,file);
+
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
+  MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
+
+  typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>       PVdagM_t;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+  typedef LittleDiracOperator::CoarseVector                          CoarseVector;
+  // Grid index contraction is positional (colour/spin/lorentz order is meaningful),
+  // so each MG projection adds one index to the tensor nest rather than reusing a slot:
+  // innerProduct(CoarseSiteObj,CoarseSiteObj) returns iScalar<vTComplex>, one level
+  // deeper than the fine-level vTComplex (same convention as Example_pvdagm_3level.cc).
+  typedef Lattice<iScalar<vTComplex> >                               CoarseCoarseScalar;
+  typedef Aggregation<vSpinColourVector,vTComplex,nbasis>            Subspace;
+
+  PVdagM_t PVdagM(Ddwf,Dpv);
+  NextToNearestStencilGeometry5D geom(Coarse5d);
+
+  //////////////////////////////////////////////////////////////////////
+  // Load the RAW subspace (cache REQUIRED; no generation here)
+  //////////////////////////////////////////////////////////////////////
+  std::string subspace_file = "/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb"
+                              + std::to_string(nbasis) + ".scidac";
+  if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
+
+  uint64_t file_exists = 0;
+  if ( UGrid->IsBoss() ) { std::ifstream f(subspace_file); file_exists = f.good() ? 1 : 0; }
+  UGrid->GlobalSum(file_exists);
+  if ( !file_exists ) {
+    std::cout << GridLogMessage << "FATAL: subspace cache not found: " << subspace_file << std::endl;
+    GRID_ASSERT(file_exists);
+  }
+
+  const int cb = 0;
+  Subspace AggregatesGCR(Coarse5d,FGrid,cb);
+  loadFields(AggregatesGCR.subspace, subspace_file);
+
+  // RAW copy before CoarsenOperator block-orthogonalises in place
+  std::vector<LatticeFermionD> rawNull(nbasis, FGrid);
+  for (int k = 0; k < nbasis; k++) rawNull[k] = AggregatesGCR.subspace[k];
+
+  LittleDiracOperator LittleDiracOpPV(geom,FGrid,Coarse5d);
+  LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesGCR);
+  NonHermitianLinearOperator<LittleDiracOperator,CoarseVector> LinOpCoarse(LittleDiracOpPV);
+
+  // Coarse images of the raw null vectors
+  std::vector<CoarseVector> psi_c(nbasis, Coarse5d);
+  for (int k = 0; k < nbasis; k++)
+    AggregatesGCR.ProjectToSubspace(psi_c[k], rawNull[k]);
+
+  //////////////////////////////////////////////////////////////////////
+  // Load census singular vectors
+  //////////////////////////////////////////////////////////////////////
+  GRID_ASSERT(getenv("CENSUS_EVEC_FILE"));
+  std::string evec_file(getenv("CENSUS_EVEC_FILE"));
+  std::vector<CoarseVector> evec(Nev, Coarse5d);
+  loadFields(evec, evec_file);
+
+  GRID_ASSERT(Nbasis2 < Nev);
+
+  //////////////////////////////////////////////////////////////////////
+  // CHECK 0: sigma re-measurement and u/v alignment per mode
+  //////////////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  std::cout << GridLogMessage << "CHECK 0: sigma and left/right alignment per mode" << std::endl;
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  {
+    CoarseVector Av(Coarse5d);
+    for(int k=0;k<Nev;k++){
+      LinOpCoarse.Op(evec[k],Av);
+      RealD nv  = norm2(evec[k]);
+      RealD nAv = norm2(Av);
+      RealD sigma = std::sqrt(nAv/nv);
+      ComplexD uv = innerProduct(Av,evec[k]);
+      RealD align = abs(uv)/std::sqrt(nAv*nv);   // |<u|v>|, u = Av/||Av||
+      std::cout << GridLogMessage << "COHERENCE: mode " << k
+                << "  sigma = " << sigma
+                << "  |<u|v>| = " << align << std::endl;
+    }
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // CHECK 1: band overlap of each evec with span(psi_c)  (GLOBAL span)
+  //////////////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  std::cout << GridLogMessage << "CHECK 1: evec overlap with GLOBAL span of 60 psi_c" << std::endl;
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  {
+    // Global (not block) Gram-Schmidt of copies of psi_c
+    std::vector<CoarseVector> Q(nbasis, Coarse5d);
+    for(int k=0;k<nbasis;k++) Q[k]=psi_c[k];
+    for(int k=0;k<nbasis;k++){
+      for(int j=0;j<k;j++){
+        ComplexD ip = innerProduct(Q[j],Q[k]);
+        Q[k] = Q[k] - ip*Q[j];
+      }
+      RealD nq = norm2(Q[k]);
+      GRID_ASSERT(nq>0.0);
+      Q[k] = Q[k] * (1.0/std::sqrt(nq));
+    }
+    for(int k=0;k<Nev;k++){
+      RealD ov = 0.0;
+      for(int j=0;j<nbasis;j++){
+        ComplexD ip = innerProduct(Q[j],evec[k]);
+        ov += real(ip*conjugate(ip));
+      }
+      std::cout << GridLogMessage << "COHERENCE: band overlap mode " << k
+                << " = " << ov/norm2(evec[k]) << std::endl;
+    }
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // CHECK A: self-coherence -- basis = evecs 0..K-1 in BLOCK2 blocks,
+  // probes = held-out evecs K..Nev-1
+  //////////////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  std::cout << GridLogMessage << "CHECK A: tail self-coherence, K=" << Nbasis2 << " basis evecs" << std::endl;
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  {
+    std::vector<CoarseVector> basisA(Nbasis2, Coarse5d);
+    for(int k=0;k<Nbasis2;k++) basisA[k]=evec[k];
+    blockGSCensus<CoarseVector,CoarseCoarseScalar>(basisA, CC5d, "A:evecs");
+
+    CoarseVector w(Coarse5d), w2(Coarse5d);
+    // Idempotency assertion on first probe
+    blockPPdag<CoarseVector,CoarseCoarseScalar>(basisA, CC5d, evec[Nbasis2], w);
+    blockPPdag<CoarseVector,CoarseCoarseScalar>(basisA, CC5d, w, w2);
+    w2 = w2 - w;
+    RealD idem = std::sqrt(norm2(w2)/norm2(w));
+    std::cout << GridLogMessage << "COHERENCE: CHECK A idempotency = " << idem
+              << "  (expect ~1e-14; O(1) => basis corrupt)" << std::endl;
+
+    RealD sum=0.0;
+    for(int k=Nbasis2;k<Nev;k++){
+      blockPPdag<CoarseVector,CoarseCoarseScalar>(basisA, CC5d, evec[k], w);
+      RealD comp = norm2(w)/norm2(evec[k]);
+      sum += comp;
+      std::cout << GridLogMessage << "COHERENCE: A completeness mode " << k
+                << " = " << comp << std::endl;
+    }
+    std::cout << GridLogMessage << "COHERENCE: A mean completeness (modes " << Nbasis2
+              << ".." << Nev-1 << ") = " << sum/(Nev-Nbasis2) << std::endl;
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // CHECK B: capture by the available basis -- basis = 60 psi_c in
+  // BLOCK2 blocks, probes = ALL evecs
+  //////////////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  std::cout << GridLogMessage << "CHECK B: capture by blocked psi_c basis" << std::endl;
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  {
+    std::vector<CoarseVector> basisB(nbasis, Coarse5d);
+    for(int k=0;k<nbasis;k++) basisB[k]=psi_c[k];
+    blockGSCensus<CoarseVector,CoarseCoarseScalar>(basisB, CC5d, "B:psi_c");
+
+    CoarseVector w(Coarse5d), w2(Coarse5d);
+    blockPPdag<CoarseVector,CoarseCoarseScalar>(basisB, CC5d, evec[0], w);
+    blockPPdag<CoarseVector,CoarseCoarseScalar>(basisB, CC5d, w, w2);
+    w2 = w2 - w;
+    RealD idem = std::sqrt(norm2(w2)/norm2(w));
+    std::cout << GridLogMessage << "COHERENCE: CHECK B idempotency = " << idem
+              << "  (expect ~1e-14; O(1) => basis corrupt)" << std::endl;
+
+    RealD sumLow=0.0, sumHigh=0.0; int nLow=0, nHigh=0;
+    for(int k=0;k<Nev;k++){
+      blockPPdag<CoarseVector,CoarseCoarseScalar>(basisB, CC5d, evec[k], w);
+      RealD comp = norm2(w)/norm2(evec[k]);
+      if (k<nbasis) { sumLow+=comp; nLow++; } else { sumHigh+=comp; nHigh++; }
+      std::cout << GridLogMessage << "COHERENCE: B completeness mode " << k
+                << " = " << comp << std::endl;
+    }
+    std::cout << GridLogMessage << "COHERENCE: B mean completeness modes 0.." << nbasis-1
+              << " = " << sumLow/nLow << std::endl;
+    if (nHigh>0)
+      std::cout << GridLogMessage << "COHERENCE: B mean completeness modes " << nbasis
+                << ".." << Nev-1 << " = " << sumHigh/nHigh << std::endl;
+  }
+
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+  std::cout << GridLogMessage << "COHERENCE SUMMARY:" << std::endl;
+  std::cout << GridLogMessage << "  CHECK 0 |<u|v>| ~ 1      : low sector effectively normal; 2-for-1 coupon worthless" << std::endl;
+  std::cout << GridLogMessage << "  CHECK 1 high for low k   : Ritz band submerged in continuum (not detached)" << std::endl;
+  std::cout << GridLogMessage << "  CHECK A ~0.98            : local coherence manifests at coarse level (3-level rescuable)" << std::endl;
+  std::cout << GridLogMessage << "  CHECK A ~0.8-0.9         : capture deficit is physical (effective-blocking law)" << std::endl;
+  std::cout << GridLogMessage << "  CHECK B << CHECK A       : psi_c basis fails to exploit coherence that exists" << std::endl;
+  std::cout << GridLogMessage << "  GS min-block remainder ~0: rank loss; orthogonalisation partly fictional" << std::endl;
+  std::cout << GridLogMessage << "=================================================" << std::endl;
+
+  std::cout << GridLogMessage << "Done" << std::endl;
+  Grid_finalize();
+  return 0;
+}

examples/Example_pvdagm_mrhs.cc

@@ -38,7 +38,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 //   precon: V-cycle -- per-RHS fine post-smoother (16-step shifted GCR),
 //           batched restriction (MultiRHSBlockProject / GEMM),
 //           ONE coarse PGCR on the 6D mrhs coarse operator
-//           (MultiGeneralCoarsenedMatrix, GEMM mults -- the ~10x win),
+//           (DeprecatedMultiGeneralCoarsenedMatrix, GEMM mults -- the ~10x win),
 //           batched prolongation.
 //
 // The coarse operator is coarsened once with the standard single-RHS
@@ -58,6 +58,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
 using namespace std;
 using namespace Grid;
@@ -183,194 +184,6 @@ public:
   }
 };
 
-//////////////////////////////////////////////////////////////////////
-// Minimal multi-RHS function interface (preconditioner slot)
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-//////////////////////////////////////////////////////////////////////
-// Single-polynomial multi-RHS PGCR (non-Hermitian).
-//
-// Verbatim adaptation of PrecGeneralisedConjugateResidualNonHermitian
-// to std::vector<Field>: every innerProduct / norm2 is SUMMED over the
-// RHS index, so one alpha/beta per step is shared by all RHS -- the
-// single GCR on the enlarged block-diagonal system.
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD   Tolerance;
-  Integer MaxIterations;
-  int mmax;
-  int nstep;
-  int steps;
-  int level;
-  int ZeroGuess = 0; int FirstCycle = 0;  // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-
-  void Level(int lv) { name = "Level " + std::to_string(lv); level=lv; };
-  void Name(std::string n) { name = n; };
-  void SetZeroGuess(int z) { ZeroGuess=z; };
-
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,
-                       LinearOperatorBase<Field> &_Linop,
-                       MrhsLinearFunction<Field> &Prec,
-                       int _mmax,int _nstep)
-    : Tolerance(tol), MaxIterations(maxit), Linop(_Linop), Preconditioner(Prec),
-      mmax(_mmax), nstep(_nstep) { level=1; }
-
-  ///////////////////////////////////////////////////////////////
-  // vector-of-fields linear algebra, reductions summed over rhs
-  ///////////////////////////////////////////////////////////////
-  static RealD vnorm2(std::vector<Field> &x){
-    RealD s=0.0; for(auto &f : x) s+=norm2(f); return s;
-  }
-  static ComplexD vinnerProduct(std::vector<Field> &x, std::vector<Field> &y){
-    ComplexD s(0.0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s;
-  }
-  static void vaxpy(std::vector<Field> &z, ComplexD a, std::vector<Field> &x, std::vector<Field> &y){
-    for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]);
-  }
-  void vOp(std::vector<Field> &in, std::vector<Field> &out){
-    for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]);
-  }
-
-  void operator() (std::vector<Field> &src, std::vector<Field> &psi){
-    RealD cp, ssq, rsq;
-    int nrhs = src.size();
-    GridBase *grid = src[0].Grid();
-
-    ssq=vnorm2(src);
-    rsq=Tolerance*Tolerance*ssq;
-
-    std::vector<Field> r(nrhs,grid);
-
-    GridStopWatch SolverTimer;
-    SolverTimer.Start();
-
-    steps=0;
-    FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-
-      cp=GCRnStep(src,psi,rsq);
-
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-               <<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-
-      if(cp<rsq){
-        SolverTimer.Stop();
-        vOp(psi,r);
-        for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-                 <<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)
-                 <<" true residual "<<std::sqrt(tr/ssq)
-                 <<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-                 <<" MrhsPGCR Time elapsed: Total "<<SolverTimer.Elapsed()<<std::endl;
-        // Per-RHS true residuals: the honest metric under the summed norm
-        for(int rr=0;rr<nrhs;rr++){
-          RealD rn = std::sqrt(norm2(r[rr])/norm2(src[rr]));
-          std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl;
-        }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-
-  RealD GCRnStep(std::vector<Field> &src, std::vector<Field> &psi, RealD rsq){
-
-    RealD cp;
-    ComplexD a, b, rq;
-    RealD zAAz;
-
-    int nrhs = src.size();
-    GridBase *grid = src[0].Grid();
-
-    std::vector<Field> r (nrhs,grid);
-    std::vector<Field> z (nrhs,grid);
-    std::vector<Field> Az(nrhs,grid);
-
-    ////////////////////////////////
-    // history for flexible orthog: [mmax][nrhs]
-    ////////////////////////////////
-    std::vector< std::vector<Field> > q(mmax, std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax, std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-
-    if (ZeroGuess && FirstCycle) {
-      for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; }
-    } else {
-      vOp(psi,Az);
-      for(int rr=0;rr<nrhs;rr++) r[rr] = src[rr]-Az[rr];
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-               <<" MrhsPGCR true residual r = src - A psi   "<<vnorm2(r)<<std::endl;
-    }
-    FirstCycle=0;
-
-    Preconditioner(r,z);
-    vOp(z,Az);
-    zAAz=vnorm2(Az);
-
-    p[0]=z;
-    q[0]=Az;
-    qq[0]=zAAz;
-
-    cp=vnorm2(r);
-
-    for(int k=0;k<nstep;k++){
-
-      steps++;
-
-      int kp     = k+1;
-      int peri_k = k %mmax;
-      int peri_kp= kp%mmax;
-
-      rq = vinnerProduct(q[peri_k],r);
-      a  = rq/qq[peri_k];
-
-      vaxpy(psi,a,p[peri_k],psi);
-      vaxpy(r,-a,q[peri_k],r);
-      cp = vnorm2(r);
-
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-               <<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-
-      if((k==nstep-1)||(cp<rsq)){
-        return cp;
-      }
-
-      Preconditioner(r,z);
-      vOp(z,Az);
-      zAAz=vnorm2(Az);
-
-      q[peri_kp]=Az;
-      p[peri_kp]=z;
-
-      int northog = ((kp)>(mmax-1))?(mmax-1):(kp);
-      for(int back=0;back<northog;back++){
-        int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-        b = -real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]);
-        vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]);
-      }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); // never reached
-    return cp;
-  }
-};
-
 //////////////////////////////////////////////////////////////////////
 // Trivial multi-RHS preconditioner
 //////////////////////////////////////////////////////////////////////
@@ -382,98 +195,6 @@ public:
   }
 };
 
-//////////////////////////////////////////////////////////////////////
-// MultiRHS two-level V-cycle.
-//
-// Mirrors MGPreconditioner in Example_pvdagm:
-//   out = in (trivial pre)                      [per rhs]
-//   r1  = in - A out                            [per rhs]
-//   batched blockProject -> pack -> ONE mrhs coarse PGCR -> unpack
-//     -> batched blockPromote; out += correction
-//   r2  = in - A out                            [per rhs]
-//   per-RHS fine post-smoother; out += smooth(r2)
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;   // same lattice type on Coarse5d and CoarseMrhs
-
-  LinearOperatorBase<FineField>            &_FineOperator;
-  FineSmoother                             &_PostSmoother;   // single-RHS smoother, looped
-  MultiRHSBlockProject<FineField>          &_Projector;
-  LinearFunction<CoarseVector>             &_CoarseSolve;    // PGCR on the 6D mrhs field
-  GridBase                                 *_CoarseGrid;     // Coarse5d (single rhs)
-  GridBase                                 *_CoarseGridMrhs; // 6D
-
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp,
-                 FineSmoother                  &Post,
-                 MultiRHSBlockProject<FineField> &Projector,
-                 LinearFunction<CoarseVector>  &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp), _PostSmoother(Post), _Projector(Projector),
-      _CoarseSolve(CoarseSolve), _CoarseGrid(CoarseGrid), _CoarseGridMrhs(CoarseGridMrhs) {}
-
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs = in.size();
-    GridBase *fgrid = in[0].Grid();
-    double t;
-
-    std::vector<FineField> vec1(nrhs,fgrid);
-    std::vector<FineField> vec2(nrhs,fgrid);
-
-    // Trivial pre-smoother: out = in (as in Example_pvdagm with simple_fine)
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-
-    // Residual
-    for(int r=0;r<nrhs;r++){
-      _FineOperator.Op(out[r],vec1[r]);
-      sub(vec1[r],in[r],vec1[r]);
-    }
-
-    // Batched fine->coarse, pack rhs into 6D field
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid);
-    std::vector<CoarseVector> Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs);
-    CoarseVector CsolMrhs(_CoarseGridMrhs);
-
-    t=-usecond();
-    _Projector.blockProject(vec1,Csrc_split);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-
-    // ONE coarse solve for all rhs (GEMM coarse mults)
-    t=-usecond();
-    CsolMrhs=Zero();
-    _CoarseSolve(CsrcMrhs,CsolMrhs);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // Unpack, batched coarse->fine, add correction
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-    _Projector.blockPromote(vec1,Csol_split);
-    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-
-    // Residual
-    for(int r=0;r<nrhs;r++){
-      _FineOperator.Op(out[r],vec1[r]);
-      sub(vec1[r],in[r],vec1[r]);
-    }
-
-    // Per-RHS post-smoother (fine level has no batching win; memory-light)
-    t=-usecond();
-    for(int r=0;r<nrhs;r++){
-      vec2[r]=Zero();
-      _PostSmoother(vec1[r],vec2[r]);
-      add(out[r],out[r],vec2[r]);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -539,8 +260,8 @@ int main (int argc, char ** argv)
 
   typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
-  typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                           CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
 

examples/Example_pvdagm_mrhs_3level.cc

@@ -23,7 +23,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 // with L3_DEFL=0 (NO deflation), applied to the enlarged block-diagonal mRHS system:
 // the coarse and coarse-coarse levels run a SINGLE Krylov (one GCR polynomial, inner
 // products summed over rhs) on the packed 6D mrhs fields, so both coarse levels batch
-// through GEMM (MultiGeneralCoarsenedMatrix) -- the valence throughput win at BOTH levels.
+// through GEMM (DeprecatedMultiGeneralCoarsenedMatrix) -- the valence throughput win at BOTH levels.
 //
 // Level structure (each coarse level is a single-field PGCR on a packed 6D mrhs field):
 //   L1 (fine)         : std::vector<LatticeFermionD>, MrhsPGCRNonHermitian on PVdagM,
@@ -50,6 +50,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
 using namespace std;
 using namespace Grid;
@@ -167,206 +168,6 @@ public:
   void HermOp  (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
 };
 
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from Example_pvdagm_mrhs.cc):
-// reductions summed over rhs -> one alpha/beta per step for the enlarged system.
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;   // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-    p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-      q[peri_kp]=Az; p[peri_kp]=z;
-      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-        b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle: a LinearFunction on the 6D mrhs COARSE field.
-// Mirrors Example_pvdagm_mrhs.cc's MrhsTwoLevelMG one level down, and the
-// single-RHS MGPreconditioner of Example_pvdagm_3level_SVDdefl.cc:
-//   out = in (trivial pre)
-//   r = in - A_coarse out
-//   restrict (unpack 6D coarse -> blockProject -> pack 6D coarse-coarse)
-//   ONE coarse-coarse solve (L3, GEMM)
-//   prolong (unpack -> blockPromote -> pack); out += correction
-//   r = in - A_coarse out
-//   coarse smoother (shifted 6D coarse op); out += smooth(r)
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;          // mrhs coarse op (6D)
-  LinearFunction<CoarseField>              &_CoarseSmoother;    // shifted 6D coarse smoother
-  MultiRHSBlockProject<CoarseField>        &_Projector;         // L2->L3 (vector-based)
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve; // L3 solve (6D cc)
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs; double t;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-
-    // trivial pre-smoother
-    out = in;
-
-    // residual (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
-    std::vector<CoarseField>       csplit(nrhs,_Coarse5d);
-    std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
-    _Projector.blockProject(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
-
-    // L3 solve (6D coarse-coarse, GEMM)
-    t=-usecond();
-    CCsol=Zero();
-    _CoarseCoarseSolve(CCsrc,CCsol);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
-    _Projector.blockPromote(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
-    add(out,out,vec1);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
-
-    // residual + coarse smoother (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-    vec2=Zero();
-    _CoarseSmoother(vec1,vec2);
-    add(out,out,vec2);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle (verbatim from Example_pvdagm_mrhs.cc):
-// per-rhs fine smoother + batched restriction + ONE coarse solve + batched prolong.
-// The coarse solve passed in is now itself three-level (preconditioned by L2->L3).
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    t=-usecond();
-    _Projector.blockProject(vec1,Csrc_split);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-    t=-usecond(); CsolMrhs=Zero(); _CoarseSolve(CsrcMrhs,CsolMrhs); t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-    _Projector.blockPromote(vec1,Csol_split);
-    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    t=-usecond();
-    for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -426,16 +227,16 @@ int main (int argc, char ** argv)
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
 
   // Level 1 tensor types
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>       LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>       LittleDiracOperator;
-  typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  MrhsLittleDiracOperator;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  MrhsLittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                                CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>                  Subspace;
 
   // Level 2 tensor types (coarsening deepens the nest by one iScalar -- see CLAUDE.md)
   typedef CoarseVector::vector_object                                      CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>          LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>          LittleDiracOperatorL2;
-  typedef MultiGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>     MrhsLittleDiracOperatorL2;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>     MrhsLittleDiracOperatorL2;
   typedef LittleDiracOperatorL2::CoarseVector                              CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,nbasis>                     SubspaceL2;
 

examples/Example_pvdagm_mrhs_3level_DenseCoarseMatrix.cc

@@ -50,6 +50,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 #include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
 
 #include <memory>
@@ -173,281 +174,6 @@ public:
   void HermOp  (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
 };
 
-//////////////////////////////////////////////////////////////////////
-// Dense CC solve on the PACKED 6D mrhs coarse-coarse field: drop-in
-// for the L3 PGCR, delegating to the library DenseCoarseMatrix.
-//////////////////////////////////////////////////////////////////////
-template<class DenseType, class CoarseCoarseField>
-class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
-public:
-  DenseType &_Dense;
-  GridBase *_CoarseCoarse5d;
-  int _nrhs;
-  MrhsDenseCCSolve(DenseType &D, GridBase *cc5d, int nrhs)
-    : _Dense(D), _CoarseCoarse5d(cc5d), _nrhs(nrhs) {}
-  using LinearFunction<CoarseCoarseField>::operator();
-  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
-    if ( getenv("DENSE_CC_CHECK") ) {
-      // Audit path: per-rhs 5D unpack so ApplyBatch can run the _Op defect
-      // check per rhs.  ~50ms/call of slice/split overhead -- audit only.
-      CoarseCoarseField tmp(in.Grid());
-      tmp = in;
-      std::vector<CoarseCoarseField> split_in (_nrhs,_CoarseCoarse5d);
-      std::vector<CoarseCoarseField> split_out(_nrhs,_CoarseCoarse5d);
-      for(int r=0;r<_nrhs;r++) ExtractSliceFast(split_in[r], tmp, r, 0);
-      _Dense.ApplyBatch(split_in, split_out);
-      for(int r=0;r<_nrhs;r++) InsertSliceFast(split_out[r], out, r, 0);
-    } else {
-      GRID_TRACE("MrhsDensCCSolve::ApplyBatch6D");
-      _Dense.ApplyBatch6D(in, out, _nrhs);
-    }
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from the
-// frozen Example_pvdagm_mrhs_3level_dense.cc)
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;   // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ GRID_TRACE("GCR-vnorm2"); RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ GRID_TRACE("GCR-vinnerProduct"); ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ GRID_TRACE("GCR-vaxpy"); for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("GCR-vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }  
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    GRID_TRACE((name+"MrhsPGCRNonHermitian").c_str());
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-    p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      {
-        GRID_TRACE("GCR-project");
-        rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-        vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      }      
-      //      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      //      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-
-      {
-        GRID_TRACE("GCR-orthog");
-        q[peri_kp]=Az; p[peri_kp]=z;
-        int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-        for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-          b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-          vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-        qq[peri_kp]=vnorm2(q[peri_kp]);
-      }      
-      //      q[peri_kp]=Az; p[peri_kp]=z;
-      //      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      //      for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-      //        b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-      //        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-      //      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle: LinearFunction on the 6D mrhs COARSE field.
-// The coarse-coarse solve slot takes EITHER the dense mrhs solve
-// (DENSE_CC=1) or the L3 PGCR (DENSE_CC=0).
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;          // mrhs coarse op (6D)
-  LinearFunction<CoarseField>              &_CoarseSmoother;    // shifted 6D coarse smoother
-  MultiRHSBlockProject<CoarseField>        &_Projector;         // L2->L3 (vector-based)
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve; // L3 solve (6D cc)
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs; double t;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-
-    // trivial pre-smoother
-    out = in;
-
-    // residual (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
-    std::vector<CoarseField>       csplit(nrhs,_Coarse5d);
-    std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-
-    t=-usecond();
-    {
-      GRID_TRACE("L2L3-Vcycle - Extract/blockProject/Insert");
-      for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
-      _Projector.blockProject(csplit,ccsplit);
-      for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
-
-    // L3 solve (dense mrhs GEMM, or PGCR)
-    t=-usecond();
-    {
-      GRID_TRACE("L2L3-Vcycle - CoarseCoarseSolve");
-      CCsol=Zero();
-      _CoarseCoarseSolve(CCsrc,CCsol);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
-    t=-usecond();
-    {
-      GRID_TRACE("L2L3-Vcycle - Ext/blockPromote/Ins");
-      for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
-      _Projector.blockPromote(csplit,ccsplit);
-      for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
-      add(out,out,vec1);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
-
-    // residual + coarse smoother (6D coarse)
-    {
-      GRID_TRACE("L2L3-Vcycle - Resid+CoarseSmooth");
-      _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-      vec2=Zero();
-      _CoarseSmoother(vec1,vec2);
-      add(out,out,vec2);
-    }
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle (verbatim from the frozen example)
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
-
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    {
-      GRID_TRACE("L1L2-Vcycle - Resid");
-      for(int r=0;r<nrhs;r++) out[r]=in[r];
-      for(int r=0;r<nrhs;r++){
-	_FineOperator.Op(out[r],vec1[r]);
-	sub(vec1[r],in[r],vec1[r]);
-      }
-    }
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - blockProject");
-      _Projector.blockProject(vec1,Csrc_split);
-      for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - CoarseSolve");
-      CsolMrhs=Zero();
-      _CoarseSolve(CsrcMrhs,CsolMrhs);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - ExtractSlice/blockPromote/add delta");
-      for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-      _Projector.blockPromote(vec1,Csol_split);
-      for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-    {
-      GRID_TRACE("L1L2-Vcycle - Residuals");
-      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    }
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - Smoothers");
-      for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
-    }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -507,16 +233,16 @@ int main (int argc, char ** argv)
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
 
   // Level 1 tensor types
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>       LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>       LittleDiracOperator;
-  typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  MrhsLittleDiracOperator;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  MrhsLittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                                CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>                  Subspace;
 
   // Level 2 tensor types (coarsening deepens the nest by one iScalar)
   typedef CoarseVector::vector_object                                      CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>          LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>          LittleDiracOperatorL2;
-  typedef MultiGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>     MrhsLittleDiracOperatorL2;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>     MrhsLittleDiracOperatorL2;
   typedef LittleDiracOperatorL2::CoarseVector                              CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,nbasis>                     SubspaceL2;
 
@@ -629,7 +355,7 @@ int main (int argc, char ** argv)
     std::cout << GridLogMessage << "**********************************************" << std::endl;
     DenseCC.reset(new DenseCC_t(CoarseCoarse5d));
     DenseCC->Import(LittleDiracOpL2);
-    MrhsDenseCC.reset(new MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector>(*DenseCC, CoarseCoarse5d, nrhs));
+    MrhsDenseCC.reset(new MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector>(*DenseCC, nrhs));
   }
 
   //////////////////////////////////////////////////////////////////////

examples/Example_pvdagm_mrhs_3level_dense.cc

@@ -60,6 +60,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
 #include <unordered_map>
 #include <memory>
@@ -236,7 +237,7 @@ public:
 #endif
   // NB: the apply GEMM (Y = slab^dag X) is a tiny-output/huge-K shape that
   // under-fills the GPU (~13ms).  The fix is a software split-K via
-  // GridBLAS.gemmBatched (see MultiRHSBlockCGLinalg.h / 2409.03904 Fig 11) —
+  // GridBLAS.gemmBatched (see MultiRHSBlockCGLinalg.h / 2409.03904 Fig 11) --
   // NOT a raw rocblas strided-batched batch, which hung on Frontier and was
   // removed.  TODO: reimplement through GridBLAS when the ~1.4 s/RHS is wanted.
 
@@ -830,226 +831,6 @@ public:
   }
 };
 
-//////////////////////////////////////////////////////////////////////
-// Dense CC solve on the PACKED 6D mrhs coarse-coarse field: unpack per
-// rhs, ONE batched dense apply, repack.  Drop-in for the L3 PGCR.
-//////////////////////////////////////////////////////////////////////
-template<class CoarseCoarseField>
-class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
-public:
-  DistributedDenseInverse<CoarseCoarseField> &_Dense;
-  GridBase *_CoarseCoarse5d;
-  int _nrhs;
-  MrhsDenseCCSolve(DistributedDenseInverse<CoarseCoarseField> &D, GridBase *cc5d, int nrhs)
-    : _Dense(D), _CoarseCoarse5d(cc5d), _nrhs(nrhs) {}
-  using LinearFunction<CoarseCoarseField>::operator();
-  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
-    if ( getenv("DENSE_CC_CHECK") ) {
-      // Audit path: per-rhs 5D unpack so ApplyBatch can run the _Op defect
-      // check per rhs.  ~50ms/call of slice/split overhead -- audit only.
-      CoarseCoarseField tmp(in.Grid());
-      tmp = in;
-      std::vector<CoarseCoarseField> split_in (_nrhs,_CoarseCoarse5d);
-      std::vector<CoarseCoarseField> split_out(_nrhs,_CoarseCoarse5d);
-      for(int r=0;r<_nrhs;r++) ExtractSliceFast(split_in[r], tmp, r, 0);
-      _Dense.ApplyBatch(split_in, split_out);
-      for(int r=0;r<_nrhs;r++) InsertSliceFast(split_out[r], out, r, 0);
-    } else {
-      _Dense.ApplyBatch6D(in, out, _nrhs);
-    }
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from Example_pvdagm_mrhs.cc)
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;   // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-    p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-      q[peri_kp]=Az; p[peri_kp]=z;
-      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-        b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle: LinearFunction on the 6D mrhs COARSE field.
-// The coarse-coarse solve slot now takes EITHER the dense mrhs solve
-// (DENSE_CC=1) or the L3 PGCR (DENSE_CC=0).
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;          // mrhs coarse op (6D)
-  LinearFunction<CoarseField>              &_CoarseSmoother;    // shifted 6D coarse smoother
-  MultiRHSBlockProject<CoarseField>        &_Projector;         // L2->L3 (vector-based)
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve; // L3 solve (6D cc)
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs; double t;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-
-    // trivial pre-smoother
-    out = in;
-
-    // residual (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
-    std::vector<CoarseField>       csplit(nrhs,_Coarse5d);
-    std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
-    _Projector.blockProject(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
-
-    // L3 solve (dense mrhs GEMM, or PGCR)
-    t=-usecond();
-    CCsol=Zero();
-    _CoarseCoarseSolve(CCsrc,CCsol);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
-    _Projector.blockPromote(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
-    add(out,out,vec1);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
-
-    // residual + coarse smoother (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-    vec2=Zero();
-    _CoarseSmoother(vec1,vec2);
-    add(out,out,vec2);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle (verbatim from Example_pvdagm_mrhs.cc)
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    t=-usecond();
-    _Projector.blockProject(vec1,Csrc_split);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-    t=-usecond(); CsolMrhs=Zero(); _CoarseSolve(CsrcMrhs,CsolMrhs); t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-    _Projector.blockPromote(vec1,Csol_split);
-    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    t=-usecond();
-    for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -1110,16 +891,16 @@ int main (int argc, char ** argv)
   typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
 
   // Level 1 tensor types
-  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>       LittleDiracOperator;
+  typedef DeprecatedGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>       LittleDiracOperator;
-  typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  MrhsLittleDiracOperator;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  MrhsLittleDiracOperator;
   typedef LittleDiracOperator::CoarseVector                                CoarseVector;
   typedef Aggregation<vSpinColourVector,vTComplex,nbasis>                  Subspace;
 
   // Level 2 tensor types (coarsening deepens the nest by one iScalar)
   typedef CoarseVector::vector_object                                      CoarseSiteObj;
   typedef iScalar<vTComplex>                                              vTTComplex;
-  typedef GeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>          LittleDiracOperatorL2;
+  typedef DeprecatedGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>          LittleDiracOperatorL2;
-  typedef MultiGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>     MrhsLittleDiracOperatorL2;
+  typedef DeprecatedMultiGeneralCoarsenedMatrix<CoarseSiteObj,vTTComplex,nbasis>     MrhsLittleDiracOperatorL2;
   typedef LittleDiracOperatorL2::CoarseVector                              CoarseCoarseVector;
   typedef Aggregation<CoarseSiteObj,vTTComplex,nbasis>                     SubspaceL2;
 
@@ -1221,13 +1002,13 @@ int main (int argc, char ** argv)
   // its memory peak; needs only the hoisted single-RHS LinOpCC5d).
   //////////////////////////////////////////////////////////////////////
   std::unique_ptr<DistributedDenseInverse<CoarseCoarseVector>> DenseCC;
-  std::unique_ptr<MrhsDenseCCSolve<CoarseCoarseVector>>        MrhsDenseCC;
+  std::unique_ptr<MrhsDenseCCSolve<DistributedDenseInverse<CoarseCoarseVector>,CoarseCoarseVector>>        MrhsDenseCC;
   if (UseDenseCC) {
     std::cout << GridLogMessage << "**********************************************" << std::endl;
     std::cout << GridLogMessage << " Dense CC inverse setup (mrhs bottom)" << std::endl;
     std::cout << GridLogMessage << "**********************************************" << std::endl;
     DenseCC.reset(new DistributedDenseInverse<CoarseCoarseVector>(LinOpCC5d, CoarseCoarse5d, nbasis));
-    MrhsDenseCC.reset(new MrhsDenseCCSolve<CoarseCoarseVector>(*DenseCC, CoarseCoarse5d, nrhs));
+    MrhsDenseCC.reset(new MrhsDenseCCSolve<DistributedDenseInverse<CoarseCoarseVector>,CoarseCoarseVector>(*DenseCC, nrhs));
   }
 
   //////////////////////////////////////////////////////////////////////

examples/Example_pvdagm_multigrid.cc

@@ -0,0 +1,189 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_pvdagm_multigrid.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    See the full license in the file "LICENSE" in the top level distribution
+    directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// The canonical three-level mrhs PVdagM multigrid driver, on the library
+// objects (Grid/algorithms/multigrid/PVdagMMultiGrid.h).
+//
+// Every parameter is in ONE serialisable struct, read from XML:
+//
+//   ./Example_pvdagm_multigrid --grid 48.48.48.96 --mpi ... \
+//                              --pvdagm-params params.xml
+//
+// With no --pvdagm-params the built-in defaults run (hot-start gauge field
+// unless Config is set in the file).  A missing file gets a template
+// written next to it and the program exits: the template documents every
+// parameter.  The effective parameters are always printed, so the log
+// describes its own run.
+//
+// Compile-time: NBASIS defaults to 60, the production basis; a subspace file
+// may hold more vectors, the load reads only the first NBASIS.  -DNBASIS=8
+// cuts it down for laptop runs.
+// examples/Makefile.am builds the fp32-coarse and fp32-dense-inversion
+// variants below as their own binaries.
+//
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/PVdagMMultiGrid.h>
+
+using namespace std;
+using namespace Grid;
+
+#ifndef NBASIS
+#define NBASIS 60
+#endif
+
+// Precision of the coarse + coarse-coarse sector is a compile-time
+// instantiation: -DCOARSE_SINGLE builds the fp32 coarse space (the dense
+// bottom's apply slab is fp32 either way; its inversion is a configure
+// option).  Both levels carry the same site type iVector<CoarseScalar,NBASIS>.
+#ifdef COARSE_SINGLE
+typedef sTComplexF CoarseScalar_t;
+#else
+typedef sTComplexD CoarseScalar_t;
+#endif
+
+struct PVdagMDriverParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(PVdagMDriverParams,
+                                  int,         Ls,
+                                  RealD,       Mass,
+                                  RealD,       M5,
+                                  RealD,       MobiusB,
+                                  RealD,       MobiusC,
+                                  std::string, Config,          // empty: hot start
+                                  int,         Nrhs,
+                                  int,         SolveSingleRHS,  // also run Nrhs=1 through the same objects
+                                  PVdagMMultiGridParams, MultiGrid);
+  PVdagMDriverParams()
+    : Ls(24), Mass(0.00078), M5(1.8), MobiusB(1.5), MobiusC(0.5),
+      Config(""), Nrhs(12), SolveSingleRHS(1) {};
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  PVdagMDriverParams P;
+  {
+    std::string pfile("");
+    if( GridCmdOptionExists(argv,argv+argc,"--pvdagm-params") )
+      pfile = GridCmdOptionPayload(argv,argv+argc,"--pvdagm-params");
+    if ( pfile.length() ) {
+      bool good; { std::ifstream f(pfile); good = f.good(); }
+      if ( !good ) {
+        if ( GlobalSharedMemory::WorldRank == 0 ) {
+          XmlWriter WR(pfile+".templ");
+          write(WR, "PVdagMDriver", P);
+          std::cout << GridLogMessage << pfile << " does not exist; template written to "
+                    << pfile << ".templ" << std::endl;
+        }
+        Grid_finalize(); return 0;
+      }
+      XmlReader RD(pfile);
+      read(RD, "PVdagMDriver", P);
+    }
+    CheckValidity(P.MultiGrid);
+    std::cout << GridLogMessage << "PVdagMDriver parameters ("
+              << (pfile.length() ? pfile : std::string("defaults")) << "):" << std::endl;
+    std::cout << P << std::endl;
+  }
+
+  Coordinate latt = GridDefaultLatt();
+  Coordinate mpi  = GridDefaultMpi();
+  Coordinate fsimd= GridDefaultSimd(Nd,vComplex::Nsimd());
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt,fsimd,mpi);
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(P.Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(P.Ls,UGrid);
+
+  GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
+  GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
+
+  LatticeGaugeField Umu(UGrid);
+  if ( P.Config.length() ) {
+    std::cout << GridLogMessage << "Reading gauge field " << P.Config << std::endl;
+    FieldMetaData header;
+    NerscIO::readConfiguration(Umu,header,P.Config);
+  } else {
+    std::cout << GridLogMessage << "Hot start gauge field" << std::endl;
+    SU<Nc>::HotConfiguration(RNG4,Umu);
+  }
+
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,P.Mass,P.M5,P.MobiusB,P.MobiusC);
+  MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0,   P.M5,P.MobiusB,P.MobiusC);
+
+  typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD> PVdagM_t;
+  PVdagM_t PVdagM(Ddwf,Dpv);
+
+  //////////////////////////////////////////////////////////////////////
+  // Grids -> coarsening -> dense bottom.  Scope order is lifetime order.
+  //
+  // The fine operator applied during setup is always fp64, on the fp64
+  // basis.  Everything the coarsening produces -- the Galerkin matrix
+  // elements and the transfer operator's store -- takes the coarse precision
+  // (CoarseScalar_t, a compile-time choice).
+  //
+  // There is exactly ONE fine transfer operator.  Its STORE follows the
+  // coarse sector, since the coarse space is what it feeds, while its import
+  // and export accept either fine precision: they are already a layout
+  // transformation, and a scalar conversion inside one is free.  So the fp64
+  // setup and an fp32 V-cycle share the same object and the same basis store.
+  // FinePrecision selects only the fine operator and smoother; the outer
+  // Krylov and its true-residual check stay fp64 throughout.
+  //////////////////////////////////////////////////////////////////////
+  typedef PVdagMMultiGridCoarsening<vSpinColourVector,CoarseScalar_t,NBASIS> Coarsening_t;
+  std::cout << GridLogMessage << "Coarse sector precision (compiled): "
+            << (sizeof(typename GridTypeMapper<CoarseScalar_t>::scalar_type)==sizeof(ComplexF) ? "fp32" : "fp64")
+            << ", nbasis " << NBASIS << std::endl;
+
+  MGCoarseGrids CGrids(FGrid, P.MultiGrid.Setup);
+  MGFineGridsF  FGridsF(FGrid);
+
+  LatticeGaugeFieldF UmuF(FGridsF.UGridF);
+  precisionChange(UmuF,Umu);
+  MobiusFermionF DdwfF(UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,P.Mass,P.M5,P.MobiusB,P.MobiusC);
+  MobiusFermionF DpvF (UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,1.0,   P.M5,P.MobiusB,P.MobiusC);
+
+  Coarsening_t  Coarsening(CGrids, FGridsF, P.MultiGrid.Setup);
+
+  Coarsening.GetSubspace(RNG5, PVdagM);
+  Coarsening.Coarsen(PVdagM);
+  Coarsening.BuildDenseBottom();
+  Coarsening.CertifyCoarsening(PVdagM);
+
+  //////////////////////////////////////////////////////////////////////
+  // Solves: mrhs then (optionally) single RHS through the SAME objects.
+  //////////////////////////////////////////////////////////////////////
+  auto RunSolve = [&](int nr)
+  {
+    PVdagMMultiGridSolver<MobiusFermionD,MobiusFermionF,Coarsening_t> Solver(Ddwf,Dpv,DdwfF,DpvF,Coarsening,P.MultiGrid,nr);
+    std::vector<LatticeFermionD> src(nr,FGrid), sol(nr,FGrid);
+    for(int r=0;r<nr;r++){ gaussian(RNG5,src[r]); sol[r]=Zero(); }
+    Solver.Solve(src,sol);
+  };
+
+  RunSolve(P.Nrhs);
+  if ( P.SolveSingleRHS && P.Nrhs != 1 ) RunSolve(1);
+
+  Grid_finalize();
+}

examples/Makefile.am

@@ -2,5 +2,29 @@ SUBDIRS = .
 
 include Make.inc
 
+# Compile-time variants of the two multigrid drivers.  The coarse-sector and
+# dense-inversion precisions are compile-time instantiations, so each is its
+# own binary built from the same source; the fine-level precision is a
+# run-time parameter and needs no variant.  The recur variant additionally
+# takes the GCR residual norm from the recurrence rather than a reduction.
+# Per-target flags give each variant its own object file.
+bin_PROGRAMS += Example_pvdagm_multigrid_fp32coarse \
+                Example_pvdagm_multigrid_fp32coarse_fp32dense \
+                Example_pvdagm_multigrid_fp32coarse_fp32dense_recur \
+                Example_hdcg_multigrid_fp32coarse
 
+Example_pvdagm_multigrid_fp32coarse_SOURCES  = Example_pvdagm_multigrid.cc
+Example_pvdagm_multigrid_fp32coarse_CPPFLAGS = -DCOARSE_SINGLE
+Example_pvdagm_multigrid_fp32coarse_LDADD    = $(top_builddir)/Grid/libGrid.a
 
+Example_pvdagm_multigrid_fp32coarse_fp32dense_SOURCES  = Example_pvdagm_multigrid.cc
+Example_pvdagm_multigrid_fp32coarse_fp32dense_CPPFLAGS = -DCOARSE_SINGLE -DGRID_DENSE_INVERSE_SINGLE
+Example_pvdagm_multigrid_fp32coarse_fp32dense_LDADD    = $(top_builddir)/Grid/libGrid.a
+
+Example_pvdagm_multigrid_fp32coarse_fp32dense_recur_SOURCES  = Example_pvdagm_multigrid.cc
+Example_pvdagm_multigrid_fp32coarse_fp32dense_recur_CPPFLAGS = -DCOARSE_SINGLE -DGRID_DENSE_INVERSE_SINGLE -DGRID_GCR_RESIDUAL_RECURRENCE
+Example_pvdagm_multigrid_fp32coarse_fp32dense_recur_LDADD    = $(top_builddir)/Grid/libGrid.a
+
+Example_hdcg_multigrid_fp32coarse_SOURCES  = Example_hdcg_multigrid.cc
+Example_hdcg_multigrid_fp32coarse_CPPFLAGS = -DCOARSE_SINGLE
+Example_hdcg_multigrid_fp32coarse_LDADD    = $(top_builddir)/Grid/libGrid.a

scripts/gcr_polynomial.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+Reconstruct the GCR smoother polynomial from PGCR coefficient logs
+(SmootherCoeffLog=1 in Example_pvdagm_3level_DenseCoarseMatrix, or
+LogCoefficients(1) on any PrecGeneralisedConjugateResidualNonHermitian).
+
+GCR with z = r (trivial preconditioner) builds  p_k = P_k(A) r0,
+r_k = R_k(A) r0:
+
+    R_0 = P_0 = 1
+    R_{k+1}(l) = R_k(l) - a_k l P_k(l)
+    P_{k+1}(l) = R_{k+1}(l) + sum_j b_{k,j} P_{k-j}(l)
+
+so after m steps  x_m = S(A) r0  with  S = sum_k a_k P_k  and residual
+polynomial  R_m = 1 - l S(l),  R_m(0) = 1.
+
+Usage
+  gcr_polynomial.py LOG [--name Fsmoother] [--lmin 0] [--lmax 3] [--n 600]
+      writes <name>_poly.dat : l  |R_mean(l)|  Re R_mean  Im R_mean  |R|min  |R|max  |S_mean(l)|
+      where mean = polynomial built from the per-step MEAN coefficients over
+      all calls, and min/max are the envelope over the individual calls.
+      Prints the mean coefficients and their relative spread per step.
+  gcr_polynomial.py --selftest
+      runs GCR in numpy on a random non-normal matrix, logs a,b the same way,
+      and checks  R_m(A) r0 == the actual residual  to rounding.
+
+gnuplot:
+  set logscale y; plot 'Fsmoother_poly.dat' u 1:2 w l t '|R_m| mean', \
+                       '' u 1:5 w l t 'min', '' u 1:6 w l t 'max'
+"""
+import re, sys, argparse
+import numpy as np
+from numpy.polynomial import polynomial as Poly   # coefficient arrays, low->high
+
+A_RE = re.compile(r'(\S+)\s+coeff\[(\d+)\]\s+a=\(([^,]+),([^)]+)\)')
+B_RE = re.compile(r'(\S+)\s+coeff\[(\d+)\]((?:\s+b\[\d+\]=\([^)]*\))+)')
+BJ_RE = re.compile(r'b\[(\d+)\]=\(([^,]+),([^)]+)\)')
+
+def parse(path, name):
+    """-> list of calls; each call = list of (a_k, [b_k0, b_k1, ...]) in step order."""
+    calls, cur = [], None
+    for line in open(path, errors='replace'):
+        m = A_RE.search(line)
+        if m and m.group(1) == name:
+            k = int(m.group(2)); a = complex(float(m.group(3)), float(m.group(4)))
+            if k == 0:
+                if cur: calls.append(cur)
+                cur = []
+            if cur is not None:
+                cur.append([a, []])
+            continue
+        m = B_RE.search(line)
+        if m and m.group(1) == name and cur is not None:
+            k = int(m.group(2))
+            bs = [complex(float(x[1]), float(x[2])) for x in BJ_RE.findall(m.group(3))]
+            if k < len(cur): cur[k][1] = bs
+    if cur: calls.append(cur)
+    return calls
+
+def polynomials(coeffs):
+    """coeffs: list of (a_k, [b_kj]) -> (S, R) as low->high complex coefficient arrays."""
+    R = [np.array([1.0+0j])]; P = [np.array([1.0+0j])]
+    S = np.array([0.0+0j])
+    for k, (a, bs) in enumerate(coeffs):
+        S = Poly.polyadd(S, a*P[k])
+        Rn = Poly.polysub(R[k], a*Poly.polymulx(P[k]))          # R_{k+1} = R_k - a l P_k
+        Pn = Rn.copy()
+        for j, b in enumerate(bs):                              # P_{k+1} = R_{k+1} + sum b_kj P_{k-j}
+            Pn = Poly.polyadd(Pn, b*P[k-j])
+        R.append(Rn); P.append(Pn)
+    return S, R[-1]
+
+def mean_coeffs(calls):
+    m = min(len(c) for c in calls)
+    out, spread = [], []
+    for k in range(m):
+        a = np.array([c[k][0] for c in calls])
+        nb = min(len(c[k][1]) for c in calls)
+        bs = [np.array([c[k][1][j] for c in calls]) for j in range(nb)]
+        out.append((a.mean(), [b.mean() for b in bs]))
+        sa = a.std()/max(abs(a.mean()),1e-300)
+        sb = [b.std()/max(abs(b.mean()),1e-300) for b in bs]
+        spread.append((sa, sb))
+    return out, spread
+
+def selftest(n=40, m=6, mmax=6, seed=1):
+    rng = np.random.default_rng(seed)
+    A = np.eye(n)*1.5 + 0.4*rng.standard_normal((n,n)) + 0.2j*rng.standard_normal((n,n))  # non-normal
+    r0 = rng.standard_normal(n) + 1j*rng.standard_normal(n)
+    r = r0.copy(); x = np.zeros(n, complex)
+    p = [r.copy()]; q = [A@r]; qq = [np.vdot(q[0],q[0]).real]
+    log = []
+    for k in range(m):
+        a = np.vdot(q[k], r)/qq[k]                               # <q,r>/<q,q>
+        x = x + a*p[k]; r = r - a*q[k]
+        z = r; Az = A@z; pn = z.copy(); qn = Az.copy(); bs = []
+        for back in range(min(k+1, mmax-1)):
+            b = -(np.vdot(q[k-back], Az)/qq[k-back]).real        # real part, as Grid does
+            pn = pn + b*p[k-back]; qn = qn + b*q[k-back]; bs.append(complex(b))
+        p.append(pn); q.append(qn); qq.append(np.vdot(qn,qn).real)
+        log.append((a, bs))
+    S, R = polynomials(log)
+    def apply(poly, v):
+        out = np.zeros_like(v); Ak = v.copy()
+        for c in poly: out = out + c*Ak; Ak = A@Ak
+        return out
+    err_r = np.linalg.norm(apply(R, r0) - r)/np.linalg.norm(r)
+    err_x = np.linalg.norm(apply(S, r0) - x)/np.linalg.norm(x)
+    print(f"selftest: |R_m(A)r0 - r_m|/|r_m| = {err_r:.2e}   |S(A)r0 - x_m|/|x_m| = {err_x:.2e}")
+    return err_r < 1e-10 and err_x < 1e-10
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser()
+    ap.add_argument("log", nargs="?")
+    ap.add_argument("--name", default="Fsmoother")
+    ap.add_argument("--lmin", type=float, default=0.0)
+    ap.add_argument("--lmax", type=float, default=3.0)
+    ap.add_argument("--n", type=int, default=600)
+    ap.add_argument("--selftest", action="store_true")
+    args = ap.parse_args()
+    if args.selftest:
+        sys.exit(0 if selftest() else 1)
+    calls = parse(args.log, args.name)
+    if not calls:
+        sys.exit(f"no '{args.name} coeff[...]' lines found in {args.log}")
+    mean, spread = mean_coeffs(calls)
+    print(f"{args.name}: {len(calls)} calls, {len(mean)} steps")
+    for k,(a,bs) in enumerate(mean):
+        print(f"  step {k}: a = ({a.real:+.5f},{a.imag:+.5f})  rel spread {spread[k][0]:.3f}   "
+              + "  ".join(f"b[{j}]={b.real:+.5f} (spread {spread[k][1][j]:.3f})" for j,b in enumerate(bs)))
+    S, R = polynomials(mean)
+    print("  S coefficients (low->high):", np.array2string(S, precision=5))
+    lam = np.linspace(args.lmin, args.lmax, args.n)
+    Rm = Poly.polyval(lam, R); Sm = Poly.polyval(lam, S)
+    per = np.array([np.abs(Poly.polyval(lam, polynomials(c)[1])) for c in calls])
+    out = f"{args.name}_poly.dat"
+    with open(out, "w") as f:
+        f.write("# lambda |R_mean| ReR ImR |R|min |R|max |S_mean|\n")
+        for i,l in enumerate(lam):
+            f.write(f"{l:.6f} {abs(Rm[i]):.6e} {Rm[i].real:.6e} {Rm[i].imag:.6e} "
+                    f"{per[:,i].min():.6e} {per[:,i].max():.6e} {abs(Sm[i]):.6e}\n")
+    print(f"  wrote {out}")

skills/multigrid-design-notes.md

@@ -221,11 +221,11 @@ judgement.
   the exact inverse of whatever matrix it was handed. Both certify the import
   and inversion, not the accuracy of `A_c`.
   The instruments that DO see it: coarsen twice and compare `BLAS_A` element by
-  element against an exact run (the comparison `Test_coarse_v2_coarsen` already
+  element against an exact run (the comparison `Test_coarse_coarsen` already
-  performs between V1 and V2), or the outer iteration count and time to
+  performs against the deprecated operator), or the outer iteration count and time to
   solution.
 - **Does not reach the coarse levels.** `SloppyComms` sets a flag on
-  `Stencil`/`StencilEven`/`StencilOdd`; V2's `M` uses `PaddedCell::Face_exchange`,
+  `Stencil`/`StencilEven`/`StencilOdd`; the coarse operator.s `M` uses `PaddedCell::Face_exchange`,
   which sends raw `vobj` bytes with no precision option. Coarse levels are less
   bandwidth sensitive, so this matters less than it sounds.
 - **Object sharing caveat:** the flag is per fermion operator. A single

systems/Frontier/config-command

@@ -2,9 +2,6 @@ CLIME=`spack find --paths c-lime@2-3-9 | grep c-lime| cut -c 15-`
 
 ../../configure --enable-comms=mpi-auto \
 --with-lime=$CLIME \
---with-slate=$SLATE \
---with-blaspp=$BLASPP \
---with-lapackpp=$LAPACKPP \
 --enable-unified=no \
 --enable-shm=nvlink \
 --enable-tracing=roctx \
@@ -16,7 +13,6 @@ CLIME=`spack find --paths c-lime@2-3-9 | grep c-lime| cut -c 15-`
 --with-openssl=$OPENSSL \
 --with-gmp=$GMP \
 --with-mpfr=$MPFR \
---disable-fermion-reps \
 CXX=hipcc MPICXX=mpicxx \
 CXXFLAGS="-fPIC -I${ROCM_PATH}/include/ -I${MPICH_DIR}/include " \
  LDFLAGS="-L${ROCM_PATH}/lib -L${MPICH_DIR}/lib -lmpi -lmpi_gtl_hsa -lhipblas -lrocsolver -lrocblas -lhipfft -lamdhip64"

systems/Frontier/nrhs_fault.job

@@ -66,7 +66,7 @@ export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU
 module load libfabric
 
-BIN=$root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix
+BIN=$root/examples/Example_pvdagm_3level_DenseCoarseMatrix
 OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1"
 # LOCAL VOLUME MATCHED to production: 48.48.48.96 on 3.6.4.4 is local 16.8.12.24; on 8 GCDs
 # (1.1.2.4) that is global 16.8.24.96 with the production blockings 2.2.3.3 / 4.4.2.4:
@@ -112,7 +112,7 @@ run_cell () {
   logopt=""; [ -n "$GRID_LOG_EXTRA" ] && logopt="--log Error,Warning,Message,$GRID_LOG_EXTRA"
   env "$@" srun -N1 -n8 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --device-mem $devmem --comms-overlap $logopt > $fname 2>&1
   echo "  exit $?"; sleep 10
-  grep -h "V2 3-level solve\|Memory access fault\|GRID_ASSERT\|Converged on iteration" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | tail -4 | cut -c1-140
+  grep -h "3-level solve\|Memory access fault\|GRID_ASSERT\|Converged on iteration" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | tail -4 | cut -c1-140
   grep -n -B3 "Memory access fault" $fname | tail -5 | cut -c1-160
 }
 
@@ -124,5 +124,5 @@ run_cell   F3_nrhs6_nodense 16000 NRHS=6 DENSE_CC=0
 run_cell   F4_nrhs6_bigmem 48000 NRHS=6                                                    # no MemoryManager eviction
 
 echo "========================================================="
-for f in log.fault.F*; do echo "$f: $(grep -c 'Memory access fault' $f) faults; $(grep -h 'V2 3-level solve' $f | tail -1 | sed 's/.*V2/V2/' | cut -c1-60)"; done
+for f in log.fault.F*; do echo "$f: $(grep -c 'Memory access fault' $f) faults; $(grep -h '3-level solve' $f | tail -1 | sed 's/.*3-level/3-level/' | cut -c1-60)"; done
 echo "========================================================="

systems/Frontier/nrhs_fault_36.job

@@ -67,8 +67,9 @@ export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU
 module load libfabric
 
-BIN=$root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix
+BIN=$root/examples/Example_pvdagm_3level_DenseCoarseMatrix
 OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
+OVERLAP="--comms-overlap"          # set empty for a no-overlap job
 vol=48.48.48.96
 MPI_GEOM=3.6.4.4
 
@@ -99,7 +100,7 @@ run_cell () {
   name=$1; shift
   echo "----- $name : $* -----"
   export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name
-  env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \
+  env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 $OVERLAP \
        --debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
   echo "  exit $?"; sleep 30
   f=$(grep -l "Memory access fault" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
@@ -111,12 +112,14 @@ run_cell () {
     tail -15 $(dirname $f)/Grid.stdout.$rank | cut -c1-160
   else
     echo "  no 'Memory access fault' in any Grid.stderr.*"
-    grep -h "V2 3-level solve\|Fouter MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -3 | cut -c1-120
+    grep -h "3-level solve\|Fouter MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -3 | cut -c1-120
   fi
 }
 
 #run_cell   A_serialised   AMD_SERIALIZE_KERNEL=3 AMD_SERIALIZE_COPY=3 AMD_LOG_LEVEL=3   # 5371703-era: OOM found; done
 #run_cell   B_plain        AMD_LOG_LEVEL=1
+# NOTE (2026-08-29): FineSloppyComms is NOT set in this job; the example defaults it to 1, so every
+# cell below ran with SLOPPY comms ON (confirmed by the PARAM line).  Only 5371826/5372414 set it to 0.
 # Ladder for the NO_TRANSLATION / hang fault (both at NRHS=12):
 #   5371703  sloppy ON,  no kdreg2            -> NO_TRANSLATION (Dhop halo) at outer 36
 #   5371826  sloppy OFF, no kdreg2            -> 12-RHS converged, HANG in single-RHS solve
@@ -146,5 +149,7 @@ run_cell   L_cache4       FI_MR_CACHE_MAX_COUNT=4              # PB: room for Pa
 # at the first exchange (as the reproducer predicts for the cache-off path); G-M all solved
 # Nrhs 6 + Nrhs 1 (NOTE: NRHS=6 above, the three failures in the ledger were at NRHS=12).
 # Missing control: the new PaddedCell, NO env knobs, at NRHS=12, twice (intermittent failure).
-run_cell   N_control      NRHS=12
+run_cell   N_control      NRHS=12       # 5374670: PASSED
-run_cell   N_control2     NRHS=12
+run_cell   N_control2     NRHS=12       # 5374670: NO_TRANSLATION, Dhop Waitall(16), 3 ranks; last logged event on the faulting rank = eviction 49 of the 62-eviction restart burst after outer step 48 (170 MB hipFree via the ring cache), the failing exchange followed it
+# Overlap on/off is a WHOLE-JOB A/B: set OVERLAP empty above and submit again.  Measured
+# 2026-08-29/30: it does not change the failure rate.