Clean up and unification of PVdagM and HDCG, mixed precision support

CLAUDE.md

@@ -106,7 +106,7 @@ Tests and benchmarks that need optional fermion representations are guarded by `
 ### GPU acceleration and the view/memory-manager discipline
 ### Multigrid (`Grid/algorithms/multigrid/`)
 
-Aggregation-based algebraic multigrid for Wilson-type fermions. Key files: `CoarsenedMatrix.h` (coarse operator), `GeneralCoarsenedMatrix.h` and `GeneralCoarsenedMatrixMultiRHS.h` (general coarsening supporting multi-RHS solves), `Aggregates.h` (near-null vector construction), `Geometry.h` (coarse-grid geometry). `MultiGrid.h` is the top-level include.
+Aggregation-based algebraic multigrid for Wilson-type fermions. Key files: `GeneralCoarsenedMatrixMultiRHSV2.h` (the multi-RHS coarse operator, "V2": coarsening by Fourier probing, batched-GEMM apply on a D+1 grid with rhs innermost and unvectorised), `MultiRHSBlockProject.h` (in `deflation/`; the batched-GEMM transfer operators), `PVdagMMultiGrid.h` (three-level non-Hermitian PVdagM chain with dense bottom) and `HDCGMultiGrid.h` (two-level Hermitian HDCG chain) with their `*Params.h` (XML-serialisable parameters), `MrhsMultiGrid.h` (mrhs V-cycle, preconditioner interface, fp64/fp32 seam), `Smoothers.h`, `MultiGridIO.h`, `Aggregates.h` (near-null vector construction), `Geometry.h` (coarse stencils: max-norm-1 boxes of Manhattan radius 1, 2 or 4), `CoarsenedMatrix.h` (the original nearest-neighbour Wilson multigrid). `deprecated/` holds the V1 coarse operators and the Aggregation-based single-RHS ADEF-2, kept only for the pre-2026 drivers. `MultiGrid.h` is the top-level include.
 
 ### GPU acceleration
 

Grid/algorithms/LinearOperator.h

@@ -214,6 +214,29 @@ public:
   }
 };
 
+////////////////////////////////////////////////////////////////////
+// Present any LinearOperatorBase as Hermitian: Op = AdjOp = HermOp.
+// For coarsening (CoarsenOperator applies Op) an HPD operator whose
+// Op is a factor and HermOp the product, e.g. SchurDiagMooee or MdagM.
+////////////////////////////////////////////////////////////////////
+template<class Field>
+class HermOpAdaptor : public LinearOperatorBase<Field> {
+  LinearOperatorBase<Field> &_Mat;
+public:
+  HermOpAdaptor(LinearOperatorBase<Field> &Mat): _Mat(Mat){};
+  void OpDiag (const Field &in, Field &out)                   { GRID_ASSERT(0); }
+  void OpDir  (const Field &in, Field &out,int dir,int disp)  { GRID_ASSERT(0); }
+  void OpDirAll(const Field &in, std::vector<Field> &out)     { GRID_ASSERT(0); }
+  void Op     (const Field &in, Field &out){ _Mat.HermOp(in,out); }
+  void AdjOp  (const Field &in, Field &out){ _Mat.HermOp(in,out); }
+  void HermOp (const Field &in, Field &out){ _Mat.HermOp(in,out); }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+    HermOp(in,out);
+    ComplexD dot = innerProduct(in,out);
+    n1=real(dot);
+    n2=norm2(out);
+  }
+};
 
 ////////////////////////////////////////////////////////////////////
 // Wrap an already herm matrix

Grid/algorithms/blas/BatchedBlas.h

@@ -1942,6 +1942,10 @@ public:
 		Cs);
     synchronise();
 
+    // Synchronise ONCE, after the whole train of calls.  A synchronise inside
+    // the loop measures launch and completion latency, which at small shapes
+    // exceeds the kernel: at M=K=60, N=1, batch 1024 it hid a factor of two
+    // between the precisions and reported them level.
     RealD t0 = usecond();
     for(int i=0;i<ncall;i++){
       gemmBatched(M,N,K,
@@ -1950,8 +1954,8 @@ public:
 		  Bs, // k x n
 		  beta, 
 		  Cs);
-      synchronise();
     }
+    synchronise();
     RealD t1 = usecond();
     RealD bytes = 1.0*sizeof(CComplex)*(M*N*2+N*K+M*K)*BATCH;
     flops = 8.0*M*N*K*BATCH*ncall;

Grid/algorithms/blas/BatchedInverse.h

@@ -111,7 +111,9 @@ public:
   // ranks wait on it.  The _64 getrf/getrs pair is the path proven in the
   // 1-rank dense coarse-coarse setup at N=69120 (DenseCoarseMatrix.h).
   // Same in-place, column-major, lda=N contract as inverseBatched.
-  // NB: written to rocSOLVER's documented z*_64 signatures; not compiled on
+  // ComplexD and ComplexF overloads (z*_64 / c*_64), so the distributed
+  // dense inverse can run in either precision (DenseInverseScalar).
+  // NB: written to rocSOLVER's documented *_64 signatures; not compiled on
   // HIP in the air-gapped loop -- verify on first hipcc build.
   ///////////////////////////////////////////////////////////////////////////
   double lastGetrfUs = 0.0, lastGetrsUs = 0.0;   // split timing of the last inverseLU (HIP path)
@@ -145,6 +147,37 @@ public:
     inverseBatched(N, bp);
 #endif
   }
+  // fp32 overload: the Schur-recursion leaf under --enable-dense-inverse-precision=single
+  void inverseLU(int64_t N, ComplexF *A)
+  {
+#ifdef GRID_HIP
+    rocblas_handle handle = Handle();
+    deviceVector<int64_t>  ipiv((uint64_t)N);
+    deviceVector<int64_t>  info(1);
+    double t0 = usecond();
+    auto st1 = rocsolver_cgetrf_64(handle, N, N, (rocblas_float_complex *)A, N, &ipiv[0], &info[0]);
+    GRID_ASSERT(st1 == rocblas_status_success);
+    accelerator_barrier();
+    lastGetrfUs = usecond()-t0;
+    int64_t info_h = -1; acceleratorCopyFromDevice(&info[0], &info_h, sizeof(int64_t));
+    GRID_ASSERT(info_h == 0);
+    deviceVector<ComplexF> X((uint64_t)N*N);
+    { ComplexF *x = &X[0]; const int64_t NN = N;
+      accelerator_for(idx, (uint64_t)N*N, 1, { int64_t j = idx/NN, i = idx - j*NN; x[idx] = (i==j) ? ComplexF(1.0,0.0) : ComplexF(0.0,0.0); });
+      accelerator_barrier(); }
+    auto st2 = rocsolver_cgetrs_64(handle, rocblas_operation_none, N, N,
+                                   (rocblas_float_complex *)A, N, &ipiv[0],
+                                   (rocblas_float_complex *)&X[0], N);
+    GRID_ASSERT(st2 == rocblas_status_success);
+    accelerator_barrier();
+    lastGetrsUs = usecond()-t0-lastGetrfUs;
+    acceleratorCopyDeviceToDevice((void *)&X[0], (void *)A, (uint64_t)N*N*sizeof(ComplexF));
+#else
+    deviceVector<ComplexF*> bp(1); std::vector<ComplexF*> ptr(1); ptr[0] = A;
+    acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexF*));
+    inverseBatched(N, bp);
+#endif
+  }
 
   void inverseBatched(int64_t N, deviceVector<ComplexF*> &Amat)
   {

Grid/algorithms/deflation/MultiRHSBlockProject.h

@@ -93,6 +93,25 @@ public:
   deviceVector<scalar> BLAS_F;      // nrhs x fine_vol * words   -- the sources
   deviceVector<scalar> BLAS_C;      // nrhs x coarse_vol * nbasis -- the coarse coeffs
 
+  // Resident device memory this object holds.  deviceVector is not
+  // evictable, so this counts against the hard budget, not the cache.
+  // Capacity, not size: a call with fewer right-hand sides shrinks the size
+  // and frees nothing.
+  uint64_t DeviceBytes(void)
+  {
+    return (uint64_t)(BLAS_V.capacity()+BLAS_F.capacity()+BLAS_C.capacity())*sizeof(scalar);
+  }
+  // The import/export scratch grows to the largest right-hand-side count
+  // any call has used and keeps that capacity.  Setup projects the whole
+  // basis in one call, so unless it is released the scratch stays as large
+  // as the basis store for the rest of the run.  The next call regrows it
+  // to the size that call needs.
+  void ReleaseScratch(void)
+  {
+    deviceVector<scalar>().swap(BLAS_F);
+    deviceVector<scalar>().swap(BLAS_C);
+  }
+
   RealD blasNorm2(deviceVector<scalar> &blas)
   {
     scalar ss(0.0);
@@ -117,9 +136,14 @@ public:
     block_vol=0;
     coarse_vol=0;
     words=0;
-    BLAS_V.resize(0);
+    // deviceVector is a std::vector with a device allocator, so resize(0)
-    BLAS_F.resize(0);
+    // drops the SIZE and keeps the CAPACITY: it returns no device memory at
-    BLAS_C.resize(0);
+    // all.  The basis store is nbasis fine vectors (10.9 GB at nbasis 64 on
+    // a 48^3x96 Ls=24 rank), so that has to be a real free.  Swap with an
+    // empty vector, which destroys the buffer.
+    deviceVector<scalar>().swap(BLAS_V);
+    deviceVector<scalar>().swap(BLAS_F);
+    deviceVector<scalar>().swap(BLAS_C);
   }
   void Allocate(int _nbasis,GridBase *_fgrid,GridBase *_cgrid)
   {
@@ -153,25 +177,39 @@ public:
 
     BLAS_V.resize (fine_vol * words * nbasis );
   }
-  void ImportFineGridVectors(std::vector <Field > &vecs, deviceVector<scalar> &blas)
+  ////////////////////////////////////////////////////////////////////////////
+  // The fine side is templated on the INCOMING vector object, not fixed to
+  // Field.  Field fixes only the STORAGE: the BLAS scalar, the word count and
+  // the full local geometry.  Import is already a layout transformation --
+  // (oSite,lane) is unpacked to full local coordinates and regathered in
+  // block order -- so a precision change costs nothing here, and neither does
+  // a different SIMD layout.  That is what lets one projector serve an fp64
+  // outer level and an fp32 coarse sector with no second basis store.
+  //
+  // The full LOCAL dimensions must agree; the SIMD layout and the precision
+  // need not.
+  ////////////////////////////////////////////////////////////////////////////
+  template<class vobj>
+  void ImportFineGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
   {
     GRID_TRACE("ImportFineGridVectors");
     int nvec = vecs.size();
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
-    //    std::cout << GridLogMessage <<" BlockProjector importing "<<nvec<< " fine grid vectors" <<std::endl;
+    typedef typename vobj::scalar_type   fine_scalar;
-
-    GRID_ASSERT(vecs[0].Grid()==fine_grid);
 
+    GridBase *fgrid = vecs[0].Grid();
     int _ndimension = coarse_grid->_ndimension;
+    for(int d=0;d<_ndimension;d++) GRID_ASSERT(fgrid->_ldimensions[d] == fine_ldimensions[d]);
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
 
     uint64_t sz = blas.size();
 
     acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
 
-    Coordinate fine_rdimensions = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions = fgrid->_rdimensions;
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
+    Coordinate fsimd    = fgrid->_simd_layout;
     int64_t bv= block_vol;
     for(int v=0;v<vecs.size();v++){
 
@@ -181,7 +219,7 @@ public:
       auto blasData_p  = &blas[0];
       auto fineData_p  = &fineData[0];
 
-      int64_t osites = fine_grid->oSites();
+      int64_t osites = fgrid->oSites();
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
@@ -213,7 +251,7 @@ public:
 	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
 	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
 
-          scalar_object data = extractLane(lane,fineData[sf]);
+          fine_scalar_object data = extractLane(lane,fineData[sf]);
 
 	  // BLAS_F[coarse_vol][nvec][block_vol][words]
 	  int64_t site = (sc*nvec + v)*bv
@@ -221,9 +259,9 @@ public:
 
 	  //	  GRID_ASSERT(site*lwords<sz);
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-
+	  const fine_scalar *dp = (const fine_scalar *)&data;
-	  *ptr = data;
+	  for(uint64_t w=0;w<lwords;w++) blasData_p[site*lwords+w] = scalar(dp[w]);
 #ifdef GRID_SIMT
 	}
 #else
@@ -245,9 +283,11 @@ public:
   // The gather of block_vol from fine_vol, and the transpose of nrhs against
   // block_vol, are the irreducible part: this is not the identity.
   ////////////////////////////////////////////////////////////////////////////
-  void ImportFineGridMrhsVectors(Field &vec_mrhs, deviceVector<scalar> &blas)
+  template<class vobj>
+  void ImportFineGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
+    typedef typename vobj::scalar_type   fine_scalar;
 
     GridBase *fine_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
@@ -255,27 +295,31 @@ public:
     GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1);
     GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1);
     GRID_ASSERT(fine_mrhs_grid->_processors[0] == 1);
+    // Full LOCAL dimensions must agree; the SIMD layout and precision need not
     for(int d=0;d<_ndimension;d++){
-      GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]);
+      GRID_ASSERT(fine_mrhs_grid->_ldimensions[d+1] == fine_ldimensions[d]);
-      GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]);
     }
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
     int nvec = fine_mrhs_grid->_rdimensions[0];   // nrhs
 
     uint64_t sz = blas.size();
     acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
 
     Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions;
-    Coordinate fine_rdimensions      = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions(_ndimension);
+    Coordinate fsimd(_ndimension);
+    for(int d=0;d<_ndimension;d++){
+      fine_rdimensions[d] = fine_mrhs_grid->_rdimensions[d+1];
+      fsimd[d]            = fine_mrhs_grid->_simd_layout[d+1];
+    }
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
     int64_t bv= block_vol;
 
     autoView( fineData   , vec_mrhs, AcceleratorRead);
     auto blasData_p  = &blas[0];
     auto fineData_p  = &fineData[0];
 
-    int64_t osites     = fine_grid->oSites();       // D dimensional
     int64_t osites_hi  = fine_mrhs_grid->oSites();  // nvec * osites
 
     const int Nsimd = vobj::Nsimd();
@@ -312,13 +356,14 @@ public:
 	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
 	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
 
-	  scalar_object data = extractLane(lane,fineData[sfr]);
+	  fine_scalar_object data = extractLane(lane,fineData[sfr]);
 
 	  int64_t site = (sc*lnvec + v)*bv
 	               + sb;
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-	  *ptr = data;
+	  const fine_scalar *dp = (const fine_scalar *)&data;
+	  for(uint64_t w=0;w<lwords;w++) blasData_p[site*lwords+w] = scalar(dp[w]);
 #ifdef GRID_SIMT
       }
 #else
@@ -339,6 +384,7 @@ public:
   void ExportCoarseGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
 
     GridBase *coarse_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
@@ -365,7 +411,7 @@ public:
     int64_t osites_hi = coarse_mrhs_grid->oSites();  // nvec * osites
 
     const int Nsimd = vobj::Nsimd();
-    uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+    uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
     GRID_ASSERT(cwords==nbasis);
     int64_t lnvec = nvec;
 
@@ -391,8 +437,11 @@ public:
 	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
 
 	  int64_t blas_site = (sc*lnvec + v)*cwords;
-	  coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	  // Element-wise, converting: the BLAS buffer is in the FINE precision,
-	  coarse_scalar_object data = *ptr;
+	  // the coarse field in its own.
+	  coarse_scalar_object data;
+	  coarse_scalar *dp = (coarse_scalar *)&data;
+	  for(uint64_t b=0;b<cwords;b++) dp[b] = coarse_scalar(blasData_p[blas_site+b]);
 	  insertLane(lane,coarseData[scr],data);
 #ifdef GRID_SIMT
       }
@@ -405,33 +454,39 @@ public:
   ////////////////////////////////////////////////////////////////////////////
   // Reverse directions: BLAS_F -> fine mrhs field, coarse mrhs field -> BLAS_C
   ////////////////////////////////////////////////////////////////////////////
-  void ExportFineGridMrhsVectors(Field &vec_mrhs, deviceVector<scalar> &blas)
+  template<class vobj>
+  void ExportFineGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
+    typedef typename vobj::scalar_type   fine_scalar;
 
     GridBase *fine_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
 
     GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1);
     GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1);
+    // Full LOCAL dimensions must agree; the SIMD layout and precision need not
     for(int d=0;d<_ndimension;d++){
-      GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]);
+      GRID_ASSERT(fine_mrhs_grid->_ldimensions[d+1] == fine_ldimensions[d]);
-      GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]);
     }
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
     int nvec = fine_mrhs_grid->_rdimensions[0];
 
     Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions;
-    Coordinate fine_rdimensions      = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions(_ndimension);
+    Coordinate fsimd(_ndimension);
+    for(int d=0;d<_ndimension;d++){
+      fine_rdimensions[d] = fine_mrhs_grid->_rdimensions[d+1];
+      fsimd[d]            = fine_mrhs_grid->_simd_layout[d+1];
+    }
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
     int64_t bv= block_vol;
 
     autoView( fineData   , vec_mrhs, AcceleratorWrite);
     auto blasData_p  = &blas[0];
     auto fineData_p  = &fineData[0];
 
-    int64_t osites     = fine_grid->oSites();
     int64_t osites_hi  = fine_mrhs_grid->oSites();
 
     const int Nsimd = vobj::Nsimd();
@@ -468,8 +523,10 @@ public:
 	  int64_t site = (sc*lnvec + v)*bv
 	               + sb;
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-	  scalar_object data = *ptr;
+	  fine_scalar_object data;
+	  fine_scalar *dp = (fine_scalar *)&data;
+	  for(uint64_t w=0;w<lwords;w++) dp[w] = fine_scalar(blasData_p[site*lwords+w]);
 	  insertLane(lane,fineData[sfr],data);
 #ifdef GRID_SIMT
       }
@@ -483,6 +540,7 @@ public:
   void ImportCoarseGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
   {
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
 
     GridBase *coarse_mrhs_grid = vec_mrhs.Grid();
     int _ndimension = coarse_grid->_ndimension;
@@ -508,7 +566,7 @@ public:
     int64_t osites_hi = coarse_mrhs_grid->oSites();
 
     const int Nsimd = vobj::Nsimd();
-    uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+    uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
     GRID_ASSERT(cwords==nbasis);
     int64_t lnvec = nvec;
 
@@ -536,8 +594,8 @@ public:
 	  coarse_scalar_object data = extractLane(lane,coarseData[scr]);
 
 	  int64_t blas_site = (sc*lnvec + v)*cwords;
-	  coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	  const coarse_scalar *dp = (const coarse_scalar *)&data;
-	  *ptr = data;
+	  for(uint64_t b=0;b<cwords;b++) blasData_p[blas_site+b] = scalar(dp[b]);
 #ifdef GRID_SIMT
       }
 #else
@@ -546,21 +604,25 @@ public:
     });
   }
 
-  void ExportFineGridVectors(std::vector <Field> &vecs, deviceVector<scalar> &blas)
+  template<class vobj>
+  void ExportFineGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
   {
     GRID_TRACE("ExportFineGridVectors");
-    typedef typename Field::vector_object vobj;
+    typedef typename vobj::scalar_object fine_scalar_object;
+    typedef typename vobj::scalar_type   fine_scalar;
 
     int nvec = vecs.size();
 
-    GRID_ASSERT(vecs[0].Grid()==fine_grid);
+    GridBase *fgrid = vecs[0].Grid();
-
     int _ndimension = coarse_grid->_ndimension;
+    // Full LOCAL dimensions must agree; the SIMD layout and precision need not
+    for(int d=0;d<_ndimension;d++) GRID_ASSERT(fgrid->_ldimensions[d] == fine_ldimensions[d]);
+    GRID_ASSERT(sizeof(fine_scalar_object)/sizeof(fine_scalar) == words);
 
-    Coordinate fine_rdimensions = fine_grid->_rdimensions;
+    Coordinate fine_rdimensions = fgrid->_rdimensions;
     Coordinate coarse_l = coarse_ldimensions;
     Coordinate block_l  = block_ldimensions;
-    Coordinate fsimd    = fine_simd;
+    Coordinate fsimd    = fgrid->_simd_layout;
 
     //    std::cout << " export fine Blas norm "<<blasNorm2(blas)<<std::endl;
 
@@ -572,7 +634,7 @@ public:
       auto blasData_p  = &blas[0];
       auto fineData_p    = &fineData[0];
 
-      int64_t osites = fine_grid->oSites();
+      int64_t osites = fgrid->oSites();
       uint64_t lwords = words;
       //      std::cout << " Nsimd is "<<vobj::Nsimd() << std::endl;
       //      std::cout << " lwords is "<<lwords << std::endl;
@@ -608,9 +670,10 @@ public:
 	  int64_t site = (sc*nvec + v)*bv
 	               + sb;
 
-	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  // element-wise: the store may differ in precision from the field
-
+	  fine_scalar_object data;
-	  scalar_object data = *ptr;
+	  fine_scalar *dp = (fine_scalar *)&data;
+	  for(uint64_t w=0;w<lwords;w++) dp[w] = fine_scalar(blasData_p[site*lwords+w]);
 
 	  insertLane(lane,fineData[sf],data);
 #ifdef GRID_SIMT
@@ -627,6 +690,7 @@ public:
     GRID_TRACE("ImportCoarseGridVectors");
     int nvec = vecs.size();
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
 
     //    std::cout << " BlockProjector importing "<<nvec<< " coarse grid vectors" <<std::endl;
 
@@ -652,7 +716,7 @@ public:
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
-      uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+      uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
       GRID_ASSERT(cwords==nbasis);
       
       accelerator_for(sc,osites,Nsimd,{
@@ -676,9 +740,8 @@ public:
 
 	    coarse_scalar_object data = extractLane(lane,coarseData[sc]);
 
-	    coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	    const coarse_scalar *dp = (const coarse_scalar *)&data;
-
+	    for(uint64_t b=0;b<cwords;b++) blasData_p[blas_site+b] = scalar(dp[b]);
-	    *ptr = data;
 #ifdef GRID_SIMT
 	}
 #else
@@ -694,6 +757,7 @@ public:
     GRID_TRACE("ExportCoarseGridVectors");
     int nvec = vecs.size();
     typedef typename vobj::scalar_object coarse_scalar_object;
+    typedef typename vobj::scalar_type   coarse_scalar;   // may differ in precision from the BLAS scalar
     //    std::cout << GridLogMessage<<" BlockProjector exporting "<<nvec<< " coarse grid vectors" <<std::endl;
 
     GRID_ASSERT(vecs[0].Grid()==coarse_grid);
@@ -719,7 +783,7 @@ public:
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
-      uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+      uint64_t cwords=sizeof(coarse_scalar_object)/sizeof(coarse_scalar);
       GRID_ASSERT(cwords==nbasis);
       
       accelerator_for(sc,osites,Nsimd,{
@@ -740,8 +804,9 @@ public:
 	    Lexicographic::IndexFromCoor(coor_c,scl,coarse_l);
 
 	    int64_t blas_site = (scl*nvec + v)*cwords;
-	    coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	    coarse_scalar_object data;
-	    coarse_scalar_object data = *ptr;
+	    coarse_scalar *dp = (coarse_scalar *)&data;
+	    for(uint64_t b=0;b<cwords;b++) dp[b] = coarse_scalar(blasData_p[blas_site+b]);
 	    insertLane(lane,coarseData[sc],data);
 #ifdef GRID_SIMT
 	}
@@ -751,18 +816,19 @@ public:
       });
     }
   }
-  void ImportBasis(std::vector < Field > &vecs)
+  template<class fobj>
+  void ImportBasis(std::vector < Lattice<fobj> > &vecs)
   {
     //    std::cout << " BlockProjector Import basis size "<<vecs.size()<<std::endl;
     ImportFineGridVectors(vecs,BLAS_V);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(std::vector<Field> &fine,std::vector< Lattice<cobj> > & coarse)
+  void blockProject(std::vector<Lattice<fobj> > &fine,std::vector< Lattice<cobj> > & coarse)
   {
     GRID_TRACE("BlockProject");
     int nrhs=fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     //    std::cout << "blockProject nbasis " <<nbasis<<" " << _nbasis<<std::endl;
     GRID_ASSERT(nbasis==_nbasis);
     
@@ -817,12 +883,12 @@ public:
 
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(std::vector<Field> &fine,std::vector<Lattice<cobj> > & coarse)
+  void blockPromote(std::vector<Lattice<fobj> > &fine,std::vector<Lattice<cobj> > & coarse)
   {
     GRID_TRACE("BlockPromote");
     int nrhs=fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     
     BLAS_F.resize (fine_vol * words * nrhs );
@@ -876,12 +942,12 @@ public:
   // multiRHS ordered interfaces.  The GEMM is identical; only the import and
   // export differ, so those are the whole of the layout question.
   ////////////////////////////////////////////////////////////////////////////
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(Field &fine_mrhs,Lattice<cobj> &coarse_mrhs)
+  void blockProject(Lattice<fobj> &fine_mrhs,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockProjectMrhs");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -893,12 +959,12 @@ public:
     ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(Field &fine_mrhs,Lattice<cobj> &coarse_mrhs)
+  void blockPromote(Lattice<fobj> &fine_mrhs,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockPromoteMrhs");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -914,12 +980,12 @@ public:
   // Mixed orderings. A single RHS fine operator produces a vector of fine
   // fields with no packing; the coarse side is still wanted in mrhs order.
   ////////////////////////////////////////////////////////////////////////////
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(std::vector<Field> &fine,Lattice<cobj> &coarse_mrhs)
+  void blockProject(std::vector<Lattice<fobj> > &fine,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockProjectMixed");
     int nrhs = fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -931,12 +997,12 @@ public:
     ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockProject(Field &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
+  void blockProject(Lattice<fobj> &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
   {
     GRID_TRACE("BlockProjectMixed");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse.size()==nrhs);
 
@@ -948,12 +1014,12 @@ public:
     ExportCoarseGridVectors(coarse,BLAS_C);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(std::vector<Field> &fine,Lattice<cobj> &coarse_mrhs)
+  void blockPromote(std::vector<Lattice<fobj> > &fine,Lattice<cobj> &coarse_mrhs)
   {
     GRID_TRACE("BlockPromoteMixed");
     int nrhs = fine.size();
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
 
@@ -965,12 +1031,12 @@ public:
     ExportFineGridVectors(fine,BLAS_F);
   }
 
-  template<class cobj>
+  template<class fobj,class cobj>
-  void blockPromote(Field &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
+  void blockPromote(Lattice<fobj> &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
   {
     GRID_TRACE("BlockPromoteMixed");
     int nrhs = fine_mrhs.Grid()->_rdimensions[0];
-    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(typename cobj::scalar_type);
     GRID_ASSERT(nbasis==_nbasis);
     GRID_ASSERT(coarse.size()==nrhs);
 

Grid/algorithms/deflation/MultiRHSDeflation.h

@@ -59,6 +59,7 @@ public:
 
   typedef typename Field::scalar_type   scalar;
   typedef typename Field::scalar_object scalar_object;
+  typedef typename Field::vector_object vobj;
 
   int nev;
   std::vector<RealD> eval;
@@ -80,10 +81,18 @@ public:
     grid=nullptr;
     vol=0;
     words=0;
-    BLAS_E.resize(0);
+    // deviceVector is a std::vector with a device allocator: resize(0) drops
-    BLAS_R.resize(0);
+    // the size and keeps the capacity, returning no device memory.  Swapping
-    BLAS_C.resize(0);
+    // with an empty vector destroys the buffer.
-    BLAS_G.resize(0);
+    deviceVector<scalar>().swap(BLAS_E);
+    deviceVector<scalar>().swap(BLAS_R);
+    deviceVector<scalar>().swap(BLAS_C);
+    deviceVector<scalar>().swap(BLAS_G);
+  }
+  // Resident (non-evictable) device memory held by this deflator.
+  uint64_t DeviceBytes(void)
+  {
+    return (BLAS_E.capacity()+BLAS_R.capacity()+BLAS_C.capacity()+BLAS_G.capacity())*sizeof(scalar);
   }
   void Allocate(int _nev,GridBase *_grid)
   {
@@ -123,31 +132,98 @@ public:
       ImportEigenVector(evec[_ev0+e],_eval[_ev0+e],e);
     }
   }
+  /////////////////////////////////////////////////////////////////////////
+  // Sources as a vector of D-dimensional fields: each is one contiguous
+  // column of BLAS_R already, so import/export are straight copies.
+  // The eigenvectors may live on a D grid or on the Nrhs=1 D+1 grid of the
+  // same local volume; the two are the same bytes, so only the site count
+  // is checked, not grid identity.
+  /////////////////////////////////////////////////////////////////////////
   void DeflateSources(std::vector<Field> &source,std::vector<Field> & guess)
   {
     int nrhs = source.size();
     GRID_ASSERT(source.size()==guess.size());
-    GRID_ASSERT(grid == guess[0].Grid());
+    GRID_ASSERT(grid->lSites() == guess[0].Grid()->lSites());
     conformable(guess[0],source[0]);
 
     int64_t vw = vol * words;
-
-    RealD t0 = usecond();
     BLAS_R.resize(nrhs * vw); // cost free if size doesn't change
     BLAS_G.resize(nrhs * vw); // cost free if size doesn't change
     BLAS_C.resize(nev * nrhs);// cost free if size doesn't change
 
-    /////////////////////////////////////////////
-    // Copy in the multi-rhs sources
-    /////////////////////////////////////////////
-    //    for(int r=0;r<nrhs;r++){
-    //      std::cout << " source["<<r<<"] = "<<norm2(source[r])<<std::endl;
-    //    }
     for(int r=0;r<nrhs;r++){
       int64_t offset = r*vw;
       autoView(v,source[r],AcceleratorRead);
       acceleratorCopyDeviceToDevice(&v[0],&BLAS_R[offset],sizeof(scalar_object)*vol);
     }
+    DeflateBLAS(nrhs);
+    for(int r=0;r<nrhs;r++){
+      int64_t offset = r*vw;
+      autoView(v,guess[r],AcceleratorWrite);
+      acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
+    }
+  }
+  /////////////////////////////////////////////////////////////////////////
+  // Sources as ONE D+1 dimensional multiRHS field, rhs innermost
+  // (dimension 0, undistributed) on an UNVECTORISED coarse space, so the
+  // site index is rhs + nrhs*x with x the D-dimensional site.  The GEMM
+  // wants x contiguous per rhs, so the import is a permutation, one fused
+  // pass over the field; likewise the export.  No slices.
+  /////////////////////////////////////////////////////////////////////////
+  void DeflateSources(const Field &source_mrhs, Field &guess_mrhs)
+  {
+    conformable(source_mrhs,guess_mrhs);
+    GridBase *hi = source_mrhs.Grid();
+    GRID_ASSERT(hi->_ndimension    == grid->_ndimension+1);
+    GRID_ASSERT(hi->_processors[0] == 1);
+    GRID_ASSERT(hi->_simd_layout[0]== 1);
+    GRID_ASSERT(vobj::Nsimd()      == 1);
+    int nrhs = hi->_fdimensions[0];
+    GRID_ASSERT(hi->lSites() == nrhs*vol);
+
+    int64_t vw = vol * words;
+    BLAS_R.resize(nrhs * vw);
+    BLAS_G.resize(nrhs * vw);
+    BLAS_C.resize(nev * nrhs);
+
+    ImportSourcesMrhs(source_mrhs,nrhs);
+    DeflateBLAS(nrhs);
+    ExportGuessMrhs(guess_mrhs,nrhs);
+  }
+  void ImportSourcesMrhs(const Field &source_mrhs,int nrhs)
+  {
+    const int64_t lwords = words, lnrhs = nrhs, vw = vol*words;
+    autoView(v,source_mrhs,AcceleratorRead);
+    auto vp = &v[0];
+    scalar *R = &BLAS_R[0];
+    accelerator_for(scr, lnrhs*vol, 1, {
+      int64_t r = scr % lnrhs;
+      int64_t x = scr / lnrhs;
+      const scalar *s = (const scalar *)&vp[scr];
+      for(int64_t w=0;w<lwords;w++) R[r*vw + x*lwords + w] = s[w];
+    });
+  }
+  void ExportGuessMrhs(Field &guess_mrhs,int nrhs)
+  {
+    const int64_t lwords = words, lnrhs = nrhs, vw = vol*words;
+    autoView(v,guess_mrhs,AcceleratorWrite);
+    auto vp = &v[0];
+    scalar *G = &BLAS_G[0];
+    accelerator_for(scr, lnrhs*vol, 1, {
+      int64_t r = scr % lnrhs;
+      int64_t x = scr / lnrhs;
+      scalar *g = (scalar *)&vp[scr];
+      for(int64_t w=0;w<lwords;w++) g[w] = G[r*vw + x*lwords + w];
+    });
+  }
+  /////////////////////////////////////////////////////////////////////////
+  // BLAS_R (nrhs columns of vol*words) -> BLAS_G, through the imported
+  // eigenbasis: C = E^dag R / lambda, G = E C.
+  /////////////////////////////////////////////////////////////////////////
+  void DeflateBLAS(int nrhs)
+  {
+    int64_t vw = vol * words;
+    RealD t0 = usecond();
 
   /*
    * in Fortran column major notation (cuBlas order)
@@ -195,7 +271,7 @@ public:
     acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
     grid->GlobalSumVector(&HOST_C[0],nev*nrhs);
     for(int e=0;e<nev;e++){
-      RealD lam(1.0/eval[e]);
+      scalar lam(1.0/eval[e]);          // in the coarse scalar: fp32 coarse spaces too
       for(int r=0;r<nrhs;r++){
 	int off = e+nev*r;
 	HOST_C[off]=HOST_C[off] * lam;
@@ -216,15 +292,6 @@ public:
 		     scalar(0.0),
 		     Gd);
     BLAS.synchronise();
-
-    ///////////////////////////////////////
-    // Copy out the multirhs
-    ///////////////////////////////////////
-    for(int r=0;r<nrhs;r++){
-      int64_t offset = r*vw;
-      autoView(v,guess[r],AcceleratorWrite);
-      acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
-    }
     RealD t1 = usecond();
     std::cout << GridLogMessage << "MultiRHSDeflation for "<<nrhs<<" sources with "<<nev<<" eigenvectors took " << (t1-t0)/1e3 <<" ms"<<std::endl;
   }

Grid/algorithms/iterative/AdefGeneric.h

@@ -480,104 +480,6 @@ class TwoLevelCG : public LinearFunction<Field>
 
 };
   
-template<class Field, class CoarseField, class Aggregation>
-class TwoLevelADEF2 : public TwoLevelCG<Field>
-{
- public:
-  ///////////////////////////////////////////////////////////////////////////////////
-  // Need something that knows how to get from Coarse to fine and back again
-  //  void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
-  //  void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
-  ///////////////////////////////////////////////////////////////////////////////////
-  GridBase *coarsegrid;
-  Aggregation &_Aggregates;                    
-  LinearFunction<CoarseField> &_CoarseSolver;
-  LinearFunction<CoarseField> &_CoarseSolverPrecise;
-  ///////////////////////////////////////////////////////////////////////////////////
-  
-  // more most opertor functions
-  TwoLevelADEF2(RealD tol,
-		Integer maxit,
-		LinearOperatorBase<Field>    &FineLinop,
-		LinearFunction<Field>        &Smoother,
-		LinearFunction<CoarseField>  &CoarseSolver,
-		LinearFunction<CoarseField>  &CoarseSolverPrecise,
-		Aggregation &Aggregates
-		) :
-      TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,Aggregates.FineGrid),
-      _CoarseSolver(CoarseSolver),
-      _CoarseSolverPrecise(CoarseSolverPrecise),
-      _Aggregates(Aggregates)
-  {
-    coarsegrid = Aggregates.CoarseGrid;
-  };
-
-  virtual void PcgM1(Field & in, Field & out)
-  {
-    GRID_TRACE("MultiGridPreconditioner ");
-    // [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
-
-    Field tmp(this->grid);
-    Field Min(this->grid);
-    CoarseField PleftProj(this->coarsegrid);
-    CoarseField PleftMss_proj(this->coarsegrid);
-
-    this->SmoothTimer.Start();
-    this->_Smoother(in,Min);
-    this->SmoothTimer.Stop();
-    this->SmoothCalls++;
-
-    this->MatrixTimer.Start();
-    this->_FineLinop.HermOp(Min,out);
-    this->MatrixTimer.Stop();
-    this->MatrixCalls++;
-    axpy(tmp,-1.0,out,in);          // tmp  = in - A Min
-
-    this->ProjectTimer.Start();
-    this->_Aggregates.ProjectToSubspace(PleftProj,tmp);
-    this->ProjectTimer.Stop();
-    this->ProjectCalls++;
-    this->CoarseTimer.Start();
-    this->_CoarseSolver(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
-    this->CoarseTimer.Stop();
-    this->CoarseCalls++;
-    this->PromoteTimer.Start();
-    this->_Aggregates.PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
-    this->PromoteTimer.Stop();
-    this->PromoteCalls++;
-
-    axpy(out,1.0,Min,tmp); // Min+tmp
-  }
-
-  virtual void Vstart(Field & x,const Field & src)
-  {
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart "<<std::endl;
-    ///////////////////////////////////
-    // Choose x_0 such that 
-    // x_0 = guess +  (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
-    //                               = [1 - Ass_inv A] Guess + Assinv src
-    //                               = P^T guess + Assinv src 
-    //                               = Vstart  [Tang notation]
-    // This gives:
-    // W^T (src - A x_0) = src_s - A guess_s - r_s
-    //                   = src_s - (A guess)_s - src_s  + (A guess)_s 
-    //                   = 0 
-    ///////////////////////////////////
-    Field r(this->grid);
-    Field mmp(this->grid);
-    CoarseField PleftProj(this->coarsegrid);
-    CoarseField PleftMss_proj(this->coarsegrid);
-
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart projecting "<<std::endl;
-    this->_Aggregates.ProjectToSubspace(PleftProj,src);     
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart coarse solve "<<std::endl;
-    this->_CoarseSolverPrecise(PleftProj,PleftMss_proj); // Ass^{-1} r_s
-    std::cout << GridLogMessage<<"HDCG: fPcg Vstart promote "<<std::endl;
-    this->_Aggregates.PromoteFromSubspace(PleftMss_proj,x);  
-
-  }
-
-};
 
   
 template<class Field>

Grid/algorithms/iterative/AdefMrhs.h

@@ -27,9 +27,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #pragma once
 
+#include <Grid/algorithms/multigrid/MrhsPreconditioner.h>
 
   /*
-   * Compared to Tang-2009:  P=Pleft. P^T = PRight Q=MssInv. 
+   * Compared to Tang-2009:  P=Pleft. P^T = PRight Q=MssInv.
    * Script A = SolverMatrix 
    * Script P = Preconditioner
    *
@@ -42,58 +43,64 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 NAMESPACE_BEGIN(Grid);
 
 
+//////////////////////////////////////////////////////////////////////
+// Two-level CG family on a vector of right-hand sides, with the
+// preconditioner (M1 and Vstart) as an MrhsPreconditioner object:
+//   fPcg           flexible PCG, one Krylov per rhs, shared M1 call
+//   PrecBlockCGrQ  preconditioned BlockCGrQ (arXiv:2409.03904 s2.2);
+//                  assumes a stationary preconditioner; not the
+//                  production choice (its linalg grows as nrhs^2)
+// Also a LinearFunction<Field>: one rhs is the nrhs=1 case.
+//////////////////////////////////////////////////////////////////////
+enum class MrhsCGAlgorithm { fPcg, PrecBlockCGrQ };
+
 template<class Field>
-class TwoLevelCGmrhs
+class TwoLevelCGmrhs : public LinearFunction<Field>
 {
  public:
+  using LinearFunction<Field>::operator();
   RealD   Tolerance;
   Integer MaxIterations;
   GridBase *grid;
+  MrhsCGAlgorithm Algorithm;
 
-  // Fine operator, Smoother, CoarseSolver
+  LinearOperatorBase<Field>    &_FineLinop;
-  LinearOperatorBase<Field>   &_FineLinop;
+  MrhsPreconditioner<Field>    &_Precon;
-  LinearFunction<Field>   &_Smoother;
+  MultiRHSBlockCGLinalg<Field>  _BlockCGLinalg;
-  MultiRHSBlockCGLinalg<Field> _BlockCGLinalg;
 
-  GridStopWatch ProjectTimer;
-  GridStopWatch PromoteTimer;
-  GridStopWatch DeflateTimer;
-  GridStopWatch CoarseTimer;
-  GridStopWatch FineTimer;
-  GridStopWatch SmoothTimer;
-  GridStopWatch InsertTimer;
-
-  /*
-    Field rrr;
-  Field sss;
-  Field qqq;
-  Field zzz;
-  */  
-  // more most opertor functions
   TwoLevelCGmrhs(RealD tol,
 		 Integer maxit,
-		 LinearOperatorBase<Field>   &FineLinop,
+		 LinearOperatorBase<Field> &FineLinop,
-		 LinearFunction<Field>       &Smoother,
+		 MrhsPreconditioner<Field> &Precon,
-		 GridBase *fine) : 
+		 GridBase *fine,
-    Tolerance(tol), 
+		 MrhsCGAlgorithm alg = MrhsCGAlgorithm::fPcg) :
+    Tolerance(tol),
     MaxIterations(maxit),
+    Algorithm(alg),
     _FineLinop(FineLinop),
-    _Smoother(Smoother)
+    _Precon(Precon)
-    /*
-    rrr(fine),
-    sss(fine),
-    qqq(fine),
-    zzz(fine)
-*/
   {
-    grid       = fine;
+    grid = fine;
   };
-  
+
-  // Vector case
+  // mrhs entry point
   virtual void operator() (std::vector<Field> &src, std::vector<Field> &x)
   {
-    SolveSingleSystem(src,x);
+    if ( Algorithm == MrhsCGAlgorithm::PrecBlockCGrQ ) SolvePrecBlockCG(src,x);
-	// SolvePrecBlockCG(src,x);
+    else                                              SolveSingleSystem(src,x);
+  }
+  // LinearFunction entry points
+  virtual void operator() (const Field &src, Field &x)
+  {
+    std::vector<Field> S(1,src);
+    std::vector<Field> X(1,x);
+    (*this)(S,X);
+    x = X[0];
+  }
+  virtual void operator() (const std::vector<Field> &src, std::vector<Field> &x)
+  {
+    std::vector<Field> S(src);
+    (*this)(S,x);
   }
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -220,7 +227,7 @@ class TwoLevelCGmrhs
     //////////////////////////
     // x0 = Vstart -- possibly modify guess
     //////////////////////////
-    Vstart(X,src);
+    _Precon.Vstart(X,src);
 
     //////////////////////////
     // R = B-AX
@@ -234,7 +241,7 @@ class TwoLevelCGmrhs
     //////////////////////////////////
     // Compute MZ = M1 Z = M1 B - M1 A x0
     //////////////////////////////////
-    PcgM1(Z,MZ);  
+    _Precon(Z,MZ);  
 
     //////////////////////////////////
     // QC = Z
@@ -248,13 +255,7 @@ class TwoLevelCGmrhs
 
     std::cout << GridLogMessage<<"PrecBlockCGrQ vec computed initial residual and QR fact " <<std::endl;
 
-    ProjectTimer.Reset();
+    _Precon.ResetTimers();
-    PromoteTimer.Reset();
-    DeflateTimer.Reset();
-    CoarseTimer.Reset();
-    SmoothTimer.Reset();
-    FineTimer.Reset();
-    InsertTimer.Reset();
 
     GridStopWatch M1Timer;
     GridStopWatch M2Timer;
@@ -278,10 +279,9 @@ class TwoLevelCGmrhs
       // MZ  = M1 Z <==== the Multigrid preconditioner
       ////////////////////
       M1Timer.Start();
-      PcgM1(Z,MZ);
+      _Precon(Z,MZ);
       M1Timer.Stop();
 
-      FineTimer.Start();
       ////////////////////
       // M  = [D^dag Z]^{-1} = (<Ddag MZ>_M)^{-1} inner prod, generalising Saad derivation of Precon CG
       ////////////////////
@@ -326,7 +326,6 @@ class TwoLevelCGmrhs
       ////////////////////////////
       m_rr = m_C.adjoint() * m_C;
       
-      FineTimer.Stop();
 
       RealD max_resid=0;
       RealD rrsum=0;
@@ -350,14 +349,7 @@ class TwoLevelCGmrhs
 	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Linalg  "<<LinalgTimer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : fine H  "<<M3Timer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
+	_Precon.ReportTimers("HDCG: mrhs PrecBlockCGrQ : ");
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Project "<<ProjectTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Coarse  "<<CoarseTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Fine    "<<FineTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Smooth  "<<SmoothTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Insert  "<<InsertTimer.Elapsed()<<std::endl;;
 
 	for(int rhs=0;rhs<nrhs;rhs++){
 
@@ -423,7 +415,7 @@ class TwoLevelCGmrhs
     //////////////////////////
     // x0 = Vstart -- possibly modify guess
     //////////////////////////
-    Vstart(x,src);
+    _Precon.Vstart(x,src);
 
     for(int rhs=0;rhs<nrhs;rhs++){
       // r0 = b -A x0
@@ -435,7 +427,7 @@ class TwoLevelCGmrhs
     // Compute z = M1 x
     //////////////////////////////////
     // This needs a multiRHS version for acceleration
-    PcgM1(r,z);
+    _Precon(r,z);
 
     std::vector<RealD> ssq(nrhs);
     std::vector<RealD> rsq(nrhs);
@@ -449,13 +441,7 @@ class TwoLevelCGmrhs
       //      std::cout << GridLogMessage<<"mrhs HDCG: "<<rhs<<" k=0 residual "<<rtzp[rhs]<<" rsq "<<rsq[rhs]<<"\n";
     }
 
-    ProjectTimer.Reset();
+    _Precon.ResetTimers();
-    PromoteTimer.Reset();
-    DeflateTimer.Reset();
-    CoarseTimer.Reset();
-    SmoothTimer.Reset();
-    FineTimer.Reset();
-    InsertTimer.Reset();
 
     GridStopWatch M1Timer;
     GridStopWatch M2Timer;
@@ -488,7 +474,7 @@ class TwoLevelCGmrhs
 
       // Compute z = M x (for *all* RHS)
       M1Timer.Start();
-      PcgM1(r,z);
+      _Precon(r,z);
       M1Timer.Stop();
       
       RealD max_rn=0.0;
@@ -530,14 +516,7 @@ class TwoLevelCGmrhs
 	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Linalg  "<<LinalgTimer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : fine M3 "<<M3Timer.Elapsed()<<std::endl;;
 	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
+	_Precon.ReportTimers("HDCG: mrhs fPcg : ");
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Project "<<ProjectTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Coarse  "<<CoarseTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Fine    "<<FineTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Smooth  "<<SmoothTimer.Elapsed()<<std::endl;;
-	std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Insert  "<<InsertTimer.Elapsed()<<std::endl;;
 
 	for(int rhs=0;rhs<nrhs;rhs++){
 	  _FineLinop.HermOp(x[rhs],mmp[rhs][0]);			  
@@ -568,167 +547,114 @@ class TwoLevelCGmrhs
       std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
     }
   }
-  
 
- public:
+  RealD PcgM3(const Field & p, Field & mmp){
-
-  virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out) = 0;
-  virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src) = 0;
-  virtual void PcgM2(const Field & in, Field & out) {
-    out=in;
-  }
-
-  virtual RealD PcgM3(const Field & p, Field & mmp){
     RealD dd;
     _FineLinop.HermOp(p,mmp);
     ComplexD dot = innerProduct(p,mmp);
     dd=real(dot);
     return dd;
   }
-
 };
 
-template<class Field, class CoarseField>
+//////////////////////////////////////////////////////////////////////
-class TwoLevelADEF2mrhs : public TwoLevelCGmrhs<Field>
+// ADEF-2 as an mrhs preconditioner object (Tang, Nabben, Vuik, Erlangga):
+//   M1     = [1 - Q A] M + Q   with  Q = P A_c^{-1} P^dag  (deflated coarse solve)
+//   Vstart = Q b               the coarse-corrected start (precise coarse solve)
+// The coarse space is the D+1 multiRHS field throughout: the projector's
+// mixed overloads take the vector of fine fields straight to it, and the
+// deflator and coarse solver act on it.  No per-rhs slices anywhere.
+// The D+1 grid follows the solver's Nrhs through SetCoarseGridMrhs.
+//////////////////////////////////////////////////////////////////////
+// Projector_t defaults to a transfer operator whose STORE matches Field.
+template<class Field, class CoarseField, class Projector_t = MultiRHSBlockProject<Field> >
+class MrhsADEF2Preconditioner : public MrhsPreconditioner<Field>
 {
 public:
-  GridBase *coarsegrid;
+  GridBase                       *grid;             // fine
-  GridBase *coarsegridmrhs;
+  GridBase                       *coarsegridmrhs;   // D+1, rhs innermost
-  LinearFunction<CoarseField> &_CoarseSolverMrhs;
+  LinearOperatorBase<Field>      &_FineLinop;
-  LinearFunction<CoarseField> &_CoarseSolverPreciseMrhs;
+  LinearFunction<Field>          &_Smoother;
-  MultiRHSBlockProject<Field>    &_Projector;
+  LinearFunction<CoarseField>    &_CoarseSolver;        // in M1
-  MultiRHSDeflation<CoarseField> &_Deflator;
+  LinearFunction<CoarseField>    &_CoarseSolverPrecise; // in Vstart
+  Projector_t                    &_Projector;   // store precision need not match Field
+  MultiRHSDeflation<CoarseField> &_Deflator;            // nev==0: no deflation, zero coarse guess
 
-  
+  MrhsADEF2Preconditioner(LinearOperatorBase<Field>      &FineLinop,
-  TwoLevelADEF2mrhs(RealD tol,
+			  LinearFunction<Field>          &Smoother,
-		    Integer maxit,
+			  LinearFunction<CoarseField>    &CoarseSolver,
-		    LinearOperatorBase<Field>    &FineLinop,
+			  LinearFunction<CoarseField>    &CoarseSolverPrecise,
-		    LinearFunction<Field>        &Smoother,
+			  Projector_t                    &Projector,
-		    LinearFunction<CoarseField>  &CoarseSolverMrhs,
+			  MultiRHSDeflation<CoarseField> &Deflator,
-		    LinearFunction<CoarseField>  &CoarseSolverPreciseMrhs,
+			  GridBase *CoarseGridMrhs,
-		    MultiRHSBlockProject<Field>    &Projector,
+			  GridBase *FineGrid) :
-		    MultiRHSDeflation<CoarseField> &Deflator,
+    _FineLinop(FineLinop), _Smoother(Smoother),
-		    GridBase *_coarsemrhsgrid) :
+    _CoarseSolver(CoarseSolver), _CoarseSolverPrecise(CoarseSolverPrecise),
-    TwoLevelCGmrhs<Field>(tol, maxit,FineLinop,Smoother,Projector.fine_grid),
+    _Projector(Projector), _Deflator(Deflator)
-    _CoarseSolverMrhs(CoarseSolverMrhs),
-    _CoarseSolverPreciseMrhs(CoarseSolverPreciseMrhs),
-    _Projector(Projector),
-    _Deflator(Deflator)
   {
-    coarsegrid = Projector.coarse_grid;
+    // The fine grid is given, NOT taken from the projector: the projector's
-    coarsegridmrhs = _coarsemrhsgrid;// Thi could be in projector
+    // grid carries its STORE's layout, which need not be this Field's (an
+    // fp32 preconditioner may project through an fp64 store, or the reverse).
+    grid           = FineGrid;
+    coarsegridmrhs = CoarseGridMrhs;
   };
+  // Store matches Field: the projector's grid IS this Field's grid.
+  MrhsADEF2Preconditioner(LinearOperatorBase<Field>      &FineLinop,
+			  LinearFunction<Field>          &Smoother,
+			  LinearFunction<CoarseField>    &CoarseSolver,
+			  LinearFunction<CoarseField>    &CoarseSolverPrecise,
+			  Projector_t                    &Projector,
+			  MultiRHSDeflation<CoarseField> &Deflator,
+			  GridBase *CoarseGridMrhs) :
+    MrhsADEF2Preconditioner(FineLinop,Smoother,CoarseSolver,CoarseSolverPrecise,
+			    Projector,Deflator,CoarseGridMrhs,Projector.fine_grid) {};
 
-  // Override Vstart
+  void SetCoarseGridMrhs(GridBase *CoarseGridMrhs) { coarsegridmrhs = CoarseGridMrhs; }
-  virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src)
+
+  // The coarse solver's starting guess: deflated, or zero
+  void CoarseGuess(CoarseField &src,CoarseField &guess)
   {
-    int nrhs=x.size();
-    ///////////////////////////////////
-    // Choose x_0 such that 
-    // x_0 = guess +  (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
-    //                               = [1 - Ass_inv A] Guess + Assinv src
-    //                               = P^T guess + Assinv src 
-    //                               = Vstart  [Tang notation]
-    // This gives:
-    // W^T (src - A x_0) = src_s - A guess_s - r_s
-    //                   = src_s - (A guess)_s - src_s  + (A guess)_s 
-    //                   = 0 
-    ///////////////////////////////////
-    std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
-    std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
-    CoarseField PleftProjMrhs(this->coarsegridmrhs);
-    CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
-
-    this->_Projector.blockProject(src,PleftProj);
-    this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
-      InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
-    }
-    
-    this->_CoarseSolverPreciseMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} r_s
-
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
-    }
-    this->_Projector.blockPromote(x,PleftMss_proj);
-  }
-
-  virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out){
-
-    int nrhs=in.size();
-
-    // [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
-    std::vector<Field> tmp(nrhs,this->grid);
-    std::vector<Field> Min(nrhs,this->grid);
-
-    std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
-    std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
-
-    CoarseField PleftProjMrhs(this->coarsegridmrhs);
-    CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
-
-    //    this->rrr=in[0];
-
-#undef SMOOTHER_BLOCK_SOLVE
-#if SMOOTHER_BLOCK_SOLVE
-    this->SmoothTimer.Start();
-    this->_Smoother(in,Min);
-    this->SmoothTimer.Stop();
-#else
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      this->SmoothTimer.Start();
-      this->_Smoother(in[rhs],Min[rhs]);
-      this->SmoothTimer.Stop();
-    }
-#endif
-    //    this->sss=Min[0];
-    
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      
-      this->FineTimer.Start();
-      this->_FineLinop.HermOp(Min[rhs],out[rhs]);
-      axpy(tmp[rhs],-1.0,out[rhs],in[rhs]);          // resid  = in - A Min
-      this->FineTimer.Stop();
-
-    }
-
-    this->ProjectTimer.Start();
-    this->_Projector.blockProject(tmp,PleftProj);
-    this->ProjectTimer.Stop();
     this->DeflateTimer.Start();
-    this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
+    if ( _Deflator.nev > 0 ) _Deflator.DeflateSources(src,guess);
+    else                     guess = Zero();
     this->DeflateTimer.Stop();
-    this->InsertTimer.Start();
+  }
-    for(int rhs=0;rhs<nrhs;rhs++) {
+  // x_0 = Q b
-      InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
+  virtual void Vstart(std::vector<Field> &x,std::vector<Field> &src)
-      InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
+  {
-    }
+    CoarseField Csrc(coarsegridmrhs), Csol(coarsegridmrhs);
-    this->InsertTimer.Stop();
+    this->ProjectTimer.Start();  _Projector.blockProject(src,Csrc);  this->ProjectTimer.Stop();
+    CoarseGuess(Csrc,Csol);
+    this->CoarseTimer.Start();   _CoarseSolverPrecise(Csrc,Csol);    this->CoarseTimer.Stop();
+    this->PromoteTimer.Start();  _Projector.blockPromote(x,Csol);    this->PromoteTimer.Stop();
+  }
+  // [1 - Q A] M in + Q in = Min + Q [in - A Min]
+  virtual void operator()(std::vector<Field> &in,std::vector<Field> &out)
+  {
+    int nrhs=in.size();
+    std::vector<Field> tmp(nrhs,grid), Min(nrhs,grid);
+    CoarseField Csrc(coarsegridmrhs), Csol(coarsegridmrhs);
 
-    this->CoarseTimer.Start();
+    this->SmoothTimer.Start();
-    this->_CoarseSolverMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} [in - A Min]_s
+    for(int r=0;r<nrhs;r++) _Smoother(in[r],Min[r]);
-    this->CoarseTimer.Stop();
+    this->SmoothTimer.Stop();
 
-    this->InsertTimer.Start();
-    for(int rhs=0;rhs<nrhs;rhs++) {
-      ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
-    }
-    this->InsertTimer.Stop();
-    this->PromoteTimer.Start();
-    this->_Projector.blockPromote(tmp,PleftMss_proj);// tmp= Q[in - A Min]  
-    this->PromoteTimer.Stop();
     this->FineTimer.Start();
-    //    this->qqq=tmp[0];
+    for(int r=0;r<nrhs;r++){
-    for(int rhs=0;rhs<nrhs;rhs++) {
+      _FineLinop.HermOp(Min[r],out[r]);
-      axpy(out[rhs],1.0,Min[rhs],tmp[rhs]); // Min+tmp
+      axpy(tmp[r],-1.0,out[r],in[r]);          // in - A Min
     }
-    //    this->zzz=out[0];
+    this->FineTimer.Stop();
+
+    this->ProjectTimer.Start();  _Projector.blockProject(tmp,Csrc);  this->ProjectTimer.Stop();
+    CoarseGuess(Csrc,Csol);
+    this->CoarseTimer.Start();   _CoarseSolver(Csrc,Csol);           this->CoarseTimer.Stop();
+    this->PromoteTimer.Start();  _Projector.blockPromote(tmp,Csol);  this->PromoteTimer.Stop();
+
+    this->FineTimer.Start();
+    for(int r=0;r<nrhs;r++) axpy(out[r],1.0,Min[r],tmp[r]);       // Min + Q[in - A Min]
     this->FineTimer.Stop();
   }
 };
 
-
 NAMESPACE_END(Grid);
-
-

Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h

@@ -57,6 +57,7 @@ public:
   GridStopWatch MatTimer;
   GridStopWatch LinalgTimer;
   std::string name;
+  std::string trace_step = "PGCR_step";   // name + step; see Name()
   int ZeroGuess  = 0;  // caller contract: guess is always zero => first-cycle r0 = src, skip the apply
   // persistent GCR history (see GCRnStep)
   GridBase           *hist_grid = nullptr;
@@ -68,7 +69,10 @@ public:
   LinearFunction<Field>     &Preconditioner;
   LinearOperatorBase<Field> &Linop;
 
-  void Name(std::string _name) { name = _name; };
+  // The name is also the trace range's, so a profile separates the four
+  // instances (Fouter, Fsmoother, Couter, Csmoother) that otherwise nest
+  // indistinguishably as one shared range.
+  void Name(std::string _name) { name = _name; trace_step = name + " PGCR_step"; };
 
   void Level(int n) { Name("Level " + std::to_string(n)); level = n; }
 
@@ -222,7 +226,7 @@ public:
     GCRLogLevel<< "PGCR true residual "<< sqrt(cp/SSQ)     <<std::endl;
 
     for(int k=0;k<nstep;k++){
-      GRID_TRACE("PGCR_step");
+      GRID_TRACE(trace_step.c_str());
       steps++;
 
       int kp     = k+1;

Grid/algorithms/multigrid/Aggregates.h

@@ -592,7 +592,7 @@ public:
     }
   }
   virtual void RefineSubspaceHDCG(LinearOperatorBase<FineField> &hermop,
-				  TwoLevelADEF2mrhs<FineField,CoarseVector> & theHDCG,
+				  TwoLevelCGmrhs<FineField> & theHDCG,
 				  int nrhs)
   {
     std::vector<FineField> src_mrhs(nrhs,FineGrid);

Grid/algorithms/multigrid/BlockCyclic.h

@@ -21,6 +21,20 @@ Author: Peter Boyle <pboyle@bnl.gov>
 
 NAMESPACE_BEGIN(Grid);
 
+///////////////////////////////////////////////////////////////////////////////
+// Scalar of the distributed dense inversion: block-cyclic redistribution,
+// SUMMA, the Schur recursion and its LU leaf.  A configure-time choice
+// (--enable-dense-inverse-precision=double|single), INDEPENDENT of the
+// coarse-space precision: the matrix elements arrive in the coarse scalar
+// and the apply slab is fp32 either way; only the factorisation's
+// arithmetic and footprint follow this type.
+///////////////////////////////////////////////////////////////////////////////
+#ifdef GRID_DENSE_INVERSE_SINGLE
+typedef ComplexF DenseInverseScalar;
+#else
+typedef ComplexD DenseInverseScalar;
+#endif
+
 ///////////////////////////////////////////////////////////////////////////////
 // BlockCyclicLayout: the index arithmetic of a 2D block-cyclic distribution
 // of an N x N matrix over a Pr x Pc logical process grid with block size nb.

Grid/algorithms/multigrid/BlockCyclicRedistribute.h

@@ -34,7 +34,7 @@ NAMESPACE_BEGIN(Grid);
 class BlockRows
 {
 public:
-  deviceVector<ComplexD> data;
+  deviceVector<DenseInverseScalar> data;
   int64_t                rows;
   int64_t                cols;
   int64_t                ld;
@@ -52,7 +52,7 @@ public:
     ld   = r;
     data.resize((uint64_t)r*c);
   }
-  ComplexD *ColumnWindow(int64_t col0)
+  DenseInverseScalar *ColumnWindow(int64_t col0)
   {
     GRID_ASSERT( col0 >= 0 );
     GRID_ASSERT( col0 <= cols );
@@ -128,10 +128,10 @@ public:
   //   buffer(a,b) = elem(rows[a], cols[b]),  a fastest.
   /////////////////////////////////////////////////////////////////////////
   static void MoveEdge(int toBuffer,
-                       ComplexD *mat, int64_t ld,
+                       DenseInverseScalar *mat, int64_t ld,
                        const std::vector<int64_t> &roff,  // per-row offset in mat
                        const std::vector<int64_t> &coff,  // per-col offset in mat
-                       ComplexD *buf)
+                       DenseInverseScalar *buf)
   {
     int64_t nr = roff.size();
     int64_t nc = coff.size();
@@ -193,7 +193,7 @@ public:
   /////////////////////////////////////////////////////////////////////////
   static void Redistribute(int dir, GridBase *grid,
                            const std::vector<int64_t> &rowStart,
-                           ComplexD *rows1d, int64_t myrows,
+                           DenseInverseScalar *rows1d, int64_t myrows,
                            BlockCyclicMatrix &A)
   {
     BlockCyclicLayout &L = A.layout;
@@ -206,7 +206,7 @@ public:
     int64_t ld1  = myrows ? myrows : 1;
 
     std::vector<int64_t> rows, cols, roff, coff;
-    deviceVector<ComplexD> sbuf(1), rbuf(1);
+    deviceVector<DenseInverseScalar> sbuf(1), rbuf(1);
 
     ///////////////////////////////////////////////////////////////////////
     // Self edge: purely local, via a bounce buffer (shares all the code).
@@ -261,7 +261,7 @@ public:
       }
       grid->SendToRecvFrom((void *)&sbuf[0], partner,
                            (void *)&rbuf[0], partner,
-                           nmax*sizeof(ComplexD));
+                           nmax*sizeof(DenseInverseScalar));
       if ( nin ){
         if ( dir > 0 ) { Offsets2D(L, irow, icol, roff, coff);
                          MoveEdge(0, &A.data[0], L.mloc, roff, coff, &rbuf[0]); }
@@ -272,11 +272,11 @@ public:
   }
 
   static void RowsToCyclic(GridBase *grid, const std::vector<int64_t> &rowStart,
-                           ComplexD *rows1d, int64_t myrows, BlockCyclicMatrix &A)
+                           DenseInverseScalar *rows1d, int64_t myrows, BlockCyclicMatrix &A)
   { Redistribute(+1, grid, rowStart, rows1d, myrows, A); }
 
   static void CyclicToRows(GridBase *grid, const std::vector<int64_t> &rowStart,
-                           BlockCyclicMatrix &A, ComplexD *rows1d, int64_t myrows)
+                           BlockCyclicMatrix &A, DenseInverseScalar *rows1d, int64_t myrows)
   { Redistribute(-1, grid, rowStart, rows1d, myrows, A); }
 };
 

Grid/algorithms/multigrid/BlockCyclicSchurInverse.h

@@ -115,7 +115,7 @@ public:
   // Window copy-scale:  Dst[i0:i1, j0:j1] = alpha * Src[same window].
   // Both share one layout, so the local bands coincide; pure local kernel.
   ///////////////////////////////////////////////////////////////////////////
-  void WindowCopyScale(ComplexD alpha,
+  void WindowCopyScale(DenseInverseScalar alpha,
                        BlockCyclicMatrix &Src, BlockCyclicMatrix &Dst,
                        int64_t i0, int64_t i1, int64_t j0, int64_t j1)
   {
@@ -127,8 +127,8 @@ public:
     L.ColRange(j0,j1, lj0,lj1);
     int64_t m = li1-li0, n = lj1-lj0;
     if ( !(m && n) ) return;
-    ComplexD *src = Src.LocalWindow(li0,lj0);
+    DenseInverseScalar *src = Src.LocalWindow(li0,lj0);
-    ComplexD *dst = Dst.LocalWindow(li0,lj0);
+    DenseInverseScalar *dst = Dst.LocalWindow(li0,lj0);
     int64_t ldS = Src.layout.mloc;
     int64_t ldD = L.mloc;
     tCopy -= usecond();
@@ -164,10 +164,10 @@ public:
     GRID_ASSERT( lc1-lc0 == w );
 
     // Pack the strided block dense (inverseBatched assumes lda == w).
-    deviceVector<ComplexD> dense((uint64_t)w*w);
+    deviceVector<DenseInverseScalar> dense((uint64_t)w*w);
     {
-      ComplexD *src = A.LocalWindow(lr0,lc0);
+      DenseInverseScalar *src = A.LocalWindow(lr0,lc0);
-      ComplexD *dst = &dense[0];
+      DenseInverseScalar *dst = &dense[0];
       int64_t   ld  = L.mloc;
       accelerator_for(idx, (uint64_t)(w*w), 1, {
         int64_t jj = idx / w;
@@ -176,15 +176,15 @@ public:
       });
     }
     {
-      deviceVector<ComplexD*> bp(1);
+      deviceVector<DenseInverseScalar*> bp(1);
-      std::vector<ComplexD*>  ptr(1);
+      std::vector<DenseInverseScalar*>  ptr(1);
       ptr[0] = &dense[0];
-      acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD*));
+      acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(DenseInverseScalar*));
       INV.inverseBatched(w, bp);
     }
     {
-      ComplexD *src = &dense[0];
+      DenseInverseScalar *src = &dense[0];
-      ComplexD *dst = A.LocalWindow(lr0,lc0);
+      DenseInverseScalar *dst = A.LocalWindow(lr0,lc0);
       int64_t   ld  = L.mloc;
       accelerator_for(idx, (uint64_t)(w*w), 1, {
         int64_t jj = idx / w;
@@ -194,8 +194,8 @@ public:
     }
     // Growth telemetry, local only.
     {
-      std::vector<ComplexD> h((uint64_t)w*w);
+      std::vector<DenseInverseScalar> h((uint64_t)w*w);
-      acceleratorCopyFromDevice(&dense[0], &h[0], h.size()*sizeof(ComplexD));
+      acceleratorCopyFromDevice(&dense[0], &h[0], h.size()*sizeof(DenseInverseScalar));
       double mx = 0.0;
       for(auto &z : h){
         double re=z.real(), im=z.imag();
@@ -232,23 +232,23 @@ public:
     int64_t lr0,lr1,lc0,lc1; L.RowRange(c0,c1,lr0,lr1); L.ColRange(c0,c1,lc0,lc1);
     const int64_t mq = lr1-lr0, nq = lc1-lc0;
 
-    deviceVector<ComplexD> dense;            // root only: W x W column major
+    deviceVector<DenseInverseScalar> dense;            // root only: W x W column major
-    deviceVector<ComplexD> piece, dummy;     // piece: my mq x nq contiguous; dummy: reverse-direction filler
+    deviceVector<DenseInverseScalar> piece, dummy;     // piece: my mq x nq contiguous; dummy: reverse-direction filler
     if ( me == root ) dense.resize((uint64_t)W*W);
 
-    auto pack_piece = [&](ComplexD *dst, int64_t m, int64_t n, int64_t r0, int64_t cc0){
+    auto pack_piece = [&](DenseInverseScalar *dst, int64_t m, int64_t n, int64_t r0, int64_t cc0){
-      ComplexD *src = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
+      DenseInverseScalar *src = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
       accelerator_for(idx,(uint64_t)(m*n),1,{ int64_t jj=idx/m, ii=idx-jj*m; dst[ii+jj*m] = src[ii+jj*ld]; });
     };
-    auto unpack_piece = [&](ComplexD *src, int64_t m, int64_t n, int64_t r0, int64_t cc0){
+    auto unpack_piece = [&](DenseInverseScalar *src, int64_t m, int64_t n, int64_t r0, int64_t cc0){
-      ComplexD *dst = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
+      DenseInverseScalar *dst = A.LocalWindow(r0,cc0); const int64_t ld = L.mloc;
       accelerator_for(idx,(uint64_t)(m*n),1,{ int64_t jj=idx/m, ii=idx-jj*m; dst[ii+jj*ld] = src[ii+jj*m]; });
     };
     // root: piece of rank q  <->  dense, via the closed-form block map
-    auto root_place = [&](ComplexD *pc, int64_t m, int64_t n, int q, int to_dense){
+    auto root_place = [&](DenseInverseScalar *pc, int64_t m, int64_t n, int q, int to_dense){
       const int pq=q/Pc, cq=q%Pc;
       const int64_t brq0=FirstBlock(b0,pq,Pr), bcq0=FirstBlock(b0,cq,Pc);
-      ComplexD *dn = &dense[0]; const int64_t WW=W, NB=nb, PR=Pr, PC=Pc, C0=c0;
+      DenseInverseScalar *dn = &dense[0]; const int64_t WW=W, NB=nb, PR=Pr, PC=Pc, C0=c0;
       accelerator_for(idx,(uint64_t)(m*n),1,{
         int64_t jj=idx/m, ii=idx-jj*m;
         int64_t gr = (brq0 + (ii/NB)*PR)*NB + ii%NB - C0;
@@ -266,18 +266,18 @@ public:
     tBigGather -= usecond();
     if ( mq*nq ) { piece.resize((uint64_t)mq*nq); dummy.resize((uint64_t)mq*nq); pack_piece(&piece[0],mq,nq,lr0,lc0); accelerator_barrier(); }
     if ( me == root ) {
-      deviceVector<ComplexD> stage;
+      deviceVector<DenseInverseScalar> stage;
       for(int q=0;q<Pr*Pc;q++){
         int64_t m,n; piece_dims(q,m,n); if ( !(m*n) ) continue;
         if ( q == root ) { root_place(&piece[0],m,n,q,1); continue; }
         if ( stage.size() < (uint64_t)(m*n) ) stage.resize((uint64_t)m*n);
-        deviceVector<ComplexD> junk((uint64_t)m*n);
+        deviceVector<DenseInverseScalar> junk((uint64_t)m*n);
-        grid->SendToRecvFrom((void *)&junk[0], q, (void *)&stage[0], q, (uint64_t)m*n*sizeof(ComplexD));
+        grid->SendToRecvFrom((void *)&junk[0], q, (void *)&stage[0], q, (uint64_t)m*n*sizeof(DenseInverseScalar));
         root_place(&stage[0],m,n,q,1);
       }
       accelerator_barrier();
     } else if ( mq*nq ) {
-      grid->SendToRecvFrom((void *)&piece[0], root, (void *)&dummy[0], root, (uint64_t)mq*nq*sizeof(ComplexD));
+      grid->SendToRecvFrom((void *)&piece[0], root, (void *)&dummy[0], root, (uint64_t)mq*nq*sizeof(DenseInverseScalar));
     }
     tBigGather += usecond();
 
@@ -289,17 +289,17 @@ public:
     // ---- scatter ----
     tBigScatter -= usecond();
     if ( me == root ) {
-      deviceVector<ComplexD> stage;
+      deviceVector<DenseInverseScalar> stage;
       for(int q=0;q<Pr*Pc;q++){
         int64_t m,n; piece_dims(q,m,n); if ( !(m*n) ) continue;
         if ( q == root ) { root_place(&piece[0],m,n,q,0); accelerator_barrier(); continue; }
         if ( stage.size() < (uint64_t)(m*n) ) stage.resize((uint64_t)m*n);
-        deviceVector<ComplexD> junk((uint64_t)m*n);
+        deviceVector<DenseInverseScalar> junk((uint64_t)m*n);
         root_place(&stage[0],m,n,q,0); accelerator_barrier();
-        grid->SendToRecvFrom((void *)&stage[0], q, (void *)&junk[0], q, (uint64_t)m*n*sizeof(ComplexD));
+        grid->SendToRecvFrom((void *)&stage[0], q, (void *)&junk[0], q, (uint64_t)m*n*sizeof(DenseInverseScalar));
       }
     } else if ( mq*nq ) {
-      grid->SendToRecvFrom((void *)&dummy[0], root, (void *)&piece[0], root, (uint64_t)mq*nq*sizeof(ComplexD));
+      grid->SendToRecvFrom((void *)&dummy[0], root, (void *)&piece[0], root, (uint64_t)mq*nq*sizeof(DenseInverseScalar));
     }
     if ( mq*nq ) { unpack_piece(&piece[0],mq,nq,lr0,lc0); accelerator_barrier(); }
     tBigScatter += usecond();
@@ -326,7 +326,7 @@ public:
     int64_t c0 = b0*L.nb;
     int64_t m  = bm*L.nb;
     int64_t c1 = std::min(L.N, b1*L.nb);
-    ComplexD one (1.0,0.0), mone(-1.0,0.0), zero(0.0,0.0);
+    DenseInverseScalar one (1.0,0.0), mone(-1.0,0.0), zero(0.0,0.0);
 
     // 1. A11 -> A11inv
     SchurNode(A,Bt,Ct,Tt,Ut, b0,bm);

Grid/algorithms/multigrid/BlockCyclicSumma.h

@@ -71,7 +71,7 @@ class BlockCyclicMatrix
 public:
   GridBase              *grid;      // borrowed, never owned
   BlockCyclicLayout      layout;
-  deviceVector<ComplexD> data;      // column major, ld = layout.mloc
+  deviceVector<DenseInverseScalar> data;      // column major, ld = layout.mloc
 
   BlockCyclicMatrix(GridBase *g, int64_t N, int64_t nb, int Pr, int Pc)
     : grid(g),
@@ -82,7 +82,7 @@ public:
     data.resize( sz ? sz : 1 );
   }
 
-  ComplexD *LocalWindow(int64_t li, int64_t lj)
+  DenseInverseScalar *LocalWindow(int64_t li, int64_t lj)
   {
     return &data[0] + li + lj*layout.mloc;
   }
@@ -93,32 +93,32 @@ public:
   // oracle only; production data enters through the direct block-cyclic
   // import, never through these.
   /////////////////////////////////////////////////////////////////////////
-  void ImportGlobal(const std::vector<ComplexD> &G)
+  void ImportGlobal(const std::vector<DenseInverseScalar> &G)
   {
     int64_t N = layout.N;
     GRID_ASSERT( (int64_t)G.size() == N*N );
-    std::vector<ComplexD> h((uint64_t)layout.mloc*layout.nloc, ComplexD(0.0,0.0));
+    std::vector<DenseInverseScalar> h((uint64_t)layout.mloc*layout.nloc, DenseInverseScalar(0.0,0.0));
     for(int64_t j=0;j<N;j++){
       for(int64_t i=0;i<N;i++){
         if ( layout.Owns(i,j) ) h[layout.LocalOffset(i,j)] = G[i + j*N];
       }
     }
     if ( h.size() )
-      acceleratorCopyToDevice(&h[0], &data[0], h.size()*sizeof(ComplexD));
+      acceleratorCopyToDevice(&h[0], &data[0], h.size()*sizeof(DenseInverseScalar));
   }
-  void ExportGlobal(std::vector<ComplexD> &G)
+  void ExportGlobal(std::vector<DenseInverseScalar> &G)
   {
     int64_t N = layout.N;
-    G.assign((uint64_t)N*N, ComplexD(0.0,0.0));
+    G.assign((uint64_t)N*N, DenseInverseScalar(0.0,0.0));
-    std::vector<ComplexD> h((uint64_t)layout.mloc*layout.nloc);
+    std::vector<DenseInverseScalar> h((uint64_t)layout.mloc*layout.nloc);
     if ( h.size() )
-      acceleratorCopyFromDevice(&data[0], &h[0], h.size()*sizeof(ComplexD));
+      acceleratorCopyFromDevice(&data[0], &h[0], h.size()*sizeof(DenseInverseScalar));
     for(int64_t j=0;j<N;j++){
       for(int64_t i=0;i<N;i++){
         if ( layout.Owns(i,j) ) G[i + j*N] = h[layout.LocalOffset(i,j)];
       }
     }
-    if ( N ) grid->GlobalSumVector((ComplexD *)&G[0], (int)(N*N)); // zero-fill: exact
+    if ( N ) grid->GlobalSumVector((DenseInverseScalar *)&G[0], (int)(N*N)); // zero-fill: exact
   }
 };
 
@@ -139,8 +139,8 @@ public:
   // per message: measured 1.26 GB/s/rank on 13 MB ring messages (production,
   // GRID_ALLOC_NCACHE_LARGE=64) against 62 s total for the same inverse when
   // hipMalloc returned a stable address.
-  deviceVector<ComplexD> Abuf;
+  deviceVector<DenseInverseScalar> Abuf;
-  deviceVector<ComplexD> Bbuf;
+  deviceVector<DenseInverseScalar> Bbuf;
   double   tAlloc=0, tPack=0, tRingA=0, tRingB=0, tGemm=0;
   uint64_t bytesRing=0, nRingMsg=0, nMultiply=0, nGemm=0;
   // Per-message-size histogram (bucket = floor(log2 bytes)): count, bytes,
@@ -156,10 +156,10 @@ public:
   static int Overlap(int64_t a0,int64_t a1,int64_t b0,int64_t b1)
   { return (a0 < b1) && (b0 < a1); }
 
-  void Multiply(ComplexD alpha,
+  void Multiply(DenseInverseScalar alpha,
                 BlockCyclicMatrix &A,
                 BlockCyclicMatrix &B,
-                ComplexD beta,
+                DenseInverseScalar beta,
                 BlockCyclicMatrix &C,
                 int64_t i0, int64_t i1,
                 int64_t j0, int64_t j1,
@@ -220,8 +220,8 @@ public:
     if ( Bbuf.size() < std::max<uint64_t>(slotB*Pr,1) ) Bbuf.resize( std::max<uint64_t>(slotB*Pr,1) );
     tAlloc += usecond();
 
-    deviceVector<ComplexD *> ap(1), bp(1), cp(1);
+    deviceVector<DenseInverseScalar *> ap(1), bp(1), cp(1);
-    std::vector<ComplexD *>  ptr(1);
+    std::vector<DenseInverseScalar *>  ptr(1);
 
     int firstblock = 1;
     for(int64_t r0=kb0; r0<kb1; r0+=Pc){       // rounds of Pc k-blocks
@@ -238,8 +238,8 @@ public:
           int64_t lc0, lc1;
           L.ColRange(s*nb, std::min(N,(s+1)*nb), lc0, lc1);
           GRID_ASSERT( lc1-lc0 == nb_s );
-          ComplexD *src = A.LocalWindow(li0, lc0);
+          DenseInverseScalar *src = A.LocalWindow(li0, lc0);
-          ComplexD *dst = &Abuf[0] + slotA*pcol;
+          DenseInverseScalar *dst = &Abuf[0] + slotA*pcol;
           int64_t   ld  = A.layout.mloc;
           int64_t   m   = mloc_i;
           accelerator_for(idx, (uint64_t)(m*nb_s), 1, {
@@ -254,8 +254,8 @@ public:
           GRID_ASSERT( lr1-lr0 == nb_s );
           int64_t idxs = (s - r0 - ((prow - r0%Pr + Pr) % Pr)) / Pr; // my panel # in round
           GRID_ASSERT( idxs >= 0 ); GRID_ASSERT( idxs < S );
-          ComplexD *src = B.LocalWindow(lr0, lj0);
+          DenseInverseScalar *src = B.LocalWindow(lr0, lj0);
-          ComplexD *dst = &Bbuf[0] + slotB*prow + slotB1*idxs;
+          DenseInverseScalar *dst = &Bbuf[0] + slotB*prow + slotB1*idxs;
           int64_t   ld  = B.layout.mloc;
           int64_t   nn  = nloc_j;
           accelerator_for(idx, (uint64_t)(nb_s*nn), 1, {
@@ -284,9 +284,9 @@ public:
           double tm = usecond();
           grid->SendToRecvFrom((void *)(&Abuf[0]+slotA*cs), dest,
                                (void *)(&Abuf[0]+slotA*cr), src,
-                               slotA*sizeof(ComplexD));
+                               slotA*sizeof(DenseInverseScalar));
-          HistAdd(slotA*sizeof(ComplexD), usecond()-tm);
+          HistAdd(slotA*sizeof(DenseInverseScalar), usecond()-tm);
-          bytesRing += slotA*sizeof(ComplexD); nRingMsg++;
+          bytesRing += slotA*sizeof(DenseInverseScalar); nRingMsg++;
         }
         tRingA += usecond();
       }
@@ -304,9 +304,9 @@ public:
           double tm = usecond();
           grid->SendToRecvFrom((void *)(&Bbuf[0]+slotB*rs), dest,
                                (void *)(&Bbuf[0]+slotB*rr), src,
-                               slotB*sizeof(ComplexD));
+                               slotB*sizeof(DenseInverseScalar));
-          HistAdd(slotB*sizeof(ComplexD), usecond()-tm);
+          HistAdd(slotB*sizeof(DenseInverseScalar), usecond()-tm);
-          bytesRing += slotB*sizeof(ComplexD); nRingMsg++;
+          bytesRing += slotB*sizeof(DenseInverseScalar); nRingMsg++;
         }
         tRingB += usecond();
       }
@@ -322,15 +322,15 @@ public:
         int cA = (int)(s%Pc);
         int rB = (int)(s%Pr);
         int64_t idxs = (s - r0 - ((rB - r0%Pr + Pr) % Pr)) / Pr;
-        ComplexD beta_use = firstblock ? beta : ComplexD(1.0,0.0);
+        DenseInverseScalar beta_use = firstblock ? beta : DenseInverseScalar(1.0,0.0);
         firstblock = 0;
 
         ptr[0] = &Abuf[0] + slotA*cA;
-        acceleratorCopyToDevice(&ptr[0], &ap[0], sizeof(ComplexD *));
+        acceleratorCopyToDevice(&ptr[0], &ap[0], sizeof(DenseInverseScalar *));
         ptr[0] = &Bbuf[0] + slotB*rB + slotB1*idxs;
-        acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(ComplexD *));
+        acceleratorCopyToDevice(&ptr[0], &bp[0], sizeof(DenseInverseScalar *));
         ptr[0] = C.LocalWindow(li0, lj0);
-        acceleratorCopyToDevice(&ptr[0], &cp[0], sizeof(ComplexD *));
+        acceleratorCopyToDevice(&ptr[0], &cp[0], sizeof(DenseInverseScalar *));
 
         BLAS.gemmBatched(GridBLAS_OP_N, GridBLAS_OP_N,
                          (int)mloc_i, (int)nloc_j, (int)nb_s,

Grid/algorithms/multigrid/DenseCoarseMatrix.h

@@ -64,7 +64,8 @@ NAMESPACE_BEGIN(Grid);
 // VERIFY ||A Ainv x - x||/||x|| certifies the DEVICE slab + split-K path at the
 // end of Import, since the single-RHS apply routes through the same core.
 //
-// Tensor-depth agnostic: site scalar objects treated as contiguous ComplexD
+// Tensor-depth agnostic: site scalar objects treated as contiguous coarse
+// scalars (ComplexF or ComplexD, following the coefficient precision)
 // (iScalar wrappers add no data), so any MG level's coarse operator imports.
 //////////////////////////////////////////////////////////////////////////////////////
 //
@@ -85,6 +86,10 @@ public:
   typedef typename vobj::scalar_object  sobj;
   typedef typename CoarseMatrix::vector_object Mvobj;
   typedef typename Mvobj::scalar_object        Msobj;
+  // Scalar of the coarse site objects (ComplexF or ComplexD): the apply slab
+  // is always fp32 and the inversion always fp64, but the SOURCE of both --
+  // the coarse operator's matrix elements -- carries this precision.
+  typedef typename GridTypeMapper<CComplex>::scalar_type CoarseScalar;
 
   GridBase *grid;
   int      nd;
@@ -115,11 +120,20 @@ public:
   std::vector<ComplexF>   hY;
   int NK;                           // split-K chunk count (divides N)
 
+  // Resident device memory: the apply slab and its staging.  deviceVector is
+  // not evictable, so this counts against the hard budget, not the cache.
+  uint64_t DeviceBytes(void)
+  {
+    return (uint64_t)(dSlab.capacity()+dX.capacity()+dY.capacity()+dG.capacity()+dPartial.capacity())*sizeof(ComplexF)
+         + (uint64_t)dLex2Rank.capacity()*sizeof(int)
+         + (uint64_t)dRm2G.capacity()*sizeof(int64_t);
+  }
+
   DenseCoarseMatrix(GridBase *g)
     : grid(g)
   {
-    GRID_ASSERT( sizeof(sobj)  == nbasis*sizeof(ComplexD) );
+    GRID_ASSERT( sizeof(sobj)  == nbasis*sizeof(CoarseScalar) );
-    GRID_ASSERT( sizeof(Msobj) == nbasis*nbasis*sizeof(ComplexD) );
+    GRID_ASSERT( sizeof(Msobj) == nbasis*nbasis*sizeof(CoarseScalar) );
     nd     = grid->_ndimension;
     N      = grid->gSites() * nbasis;
     lsites = grid->lSites();
@@ -228,7 +242,7 @@ public:
     ////////////////////////////////////////////////////////////////////
     {
       Field x(grid); Field y(grid); Field z(grid);
-      x = ComplexD(1.0,0.0);
+      x = CoarseScalar(1.0,0.0);
       double ta = usecond();
       (*this)(x, y);
       double tb = usecond();
@@ -263,7 +277,7 @@ public:
     Field min(mgrid), mout(mgrid);
     for(int r=0;r<nr;r++){
       Field scaled(grid);
-      scaled = ComplexD(r+1.0,0.0)*in;
+      scaled = CoarseScalar(r+1.0,0.0)*in;
       InsertSliceFast(scaled,min,r,0);
     }
 
@@ -273,13 +287,18 @@ public:
     for(int r=1;r<nr;r++){
       Field sr(grid),d(grid);
       ExtractSliceFast(sr,mout,r,0);
-      d = sr - ComplexD(r+1.0,0.0)*out;
+      d = sr - CoarseScalar(r+1.0,0.0)*out;
       RealD rel = std::sqrt(norm2(d)/norm2(sr));
-      if ( rel >= 1.0e-6 ) {
+      // Slices differ by rounding amplified by the operator's conditioning
+      // when the product cancels (A applied to A^-1 x): the tolerance follows
+      // the coarse precision (fp32 measured ~5e-6 here).  A genuine rhs
+      // mix-up is O(1).
+      const RealD otol = (sizeof(CoarseScalar)==sizeof(ComplexF)) ? 1.0e-4 : 1.0e-6;
+      if ( rel >= otol ) {
         std::cout << GridLogMessage << "DenseCoarseMatrix: oracle rhs "<<r
                   <<" inconsistent with rhs 0, rel "<<rel<<std::endl;
       }
-      GRID_ASSERT( rel < 1.0e-6 );
+      GRID_ASSERT( rel < otol );
     }
   }
 
@@ -313,7 +332,7 @@ public:
         Lexicographic::IndexFromCoor(ncoor, nsite, gdims);
         Msobj m;
         peekLocalSite(m, Av, myLcoor[ss]);
-        ComplexD *md = (ComplexD *)&m;
+        CoarseScalar *md = (CoarseScalar *)&m;
         // The operator contracts out(s,b) = sum_a A[p](s)(a,b) in(nbr,a)
         // (GeneralCoarsenedMatrix.h Mult kernel): the stored site matrix
         // acts TRANSPOSED, so element (a,b) lands at dense row (s,b),
@@ -382,7 +401,7 @@ public:
       sobj s;
       for(int b=0; b<nbasis; b++){
         double ph = 0.37*(double)(myGsite[ss]*nbasis+b);
-        ((ComplexD *)&s)[b] = ComplexD(std::cos(ph),std::sin(0.61*ph));
+        ((CoarseScalar *)&s)[b] = CoarseScalar(std::cos(ph),std::sin(0.61*ph));
       }
       pokeLocalSite(s, x, myLcoor[ss]);
     }
@@ -391,7 +410,7 @@ public:
     for(int ss=0; ss<lsites; ss++){
       sobj s;
       peekLocalSite(s, x, myLcoor[ss]);
-      for(int b=0; b<nbasis; b++) xh[ myGsite[ss]*nbasis + b ] = ((ComplexD *)&s)[b];
+      for(int b=0; b<nbasis; b++) xh[ myGsite[ss]*nbasis + b ] = ComplexD(((CoarseScalar *)&s)[b]);
     }
     grid->GlobalSumVector(&xh[0], (int)N);
     std::vector<ComplexD> yh(nrows);
@@ -403,7 +422,7 @@ public:
     });
     for(int ss=0; ss<lsites; ss++){
       sobj s;
-      for(int b=0; b<nbasis; b++) ((ComplexD *)&s)[b] = yh[ss*nbasis+b];
+      for(int b=0; b<nbasis; b++) ((CoarseScalar *)&s)[b] = CoarseScalar(yh[ss*nbasis+b]);
       pokeLocalSite(s, Dx, myLcoor[ss]);
     }
     ApplyOracle(Op, x, Ax);
@@ -414,7 +433,7 @@ public:
     if ( rel >= 1.0e-3 ) {
       std::cout << GridLogMessage << "DenseCoarseMatrix: IMPORT CERTIFICATE FAILED. If O(1), the "
                 << "stencil shift-sign convention of the coarse operator has changed: "
-                << "the import in ImportDense/ImportDenseFP64 must change with it"
+                << "the import in ImportDense/ImportDenseForInversion must change with it"
                 << std::endl;
     }
     GRID_ASSERT(rel < 1.0e-3);
@@ -468,24 +487,25 @@ public:
   }
 
   ////////////////////////////////////////////////////////////////////
-  // 3b. Direct stencil -> fp64 rank-major import of MY ROWS of A (the
+  // 3b. Direct stencil -> rank-major import of MY ROWS of A in the
-  //    end-to-end fp64 path: the stencil source IS ComplexD; nothing is
+  //    inversion precision (DenseInverseScalar): the stencil source is
-  //    rounded through fp32 on the way into the inversion).  Same
+  //    read once, straight into the buffer the Schur recursion factorises,
+  //    never via the fp32 apply slab.  Same
   //    loop/sign/accumulate/transposed-contraction discipline as
   //    ImportDense; output is column-major rows x N with columns in
   //    rank-major order (g2rm).
-  //    ALWAYS-ON CERTIFICATE: the fp64 import, rounded, must agree with
+  //    ALWAYS-ON CERTIFICATE: this import, rounded, must agree with
   //    the fp32 slab entry at the corresponding global column, over the
   //    WHOLE of my rows (few ulp: wrapped-shift collisions accumulate in
   //    different precision order).  NaN-proof: non-finite entries are
   //    counted explicitly since max() silently masks NaN.
   ////////////////////////////////////////////////////////////////////
   template<class CoarseOp>
-  void ImportDenseFP64(CoarseOp &Op, BlockRows &S, std::vector<int64_t> &g2rm)
+  void ImportDenseForInversion(CoarseOp &Op, BlockRows &S, std::vector<int64_t> &g2rm)
   {
     Coordinate gdims = grid->GlobalDimensions();
 
-    std::vector<ComplexD> h((uint64_t)nrows*N, ComplexD(0.0,0.0));
+    std::vector<DenseInverseScalar> h((uint64_t)nrows*N, DenseInverseScalar(0.0,0.0));
     for(int p=0; p<Op.Geometry().npoint; p++)
     {
       Coordinate shift = Op.Geometry().shifts[p];
@@ -502,7 +522,7 @@ public:
         Lexicographic::IndexFromCoor(ncoor, nsite, gdims);
         Msobj m;
         peekLocalSite(m, Av, myLcoor[ss]);
-        ComplexD *md = (ComplexD *)&m;
+        CoarseScalar *md = (CoarseScalar *)&m;
         // Transposed contraction as ImportDense: (a,b) lands at
         // row (s,b), column (nbr,a); column index in rank-major order.
         for(int a=0; a<nbasis; a++)
@@ -510,7 +530,7 @@ public:
           int64_t jj = g2rm[ nsite*nbasis + a ];
           for(int b=0; b<nbasis; b++)
           {
-            h[(uint64_t)(ss*nbasis+b) + (uint64_t)jj*nrows] += md[a*nbasis+b];
+            h[(uint64_t)(ss*nbasis+b) + (uint64_t)jj*nrows] += DenseInverseScalar(md[a*nbasis+b]);
           }
         }
       });
@@ -523,7 +543,7 @@ public:
     {
       for(int64_t gcol=0; gcol<N; gcol++)
       {
-        ComplexD d64 = h[(uint64_t)(i + g2rm[gcol]*nrows)];
+        ComplexD d64 = ComplexD(h[(uint64_t)(i + g2rm[gcol]*nrows)]);
         ComplexF f32 = slab[(uint64_t)i*N + gcol];
         double dev = abs(ComplexD(f32) - d64);
         if ( !std::isfinite(dev) ) nbad++;
@@ -534,21 +554,22 @@ public:
     RealD gbad = (RealD)nbad;
     grid->GlobalMax(gmx);
     grid->GlobalSumVector(&gbad, 1);
-    std::cout << GridLogMessage << "DenseCoarseMatrix: fp64 import certificate "
+    std::cout << GridLogMessage << "DenseCoarseMatrix: inversion-source import certificate "
-              << "max|A64 - A32| = " << gmx
+              << "max|A_inv - A_slab| = " << gmx
               << "  non-finite entries " << (int64_t)gbad << std::endl;
     GRID_ASSERT( gbad == 0 );
     GRID_ASSERT( gmx < 1.0e-5 );
 
     S.Resize(nrows, N);
-    acceleratorCopyToDevice(&h[0], &S.data[0], (uint64_t)nrows*N*sizeof(ComplexD));
+    acceleratorCopyToDevice(&h[0], &S.data[0], (uint64_t)nrows*N*sizeof(DenseInverseScalar));
   }
 
   ////////////////////////////////////////////////////////////////////
-  // 3c. The inverse: distributed recursive Schur, END-TO-END fp64.
+  // 3c. The inverse: distributed recursive Schur, end to end in the
-  //    stencil (ComplexD) -> fp64 rank-major import -> fp64 recursion ->
+  //    inversion precision (DenseInverseScalar, a configure-time choice):
-  //    ONE terminal rounding into the fp32 apply slab.  Everything
+  //    stencil -> rank-major import -> recursion -> ONE terminal rounding
-  //    downstream (device residency, split-K apply, VERIFY) is fp32.
+  //    into the fp32 apply slab.  Everything downstream (device
+  //    residency, split-K apply, VERIFY) is fp32 regardless.
   ////////////////////////////////////////////////////////////////////
   template<class CoarseOp>
   void InvertDense(CoarseOp &Op)
@@ -578,7 +599,7 @@ public:
     }
 
     BlockRows S;
-    ImportDenseFP64(Op, S, g2rm);
+    ImportDenseForInversion(Op, S, g2rm);
 
     ////////////////////////////////////////////////////////////////
     // The 2D block-cyclic recursion (BlockCyclicSchurInverse).
@@ -609,8 +630,8 @@ public:
     // The single terminal rounding: fp64 inverse -> fp32 apply slab
     // (row-major, global columns)
     {
-      std::vector<ComplexD> h((uint64_t)nrows*N);
+      std::vector<DenseInverseScalar> h((uint64_t)nrows*N);
-      acceleratorCopyFromDevice(&S.data[0], &h[0], (uint64_t)nrows*N*sizeof(ComplexD));
+      acceleratorCopyFromDevice(&S.data[0], &h[0], (uint64_t)nrows*N*sizeof(DenseInverseScalar));
       thread_for(gcol, N, {
         int64_t jj = g2rm[gcol];
         for(int64_t i=0; i<nrows; i++)
@@ -620,7 +641,9 @@ public:
       });
     }
     double t4 = usecond();
-    std::cout << GridLogMessage << "DenseCoarseMatrix: SCHUR fp64 distributed invert took "
+    std::cout << GridLogMessage << "DenseCoarseMatrix: SCHUR "
+              << (sizeof(DenseInverseScalar)==sizeof(ComplexF) ? "fp32" : "fp64")
+              << " distributed invert took "
               << (t4-t1)/1.0e6 << " s (recursion " << (t3-t2)/1.0e6 << " s)" << std::endl;
   }
 
@@ -717,7 +740,7 @@ public:
         sobj s;
         peekLocalSite(s, src, myLcoor[ss]);
         for(int b=0; b<nbasis; b++)
-          hX[ myGsite[ss]*nbasis + b ] = ComplexF(((ComplexD *)&s)[b]);
+          hX[ myGsite[ss]*nbasis + b ] = ComplexF(((CoarseScalar *)&s)[b]);
       }
     }
     SlabApplyPacked(1, nullptr);
@@ -725,7 +748,7 @@ public:
       for(int ss=0; ss<lsites; ss++){
         sobj s;
         for(int b=0; b<nbasis; b++)
-          ((ComplexD *)&s)[b] = ComplexD(hY[ss*nbasis + b]);
+          ((CoarseScalar *)&s)[b] = CoarseScalar(hY[ss*nbasis + b]);
         pokeLocalSite(s, psi, myLcoor[ss]);
       }
     }
@@ -762,7 +785,7 @@ public:
           sobj s;
           peekLocalSite(s, src[rr], myLcoor[ss]);
           for(int b=0; b<nbasis; b++)
-            hX[ (uint64_t)rr*N + myGsite[ss]*nbasis + b ] = ComplexF(((ComplexD *)&s)[b]);
+            hX[ (uint64_t)rr*N + myGsite[ss]*nbasis + b ] = ComplexF(((CoarseScalar *)&s)[b]);
         }
       }
     }
@@ -772,7 +795,7 @@ public:
         for(int ss=0; ss<lsites; ss++){
           sobj s;
           for(int b=0; b<nbasis; b++)
-            ((ComplexD *)&s)[b] = ComplexD(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);
+            ((CoarseScalar *)&s)[b] = CoarseScalar(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);
           pokeLocalSite(s, psi[rr], myLcoor[ss]);
         }
       }
@@ -815,7 +838,7 @@ public:
           sobj s;
           peekLocalSite(s, iv, c6);
           for(int b=0; b<nbasis; b++)
-            hX[(uint64_t)rr*N + myGsite[ss]*nbasis + b] = ComplexF(((ComplexD *)&s)[b]);
+            hX[(uint64_t)rr*N + myGsite[ss]*nbasis + b] = ComplexF(((CoarseScalar *)&s)[b]);
         }
       }
     }
@@ -832,7 +855,7 @@ public:
           c6[0] = rr;
           sobj s;
           for(int b=0; b<nbasis; b++)
-            ((ComplexD *)&s)[b] = ComplexD(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);  // Y col-major
+            ((CoarseScalar *)&s)[b] = CoarseScalar(hY[(uint64_t)rr*nrows + (ss*nbasis+b)]);  // Y col-major
           pokeLocalSite(s, ov, c6);
         }
       }

Grid/algorithms/multigrid/GeneralCoarsenedMatrix.h

@@ -1,704 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/GeneralCoarsenedMatrix.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <pboyle@bnl.gov>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#pragma once
-
-#include <Grid/qcd/QCD.h> // needed for Dagger(Yes|No), Inverse(Yes|No)
-
-#include <Grid/lattice/PaddedCell.h>
-#include <Grid/stencil/GeneralLocalStencil.h>
-#include <Grid/algorithms/deflation/MultiRHSBlockProject.h>
-
-NAMESPACE_BEGIN(Grid);
-
-// Fine Object == (per site) type of fine field
-// nbasis      == number of deflation vectors
-template<class Fobj,class CComplex,int nbasis>
-class GeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
-public:
-
-  typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
-  typedef iVector<CComplex,nbasis >           siteVector;
-  typedef iMatrix<CComplex,nbasis >           siteMatrix;
-  typedef Lattice<iScalar<CComplex> >         CoarseComplexField;
-  typedef Lattice<siteVector>                 CoarseVector;
-  typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
-  typedef iMatrix<CComplex,nbasis >  Cobj;
-  typedef iVector<CComplex,nbasis >  Cvec;
-  typedef Lattice< CComplex >   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<Fobj >        FineField;
-  typedef Lattice<CComplex >    FineComplexField;
-  typedef CoarseVector Field;
-  ////////////////////
-  // Data members
-  ////////////////////
-  int hermitian;
-  GridBase      *       _FineGrid; 
-  GridCartesian *       _CoarseGrid; 
-  NonLocalStencilGeometry &geom;
-  PaddedCell Cell;
-  GeneralLocalStencil Stencil;
-  
-  std::vector<CoarseMatrix> _A;
-  std::vector<CoarseMatrix> _Adag;
-  std::vector<CoarseVector> MultTemporaries;
-
-  int64_t MultCalls;
-  double  MultFlopsAccum;
-  double  MultUsecAccum;
-
-  ///////////////////////
-  // Interface
-  ///////////////////////
-  //////////////////////////////////////////////////////////////////////////
-  // Bilingual accessors: everything a consumer needs to read the operator
-  // without knowing which of the three coarse classes it holds. The D
-  // dimensional grid the elements live on, the geometry they are indexed by,
-  // and one unpadded point at a time (a whole npoint vector is too much
-  // memory at production nbasis).
-  //////////////////////////////////////////////////////////////////////////
-  GridCartesian * CoarseGridD(void)             { return _CoarseGrid; };
-  NonLocalStencilGeometry & Geometry(void)      { return geom; };
-  void ExtractMatrix(int p,CoarseMatrix &A)     { A = Cell.Extract(_A[p]); };
-
-  GridBase      * Grid(void)           { return _CoarseGrid; };   // this is all the linalg routines need to know
-  GridBase      * FineGrid(void)       { return _FineGrid; };   // this is all the linalg routines need to know
-  GridCartesian * CoarseGrid(void)     { return _CoarseGrid; };   // this is all the linalg routines need to know
-
-  /*  void ShiftMatrix(RealD shift)
-  {
-    int Nd=_FineGrid->Nd(); 
-    Coordinate zero_shift(Nd,0);
-    for(int p=0;p<geom.npoint;p++){
-      if ( zero_shift==geom.shifts[p] ) {
-	_A[p] = _A[p]+shift;
-	//	_Adag[p] = _Adag[p]+shift;
-      }
-    }    
-  }
-  void ProjectNearestNeighbour(RealD shift, GeneralCoarseOp &CopyMe)
-  {
-    int nfound=0;
-    std::cout << GridLogMessage <<"GeneralCoarsenedMatrix::ProjectNearestNeighbour "<< CopyMe._A[0].Grid()<<std::endl;
-    for(int p=0;p<geom.npoint;p++){
-      for(int pp=0;pp<CopyMe.geom.npoint;pp++){
- 	// Search for the same relative shift
-	// Avoids brutal handling of Grid pointers
-	if ( CopyMe.geom.shifts[pp]==geom.shifts[p] ) {
-	  _A[p] = CopyMe.Cell.Extract(CopyMe._A[pp]);
-	  //	  _Adag[p] = CopyMe.Cell.Extract(CopyMe._Adag[pp]);
-	  nfound++;
-	}
-      }
-    }
-    GRID_ASSERT(nfound==geom.npoint);
-    ExchangeCoarseLinks();
-  }
-  */
-  
-  GeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridBase *FineGrid, GridCartesian * CoarseGrid,int _herm=1)
-    : geom(_geom),
-      _FineGrid(FineGrid),
-      _CoarseGrid(CoarseGrid),
-      hermitian(_herm),
-      Cell(_geom.Depth(),_CoarseGrid),
-      Stencil(Cell.grids.back(),geom.shifts),
-      MultCalls(0), MultFlopsAccum(0.0), MultUsecAccum(0.0)
-  {
-    {
-      int npoint = _geom.npoint;
-    }
-    _A.resize(geom.npoint,CoarseGrid);
-    if ( !hermitian ) _Adag.resize(geom.npoint,CoarseGrid);
-  }
-  void M (const CoarseVector &in, CoarseVector &out)
-  {
-    Mult(_A,in,out);
-  }
-  void Mdag (const CoarseVector &in, CoarseVector &out)
-  {
-    if(hermitian)
-      Mult(_A,in,out);
-    else
-      Mult(_Adag,in,out);
-  }
-  void Mult (std::vector<CoarseMatrix> &A,const CoarseVector &in, CoarseVector &out)
-  {
-    RealD tviews=0;    RealD ttot=0;    RealD tmult=0;   RealD texch=0;    RealD text=0; RealD ttemps=0; RealD tcopy=0;
-    RealD tmult2=0;
-
-    ttot=-usecond();
-    conformable(CoarseGrid(),in.Grid());
-    conformable(in.Grid(),out.Grid());
-    out.Checkerboard() = in.Checkerboard();
-    CoarseVector tin=in;
-
-    texch-=usecond();
-    CoarseVector pin = Cell.ExchangePeriodic(tin);
-    texch+=usecond();
-
-    CoarseVector pout(pin.Grid());
-
-    int npoint = geom.npoint;
-    typedef LatticeView<Cobj> Aview;
-    typedef LatticeView<Cvec> Vview;
-      
-    const int Nsimd = CComplex::Nsimd();
-    
-    int64_t osites=pin.Grid()->oSites();
-
-    RealD flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
-    RealD bytes = 1.0*osites*sizeof(siteMatrix)*npoint
-                + 2.0*osites*sizeof(siteVector)*npoint;
-      
-    {
-      tviews-=usecond();
-      autoView( in_v , pin, AcceleratorRead);
-      autoView( out_v , pout, AcceleratorWriteDiscard);
-      autoView( Stencil_v  , Stencil, AcceleratorRead);
-      tviews+=usecond();
-
-      // Static and prereserve to keep UVM region live and not resized across multiple calls
-      ttemps-=usecond();
-      MultTemporaries.resize(npoint,pin.Grid());       
-      ttemps+=usecond();
-      std::vector<Aview> AcceleratorViewContainer_h;
-      std::vector<Vview> AcceleratorVecViewContainer_h; 
-
-      tviews-=usecond();
-      for(int p=0;p<npoint;p++) {
-	AcceleratorViewContainer_h.push_back(      A[p].View(AcceleratorRead));
-	AcceleratorVecViewContainer_h.push_back(MultTemporaries[p].View(AcceleratorWrite));
-      }
-      tviews+=usecond();
-
-      static deviceVector<Aview> AcceleratorViewContainer; AcceleratorViewContainer.resize(npoint);
-      static deviceVector<Vview> AcceleratorVecViewContainer; AcceleratorVecViewContainer.resize(npoint); 
-      
-      auto Aview_p = &AcceleratorViewContainer[0];
-      auto Vview_p = &AcceleratorVecViewContainer[0];
-      tcopy-=usecond();
-      acceleratorCopyToDevice(&AcceleratorViewContainer_h[0],&AcceleratorViewContainer[0],npoint *sizeof(Aview));
-      acceleratorCopyToDevice(&AcceleratorVecViewContainer_h[0],&AcceleratorVecViewContainer[0],npoint *sizeof(Vview));
-      tcopy+=usecond();
-
-      tmult-=usecond();
-      accelerator_for(spb, osites*nbasis*npoint, Nsimd, {
-	  typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
-	  int32_t ss   = spb/(nbasis*npoint);
-	  int32_t bp   = spb%(nbasis*npoint);
-	  int32_t point= bp/nbasis;
-	  int32_t b    = bp%nbasis;
-	  auto SE  = Stencil_v.GetEntry(point,ss);
-	  auto nbr = coalescedReadGeneralPermute(in_v[SE->_offset],SE->_permute,Nd);
-	  auto res = coalescedRead(Aview_p[point][ss](0,b))*nbr(0);
-	  for(int bb=1;bb<nbasis;bb++) {
-	    res = res + coalescedRead(Aview_p[point][ss](bb,b))*nbr(bb);
-	  }
-	  coalescedWrite(Vview_p[point][ss](b),res);
-      });
-      tmult2-=usecond();
-      accelerator_for(sb, osites*nbasis, Nsimd, {
-	  int ss = sb/nbasis;
-	  int b  = sb%nbasis;
-	  auto res = coalescedRead(Vview_p[0][ss](b));
-	  for(int point=1;point<npoint;point++){
-	    res = res + coalescedRead(Vview_p[point][ss](b));
-	  }
-	  coalescedWrite(out_v[ss](b),res);
-      });
-      tmult2+=usecond();
-      tmult+=usecond();
-      for(int p=0;p<npoint;p++) {
-	AcceleratorViewContainer_h[p].ViewClose();
-	AcceleratorVecViewContainer_h[p].ViewClose();
-      }
-    }
-
-    text-=usecond();
-    out = Cell.Extract(pout);
-    text+=usecond();
-    ttot+=usecond();
-    
-    MultCalls++;
-    MultFlopsAccum += flops;
-    MultUsecAccum  += ttot;
-    std::cout << GridLogPerformance
-	      << "Coarse Mult call " << MultCalls
-	      << "  tot " << ttot << " us"
-	      << "  kernel " << tmult << " us"
-	      << "  kernel " << flops/tmult*1e-3 << " GFlop/s"
-	      << "  overall " << MultFlopsAccum/MultUsecAccum*1e-3 << " GFlop/s (cumul)"
-	      << "  bw " << bytes/tmult*1e-3 << " GB/s"
-	      << std::endl;
-
-  };
-  
-  void PopulateAdag(void)
-  {
-#if 0
-    // Serial global peek/poke reference implementation
-    for(int64_t bidx=0;bidx<CoarseGrid()->gSites() ;bidx++){
-      Coordinate bcoor;
-      CoarseGrid()->GlobalIndexToGlobalCoor(bidx,bcoor);
-
-      for(int p=0;p<geom.npoint;p++){
-	Coordinate scoor = bcoor;
-	for(int mu=0;mu<bcoor.size();mu++){
-	  int L = CoarseGrid()->GlobalDimensions()[mu];
-	  scoor[mu] = (bcoor[mu] - geom.shifts[p][mu] + L) % L; // Modulo arithmetic
-	}
-	// Flip to poke/peekLocalSite and not too bad
-	auto link = peekSite(_A[p],scoor);
-	int pp = geom.Reverse(p);
-	pokeSite(adj(link),_Adag[pp],bcoor);
-      }
-    }
-#else
-    // Parallel: _Adag[pp](x) = adj( _A[p](x + s_pp) ),  pp = Reverse(p), s_pp = -s_p.
-    // The neighbour fetch reuses the same padded-cell + stencil machinery as Mult,
-    // reading one matrix element per coalesced access so no whole site matrix
-    // (230KB at nbasis=60) ever lands on a GPU thread stack (HIP limit 128KB).
-    // Halo sites compute garbage neighbours; Cell.Extract discards them.
-    // Must run on the unpadded _A, i.e. before ExchangeCoarseLinks.
-    const int Nsimd = CComplex::Nsimd();
-    for(int p=0;p<geom.npoint;p++){
-      int pp = geom.Reverse(p);
-      CoarseMatrix Apad = Cell.ExchangePeriodic(_A[p]);
-      CoarseMatrix Dpad(Apad.Grid());
-      int64_t osites = Apad.Grid()->oSites();
-      {
-	autoView( Apad_v   , Apad, AcceleratorRead);
-	autoView( Dpad_v   , Dpad, AcceleratorWriteDiscard);
-	autoView( Stencil_v, Stencil, AcceleratorRead);
-	accelerator_for(sj, osites*nbasis, Nsimd, {
-	    int32_t ss = sj/nbasis;
-	    int32_t j  = sj%nbasis;
-	    auto SE = Stencil_v.GetEntry(pp,ss);
-	    for(int i=0;i<nbasis;i++){
-	      auto z = coalescedReadGeneralPermute(Apad_v[SE->_offset](i,j),SE->_permute,Nd);
-	      coalescedWrite(Dpad_v[ss](j,i),conjugate(z));
-	    }
-	});
-      }
-      _Adag[pp] = Cell.Extract(Dpad);
-    }
-#endif
-  }
-  /////////////////////////////////////////////////////////////
-  // 
-  // A) Only reduced flops option is to use a padded cell of depth 4
-  // and apply MpcDagMpc in the padded cell.
-  //
-  // Makes for ONE application of MpcDagMpc per vector instead of 30 or 80.
-  // With the effective cell size around (B+8)^4 perhaps 12^4/4^4 ratio
-  // Cost is 81x more, same as stencil size.
-  //
-  // But: can eliminate comms and do as local dirichlet.
-  //
-  // Local exchange gauge field once.
-  // Apply to all vectors, local only computation.
-  // Must exchange ghost subcells in reverse process of PaddedCell to take inner products
-  //
-  // B) Can reduce cost: pad by 1, apply Deo      (4^4+6^4+8^4+8^4 )/ (4x 4^4)
-  //                     pad by 2, apply Doe
-  //                     pad by 3, apply Deo
-  //                     then break out 8x directions; cost is ~10x MpcDagMpc per vector
-  //
-  // => almost factor of 10 in setup cost, excluding data rearrangement
-  //
-  // Intermediates -- ignore the corner terms, leave approximate and force Hermitian
-  // Intermediates -- pad by 2 and apply 1+8+24 = 33 times.
-  /////////////////////////////////////////////////////////////
-
-    //////////////////////////////////////////////////////////
-    // BFM HDCG style approach: Solve a system of equations to get Aij
-    //////////////////////////////////////////////////////////
-    /*
-     *     Here, k,l index which possible shift within the 3^Nd "ball" connected by MdagM.
-     *
-     *     conj(phases[block]) proj[k][ block*Nvec+j ] =  \sum_ball  e^{i q_k . delta} < phi_{block,j} | MdagM | phi_{(block+delta),i} > 
-     *                                                 =  \sum_ball e^{iqk.delta} A_ji
-     *
-     *     Must invert matrix M_k,l = e^[i q_k . delta_l]
-     *
-     *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
-     */
-#if 0
-  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
-		       Aggregation<Fobj,CComplex,nbasis> & Subspace)
-  {
-    std::cout << GridLogMessage<< "GeneralCoarsenMatrix "<< std::endl;
-    GridBase *grid = FineGrid();
-
-    RealD tproj=0.0;
-    RealD teigen=0.0;
-    RealD tmat=0.0;
-    RealD tphase=0.0;
-    RealD tinv=0.0;
-
-    /////////////////////////////////////////////////////////////
-    // Orthogonalise the subblocks over the basis
-    /////////////////////////////////////////////////////////////
-    CoarseScalar InnerProd(CoarseGrid()); 
-    blockOrthogonalise(InnerProd,Subspace.subspace);
-
-    const int npoint = geom.npoint;
-      
-    Coordinate clatt = CoarseGrid()->GlobalDimensions();
-    int Nd = CoarseGrid()->Nd();
-
-      /*
-       *     Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
-       *     Matrix index i is mapped to this shift via 
-       *               geom.shifts[i]
-       *
-       *     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block] 
-       *       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} > 
-       *       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
-       *       = M_{kl} A_ji^{b.b+l}
-       *
-       *     Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
-       *  
-       *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
-       *
-       *     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
-       */
-    teigen-=usecond();
-    Eigen::MatrixXcd Mkl    = Eigen::MatrixXcd::Zero(npoint,npoint);
-    Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
-    ComplexD ci(0.0,1.0);
-    for(int k=0;k<npoint;k++){ // Loop over momenta
-
-      for(int l=0;l<npoint;l++){ // Loop over nbr relative
-	ComplexD phase(0.0,0.0);
-	for(int mu=0;mu<Nd;mu++){
-	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  phase=phase+TwoPiL*geom.shifts[k][mu]*geom.shifts[l][mu];
-	}
-	phase=exp(phase*ci);
-	Mkl(k,l) = phase;
-      }
-    }
-    invMkl = Mkl.inverse();
-    teigen+=usecond();
-
-    ///////////////////////////////////////////////////////////////////////
-    // Now compute the matrix elements of linop between the orthonormal
-    // set of vectors.
-    ///////////////////////////////////////////////////////////////////////
-    FineField phaV(grid); // Phased block basis vector
-    FineField MphaV(grid);// Matrix applied
-    CoarseVector coarseInner(CoarseGrid());
-
-    std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid());
-    std::vector<CoarseVector>          FT(npoint,CoarseGrid());
-    for(int i=0;i<nbasis;i++){// Loop over basis vectors
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
-      for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
-	/////////////////////////////////////////////////////
-	// Stick a phase on every block
-	/////////////////////////////////////////////////////
-	tphase-=usecond();
-	CoarseComplexField coor(CoarseGrid());
-	CoarseComplexField pha(CoarseGrid());	pha=Zero();
-	for(int mu=0;mu<Nd;mu++){
-	  LatticeCoordinate(coor,mu);
-	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  pha = pha + (TwoPiL * geom.shifts[p][mu]) * coor;
-	}
-	pha  =exp(pha*ci);
-	phaV=Zero();
-	blockZAXPY(phaV,pha,Subspace.subspace[i],phaV);
-	tphase+=usecond();
-
-	/////////////////////////////////////////////////////////////////////
-	// Multiple phased subspace vector by matrix and project to subspace
-	// Remove local bulk phase to leave relative phases
-	/////////////////////////////////////////////////////////////////////
-	tmat-=usecond();
-	linop.Op(phaV,MphaV);
-	tmat+=usecond();
-
-	tproj-=usecond();
-	blockProject(coarseInner,MphaV,Subspace.subspace);
-	coarseInner = conjugate(pha) * coarseInner;
-
-	ComputeProj[p] = coarseInner;
-	tproj+=usecond();
-
-      }
-
-      tinv-=usecond();
-      for(int k=0;k<npoint;k++){
-	FT[k] = Zero();
-	for(int l=0;l<npoint;l++){
-	  FT[k]= FT[k]+ invMkl(l,k)*ComputeProj[l];
-	}
-      
-	int osites=CoarseGrid()->oSites();
-	autoView( A_v  , _A[k], AcceleratorWrite);
-	autoView( FT_v  , FT[k], AcceleratorRead);
-	accelerator_for(sss, osites, nbasis, {
-#ifdef GRID_SIMT
-	    int j = acceleratorSIMTlane(nbasis);
-	    A_v[sss](i,j) = FT_v[sss](j);
-#else
-	    // CPU build: acceleratorSIMTlane()==0 -- an un-looped SIMT tensor
-	    // index writes ONLY j=0 and silently drops the other nbasis-1
-	    // columns (caught by Test_schur_dense_coarse import certificate,
-	    // 2026-08-14).  Loop explicitly.
-	    for(int j=0;j<nbasis;j++){
-	      A_v[sss](i,j) = FT_v[sss](j);
-	    }
-#endif
-        });
-      }
-      tinv+=usecond();
-    }
-
-    // Only needed if nonhermitian
-    if ( ! hermitian ) {
-      std::cout << GridLogMessage<<"PopulateAdag  "<<std::endl;
-      PopulateAdag();
-    }
-
-    // Need to write something to populate Adag from A
-    ExchangeCoarseLinks();
-    std::cout << GridLogMessage<<"CoarsenOperator eigen  "<<teigen<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator phase  "<<tphase<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator mat    "<<tmat <<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator proj   "<<tproj<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator inv    "<<tinv<<" us"<<std::endl;
-  }
-#else
-  //////////////////////////////////////////////////////////////////////
-  // Galerkin projection of matrix
-  //////////////////////////////////////////////////////////////////////
-  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
-		       Aggregation<Fobj,CComplex,nbasis> & Subspace)
-  {
-    CoarsenOperator(linop,Subspace,Subspace);
-  }
-  //////////////////////////////////////////////////////////////////////
-  // Petrov - Galerkin projection of matrix
-  //////////////////////////////////////////////////////////////////////
-  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
-		       Aggregation<Fobj,CComplex,nbasis> & U,
-		       Aggregation<Fobj,CComplex,nbasis> & V)
-  {
-    std::cout << GridLogMessage<< "GeneralCoarsenMatrix "<< std::endl;
-    GridBase *grid = FineGrid();
-
-    RealD tproj=0.0;
-    RealD teigen=0.0;
-    RealD tmat=0.0;
-    RealD tphase=0.0;
-    RealD tphaseBZ=0.0;
-    RealD tinv=0.0;
-
-    /////////////////////////////////////////////////////////////
-    // Orthogonalise the subblocks over the basis
-    /////////////////////////////////////////////////////////////
-    CoarseScalar InnerProd(CoarseGrid()); 
-    blockOrthogonalise(InnerProd,V.subspace);
-    blockOrthogonalise(InnerProd,U.subspace);
-
-    const int npoint = geom.npoint;
-      
-    Coordinate clatt = CoarseGrid()->GlobalDimensions();
-    int Nd = CoarseGrid()->Nd();
-
-      /*
-       *     Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
-       *     Matrix index i is mapped to this shift via 
-       *               geom.shifts[i]
-       *
-       *     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block] 
-       *       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} > 
-       *       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
-       *       = M_{kl} A_ji^{b.b+l}
-       *
-       *     Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
-       *  
-       *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
-       *
-       *     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
-       */
-    teigen-=usecond();
-    Eigen::MatrixXcd Mkl    = Eigen::MatrixXcd::Zero(npoint,npoint);
-    Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
-    ComplexD ci(0.0,1.0);
-    for(int k=0;k<npoint;k++){ // Loop over momenta
-
-      for(int l=0;l<npoint;l++){ // Loop over nbr relative
-	ComplexD phase(0.0,0.0);
-	for(int mu=0;mu<Nd;mu++){
-	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  phase=phase+TwoPiL*geom.shifts[k][mu]*geom.shifts[l][mu];
-	}
-	phase=exp(phase*ci);
-	Mkl(k,l) = phase;
-      }
-    }
-    invMkl = Mkl.inverse();
-    teigen+=usecond();
-
-    ///////////////////////////////////////////////////////////////////////
-    // Now compute the matrix elements of linop between the orthonormal
-    // set of vectors.
-    ///////////////////////////////////////////////////////////////////////
-    std::vector<FineComplexField> phaF(npoint,grid);
-    std::vector<CoarseComplexField> pha(npoint,CoarseGrid());
-
-    typedef typename CComplex::scalar_type SComplex;
-    FineComplexField one(grid); one=SComplex(1.0);
-    FineComplexField zz(grid); zz = Zero();
-    tphase=-usecond();
-    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
-      /////////////////////////////////////////////////////
-      // Stick a phase on every block
-      /////////////////////////////////////////////////////
-      CoarseComplexField coor(CoarseGrid());
-      pha[p]=Zero();
-      for(int mu=0;mu<Nd;mu++){
-	LatticeCoordinate(coor,mu);
-	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	pha[p] = pha[p] + (TwoPiL * geom.shifts[p][mu]) * coor;
-      }
-      pha[p]  =exp(pha[p]*ci);
-
-      blockZAXPY(phaF[p],pha[p],one,zz);
-
-    }
-    tphase+=usecond();
-
-    // Import basis into BLAS layout once; blockProject then reads it once per
-    // basis vector rather than once per (i,p) as in scalar blockProject.
-    // Process all npoint in a single batch.
-    MultiRHSBlockProject<FineField> Projector;
-    Projector.Allocate(nbasis, grid, CoarseGrid());
-    Projector.ImportBasis(U.subspace);
-
-    std::vector<FineField>    phaV_batch(npoint, grid);
-    std::vector<FineField>    MphaV_batch(npoint, grid);
-    std::vector<CoarseVector> proj_batch(npoint, CoarseGrid());
-    std::vector<CoarseVector> ComputeProj(npoint, CoarseGrid());
-    std::vector<CoarseVector>          FT(npoint, CoarseGrid());
-
-    // Pre-allocate BLAS_F and BLAS_C to avoid repeated hipMalloc/hipFree of
-    // ~5.6 GB per blockProject call, which hangs on ROCm for large allocations.
-    Projector.BLAS_F.resize(Projector.fine_vol * Projector.words * npoint);
-    Projector.BLAS_C.resize(Projector.coarse_vol * nbasis * npoint);
-
-    for(int i=0;i<nbasis;i++){// Loop over basis vectors
-      accelerator_barrier(); // ensure prior iteration's async writes are retired
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
-
-      tphaseBZ-=usecond();
-      for(int p=0;p<npoint;p++)
-	phaV_batch[p] = phaF[p] * V.subspace[i];
-      tphaseBZ+=usecond();
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<" phaseBZ done"<< std::endl;
-
-      tmat-=usecond();
-      for(int p=0;p<npoint;p++)
-	linop.Op(phaV_batch[p], MphaV_batch[p]);
-      tmat+=usecond();
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<" mat done"<< std::endl;
-
-      // One batched GEMM reads BLAS_V once for all npoint vectors.
-      tproj-=usecond();
-      Projector.blockProject(MphaV_batch, proj_batch);
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<" blockProject done"<< std::endl;
-      for(int p=0;p<npoint;p++)
-	ComputeProj[p] = conjugate(pha[p]) * proj_batch[p];
-      tproj+=usecond();
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<" proj done"<< std::endl;
-
-      tinv-=usecond();
-      for(int k=0;k<npoint;k++){
-	FT[k] = Zero();
-	for(int l=0;l<npoint;l++){
-	  FT[k]= FT[k]+ invMkl(l,k)*ComputeProj[l];
-	}
-
-	int osites=CoarseGrid()->oSites();
-	autoView( A_v  , _A[k], AcceleratorWrite);
-	autoView( FT_v  , FT[k], AcceleratorRead);
-	accelerator_for(sss, osites, nbasis, {
-#ifdef GRID_SIMT
-	    int j = acceleratorSIMTlane(nbasis);
-	    A_v[sss](i,j) = FT_v[sss](j);
-#else
-	    // CPU build: acceleratorSIMTlane()==0 -- an un-looped SIMT tensor
-	    // index writes ONLY j=0 and silently drops the other nbasis-1
-	    // columns (caught by Test_schur_dense_coarse import certificate,
-	    // 2026-08-14).  Loop explicitly.
-	    for(int j=0;j<nbasis;j++){
-	      A_v[sss](i,j) = FT_v[sss](j);
-	    }
-#endif
-        });
-      }
-      tinv+=usecond();
-    }
-
-    // Only needed if nonhermitian
-    if ( ! hermitian ) {
-      std::cout << GridLogMessage<<"PopulateAdag  "<<std::endl;
-      PopulateAdag();
-    }
-
-    //    for(int p=0;p<geom.npoint;p++){
-    //      std::cout << " _A["<<p<<"] "<<norm2(_A[p])<<std::endl;
-    //    }
-
-    // Need to write something to populate Adag from A
-    ExchangeCoarseLinks();
-    std::cout << GridLogMessage<<"CoarsenOperator eigen  "<<teigen<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator phase  "<<tphase<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator mat    "<<tmat <<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator proj   "<<tproj<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator inv    "<<tinv<<" us"<<std::endl;
-  }
-#endif  
-  void ExchangeCoarseLinks(void){
-    for(int p=0;p<geom.npoint;p++){
-      _A[p] = Cell.ExchangePeriodic(_A[p]);
-      if ( !hermitian ) _Adag[p]= Cell.ExchangePeriodic(_Adag[p]);
-    }
-  }
-  virtual  void Mdiag    (const Field &in, Field &out){ GRID_ASSERT(0);};
-  virtual  void Mdir     (const Field &in, Field &out,int dir, int disp){assert(0);};
-  virtual  void MdirAll  (const Field &in, std::vector<Field> &out){assert(0);};
-};
-
-
-  
-NAMESPACE_END(Grid);

Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHS.h

@@ -1,738 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/GeneralCoarsenedMatrixMultiRHS.h
-
-    Copyright (C) 2015
-
-Author: Peter Boyle <pboyle@bnl.gov>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#pragma once
-
-
-NAMESPACE_BEGIN(Grid);
-
-
-// Fine Object == (per site) type of fine field
-// nbasis      == number of deflation vectors
-template<class Fobj,class CComplex,int nbasis>
-class MultiGeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
-public:
-  typedef typename CComplex::scalar_object SComplex;
-  typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
-  typedef MultiGeneralCoarsenedMatrix<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
-
-  typedef iVector<CComplex,nbasis >           siteVector;
-  typedef iMatrix<CComplex,nbasis >           siteMatrix;
-  typedef iVector<SComplex,nbasis >           calcVector;
-  typedef iMatrix<SComplex,nbasis >           calcMatrix;
-  typedef Lattice<iScalar<CComplex> >         CoarseComplexField;
-  typedef Lattice<siteVector>                 CoarseVector;
-  typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
-  typedef iMatrix<CComplex,nbasis >  Cobj;
-  typedef iVector<CComplex,nbasis >  Cvec;
-  typedef Lattice< CComplex >   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<Fobj >        FineField;
-  typedef Lattice<CComplex >    FineComplexField;
-  typedef CoarseVector Field;
-
-  ////////////////////
-  // Data members
-  ////////////////////
-  GridCartesian *       _CoarseGridMulti; 
-  NonLocalStencilGeometry geom;
-  NonLocalStencilGeometry geom_srhs;
-  PaddedCell Cell;
-  GeneralLocalStencil Stencil;
-
-  deviceVector<calcVector> BLAS_B;
-  deviceVector<calcVector> BLAS_C;
-  std::vector<deviceVector<calcMatrix> > BLAS_A;
-
-  std::vector<deviceVector<ComplexD *> > BLAS_AP;
-  std::vector<deviceVector<ComplexD *> > BLAS_BP;
-  deviceVector<ComplexD *>               BLAS_CP;
-
-  ///////////////////////
-  // Interface
-  ///////////////////////
-  GridBase      * Grid(void)           { return _CoarseGridMulti; };   // this is all the linalg routines need to know
-  GridCartesian * CoarseGrid(void)     { return _CoarseGridMulti; };   // this is all the linalg routines need to know
-
-  //////////////////////////////////////////////////////////////////////////
-  // Bilingual accessors, matching GeneralCoarsenedMatrix. Grid() here is the
-  // D+1 multiRHS grid and this class never holds the D dimensional one, so
-  // ExtractMatrix writes into whatever grid the caller's lattice is on.
-  //////////////////////////////////////////////////////////////////////////
-  NonLocalStencilGeometry & Geometry(void)      { return geom_srhs; };
-  void ExtractMatrix(int p,CoarseMatrix &A)     { BLAStoGrid(A,BLAS_A[p]); };
-
-  // I/O on the operator matrices, via the BLAS layout array. The parameter is
-  // a vector over the geometry points; the body indexes A[p].
-  void SetMatrix (int p,std::vector<CoarseMatrix> & A)
-  {
-    GRID_ASSERT(A.size()==geom_srhs.npoint);
-    GridtoBLAS(A[p],BLAS_A[p]);
-  }
-  void GetMatrix (int p,std::vector<CoarseMatrix> & A)
-  {
-    GRID_ASSERT(A.size()==geom_srhs.npoint);
-    BLAStoGrid(A[p],BLAS_A[p]);
-  }
-  void CopyMatrix (GeneralCoarseOp &_Op)
-  {
-    for(int p=0;p<geom.npoint;p++){
-      auto Aup = _Op.Cell.Extract(_Op._A[p]);
-      //Unpadded
-      GridtoBLAS(Aup,BLAS_A[p]);
-    }
-  }
-  /*
-  void CheckMatrix (GeneralCoarseOp &_Op)
-  {
-    std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
-    for(int p=0;p<geom.npoint;p++){
-      //Unpadded
-      auto Aup = _Op.Cell.Extract(_Op._A[p]);
-      auto Ack = Aup;
-      BLAStoGrid(Ack,BLAS_A[p]);
-      std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
-      std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
-    }
-    std::cout <<"************* "<<std::endl;
-  }
-  */
-  
-  MultiGeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridCartesian *CoarseGridMulti) :
-    _CoarseGridMulti(CoarseGridMulti),
-    geom_srhs(_geom),
-    geom(_CoarseGridMulti,_geom.hops,_geom.skip+1),
-    Cell(geom.Depth(),_CoarseGridMulti),
-    Stencil(Cell.grids.back(),geom.shifts) // padded cell stencil
-  {
-    int32_t padded_sites   = Cell.grids.back()->lSites();
-    int32_t unpadded_sites = CoarseGridMulti->lSites();
-    
-    int32_t nrhs  = CoarseGridMulti->FullDimensions()[0];  // # RHS
-    int32_t orhs  = nrhs/CComplex::Nsimd();
-
-    padded_sites   = padded_sites/nrhs;
-    unpadded_sites = unpadded_sites/nrhs;
-    
-    /////////////////////////////////////////////////
-    // Device data vector storage
-    /////////////////////////////////////////////////
-    BLAS_A.resize(geom.npoint);
-    for(int p=0;p<geom.npoint;p++){
-      BLAS_A[p].resize (unpadded_sites); // no ghost zone, npoint elements
-    }
-    
-    BLAS_B.resize(nrhs *padded_sites);   // includes ghost zone
-    BLAS_C.resize(nrhs *unpadded_sites); // no ghost zone
-    BLAS_AP.resize(geom.npoint);
-    BLAS_BP.resize(geom.npoint);
-    for(int p=0;p<geom.npoint;p++){
-      BLAS_AP[p].resize(unpadded_sites);
-      BLAS_BP[p].resize(unpadded_sites);
-    }
-    BLAS_CP.resize(unpadded_sites);
-
-    /////////////////////////////////////////////////
-    // Pointers to data
-    /////////////////////////////////////////////////
-
-    // Site identity mapping for A
-    for(int p=0;p<geom.npoint;p++){
-      for(int ss=0;ss<unpadded_sites;ss++){
-	ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
-	acceleratorPut(BLAS_AP[p][ss],ptr);
-      }
-    }
-    // Site identity mapping for C
-    for(int ss=0;ss<unpadded_sites;ss++){
-      ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
-      acceleratorPut(BLAS_CP[ss],ptr);
-    }
-
-    // Neighbour table is more complicated
-    int32_t j=0; // Interior point counter (unpadded)
-    for(int32_t s=0;s<padded_sites;s++){ // 4 volume, padded
-      int ghost_zone=0;
-      for(int32_t point = 0 ; point < geom.npoint; point++){
-	int i=s*orhs*geom.npoint+point;
-	if( Stencil._entries[i]._wrap ) { // stencil is indexed by the oSite of the CoarseGridMulti, hence orhs factor
-	  ghost_zone=1; // If general stencil wrapped in any direction, wrap=1
-	}
-      }
-
-      if( ghost_zone==0) {
-	for(int32_t point = 0 ; point < geom.npoint; point++){
-	  int i=s*orhs*geom.npoint+point;
- 	  int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite
-	  GRID_ASSERT(nbr<BLAS_B.size());
-	  ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
-	  acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
-	}
-	j++;
-      }
-    }
-    GRID_ASSERT(j==unpadded_sites);
-  }
-  template<class vobj> void GridtoBLAS(const Lattice<vobj> &from,deviceVector<typename vobj::scalar_object> &to)
-  {
-  typedef typename vobj::scalar_object sobj;
-  typedef typename vobj::scalar_type scalar_type;
-  typedef typename vobj::vector_type vector_type;
-
-  GridBase *Fg = from.Grid();
-  GRID_ASSERT(!Fg->_isCheckerBoarded);
-  int nd = Fg->_ndimension;
-
-  to.resize(Fg->lSites());
-
-  Coordinate LocalLatt = Fg->LocalDimensions();
-  size_t nsite = 1;
-  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
-
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  // do the index calc on the GPU
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  Coordinate f_ostride = Fg->_ostride;
-  Coordinate f_istride = Fg->_istride;
-  Coordinate f_rdimensions = Fg->_rdimensions;
-
-  autoView(from_v,from,AcceleratorRead);
-  auto to_v = &to[0];
-
-  const int words=sizeof(vobj)/sizeof(vector_type);
-  accelerator_for(idx,nsite,1,{
-      
-      Coordinate from_coor, base;
-      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
-      for(int i=0;i<nd;i++){
-	from_coor[i] = base[i];
-      }
-      int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*(from_coor[d]%f_rdimensions[d]);
-      int from_lane = 0; for(int d=0;d<nd;d++) from_lane+=f_istride[d]*(from_coor[d]/f_rdimensions[d]);
-
-      const vector_type* from = (const vector_type *)&from_v[from_oidx];
-      scalar_type* to = (scalar_type *)&to_v[idx];
-      
-      scalar_type stmp;
-      for(int w=0;w<words;w++){
-	stmp = getlane(from[w], from_lane);
-	to[w] = stmp;
-      }
-    });
-  }    
-  template<class vobj> void BLAStoGrid(Lattice<vobj> &grid,deviceVector<typename vobj::scalar_object> &in)
-  {
-  typedef typename vobj::scalar_object sobj;
-  typedef typename vobj::scalar_type scalar_type;
-  typedef typename vobj::vector_type vector_type;
-
-  GridBase *Tg = grid.Grid();
-  GRID_ASSERT(!Tg->_isCheckerBoarded);
-  int nd = Tg->_ndimension;
-  
-  GRID_ASSERT(in.size()==Tg->lSites());
-
-  Coordinate LocalLatt = Tg->LocalDimensions();
-  size_t nsite = 1;
-  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
-
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  // do the index calc on the GPU
-  ////////////////////////////////////////////////////////////////////////////////////////////////
-  Coordinate t_ostride = Tg->_ostride;
-  Coordinate t_istride = Tg->_istride;
-  Coordinate t_rdimensions = Tg->_rdimensions;
-
-  autoView(to_v,grid,AcceleratorWrite);
-  auto from_v = &in[0];
-
-  const int words=sizeof(vobj)/sizeof(vector_type);
-  accelerator_for(idx,nsite,1,{
-      
-      Coordinate to_coor, base;
-      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
-      for(int i=0;i<nd;i++){
-	to_coor[i] = base[i];
-      }
-      int to_oidx = 0; for(int d=0;d<nd;d++) to_oidx+=t_ostride[d]*(to_coor[d]%t_rdimensions[d]);
-      int to_lane = 0; for(int d=0;d<nd;d++) to_lane+=t_istride[d]*(to_coor[d]/t_rdimensions[d]);
-
-      vector_type* to = (vector_type *)&to_v[to_oidx];
-      scalar_type* from = (scalar_type *)&from_v[idx];
-      
-      scalar_type stmp;
-      for(int w=0;w<words;w++){
-	stmp=from[w];
-	putlane(to[w], stmp, to_lane);
-      }
-    });
-  }
-  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
-		       Aggregation<Fobj,CComplex,nbasis> & Subspace,
-		       GridBase *CoarseGrid)
-  {
-#if 0
-    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
-
-    GridBase *grid = Subspace.FineGrid;
-
-    /////////////////////////////////////////////////////////////
-    // Orthogonalise the subblocks over the basis
-    /////////////////////////////////////////////////////////////
-    CoarseScalar InnerProd(CoarseGrid); 
-    blockOrthogonalise(InnerProd,Subspace.subspace);
-
-    const int npoint = geom_srhs.npoint;
-
-    Coordinate clatt = CoarseGrid->GlobalDimensions();
-    int Nd = CoarseGrid->Nd();
-      /*
-       *     Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
-       *     Matrix index i is mapped to this shift via 
-       *               geom.shifts[i]
-       *
-       *     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block] 
-       *       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} > 
-       *       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
-       *       = M_{kl} A_ji^{b.b+l}
-       *
-       *     Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
-       *  
-       *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
-       *
-       *     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
-       */
-    Eigen::MatrixXcd Mkl    = Eigen::MatrixXcd::Zero(npoint,npoint);
-    Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
-    ComplexD ci(0.0,1.0);
-    for(int k=0;k<npoint;k++){ // Loop over momenta
-
-      for(int l=0;l<npoint;l++){ // Loop over nbr relative
-	ComplexD phase(0.0,0.0);
-	for(int mu=0;mu<Nd;mu++){
-	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
-	}
-	phase=exp(phase*ci);
-	Mkl(k,l) = phase;
-      }
-    }
-    invMkl = Mkl.inverse();
-
-    ///////////////////////////////////////////////////////////////////////
-    // Now compute the matrix elements of linop between the orthonormal
-    // set of vectors.
-    ///////////////////////////////////////////////////////////////////////
-    FineField phaV(grid); // Phased block basis vector
-    FineField MphaV(grid);// Matrix applied
-    std::vector<FineComplexField> phaF(npoint,grid);
-    std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
-    
-    CoarseVector coarseInner(CoarseGrid);
-    
-    typedef typename CComplex::scalar_type SComplex;
-    FineComplexField one(grid); one=SComplex(1.0);
-    FineComplexField zz(grid); zz = Zero();
-    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
-      /////////////////////////////////////////////////////
-      // Stick a phase on every block
-      /////////////////////////////////////////////////////
-      CoarseComplexField coor(CoarseGrid);
-      pha[p]=Zero();
-      for(int mu=0;mu<Nd;mu++){
-	LatticeCoordinate(coor,mu);
-	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
-      }
-      pha[p]  =exp(pha[p]*ci);	
-
-      blockZAXPY(phaF[p],pha[p],one,zz);
-    }
-
-    // Could save on temporary storage here
-    std::vector<CoarseMatrix> _A;
-    _A.resize(geom_srhs.npoint,CoarseGrid);
-
-    std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
-    CoarseVector          FT(CoarseGrid);
-    for(int i=0;i<nbasis;i++){// Loop over basis vectors
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
-      for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
-
-	phaV = phaF[p]*Subspace.subspace[i];
-
-	/////////////////////////////////////////////////////////////////////
-	// Multiple phased subspace vector by matrix and project to subspace
-	// Remove local bulk phase to leave relative phases
-	/////////////////////////////////////////////////////////////////////
-	linop.Op(phaV,MphaV);
-
-	// Fixme, could use batched block projector here
-	blockProject(coarseInner,MphaV,Subspace.subspace);
-
-	coarseInner = conjugate(pha[p]) * coarseInner;
-
-	ComputeProj[p] = coarseInner;
-      }
-
-      // Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
-      for(int k=0;k<npoint;k++){
-
-	FT = Zero();
-	for(int l=0;l<npoint;l++){
-	  FT= FT+ invMkl(l,k)*ComputeProj[l];
-	}
-      
-	int osites=CoarseGrid->oSites();
-	autoView( A_v  , _A[k], AcceleratorWrite);
-	autoView( FT_v  , FT, AcceleratorRead);
-	accelerator_for(sss, osites, 1, {
-	    for(int j=0;j<nbasis;j++){
-	      A_v[sss](i,j) = FT_v[sss](j);
-	    }
-        });
-      }
-    }
-
-    // Only needed if nonhermitian
-    //    if ( ! hermitian ) {
-    //      std::cout << GridLogMessage<<"PopulateAdag  "<<std::endl;
-    //      PopulateAdag();
-    //    }
-    // Need to write something to populate Adag from A
-
-    for(int p=0;p<geom_srhs.npoint;p++){
-      GridtoBLAS(_A[p],BLAS_A[p]);
-    }
-    /*
-Grid : Message : 11698.730546 s : CoarsenOperator eigen  1334 us
-Grid : Message : 11698.730563 s : CoarsenOperator phase  34729 us
-Grid : Message : 11698.730565 s : CoarsenOperator phaseBZ 2423814 us
-Grid : Message : 11698.730566 s : CoarsenOperator mat    127890998 us
-Grid : Message : 11698.730567 s : CoarsenOperator proj   515840840 us
-Grid : Message : 11698.730568 s : CoarsenOperator inv    103948313 us
-Takes 600s to compute matrix elements, DOMINATED by the block project.
-Easy to speed up with the batched block project.
-Store npoint vectors, get npoint x Nbasis block projection, and 81 fold faster.
-
-// Block project below taks to 240s
-Grid : Message : 328.193418 s : CoarsenOperator phase      38338 us
-Grid : Message : 328.193434 s : CoarsenOperator phaseBZ  1711226 us
-Grid : Message : 328.193436 s : CoarsenOperator mat    122213270 us
-//Grid : Message : 328.193438 s : CoarsenOperator proj   1181154 us <-- this is mistimed
-//Grid : Message : 11698.730568 s : CoarsenOperator inv  103948313 us <-- Cut this ~10x if lucky by loop fusion
-     */
-#else
-    RealD tproj=0.0;
-    RealD tmat=0.0;
-    RealD tphase=0.0;
-    RealD tphaseBZ=0.0;
-    RealD tinv=0.0;
-
-    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
-
-    GridBase *grid = Subspace.FineGrid;
-
-    /////////////////////////////////////////////////////////////
-    // Orthogonalise the subblocks over the basis
-    /////////////////////////////////////////////////////////////
-    CoarseScalar InnerProd(CoarseGrid); 
-    blockOrthogonalise(InnerProd,Subspace.subspace);
-
-
-    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
-    Projector.Allocate(nbasis,grid,CoarseGrid);
-    Projector.ImportBasis(Subspace.subspace);
-    
-    const int npoint = geom_srhs.npoint;
-
-    Coordinate clatt = CoarseGrid->GlobalDimensions();
-    int Nd = CoarseGrid->Nd();
-      /*
-       *     Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
-       *     Matrix index i is mapped to this shift via 
-       *               geom.shifts[i]
-       *
-       *     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block] 
-       *       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} > 
-       *       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
-       *       = M_{kl} A_ji^{b.b+l}
-       *
-       *     Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
-       *  
-       *     Where q_k = delta_k . (2*M_PI/global_nb[mu])
-       *
-       *     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
-       */
-    Eigen::MatrixXcd Mkl    = Eigen::MatrixXcd::Zero(npoint,npoint);
-    Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
-    ComplexD ci(0.0,1.0);
-    for(int k=0;k<npoint;k++){ // Loop over momenta
-
-      for(int l=0;l<npoint;l++){ // Loop over nbr relative
-	ComplexD phase(0.0,0.0);
-	for(int mu=0;mu<Nd;mu++){
-	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
-	}
-	phase=exp(phase*ci);
-	Mkl(k,l) = phase;
-      }
-    }
-    invMkl = Mkl.inverse();
-
-    ///////////////////////////////////////////////////////////////////////
-    // Now compute the matrix elements of linop between the orthonormal
-    // set of vectors.
-    ///////////////////////////////////////////////////////////////////////
-    FineField phaV(grid); // Phased block basis vector
-    FineField MphaV(grid);// Matrix applied
-    std::vector<FineComplexField> phaF(npoint,grid);
-    std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
-    
-    CoarseVector coarseInner(CoarseGrid);
-    
-    tphase=-usecond();
-    typedef typename CComplex::scalar_type SComplex;
-    FineComplexField one(grid); one=SComplex(1.0);
-    FineComplexField zz(grid); zz = Zero();
-    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
-      /////////////////////////////////////////////////////
-      // Stick a phase on every block
-      /////////////////////////////////////////////////////
-      CoarseComplexField coor(CoarseGrid);
-      pha[p]=Zero();
-      for(int mu=0;mu<Nd;mu++){
-	LatticeCoordinate(coor,mu);
-	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
-      }
-      pha[p]  =exp(pha[p]*ci);	
-
-      blockZAXPY(phaF[p],pha[p],one,zz);
-    }
-    tphase+=usecond();
-
-    // Could save on temporary storage here
-    std::vector<CoarseMatrix> _A;
-    _A.resize(geom_srhs.npoint,CoarseGrid);
-
-    // Count use small chunks than npoint == 81 and save memory
-    int batch = 9;
-    std::vector<FineField>    _MphaV(batch,grid);
-    std::vector<CoarseVector> TmpProj(batch,CoarseGrid);
-
-    std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
-    CoarseVector          FT(CoarseGrid);
-    for(int i=0;i<nbasis;i++){// Loop over basis vectors
-      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
-
-      //      std::cout << GridLogMessage << " phasing the fine vector "<<std::endl;
-      // Fixme : do this in batches
-      for(int p=0;p<npoint;p+=batch){ // Loop over momenta in npoint
-
-	for(int b=0;b<MIN(batch,npoint-p);b++){
-	  tphaseBZ-=usecond();
-	  phaV = phaF[p+b]*Subspace.subspace[i];
-	  tphaseBZ+=usecond();
-
-	  /////////////////////////////////////////////////////////////////////
-	  // Multiple phased subspace vector by matrix and project to subspace
-	  // Remove local bulk phase to leave relative phases
-	  /////////////////////////////////////////////////////////////////////
-	  // Memory footprint was an issue
-	  tmat-=usecond();
-	  linop.Op(phaV,MphaV);
-	  _MphaV[b] = MphaV;
-	  tmat+=usecond();
-	}      
-
-	//	std::cout << GridLogMessage << " Calling block project "<<std::endl;
-	tproj-=usecond();
-	Projector.blockProject(_MphaV,TmpProj);
-	tproj+=usecond();
-	
-	//	std::cout << GridLogMessage << " conj phasing the coarse vectors "<<std::endl;
-	for(int b=0;b<MIN(batch,npoint-p);b++){
-	  ComputeProj[p+b] = conjugate(pha[p+b])*TmpProj[b];
-	}
-      }
-
-      // Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
-      
-      // std::cout << GridLogMessage << " Starting FT inv "<<std::endl;
-      tinv-=usecond();
-      for(int k=0;k<npoint;k++){
-	FT = Zero();
-	// 81 kernel calls as many ComputeProj vectors
-	// Could fuse with a vector of views, but ugly
-	// Could unroll the expression and run fewer kernels -- much more attractive
-	// Could also do non blocking.
-#if 0	
-	for(int l=0;l<npoint;l++){
-	  FT= FT+ invMkl(l,k)*ComputeProj[l];
-	}
-#else
-	const int radix = 9;
-	int ll;
-	for(ll=0;ll+radix-1<npoint;ll+=radix){
-	  // When ll = npoint-radix, ll+radix-1 = npoint-1, and we do it all.
-	  FT = FT 
-	    + invMkl(ll+0,k)*ComputeProj[ll+0]
-	    + invMkl(ll+1,k)*ComputeProj[ll+1]
-	    + invMkl(ll+2,k)*ComputeProj[ll+2]
-	    + invMkl(ll+3,k)*ComputeProj[ll+3]
-	    + invMkl(ll+4,k)*ComputeProj[ll+4]
-	    + invMkl(ll+5,k)*ComputeProj[ll+5]
-	    + invMkl(ll+6,k)*ComputeProj[ll+6]
-	    + invMkl(ll+7,k)*ComputeProj[ll+7]
-	    + invMkl(ll+8,k)*ComputeProj[ll+8];
-	}
-	for(int l=ll;l<npoint;l++){
-	  FT= FT+ invMkl(l,k)*ComputeProj[l];
-	}
-#endif
-      
-	// 1 kernel call -- must be cheaper
-	int osites=CoarseGrid->oSites();
-	autoView( A_v  , _A[k], AcceleratorWrite);
-	autoView( FT_v  , FT, AcceleratorRead);
-	accelerator_for(sss, osites, 1, {
-	    for(int j=0;j<nbasis;j++){
-	      A_v[sss](i,j) = FT_v[sss](j);
-	    }
-        });
-      }
-      tinv+=usecond();
-    }
-
-    // Only needed if nonhermitian
-    //    if ( ! hermitian ) {
-    //      std::cout << GridLogMessage<<"PopulateAdag  "<<std::endl;
-    //      PopulateAdag();
-    //    }
-    // Need to write something to populate Adag from A
-    //    std::cout << GridLogMessage << " Calling GridtoBLAS "<<std::endl;
-    for(int p=0;p<geom_srhs.npoint;p++){
-      GridtoBLAS(_A[p],BLAS_A[p]);
-    }
-    std::cout << GridLogMessage<<"CoarsenOperator phase  "<<tphase<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator mat    "<<tmat <<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator proj   "<<tproj<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"CoarsenOperator inv    "<<tinv<<" us"<<std::endl;
-#endif
-  }
-  void Mdag(const CoarseVector &in, CoarseVector &out)
-  {
-    this->M(in,out);
-  }
-  void M (const CoarseVector &in, CoarseVector &out)
-  {
-    //    std::cout << GridLogMessage << "New Mrhs coarse"<<std::endl;
-    conformable(CoarseGrid(),in.Grid());
-    conformable(in.Grid(),out.Grid());
-    out.Checkerboard() = in.Checkerboard();
-
-    RealD t_tot;
-    RealD t_exch;
-    RealD t_GtoB;
-    RealD t_BtoG;
-    RealD t_mult;
-
-    t_tot=-usecond();
-    CoarseVector tin=in;
-    t_exch=-usecond();
-    CoarseVector pin = Cell.ExchangePeriodic(tin); //padded input
-    t_exch+=usecond();
-
-    CoarseVector pout(pin.Grid());
-
-    int npoint = geom.npoint;
-    typedef calcMatrix* Aview;
-    typedef LatticeView<Cvec> Vview;
-      
-    const int Nsimd = CComplex::Nsimd();
-
-    int64_t nrhs  =pin.Grid()->GlobalDimensions()[0];
-    GRID_ASSERT(nrhs>=1);
-
-    RealD flops,bytes;
-    int64_t osites=in.Grid()->oSites(); // unpadded
-    int64_t unpadded_vol = CoarseGrid()->lSites()/nrhs;
-    
-    flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
-    bytes = 1.0*osites*sizeof(siteMatrix)*npoint/pin.Grid()->GlobalDimensions()[0]
-          + 2.0*osites*sizeof(siteVector)*npoint;
-    
-
-    t_GtoB=-usecond();
-    GridtoBLAS(pin,BLAS_B);
-    t_GtoB+=usecond();
-
-    GridBLAS BLAS;
-
-    t_mult=-usecond();
-    for(int p=0;p<geom.npoint;p++){
-      RealD c = 1.0;
-      if (p==0) c = 0.0;
-      ComplexD beta(c);
-
-      BLAS.gemmBatched(nbasis,nrhs,nbasis,
-		       ComplexD(1.0),
-		       BLAS_AP[p], 
-		       BLAS_BP[p], 
-		       ComplexD(c), 
-		       BLAS_CP);
-    }
-    BLAS.synchronise();
-    t_mult+=usecond();
-
-    t_BtoG=-usecond();
-    BLAStoGrid(out,BLAS_C);
-    t_BtoG+=usecond();
-    t_tot+=usecond();
-    /*
-    std::cout << GridLogMessage << "New Mrhs coarse DONE "<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult exch "<<t_exch<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult mult "<<t_mult<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult GtoB  "<<t_GtoB<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult BtoG  "<<t_BtoG<<" us"<<std::endl;
-    std::cout << GridLogMessage<<"Coarse Mult tot  "<<t_tot<<" us"<<std::endl;
-    */
-    //    std::cout << GridLogMessage<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse Kernel flops "<< flops<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse Kernel flop/s "<< flops/t_mult<<" mflop/s"<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse Kernel bytes/s "<< bytes/t_mult/1000<<" GB/s"<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse overall flops/s "<< flops/t_tot<<" mflop/s"<<std::endl;
-    //    std::cout << GridLogMessage<<"Coarse total bytes   "<< bytes/1e6<<" MB"<<std::endl;
-  };
-  virtual  void Mdiag    (const Field &in, Field &out){ GRID_ASSERT(0);};
-  virtual  void Mdir     (const Field &in, Field &out,int dir, int disp){assert(0);};
-  virtual  void MdirAll  (const Field &in, std::vector<Field> &out){assert(0);};
-};
-  
-NAMESPACE_END(Grid);

Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHSV2.h

@@ -37,7 +37,9 @@ template<class Fobj,class CComplex,int nbasis>
 class MultiGeneralCoarsenedOperatorV2 : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
 public:
   typedef typename CComplex::scalar_object SComplex;
-  typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
+  // The BLAS scalar of this level follows the coefficient precision:
+  // ComplexF for an fp32 coarse space, ComplexD for fp64.
+  typedef typename GridTypeMapper<CComplex>::scalar_type CoarseBLASScalar;
   typedef MultiGeneralCoarsenedOperatorV2<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
 
   typedef iVector<CComplex,nbasis >           siteVector;
@@ -82,9 +84,80 @@ public:
   deviceVector<calcVector> BLAS_C;
   std::vector<deviceVector<calcMatrix> > BLAS_A;
 
-  std::vector<deviceVector<ComplexD *> > BLAS_AP;
+  std::vector<deviceVector<CoarseBLASScalar *> > BLAS_AP;
-  std::vector<deviceVector<ComplexD *> > BLAS_BP;
+  std::vector<deviceVector<CoarseBLASScalar *> > BLAS_BP;
-  deviceVector<ComplexD *>               BLAS_CP;
+  deviceVector<CoarseBLASScalar *>               BLAS_CP;
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Stencil legs carried in the BATCH dimension, LegGroup at a time.
+  //
+  // One call per stencil point batches over the local coarse volume alone,
+  // which at the production point is 1024 sites.  That is too few for the
+  // single-precision kernel: it then runs at half the bandwidth the double
+  // one reaches and takes the same time, so an fp32 coarse space buys
+  // nothing in the operator.  The same shape at four times the batch runs
+  // 1.7x faster in fp32 than fp64, so the fix is more batch, not fewer
+  // bytes.  Grouping G legs multiplies the batch by G and divides the launch
+  // count by G, at the price of G partial outputs summed at the end.
+  //
+  // G must divide npoint; 3 divides both the 33 point (next-to-nearest) and
+  // 81 point (full box) stencils.  G=1 is the ungrouped form.  The only
+  // numerical difference is the ORDER of the sum over stencil points.
+  ///////////////////////////////////////////////////////////////////////////
+  // The group need not divide npoint: the last call carries the remainder,
+  // with its own smaller tables.  The batch a call presents is LegGroup times
+  // the local coarse volume, so the group that saturates the kernel depends
+  // on the decomposition, and the default is chosen from it rather than
+  // fixed.  TargetBatch is where the measured single-precision curve flattens
+  // (530 GB/s at 1024, 695 at 9216, 728 at 33792 for 60x12x60 on MI250X).
+  //
+  // 9216 is nine times the production local coarse volume, so the default
+  // lands on nine legs per call: 81 = 9x9 for the full-box stencil and
+  // 33 = 9+9+9+6 for the next-to-nearest one, which is the grouping the
+  // earlier HDCG coarse operator used.
+  static const int TargetBatch = 9216;
+
+  int LegGroup = 1;
+  std::vector<deviceVector<CoarseBLASScalar *> > BLAS_APg; // per group
+  std::vector<deviceVector<CoarseBLASScalar *> > BLAS_BPg;
+  deviceVector<CoarseBLASScalar *>               BLAS_CPg; // LegGroup*sites
+  deviceVector<CoarseBLASScalar *>               BLAS_CPr; // remainder*sites
+  deviceVector<calcVector>                       BLAS_Cg;  // LegGroup partials
+
+  int LegGroups(void)    const { return geom.npoint/LegGroup; }          // full
+  int LegRemainder(void) const { return geom.npoint%LegGroup; }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Matrix pointer table for the grouped call.  Group g holds legs
+  // g*LegGroup .. and the batch index runs site fastest within a leg.  The
+  // last group is short when LegGroup does not divide npoint.
+  ///////////////////////////////////////////////////////////////////////////
+  void BuildGroupedA(void)
+  {
+    int32_t sites = _CoarseGrid->lSites();
+    int ng = LegGroups() + (LegRemainder()?1:0);
+    BLAS_APg.resize(ng);
+    for(int g=0;g<ng;g++){
+      int legs = (g<LegGroups()) ? LegGroup : LegRemainder();
+      BLAS_APg[g].resize(legs*sites);
+      for(int l=0;l<legs;l++){
+	int p = g*LegGroup+l;
+	for(int32_t ss=0;ss<sites;ss++){
+	  CoarseBLASScalar *ptr = (CoarseBLASScalar *)&BLAS_A[p][ss];
+	  acceleratorPut(BLAS_APg[g][l*sites+ss],ptr);
+	}
+      }
+    }
+  }
+
+  void SetLegGroup(int G)
+  {
+    GRID_ASSERT( G>=1 );
+    GRID_ASSERT( G<=geom.npoint );   // every partial must get an initialising call
+    LegGroup = G;
+    BuildGroupedA();
+    if ( _CoarseGridMulti ) SetGrid(_CoarseGridMulti);   // rebuild the rest
+  }
 
   ///////////////////////
   // Interface
@@ -115,6 +188,15 @@ public:
   // D+1 multiRHS grid here, so a consumer wanting the space the elements live
   // on must ask for CoarseGridD().
   //////////////////////////////////////////////////////////////////////////
+  // Resident device memory this operator holds (matrix elements and the
+  // Nrhs-dependent BLAS buffers).  Not evictable.
+  uint64_t DeviceBytes(void)
+  {
+    uint64_t b = (uint64_t)(BLAS_B.capacity()+BLAS_C.capacity()+BLAS_Cg.capacity())*sizeof(calcVector);
+    for(int p=0;p<(int)BLAS_A.size();p++) b += (uint64_t)BLAS_A[p].capacity()*sizeof(calcMatrix);
+    return b;
+  }
+
   NonLocalStencilGeometry & Geometry(void)      { return geom_srhs; };
   void ExtractMatrix(int p,CoarseMatrix &A)     { BLAStoGrid(A,BLAS_A[p]); };
 
@@ -130,30 +212,7 @@ public:
     GRID_ASSERT(A.size()==geom_srhs.npoint);
     BLAStoGrid(A[p],BLAS_A[p]);
   }
-  void CopyMatrix (GeneralCoarseOp &_Op)
+
-  {
-    for(int p=0;p<geom.npoint;p++){
-      auto Aup = _Op.Cell.Extract(_Op._A[p]);
-      //Unpadded
-      GridtoBLAS(Aup,BLAS_A[p]);
-    }
-  }
-  /*
-  void CheckMatrix (GeneralCoarseOp &_Op)
-  {
-    std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
-    for(int p=0;p<geom.npoint;p++){
-      //Unpadded
-      auto Aup = _Op.Cell.Extract(_Op._A[p]);
-      auto Ack = Aup;
-      BLAStoGrid(Ack,BLAS_A[p]);
-      std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
-      std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
-    }
-    std::cout <<"************* "<<std::endl;
-  }
-  */
-  
   ///////////////////////////////////////////////////////////////////////////
   // Constructor takes the D dimensional coarse grid. Everything built here
   // is independent of Nrhs, in particular the matrix elements, which must
@@ -184,10 +243,24 @@ public:
     // Site identity mapping for A
     for(int p=0;p<geom.npoint;p++){
       for(int ss=0;ss<unpadded_sites;ss++){
-	ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
+	CoarseBLASScalar *ptr = (CoarseBLASScalar *)&BLAS_A[p][ss];
 	acceleratorPut(BLAS_AP[p][ss],ptr);
       }
     }
+
+    // Enough legs to bring the batch up to where the kernel saturates, and
+    // never more legs than the stencil has.
+    LegGroup = (TargetBatch + unpadded_sites - 1)/unpadded_sites;
+    if ( LegGroup < 1 )            LegGroup = 1;
+    if ( LegGroup > geom.npoint )  LegGroup = geom.npoint;
+    BuildGroupedA();
+    std::cout << GridLogMessage << "MultiGeneralCoarsenedOperatorV2: stencil "
+	      << geom.npoint << " points, local coarse volume " << unpadded_sites
+	      << ", legs per GEMM " << LegGroup
+	      << " (batch " << LegGroup*unpadded_sites << ", "
+	      << LegGroups() << " full call(s)"
+	      << (LegRemainder() ? " + a remainder of "+std::to_string(LegRemainder()) : "")
+	      << ")" << std::endl;
   }
 
   virtual ~MultiGeneralCoarsenedOperatorV2()
@@ -209,6 +282,11 @@ public:
 
     BLAS_B.resize(0);
     BLAS_C.resize(0);
+    BLAS_Cg.resize(0);
+    BLAS_CPg.resize(0);
+    BLAS_CPr.resize(0);
+    for(int g=0;g<(int)BLAS_BPg.size();g++){ BLAS_BPg[g].resize(0); }
+    BLAS_BPg.resize(0);
     for(int p=0;p<BLAS_BP.size();p++){
       BLAS_BP[p].resize(0);
     }
@@ -277,10 +355,33 @@ public:
 
     // Site identity mapping for C
     for(int ss=0;ss<unpadded_sites;ss++){
-      ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
+      CoarseBLASScalar *ptr = (CoarseBLASScalar *)&BLAS_C[ss*nrhs];
       acceleratorPut(BLAS_CP[ss],ptr);
     }
 
+    // Grouped output: LegGroup partial results, each the full C volume, and
+    // their pointer table.  The batch index runs site fastest within a leg,
+    // matching the grouped A and B tables.  The remainder call writes into
+    // the first few partials, so it needs a truncated copy of the table.
+    int nfull = LegGroups();
+    int rem   = LegRemainder();
+    int ng    = nfull + (rem?1:0);
+    BLAS_Cg.resize (LegGroup*nrhs*unpadded_sites);
+    BLAS_CPg.resize(LegGroup*unpadded_sites);
+    BLAS_CPr.resize(rem*unpadded_sites);
+    for(int l=0;l<LegGroup;l++){
+      for(int ss=0;ss<unpadded_sites;ss++){
+	CoarseBLASScalar *ptr = (CoarseBLASScalar *)&BLAS_Cg[(l*unpadded_sites+ss)*nrhs];
+	acceleratorPut(BLAS_CPg[l*unpadded_sites+ss],ptr);
+	if ( l<rem ) acceleratorPut(BLAS_CPr[l*unpadded_sites+ss],ptr);
+      }
+    }
+    BLAS_BPg.resize(ng);
+    for(int g=0;g<ng;g++){
+      int legs = (g<nfull) ? LegGroup : rem;
+      BLAS_BPg[g].resize(legs*unpadded_sites);
+    }
+
     // Neighbour table is more complicated
     int32_t j=0; // Interior point counter (unpadded)
     for(int32_t s=0;s<padded_sites;s++){ // D volume, padded
@@ -298,8 +399,9 @@ public:
  	  int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite, D dim
 	  nbr = nbr*nrhs;                                              // D -> D+1, rhs innermost
 	  GRID_ASSERT(nbr<BLAS_B.size());
-	  ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
+	  CoarseBLASScalar * ptr = (CoarseBLASScalar *)&BLAS_B[nbr];
 	  acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
+	  acceleratorPut(BLAS_BPg[point/LegGroup][(point%LegGroup)*unpadded_sites+j],ptr);
 	}
 	j++;
       }
@@ -411,11 +513,43 @@ public:
   //     Where q_k = delta_k . (2*M_PI/global_nb[mu])
   //     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
   ///////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////////////////////////////////////////////////
+  // Probe momenta.  The stencil DISPLACEMENTS are the +-1 box; the momenta
+  // used to separate them are free, and are chosen here to span the
+  // Brillouin zone: k_mu = K_mu * shift_mu with K_mu ~ L_mu/3, so a phase
+  // is ~2pi/3 per unit displacement rather than 2pi/L_mu.
+  //
+  // This is what conditions the extraction.  With K_mu = 1 every entry of
+  // the phase matrix tends to 1 as the coarse lattice grows, the matrix
+  // tends to rank one, and its inverse amplifies any error in the measured
+  // projections: at 24.24.16.32 the condition number is 2.7e5, so fp32
+  // projections give coarse matrix elements wrong by ~1e-3.  Spread momenta
+  // bring it to ~20, independent of the lattice size.
+  //
+  // K_mu is stepped away from L_mu/2, where +k and -k alias into the same
+  // momentum and the matrix is singular.
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenMomenta(GridBase *CoarseGrid,Coordinate &K)
+  {
+    Coordinate clatt = CoarseGrid->GlobalDimensions();
+    int Nd = CoarseGrid->Nd();
+    K.resize(Nd);
+    for(int mu=0;mu<Nd;mu++){
+      int L = clatt[mu];
+      int k = (L>=3) ? (int)std::lround(L/3.0) : 1;
+      if ( k < 1 ) k = 1;
+      if ( (L>2) && ((2*k)%L == 0) ) k = k-1;     // +k and -k must differ
+      if ( k < 1 ) k = 1;
+      K[mu] = k;
+    }
+  }
+
   void CoarsenFourierMatrix(GridBase *CoarseGrid,Eigen::MatrixXcd &invMkl)
   {
     const int npoint = geom_srhs.npoint;
     Coordinate clatt = CoarseGrid->GlobalDimensions();
     int Nd = CoarseGrid->Nd();
+    Coordinate K;  CoarsenMomenta(CoarseGrid,K);
 
     Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
     ComplexD ci(0.0,1.0);
@@ -424,13 +558,21 @@ public:
 	ComplexD phase(0.0,0.0);
 	for(int mu=0;mu<Nd;mu++){
 	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
-	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
+	  phase=phase+TwoPiL*K[mu]*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
 	}
 	phase=exp(phase*ci);
 	Mkl(k,l) = phase;
       }
     }
     invMkl = Mkl.inverse();
+
+    // The extraction's error amplification, printed because it is the thing
+    // that decides whether an fp32 coarse sector is usable.
+    Eigen::JacobiSVD<Eigen::MatrixXcd> svd(Mkl);
+    RealD cond = svd.singularValues()(0)/svd.singularValues()(npoint-1);
+    std::cout << GridLogMessage << "CoarsenOperator: probe momenta "<<K
+	      <<" on coarse lattice "<<clatt
+	      <<" ; Fourier matrix condition number "<<cond<<std::endl;
   }
 
   ///////////////////////////////////////////////////////////////////////////
@@ -477,10 +619,13 @@ public:
     const int npoint = geom_srhs.npoint;
     Coordinate clatt = CoarseGrid->GlobalDimensions();
     int Nd = CoarseGrid->Nd();
+    Coordinate K;  CoarsenMomenta(CoarseGrid,K);
     ComplexD ci(0.0,1.0);
 
-    typedef typename CComplex::scalar_type SComplex;
+    // The fine-side scratch carries the FINE level's precision, which need
-    FineComplexField one(grid); one=SComplex(1.0);
+    // not be the coarse one (fp64 fine, fp32 coarse at L1).
+    typedef typename GridTypeMapper<FineInner>::scalar_type FineScalar;
+    FineComplexField one(grid); one=FineScalar(1.0);
     FineComplexField zz(grid);  zz = Zero();
     BlockComplexField pha_blk (BlockGrid);
     BlockComplexField blk_coor(BlockGrid);
@@ -492,9 +637,9 @@ public:
       for(int mu=0;mu<Nd;mu++){
 	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
 	LatticeCoordinate(coor,mu);
-	pha[p]  = pha[p]  + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
+	pha[p]  = pha[p]  + (TwoPiL * K[mu] * geom_srhs.shifts[p][mu]) * coor;
 	LatticeCoordinate(blk_coor,mu);
-	pha_blk = pha_blk + (TwoPiL * geom_srhs.shifts[p][mu]) * blk_coor;
+	pha_blk = pha_blk + (TwoPiL * K[mu] * geom_srhs.shifts[p][mu]) * blk_coor;
       }
       pha[p] =exp(pha[p] *ci);
       pha_blk=exp(pha_blk*ci);
@@ -558,10 +703,25 @@ public:
   // ExtractSlice/InsertSlice pair per rhs; prefer the single RHS variant
   // below in that case.
   ///////////////////////////////////////////////////////////////////////////
+  // Owning the transfer operator: the caller passes one, this imports the
+  // orthonormalised basis into it, and the caller keeps it for the solve.
+  // The overload below makes its own and is kept for the pre-2026 drivers;
+  // it costs a second full basis store, which is why nothing new should use
+  // it (documentation/MultiGridMemoryAudit.md).
   void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
 		       GridCartesian *FineGridMulti,
 		       std::vector<FineField> &Subspace,
 		       GridBase *CoarseGrid)
+  {
+    MultiRHSBlockProject<Lattice<Fobj> > Projector;
+    CoarsenOperator(linop,FineGridMulti,Subspace,CoarseGrid,Projector);
+  }
+  template<class Projector_t>
+  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
+		       GridCartesian *FineGridMulti,
+		       std::vector<FineField> &Subspace,
+		       GridBase *CoarseGrid,
+		       Projector_t &Projector)
   {
     RealD tproj=0.0, tmat=0.0, tphase=0.0, tphaseBZ=0.0, tslice=0.0, tinv=0.0;
 
@@ -585,7 +745,7 @@ public:
     BlockComplexField InnerProd(&BlockGrid);
     blockOrthogonalise(InnerProd,Subspace);
 
-    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
+    // the caller owns it; import the basis we have just orthonormalised
     Projector.Allocate(nbasis,grid,CoarseGrid);
     Projector.ImportBasis(Subspace);
 
@@ -674,6 +834,16 @@ public:
 		       std::vector<FineField> &Subspace,
 		       GridBase *CoarseGrid,
 		       int batch)
+  {
+    MultiRHSBlockProject<Lattice<Fobj> > Projector;
+    CoarsenOperator(linop,Subspace,CoarseGrid,batch,Projector);
+  }
+  template<class Projector_t>
+  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
+		       std::vector<FineField> &Subspace,
+		       GridBase *CoarseGrid,
+		       int batch,
+		       Projector_t &Projector)
   {
     RealD tproj=0.0, tmat=0.0, tphase=0.0, tphaseBZ=0.0, tslice=0.0, tinv=0.0;
 
@@ -690,7 +860,7 @@ public:
     BlockComplexField InnerProd(&BlockGrid);
     blockOrthogonalise(InnerProd,Subspace);
 
-    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
+    // the caller owns it; import the basis we have just orthonormalised
     Projector.Allocate(nbasis,grid,CoarseGrid);
     Projector.ImportBasis(Subspace);
 
@@ -787,16 +957,15 @@ public:
 
     GRID_TRACE("CoarseV2Mult");
     t_tot=-usecond();
-    CoarseVector tin=in;
     t_exch=-usecond();
-    // lambda scope so the roctx range covers exactly the exchange; the
+    // The exchange takes its input by const reference, so it reads the
-    // PaddedCellFwd/BwdMPI markers inside it then nest properly.
+    // caller's field directly.  lambda scope so the roctx range covers
+    // exactly the exchange; the PaddedCellFwd/BwdMPI markers inside it then
+    // nest properly.
     CoarseVector pin = [&](){ GRID_TRACE("CoarseV2Exchange");
-                              return CellMulti->ExchangePeriodic(tin); }(); //padded input
+                              return CellMulti->ExchangePeriodic(in); }(); //padded input
     t_exch+=usecond();
 
-    CoarseVector pout(pin.Grid());
-
     int npoint = geom.npoint;
     typedef calcMatrix* Aview;
     typedef LatticeView<Cvec> Vview;
@@ -825,22 +994,64 @@ public:
 
     t_mult=-usecond();
     { GRID_TRACE("CoarseV2StencilGEMM");
-    for(int p=0;p<geom.npoint;p++){
+    // The scalar type selects the GEMM: Cgemm for an fp32 coarse space,
-      RealD c = 1.0;
+    // Zgemm for fp64.
-      if (p==0) c = 0.0;
+    if ( LegGroup == 1 ) {
-      ComplexD beta(c);
+      for(int p=0;p<geom.npoint;p++){
-
+	RealD c = (p==0) ? 0.0 : 1.0;
-      BLAS.gemmBatched(nbasis,nrhs,nbasis,
+	BLAS.gemmBatched(nbasis,nrhs,nbasis,
-		       ComplexD(1.0),
+			 CoarseBLASScalar(1.0),
-		       BLAS_AP[p], 
+			 BLAS_AP[p],
-		       BLAS_BP[p], 
+			 BLAS_BP[p],
-		       ComplexD(c), 
+			 CoarseBLASScalar(c),
-		       BLAS_CP);
+			 BLAS_CP);
+      }
+    } else {
+      // LegGroup legs per call: the batch is LegGroup times the local volume
+      // and there are LegGroup partial outputs, accumulated across the
+      // groups and summed below.  A final short call carries the remainder
+      // when the group does not divide the stencil; it writes into the first
+      // few partials, which the full calls have already initialised.
+      int nfull = LegGroups();
+      int rem   = LegRemainder();
+      for(int g=0;g<nfull;g++){
+	RealD c = (g==0) ? 0.0 : 1.0;
+	BLAS.gemmBatched(nbasis,nrhs,nbasis,
+			 CoarseBLASScalar(1.0),
+			 BLAS_APg[g],
+			 BLAS_BPg[g],
+			 CoarseBLASScalar(c),
+			 BLAS_CPg);
+      }
+      if ( rem ) {
+	RealD c = (nfull==0) ? 0.0 : 1.0;
+	BLAS.gemmBatched(nbasis,nrhs,nbasis,
+			 CoarseBLASScalar(1.0),
+			 BLAS_APg[nfull],
+			 BLAS_BPg[nfull],
+			 CoarseBLASScalar(c),
+			 BLAS_CPr);
+      }
     }
     BLAS.synchronise();
     }
     t_mult+=usecond();
 
+    if ( LegGroup > 1 ) { GRID_TRACE("CoarseV2LegSum");
+      // Sum the LegGroup partial results into the single output buffer.
+      // Flat over scalars: a calcVector is nbasis of them and the partials
+      // are contiguous, one whole C volume after another.
+      int64_t nscalar = (int64_t)BLAS_C.size()*nbasis;   // one whole C volume
+      const int G = LegGroup;
+      CoarseBLASScalar *dst = (CoarseBLASScalar *)&BLAS_C[0];
+      CoarseBLASScalar *src = (CoarseBLASScalar *)&BLAS_Cg[0];
+      accelerator_for(i,nscalar,1,{
+	  CoarseBLASScalar sum = src[i];
+	  for(int l=1;l<G;l++) sum = sum + src[(int64_t)l*nscalar + i];
+	  dst[i] = sum;
+	});
+    }
+
     t_BtoG=-usecond();
     { GRID_TRACE("CoarseV2BLASToGrid");
       BLAStoGrid(out,BLAS_C);

Grid/algorithms/multigrid/MrhsMultiGrid.h

@@ -20,20 +20,10 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #pragma once
 
 #include <Grid/algorithms/deflation/MultiRHSBlockProject.h>
+#include <Grid/algorithms/multigrid/MrhsPreconditioner.h>
 
 NAMESPACE_BEGIN(Grid);
 
-//////////////////////////////////////////////////////////////////////
-// mrhs LinearFunction interface: vector-of-fields in, vector out.
-// The outer level carries mrhs as std::vector<Field>; below it mrhs
-// is PACKED into a single D+1 field (rhs = dim 0) and the coarse
-// classes are plain LinearFunctions on that.
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
 
 //////////////////////////////////////////////////////////////////////
 // Single-polynomial mrhs PGCR: one step length and one set of
@@ -54,18 +44,21 @@ public:
   RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
   int ZeroGuess = 0; int FirstCycle = 0;
   std::string name = "Level 1";
+  // Trace range names carry the instance name, so a profile separates the
+  // solvers that otherwise nest as one shared range.
+  std::string trace_op = "MrhsPGCR::vOp", trace_orthog = "MrhsPGCR orthog";
   LinearOperatorBase<Field> &Linop;
   MrhsLinearFunction<Field> &Preconditioner;
   std::function<void(int)> OnStep;      // called with the outer step count after every step
   void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
+  void Name(std::string n){ name = n; trace_op = name+" MrhsPGCR::vOp"; trace_orthog = name+" MrhsPGCR orthog"; }
   void SetZeroGuess(int z){ ZeroGuess=z; }
   MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
     : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
   static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
   static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
   static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("MrhsPGCR::vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
+  void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE(trace_op.c_str()); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
   void operator()(std::vector<Field> &src,std::vector<Field> &psi){
     RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
     ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
@@ -108,7 +101,7 @@ public:
       vOp(p[peri_kp],q[peri_kp]);
       int northog=((kp)>(mmax-1))?(mmax-1):(kp);
       {
-        GRID_TRACE("MrhsPGCR orthog");
+        GRID_TRACE(trace_orthog.c_str());
         // Classical Gram-Schmidt: all coefficients against the UN-updated new q
         // (independent, batchable), then apply.  Complex coefficient: the
         // operator is non-Hermitian, real(<q_j,Aq>) alone left q's non-orthogonal.
@@ -203,19 +196,23 @@ public:
 // is wired by the composer to PVdagMLinearOperator::SloppyComms (a
 // no-op by default), replacing the file-scope global the example used.
 //////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
+// Projector_t defaults to a transfer operator whose STORE matches FineField,
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
+// which is the case whenever the coarse sector and the fine level share a
+// precision; pass it explicitly when they differ.
+template<class FineField, class MrhsCoarseVector, class FineSmoother,
+         class Projector_t = MultiRHSBlockProject<FineField> >
+class MrhsTwoLevelMG : public MrhsPreconditioner<FineField> {
 public:
   typedef MrhsCoarseVector CoarseVector;
   LinearOperatorBase<FineField>   &_FineOperator;
   FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
+  Projector_t                     &_Projector;   // store precision need not match FineField
   LinearFunction<CoarseVector>    &_CoarseSolve;
   GridBase *_CoarseGrid, *_CoarseGridMrhs;
   std::function<void(int)> SetSloppy = [](int){};
   int SloppyComms = 0;                 // value passed to SetSloppy on entry
   MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
+                 Projector_t &Projector, LinearFunction<CoarseVector> &CoarseSolve,
                  GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
     : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
       _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
@@ -253,4 +250,52 @@ public:
   }
 };
 
+//////////////////////////////////////////////////////////////////////
+// The fp64/fp32 seam of the solve chain.  The outer Krylov hands fp64
+// residuals to its preconditioner; this adapter converts them to fp32,
+// runs an fp32 preconditioner (the whole V-cycle), and converts the
+// correction back.  Two precisionChange per outer step per rhs, on
+// workspaces built once.  The outer operator never sees fp32.
+//
+// The fp32 scratch is allocated ON FIRST USE, not in the constructor: the
+// solver builds this seam whichever precision is selected, and at Nrhs 12
+// on a 48^3x96 Ls=24 rank the scratch is 2 GB of device memory.  A seam
+// that is never called must cost nothing.  Same idiom as the GCR history
+// vectors: re-made only if the grid or the rhs count changes.
+//////////////////////////////////////////////////////////////////////
+template<class FieldD, class FieldF>
+class MrhsMixedPrecPreconditioner : public MrhsPreconditioner<FieldD> {
+public:
+  MrhsPreconditioner<FieldF> &_Inner;
+  precisionChangeWorkspace    _ws_d2f;   // out fp32, in fp64
+  precisionChangeWorkspace    _ws_f2d;   // out fp64, in fp32
+  GridBase                   *_gridF;
+  std::vector<FieldF>         _in_f, _out_f;
+  MrhsMixedPrecPreconditioner(MrhsPreconditioner<FieldF> &Inner, GridBase *gridD, GridBase *gridF, int nrhs)
+    : _Inner(Inner), _ws_d2f(gridF,gridD), _ws_f2d(gridD,gridF), _gridF(gridF) {}
+  void Scratch(int nrhs){
+    if ( (int)_in_f.size() == nrhs ) return;
+    _in_f.clear();  _in_f.reserve(nrhs);
+    _out_f.clear(); _out_f.reserve(nrhs);
+    for(int r=0;r<nrhs;r++){ _in_f.emplace_back(_gridF); _out_f.emplace_back(_gridF); }
+  }
+  virtual void operator()(std::vector<FieldD> &in, std::vector<FieldD> &out){
+    GRID_TRACE("MGPrecisionSeam");
+    int nrhs=in.size(); Scratch(nrhs);
+    for(int r=0;r<nrhs;r++) precisionChange(_in_f[r],in[r],_ws_d2f);
+    _Inner(_in_f,_out_f);
+    for(int r=0;r<nrhs;r++) precisionChange(out[r],_out_f[r],_ws_f2d);
+  }
+  // The start and the timers are the inner preconditioner's
+  virtual void Vstart(std::vector<FieldD> &x, std::vector<FieldD> &src){
+    GRID_TRACE("MGPrecisionSeamVstart");
+    int nrhs=src.size(); Scratch(nrhs);
+    for(int r=0;r<nrhs;r++) precisionChange(_in_f[r],src[r],_ws_d2f);
+    _Inner.Vstart(_out_f,_in_f);
+    for(int r=0;r<nrhs;r++) precisionChange(x[r],_out_f[r],_ws_f2d);
+  }
+  virtual void ResetTimers(void)                       { _Inner.ResetTimers(); }
+  virtual void ReportTimers(const std::string &prefix) { _Inner.ReportTimers(prefix); }
+};
+
 NAMESPACE_END(Grid);

Grid/algorithms/multigrid/MultiGrid.h

@@ -30,13 +30,22 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #include <Grid/algorithms/multigrid/Aggregates.h>
 #include <Grid/algorithms/multigrid/Geometry.h>
 #include <Grid/algorithms/multigrid/CoarsenedMatrix.h>
-#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrix.h>
-#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHS.h>
 #include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHSV2.h>
+// DEPRECATED: the V1 coarse operators and the Aggregation-based single-RHS
+// ADEF-2.  Nothing in the library uses them; the PVdagM and HDCG chains are
+// on V2 (PVdagMMultiGrid.h, HDCGMultiGrid.h).  Kept so the pre-2026 drivers
+// in tests/debug and examples still build; removing this block is the
+// deletion gate for them.
+#include <Grid/algorithms/multigrid/deprecated/GeneralCoarsenedMatrix.h>
+#include <Grid/algorithms/multigrid/deprecated/GeneralCoarsenedMatrixMultiRHS.h>
+#include <Grid/algorithms/multigrid/deprecated/TwoLevelADEF2.h>
 #include <Grid/algorithms/multigrid/MrhsPromotedOperator.h>
 #include <Grid/algorithms/multigrid/Smoothers.h>
 #include <Grid/algorithms/multigrid/PVdagMMultiGridParams.h>
 // PVdagMOperators.h / MrhsMultiGrid.h / PVdagMMultiGrid.h /
-// DenseCoarseMatrix.h are NOT in this umbrella: consumers of the PVdagM
+// DenseCoarseMatrix.h / MultiGridIO.h / HDCGMultiGrid.h are NOT in this
-// chain include PVdagMMultiGrid.h explicitly (it pulls the dense stack
+// umbrella: consumers include PVdagMMultiGrid.h or HDCGMultiGrid.h
-// and BLAS).
+// explicitly (they pull the dense stack, BLAS, and the scidac I/O, which
+// is declared after Algorithms.h in Grid.h).  Keeping MrhsMultiGrid.h out
+// of the Algorithms.h chain also keeps its class names away from the
+// pre-2026 drivers that define their own copies.

Grid/algorithms/multigrid/PVdagMMultiGrid.h

@@ -20,6 +20,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #pragma once
 
 #include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
+#include <Grid/algorithms/multigrid/MultiGridIO.h>
 #include <Grid/algorithms/multigrid/PVdagMOperators.h>
 #include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
@@ -48,36 +49,13 @@ NAMESPACE_BEGIN(Grid);
 //////////////////////////////////////////////////////////////////////////////////////
 
 //////////////////////////////////////////////////////////////////////
-// Subspace I/O: bare-vector scidac records.  Loaded vectors are RAW --
+// Subspace I/O is in MultiGridIO.h (shared with the HDCG chain).
-// deliberately NOT re-orthogonalised: CoarsenOperator block
+// Loaded vectors are RAW -- deliberately NOT re-orthogonalised:
-// orthonormalises in place, and projecting a block-orthonormal vector
+// CoarsenOperator block orthonormalises in place, and projecting a
-// onto its own block-orthonormalised aggregation gives e_k with the
+// block-orthonormal vector onto its own block-orthonormalised
-// near-null content silently gone.  GramGuard below catches that.
+// aggregation gives e_k with the near-null content silently gone.
+// GramGuard below catches that.
 //////////////////////////////////////////////////////////////////////
-template <class Field>
-void saveSubspace(std::vector<Field> &subspace, std::string const fname){
-#ifdef HAVE_LIME
-  Grid::emptyUserRecord record;
-  Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
-  SW.open(fname);
-  for (int k = 0; k < (int)subspace.size(); k++) {
-    SW.writeScidacFieldRecord(subspace[k], record);
-  }
-  SW.close();
-#endif
-}
-template <class Field>
-void loadSubspace(std::vector<Field> &subspace, std::string const fname){
-#ifdef HAVE_LIME
-  Grid::emptyUserRecord record;
-  Grid::ScidacReader SR;
-  SR.open(fname);
-  for (int k = 0; k < (int)subspace.size(); k++) {
-    SR.readScidacFieldRecord(subspace[k], record);
-  }
-  SR.close();
-#endif
-}
 
 //////////////////////////////////////////////////////////////////////
 // ||<v|v> - I||_F over a set of coarse vectors.  Small means the raw
@@ -146,14 +124,14 @@ public:
 
     clatt.resize(4); cclatt.resize(4);
     for(int d=0;d<4;d++){
-      GRID_ASSERT( fdims[d+1] % P.Block[d] == 0 );
+      GRID_ASSERT( fdims[d+1] % P.Block1[d] == 0 );
-      clatt[d] = fdims[d+1] / P.Block[d];
+      clatt[d] = fdims[d+1] / P.Block1[d];
     }
     for(int d=0;d<4;d++){
       GRID_ASSERT( clatt[d] % P.Block2[d] == 0 );
       cclatt[d] = clatt[d] / P.Block2[d];
     }
-    std::cout << GridLogMessage << "MGCoarseGrids: Block  " << P.Block  << "  coarse lattice        " << clatt  << std::endl;
+    std::cout << GridLogMessage << "MGCoarseGrids: Block1 " << P.Block1 << "  coarse lattice        " << clatt  << std::endl;
     std::cout << GridLogMessage << "MGCoarseGrids: Block2 " << P.Block2 << "  coarse-coarse lattice " << cclatt << std::endl;
 
     Coordinate c5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
@@ -181,6 +159,41 @@ public:
   }
 };
 
+//////////////////////////////////////////////////////////////////////
+// The fp32 fine grids: same lattice and decomposition as the fp64 fine
+// grid, vComplexF SIMD layout.  For the fp32 fine level inside the
+// preconditioner (the fp32 fermion operators and the fp32 transfer
+// operator borrow these).  Owned here; declare BEFORE the coarsening.
+//////////////////////////////////////////////////////////////////////
+class MGFineGridsF {
+public:
+  GridCartesian         *UGridF;
+  GridRedBlackCartesian *UrbGridF;
+  GridCartesian         *FGridF;
+  GridRedBlackCartesian *FrbGridF;
+
+  MGFineGridsF(GridCartesian *FGrid)
+  {
+    Coordinate fdims = FGrid->FullDimensions();      // {Ls, x,y,z,t}
+    Coordinate fmpi  = FGrid->_processors;
+    GRID_ASSERT( fdims.size() == 5 );
+    int Ls = fdims[0];
+    Coordinate latt4({fdims[1],fdims[2],fdims[3],fdims[4]});
+    Coordinate mpi4 ({fmpi[1], fmpi[2], fmpi[3], fmpi[4]});
+    UGridF   = SpaceTimeGrid::makeFourDimGrid(latt4,GridDefaultSimd(Nd,vComplexF::Nsimd()),mpi4);
+    UrbGridF = SpaceTimeGrid::makeFourDimRedBlackGrid(UGridF);
+    FGridF   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGridF);
+    FrbGridF = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGridF);
+  }
+  ~MGFineGridsF()
+  {
+    delete FrbGridF;
+    delete FGridF;
+    delete UrbGridF;
+    delete UGridF;
+  }
+};
+
 //////////////////////////////////////////////////////////////////////
 // ALL the coarsening information for one gauge configuration.
 //
@@ -202,19 +215,41 @@ public:
   typedef MultiGeneralCoarsenedOperatorV2<Fobj,CComplex,nbasis>         CoarseOperator;
   typedef typename CoarseOperator::CoarseVector                         CoarseVector;
   typedef typename CoarseVector::vector_object                          CoarseSiteObj;
-  typedef iScalar<CComplex>                                             CComplex2;  // coarsening deepens the nest by one iScalar
+  // Every coarse level carries the same site type iVector<CComplex,nbasis>:
-  typedef MultiGeneralCoarsenedOperatorV2<CoarseSiteObj,CComplex2,nbasis> CoarseCoarseOperator;
+  // the coefficient scalar does not deepen with the level (the operator keeps
+  // its own deeper scratch type for the block inner products).  Levels are
+  // told apart by their grids, not their C++ types.
+  typedef MultiGeneralCoarsenedOperatorV2<CoarseSiteObj,CComplex,nbasis> CoarseCoarseOperator;
   typedef typename CoarseCoarseOperator::CoarseVector                   CoarseCoarseVector;
-  typedef DenseCoarseMatrix<CComplex2,nbasis>                           DenseBottom;
+  typedef DenseCoarseMatrix<CComplex,nbasis>                            DenseBottom;
+  // The fp32 fine field, derived from the fp64 one: the fp32 fine level of
+  // the preconditioner projects through the SAME basis in fp32 layout.
+  typedef typename GridTypeMapper<Fobj>::SinglePrecision                FobjF;
+  typedef Lattice<FobjF>                                                FineFieldF;
+
+  //////////////////////////////////////////////////////////////////////
+  // ONE fine transfer operator.  Its STORE follows the coarse sector --
+  // the coarse space is what it feeds -- while its import and export
+  // accept EITHER fine precision, because they are already a layout
+  // transformation and a scalar conversion inside one costs nothing.
+  // So an fp64 outer level and an fp32 V-cycle share this single object,
+  // and no second basis store, handover or retained basis is needed.
+  //////////////////////////////////////////////////////////////////////
+  static const bool CoarseIsSingle =
+    ( sizeof(typename GridTypeMapper<CComplex>::scalar_type) == sizeof(ComplexF) );
+  typedef typename std::conditional<CoarseIsSingle,FineFieldF,FineField>::type ProjectorField;
+  typedef MultiRHSBlockProject<ProjectorField>                          ProjectorL1_t;
+  typedef MultiRHSBlockProject<CoarseVector>                            ProjectorL2_t;
 
   MGCoarseGrids                     &Grids;      // borrowed
+  MGFineGridsF                      &GridsF;     // borrowed
   MGSetupParams                      Params;
   NextToNearestStencilGeometry5D     geom;
   NextToNearestStencilGeometry5D     geom2;
   CoarseOperator                     CoarseOpPV;
   CoarseCoarseOperator               CoarseOpL2;
-  MultiRHSBlockProject<FineField>    MrhsProjector;
+  ProjectorL1_t                      MrhsProjectorL1;  // store follows the coarse sector
-  MultiRHSBlockProject<CoarseVector> MrhsProjectorL2;
+  ProjectorL2_t                      MrhsProjectorL2;
   DenseBottom                       *DenseCC;
   std::vector<FineField>             rawNull;    // RAW fine near-null basis
   std::vector<CoarseVector>          rawPsi;     // RAW coarse near-null basis
@@ -222,8 +257,9 @@ public:
   GridCartesian                     *CCMrhs;
   int                                nrhs;
 
-  PVdagMMultiGridCoarsening(MGCoarseGrids &_Grids, const MGSetupParams &P)
+  PVdagMMultiGridCoarsening(MGCoarseGrids &_Grids, MGFineGridsF &_GridsF, const MGSetupParams &P)
     : Grids(_Grids),
+      GridsF(_GridsF),
       Params(P),
       geom (_Grids.Coarse5d),
       geom2(_Grids.CoarseCoarse5d),
@@ -289,7 +325,7 @@ public:
   // halos follow the FineSloppyComms policy, restored EXACT on exit).
   ////////////////////////////////////////////////////////////////////
   template<class PVdagMOp>
-  void Coarsen(PVdagMOp &FineOp)
+  void Coarsen(PVdagMOp &FineOp, int retain_basis=0)
   {
     GRID_ASSERT( rawNull.size() == nbasis );
 
@@ -301,18 +337,22 @@ public:
     std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L1 CoarsenOperator, batch "
               << Grids.batch << std::endl;
 
+    // CoarsenOperator orthonormalises sub in place and imports it into OUR
+    // transfer operator, which it then uses.  One basis store, imported once.
+    // The projector's grid carries the STORE's SIMD layout, which is the fp32
+    // fine grid when the coarse sector is fp32; fp64 vectors convert on import.
+    MrhsProjectorL1.Allocate(nbasis,
+                             CoarseIsSingle ? (GridBase *)GridsF.FGridF : (GridBase *)Grids.FGrid,
+                             Grids.Coarse5d);
     FineOp.SloppyComms(Params.FineSloppyComms);
-    CoarseOpPV.CoarsenOperator(FineOp,sub,Grids.Coarse5d,Grids.batch);
+    CoarseOpPV.CoarsenOperator(FineOp,sub,Grids.Coarse5d,Grids.batch,MrhsProjectorL1);
     FineOp.SloppyComms(0);
-
+    // The Lattice copy of the basis has served its purpose: the transfer
-    // Transfer operators from the orthonormalised basis; then the
+    // operator holds it in BLAS layout from here on.
-    // Galerkin images of the RAW basis define the L2 null space.
-    MrhsProjector.Allocate(nbasis,Grids.FGrid,Grids.Coarse5d);
-    MrhsProjector.ImportBasis(sub);
     sub.clear(); sub.shrink_to_fit();
 
     std::vector<CoarseVector> psi(nbasis,Grids.Coarse5d);
-    MrhsProjector.blockProject(rawNull,psi);
+    MrhsProjectorL1.blockProject(rawNull,psi);
     GramGuard("psi_coarse",psi,Grids.Coarse5d);
 
     rawPsi.clear();
@@ -324,16 +364,25 @@ public:
     NonHermitianLinearOperator<CoarseOperator,CoarseVector> LinOpCoarse(CoarseOpPV);
     std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L2 CoarsenOperator, batch "
               << Grids.batch << std::endl;
-    CoarseOpL2.CoarsenOperator(LinOpCoarse,Grids.CoarseBatch,psi,Grids.CoarseCoarse5d);
-
     MrhsProjectorL2.Allocate(nbasis,Grids.Coarse5d,Grids.CoarseCoarse5d);
-    MrhsProjectorL2.ImportBasis(psi);              // now block orthonormal
+    CoarseOpL2.CoarsenOperator(LinOpCoarse,Grids.CoarseBatch,psi,Grids.CoarseCoarse5d,MrhsProjectorL2);
     {
       std::vector<CoarseCoarseVector> psi_cc(nbasis,Grids.CoarseCoarse5d);
       MrhsProjectorL2.blockProject(rawPsi,psi_cc); // RAW vectors in
       GramGuard("psi_cc",psi_cc,Grids.CoarseCoarse5d);
     }
+    // The RAW basis has done its work (the L2 null space above was its last
+    // use) and is the single largest resident block of the setup, so it is
+    // freed.  retain_basis keeps it for a caller that coarsens again from the
+    // same basis; Setup.RetainSubspace keeps it for the object's whole life,
+    // which is what a fixed-basis rebuild on a changed gauge field (HMC)
+    // needs.  A valence solve never rebuilds.
+    if ( !retain_basis && !Params.RetainSubspace ) DiscardBasis();
+    MrhsProjectorL1.ReleaseScratch();
+    MrhsProjectorL2.ReleaseScratch();
+
     nrhs = -1;                                     // both operators left at the batch grids
+    ReportDeviceFootprint("after Coarsen");
   }
 
   ////////////////////////////////////////////////////////////////////
@@ -373,6 +422,7 @@ public:
     CoarseOpL2.ReleaseGrid();           // let go before the grid dies
     delete CoarseCoarseOne;
     nrhs = -1;
+    ReportDeviceFootprint("after BuildDenseBottom");
   }
 
   ////////////////////////////////////////////////////////////////////
@@ -398,6 +448,66 @@ public:
     std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: operators at Nrhs " << nr << std::endl;
   }
 
+  ////////////////////////////////////////////////////////////////////
+  // Galerkin certificate for the L1 coarsening: ||A_c x - P^dag A P x||
+  // over ||A_c x|| on a random coarse vector, through the SAME transfer
+  // operators the solve uses.  The coarse operator is compared with its
+  // own definition, so this reads the ROUNDING level of the coarsening
+  // (fp64 coarse ~1e-13, fp32 coarse ~1e-6) independently of how the
+  // solve converges.  Drives the fine operator with EXACT halos; at more
+  // than one rank the sloppy-halo coarsening error is included in the
+  // reading.  Nrhs 1; the caller's Nrhs is restored by its own SetNrhs.
+  ////////////////////////////////////////////////////////////////////
+  template<class PVdagMOp>
+  RealD CertifyCoarsening(PVdagMOp &FineOp)
+  {
+    SetNrhs(1);
+    CoarseVector xc(CMrhs), Acx(CMrhs), PtAPx(CMrhs);
+    GridParallelRNG cRNG(CMrhs); cRNG.SeedFixedIntegers(std::vector<int>({21,22,23,24}));
+    random(cRNG,xc);
+
+    CoarseOpPV.M(xc,Acx);                                   // A_c x
+
+    std::vector<FineField> Px(1,Grids.FGrid), APx(1,Grids.FGrid);
+    MrhsProjectorL1.blockPromote(Px,xc);                      // P x
+    FineOp.SloppyComms(0);
+    FineOp.Op(Px[0],APx[0]);                                // A P x
+    MrhsProjectorL1.blockProject(APx,PtAPx);                  // P^dag A P x
+
+    CoarseVector d(CMrhs); d = Acx - PtAPx;
+    RealD rel = std::sqrt(norm2(d)/norm2(Acx));
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening: L1 GALERKIN CERTIFICATE "
+              << "||A_c x - P^dag A P x||/||A_c x|| = " << rel << std::endl;
+    return rel;
+  }
+
+
+  ////////////////////////////////////////////////////////////////////
+  // The device memory this coarsening holds that the memory manager
+  // CANNOT evict: BLAS stores, which are plain device allocations.
+  // Lattice fields are omitted on purpose -- they are evictable, so they
+  // cost eviction traffic, not failure.  What is reported is what has to
+  // fit alongside the manager's cache, the comms buffers and the shm
+  // segment, which is the budget an out-of-memory run actually breaks.
+  ////////////////////////////////////////////////////////////////////
+  void ReportDeviceFootprint(const std::string &when)
+  {
+    uint64_t p1 = MrhsProjectorL1.DeviceBytes();
+    uint64_t p2 = MrhsProjectorL2.DeviceBytes();
+    uint64_t o1 = CoarseOpPV.DeviceBytes();
+    uint64_t o2 = CoarseOpL2.DeviceBytes();
+    uint64_t dn = DenseCC ? DenseCC->DeviceBytes() : 0;
+    uint64_t tot = p1+p2+o1+o2+dn;
+    const double G = 1024.*1024.*1024.;
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening (" << when << "): resident device memory/rank "
+              << tot/G << " GB = transfer L1 " << p1/G << " + L2 " << p2/G
+              << " + operator L1 " << o1/G << " + L2 " << o2/G
+              << " + dense " << dn/G << " GB" << std::endl;
+    std::cout << GridLogMessage << "PVdagMMultiGridCoarsening (" << when << "): not evictable; must fit alongside the "
+              << MemoryManager::DeviceMaxBytes/G
+              << " GB manager cache, the comms buffers and the shm segment" << std::endl;
+  }
+
   ////////////////////////////////////////////////////////////////////
   // Free the retained RAW bases (valence use: coarsening is final for
   // this configuration and the fine basis is ~GB-scale host memory).
@@ -420,16 +530,18 @@ public:
 // (its applications define what "converged" means), asserted by the
 // FINAL true-residual report in Solve.
 //////////////////////////////////////////////////////////////////////
-template<class Matrix,class Coarsening>
+template<class Matrix,class MatrixF,class Coarsening>
 class PVdagMMultiGridSolver {
 public:
   typedef typename Coarsening::FineField            FineField;
+  typedef typename Coarsening::FineFieldF           FineFieldF;
   typedef typename Coarsening::CoarseOperator       CoarseOperator;
   typedef typename Coarsening::CoarseVector         CoarseVector;
   typedef typename Coarsening::CoarseCoarseOperator CoarseCoarseOperator;
   typedef typename Coarsening::CoarseCoarseVector   CoarseCoarseVector;
   typedef typename Coarsening::DenseBottom          DenseBottom;
   typedef PrecGeneralisedConjugateResidualNonHermitian<FineField>    FineSmoother_t;
+  typedef PrecGeneralisedConjugateResidualNonHermitian<FineFieldF>   FineSmootherF_t;
   typedef PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> CoarseKrylov_t;
 
   Coarsening            &C;
@@ -447,11 +559,20 @@ public:
   CoarseKrylov_t                                CoarseSmootherGCR;
   MrhsCoarseThreeLevelPrec<CoarseVector,CoarseCoarseVector> L2to3Precon;
   CoarseKrylov_t                                L2PGCR;
+  // The fp64 fine level of the preconditioner
   FineSmoother_t                                SmootherGCR;
-  MrhsTwoLevelMG<FineField,CoarseVector,FineSmoother_t> ThreeLevelPrecon;
+  MrhsTwoLevelMG<FineField,CoarseVector,FineSmoother_t,typename Coarsening::ProjectorL1_t> ThreeLevelPrecon;
+  // The fp32 fine level of the preconditioner, behind the fp64/fp32 seam.
+  // Both share the coarse chain (L2PGCR); the outer Krylov picks one.
+  PVdagMLinearOperator<MatrixF,FineFieldF>        PVdagMF;
+  ShiftedPVdagMLinearOperator<MatrixF,FineFieldF> ShiftedPVdagMF;
+  TrivialPrecon<FineFieldF>                       simple_fine_f;
+  FineSmootherF_t                                 SmootherGCRF;
+  MrhsTwoLevelMG<FineFieldF,CoarseVector,FineSmootherF_t,typename Coarsening::ProjectorL1_t> ThreeLevelPreconF;
+  MrhsMixedPrecPreconditioner<FineField,FineFieldF> PreconSeam;
   MrhsPGCRNonHermitian<FineField>               L1PGCR;
 
-  PVdagMMultiGridSolver(Matrix &Ddwf, Matrix &Dpv,
+  PVdagMMultiGridSolver(Matrix &Ddwf, Matrix &Dpv, MatrixF &DdwfF, MatrixF &DpvF,
                         Coarsening &_C, const PVdagMMultiGridParams &P, int nr)
     : C(_C), Params(P), nrhs(nr),
       _regrid((C.SetNrhs(nr),0)),              // members below capture C.CMrhs/C.CCMrhs
@@ -468,11 +589,20 @@ public:
       L2PGCR(P.CoarseSolver.Tol,P.CoarseSolver.Order/16,LinOpC,L2to3Precon,P.CoarseSolver.Mmax,16),
       // Fine smoother: one restart of nstep GCR steps, tolerance 0 = fixed work.
       SmootherGCR(0.0,1,ShiftedPVdagM,simple_fine,P.FineSmoother.Mmax,P.FineSmoother.Nstep),
-      ThreeLevelPrecon(PVdagM,SmootherGCR,C.MrhsProjector,L2PGCR,C.Grids.Coarse5d,C.CMrhs),
+      ThreeLevelPrecon(PVdagM,SmootherGCR,C.MrhsProjectorL1,L2PGCR,C.Grids.Coarse5d,C.CMrhs),
-      L1PGCR(P.Outer.Tol,P.Outer.MaxIterations,PVdagM,ThreeLevelPrecon,P.Outer.Mmax,P.Outer.Nstep)
+      PVdagMF(DdwfF,DpvF),
+      ShiftedPVdagMF(P.FineSmoother.Shift,DdwfF,DpvF),
+      SmootherGCRF(0.0,1,ShiftedPVdagMF,simple_fine_f,P.FineSmoother.Mmax,P.FineSmoother.Nstep),
+      ThreeLevelPreconF(PVdagMF,SmootherGCRF,C.MrhsProjectorL1,L2PGCR,C.Grids.Coarse5d,C.CMrhs),
+      PreconSeam(ThreeLevelPreconF,Ddwf.FermionGrid(),DdwfF.FermionGrid(),nr),
+      L1PGCR(P.Outer.Tol,P.Outer.MaxIterations,PVdagM,
+             (P.Setup.FinePrecision==MGPrecision::fp32)
+               ? static_cast<MrhsLinearFunction<FineField>&>(PreconSeam)
+               : static_cast<MrhsLinearFunction<FineField>&>(ThreeLevelPrecon),
+             P.Outer.Mmax,P.Outer.Nstep)
   {
     GRID_ASSERT( C.DenseCC != nullptr );
-    
+
     CoarseSmootherGCR.Level(2);
     CoarseSmootherGCR.Name("Csmoother");
     CoarseSmootherGCR.SetZeroGuess(1);
@@ -480,23 +610,39 @@ public:
     L2PGCR.Level(2);
     L2PGCR.Name("Couter");
     L2PGCR.SetZeroGuess(1);
-    
+
     SmootherGCR.Level(1);
     SmootherGCR.Name("Fsmoother");
     SmootherGCR.SetZeroGuess(1);
 
+    SmootherGCRF.Level(1);
+    SmootherGCRF.Name("Fsmoother");
+    SmootherGCRF.SetZeroGuess(1);
+
     L1PGCR.Level(1);
     L1PGCR.Name("Fouter");
     L1PGCR.SetZeroGuess(1);
-    
+
     // The V-cycle is preconditioner: sloppy halos inside, exact restored on exit.
     ThreeLevelPrecon.SetSloppy    = [this](int s){ PVdagM.SloppyComms(s); };
     ThreeLevelPrecon.SloppyComms  = Params.Setup.FineSloppyComms;
+    ThreeLevelPreconF.SetSloppy   = [this](int s){ PVdagMF.SloppyComms(s); };
+    ThreeLevelPreconF.SloppyComms = Params.Setup.FineSloppyComms;
     PVdagM.SloppyComms(0);
-    std::cout << GridLogMessage << "PVdagMMultiGridSolver: Nrhs " << nr
+    PVdagMF.SloppyComms(0);
-              << ", fine halo policy: preconditioner+coarsening "
+    // Three stages, three arithmetics, and a wire format that follows the
-              << (Params.Setup.FineSloppyComms ? "SLOPPY (fp32 wire)" : "exact")
+    // ARITHMETIC of the stage it serves: fp32 on an fp64 operator, bf16 on
-              << ", outer Krylov EXACT" << std::endl;
+    // an fp32 one.  The coarsening always applies the fp64 operator, so its
+    // wire is fp32 whatever the preconditioner runs in.
+    const bool sloppy = Params.Setup.FineSloppyComms;
+    const bool pfp32  = (Params.Setup.FinePrecision==MGPrecision::fp32);
+    std::cout << GridLogMessage << "PVdagMMultiGridSolver: Nrhs " << nr << std::endl;
+    std::cout << GridLogMessage << "  outer Krylov       : fp64, halos EXACT (the true residual is measured here)" << std::endl;
+    std::cout << GridLogMessage << "  preconditioner fine: " << (pfp32 ? "fp32 (seam at the outer Krylov)" : "fp64")
+              << ", halos " << (sloppy ? (pfp32 ? "SLOPPY bf16 wire" : "SLOPPY fp32 wire") : "exact") << std::endl;
+    std::cout << GridLogMessage << "  coarsening (done)  : fp64 operator, halos "
+              << (sloppy ? "SLOPPY fp32 wire" : "exact")
+              << " -- the coarse operator does not follow FinePrecision" << std::endl;
   }
 
   void Solve(std::vector<FineField> &src, std::vector<FineField> &sol)

Grid/algorithms/multigrid/PVdagMMultiGridParams.h

@@ -42,6 +42,12 @@ NAMESPACE_BEGIN(Grid);
 // not consumer knobs.
 //////////////////////////////////////////////////////////////////////////////////////
 
+// Arithmetic precision of the fine level INSIDE the preconditioner -- the
+// smoother and the V-cycle's own fine residuals.  The outer Krylov is
+// always fp64: the fp64/fp32 seam sits at its preconditioner call.
+// Orthogonal to FineSloppyComms, which is the halo WIRE format.
+GRID_SERIALIZABLE_ENUM(MGPrecision, undef, fp64, 1, fp32, 2);
+
 // The smoother is the adaptive shifted PGCR -- the one correct route for
 // this non-Hermitian chain (stationary replay and Chebyshev were explored
 // and did not win here; Chebyshev remains the HERMITIAN chain's smoother).
@@ -79,14 +85,17 @@ struct MGDenseParams : Serializable {
 
 struct MGSetupParams : Serializable {
   GRID_SERIALIZABLE_CLASS_MEMBERS(MGSetupParams,
-                                  std::vector<int>, Block,           // fine -> coarse blocking
+                                  std::vector<int>, Block1,          // fine -> coarse blocking
                                   std::vector<int>, Block2,          // coarse -> coarse-coarse blocking
                                   int,              CoarsenBatch,
                                   std::string,      SubspaceFile,    // scidac; empty = create from noise, no I/O
-                                  int,              FineSloppyComms);// fp32 wire INSIDE the preconditioner only
+                                  int,              FineSloppyComms, // reduced-precision halo WIRE inside the preconditioner only: fp32 on the fp64 operator, bf16 on the fp32 operator
+                                  MGPrecision,      FinePrecision,   // fine-level ARITHMETIC inside the preconditioner
+                                  int,              RetainSubspace); // keep the RAW basis after setup (nbasis fine vectors); needed ONLY for a fixed-basis rebuild on a changed gauge field (HMC).  A valence solve never rebuilds, so 0 frees it.
   MGSetupParams()
-    : Block({2,2,3,3}), Block2({4,4,2,4}), CoarsenBatch(9),
+    : Block1({2,2,3,3}), Block2({4,4,2,4}), CoarsenBatch(9),
-      SubspaceFile(""), FineSloppyComms(1) {};
+      SubspaceFile(""), FineSloppyComms(1), FinePrecision(MGPrecision::fp64),
+      RetainSubspace(0) {};
 };
 
 struct PVdagMMultiGridParams : Serializable {
@@ -104,9 +113,10 @@ struct PVdagMMultiGridParams : Serializable {
 
 inline void CheckValidity(const PVdagMMultiGridParams &P)
 {
-  GRID_ASSERT( P.Setup.Block.size()  == 4 );
+  GRID_ASSERT( P.Setup.Block1.size() == 4 );
   GRID_ASSERT( P.Setup.Block2.size() == 4 );
   GRID_ASSERT( P.Setup.CoarsenBatch  >= 1 );
+  GRID_ASSERT( P.Setup.FinePrecision != MGPrecision::undef );
   GRID_ASSERT( P.FineSmoother.Nstep   > 0 );
   GRID_ASSERT( P.CoarseSmoother.Nstep > 0 );
   GRID_ASSERT( P.CoarseSolver.Tol     > 0.0 );

Grid/algorithms/multigrid/Smoothers.h

@@ -36,6 +36,8 @@ NAMESPACE_BEGIN(Grid);
 //                              smoother operator such as shifted PVdagM.
 //   ChebyshevInverter          one Chebyshev-corrected step with residual print.
 //   MirsSmoother               shifted-MdagM CG, HDCG arXiv:1402.2585.
+//   CGSmoother                 fixed-iteration CG on an already-shifted
+//                              Hermitian operator: the mrhs HDCG smoother.
 //   GCRReplaySmoother          replays a GCR's recorded step lengths a_k and
 //                              orthogonalisation coefficients b_kj with NO
 //                              inner products: one matvec per step, zero
@@ -191,6 +193,30 @@ public:
   }
 };
 
+//////////////////////////////////////////////////////////////////////////////
+// Fixed-iteration CG on an (already shifted) Hermitian operator, tolerance
+// zero so the work is fixed.  NOT a stationary preconditioner: the CG
+// coefficients depend on the input, so the map is nonlinear.  Use under a
+// flexible outer (fPcg); BlockCGrQ needs the Chebyshev smoother.
+//////////////////////////////////////////////////////////////////////////////
+template<class Field> class CGSmoother : public LinearFunction<Field>
+{
+public:
+  using LinearFunction<Field>::operator();
+  LinearOperatorBase<Field> &_Op;
+  int iters;
+  CGSmoother(int _iters, LinearOperatorBase<Field> &Op) : _Op(Op), iters(_iters)
+  {
+    std::cout << GridLogMessage << "CGSmoother order " << iters << std::endl;
+  }
+  void operator() (const Field &in, Field &out)
+  {
+    ConjugateGradient<Field> CG(0.0,iters,false);   // never converges: by design
+    out = Zero();
+    CG(_Op,in,out);
+  }
+};
+
 //////////////////////////////////////////////////////////////////////////////
 // Replay of a recorded GCR with a trivial preconditioner:
 //   p_0 = r_0 ;  x_{k+1} = x_k + a_k p_k ;  r_{k+1} = r_k - a_k A p_k ;

Grid/lattice/Lattice_transfer.h

@@ -210,6 +210,14 @@ accelerator_inline void convertType(vComplexD2 & out, const vComplexF & in) {
   precisionChange(out,in);
 }
 
+// Precision change in the lex (unvectorised) chart: one lane, so a plain
+// scalar conversion.  Reached by the block operations on an fp32 coarse
+// space, whose inner products accumulate in double and convert back.
+accelerator_inline void convertType(sComplexF & out, const sComplexD & in) { out.v = ComplexF(in.v); }
+accelerator_inline void convertType(sComplexD & out, const sComplexF & in) { out.v = ComplexD(in.v); }
+accelerator_inline void convertType(sRealF    & out, const sRealD    & in) { out.v = RealF(in.v); }
+accelerator_inline void convertType(sRealD    & out, const sRealF    & in) { out.v = RealD(in.v); }
+
 template<typename T1,typename T2>
 accelerator_inline void convertType(iScalar<T1> & out, const iScalar<T2> & in) {
   convertType(out._internal,in._internal);

Grid/tensors/Tensor_traits.h

@@ -110,6 +110,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef RealD DoublePrecision;
     typedef RealD DoublePrecision2;
+    typedef RealF SinglePrecision;
   };
   template<> struct GridTypeMapper<RealD> : public GridTypeMapper_Base {
     typedef RealD scalar_type;
@@ -124,6 +125,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef RealD DoublePrecision;
     typedef RealD DoublePrecision2;
+    typedef RealF SinglePrecision;
   };
   template<> struct GridTypeMapper<ComplexF> : public GridTypeMapper_Base {
     typedef ComplexF scalar_type;
@@ -138,6 +140,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
+    typedef ComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<ComplexD> : public GridTypeMapper_Base {
     typedef ComplexD scalar_type;
@@ -152,6 +155,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
+    typedef ComplexF SinglePrecision;
   };
 
 #if defined(GRID_CUDA) || defined(GRID_HIP)  
@@ -168,6 +172,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef scalar_typeD DoublePrecision;
     typedef scalar_typeD DoublePrecision2;
+    typedef std::complex<float> SinglePrecision;
   };
   template<> struct GridTypeMapper<std::complex<double> > : public GridTypeMapper_Base {
     typedef std::complex<double> scalar_type;
@@ -182,6 +187,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef scalar_typeD DoublePrecision;
     typedef scalar_typeD DoublePrecision2;
+    typedef std::complex<float> SinglePrecision;
   };
 #endif
 
@@ -198,6 +204,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
+    typedef void SinglePrecision;
   };
 
   template<> struct GridTypeMapper<vRealF> : public GridTypeMapper_Base {
@@ -213,6 +220,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD2 DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vRealD> : public GridTypeMapper_Base {
     typedef RealD  scalar_type;
@@ -227,6 +235,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vRealD2> : public GridTypeMapper_Base {
     typedef RealD  scalar_type;
@@ -241,6 +250,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD2 DoublePrecision;
     typedef vRealD2 DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vRealH> : public GridTypeMapper_Base {
     // Fixme this is incomplete until Grid supports fp16 or bfp16 arithmetic types
@@ -256,6 +266,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD DoublePrecision2;
+    typedef vRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexH> : public GridTypeMapper_Base {
     // Fixme this is incomplete until Grid supports fp16 or bfp16 arithmetic types
@@ -271,6 +282,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexF> : public GridTypeMapper_Base {
     typedef ComplexF  scalar_type;
@@ -285,6 +297,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD2 DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexD> : public GridTypeMapper_Base {
     typedef ComplexD  scalar_type;
@@ -299,6 +312,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vComplexD2> : public GridTypeMapper_Base {
     typedef ComplexD  scalar_type;
@@ -313,6 +327,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef vComplexD2 DoublePrecision;
     typedef vComplexD2 DoublePrecision2;
+    typedef vComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<vInteger> : public GridTypeMapper_Base {
     typedef  Integer scalar_type;
@@ -327,6 +342,7 @@ NAMESPACE_BEGIN(Grid);
     typedef vInteger Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
+    typedef void SinglePrecision;
   };
   template<> struct GridTypeMapper<sRealF> : public GridTypeMapper_Base {
     typedef RealF  scalar_type;
@@ -341,6 +357,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sRealD DoublePrecision;
     typedef sRealD DoublePrecision2;
+    typedef sRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<sRealD> : public GridTypeMapper_Base {
     typedef RealD  scalar_type;
@@ -355,6 +372,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sRealD DoublePrecision;
     typedef sRealD DoublePrecision2;
+    typedef sRealF SinglePrecision;
   };
   template<> struct GridTypeMapper<sComplexF> : public GridTypeMapper_Base {
     typedef ComplexF  scalar_type;
@@ -369,6 +387,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sComplexD DoublePrecision;
     typedef sComplexD DoublePrecision2;
+    typedef sComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<sComplexD> : public GridTypeMapper_Base {
     typedef ComplexD  scalar_type;
@@ -383,6 +402,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef sComplexD DoublePrecision;
     typedef sComplexD DoublePrecision2;
+    typedef sComplexF SinglePrecision;
   };
   template<> struct GridTypeMapper<sInteger> : public GridTypeMapper_Base {
     typedef Integer  scalar_type;
@@ -397,6 +417,7 @@ NAMESPACE_BEGIN(Grid);
     typedef sInteger Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
+    typedef void SinglePrecision;
   };
 
 #define GridTypeMapper_RepeatedTypes \
@@ -417,6 +438,7 @@ NAMESPACE_BEGIN(Grid);
     using Integerified    = iScalar<typename BaseTraits::Integerified>;
     using DoublePrecision = iScalar<typename BaseTraits::DoublePrecision>;
     using DoublePrecision2= iScalar<typename BaseTraits::DoublePrecision2>;
+    using SinglePrecision = iScalar<typename BaseTraits::SinglePrecision>;
     static constexpr int Rank = BaseTraits::Rank + 1;
     static constexpr std::size_t count = BaseTraits::count;
     static constexpr int Dimension(int dim) {
@@ -433,6 +455,7 @@ NAMESPACE_BEGIN(Grid);
     using Integerified    = iVector<typename BaseTraits::Integerified,    N>;
     using DoublePrecision = iVector<typename BaseTraits::DoublePrecision, N>;
     using DoublePrecision2= iVector<typename BaseTraits::DoublePrecision2, N>;
+    using SinglePrecision = iVector<typename BaseTraits::SinglePrecision, N>;
     static constexpr int Rank = BaseTraits::Rank + 1;
     static constexpr std::size_t count = BaseTraits::count * N;
     static constexpr int Dimension(int dim) {
@@ -449,6 +472,7 @@ NAMESPACE_BEGIN(Grid);
     using Integerified    = iMatrix<typename BaseTraits::Integerified,    N>;
     using DoublePrecision = iMatrix<typename BaseTraits::DoublePrecision, N>;
     using DoublePrecision2= iMatrix<typename BaseTraits::DoublePrecision2, N>;
+    using SinglePrecision = iMatrix<typename BaseTraits::SinglePrecision, N>;
     static constexpr int Rank = BaseTraits::Rank + 2;
     static constexpr std::size_t count = BaseTraits::count * N * N;
     static constexpr int Dimension(int dim) {

benchmarks/Benchmark_usqcd.cc

@@ -276,14 +276,31 @@ public:
 
     GridBLAS blas;
 
+    // The batched GEMMs of a coarse multigrid level are memory bound, not
+    // flop bound: each call streams the matrices once and does O(1) flops per
+    // element.  So report the effective bandwidth beside the flop rate --
+    // measured against the device's peak it says whether a shape is running
+    // well, which the flop rate alone does not.  beta=1, so C is read as well
+    // as written.
+    auto Report = [&](int M,int N,int K,int BATCH)
+    {
+      double gf    = blas.benchmark<CComplex>(M,N,K,BATCH);
+      double flops = 8.0*M*N*K*BATCH;
+      double bytes = 1.0*sizeof(CComplex)*(1.0*M*K + 1.0*K*N + 2.0*M*N)*BATCH;
+      double gbs   = gf*bytes/flops;
+      fprintf(FP,"%d, %d, %d, %d, %f, %f\n", M, N, K, BATCH, gf, gbs);
+      std::cout<<GridLogMessage<<std::setprecision(3)
+	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<gf<<"\t\t"<<gbs<<std::endl;
+    };
+
     int fpbits = sizeof(CComplex)*4;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << "= batched GEMM fp"<<fpbits<<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"Gflop/s / rank (coarse mrhs)"<<std::endl;
+    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (coarse mrhs)"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
-    fprintf(FP,"GEMM\n\n M, N, K, BATCH, GF/s per rank fp%d\n",fpbits);
+    fprintf(FP,"GEMM\n\n M, N, K, BATCH, GF/s per rank, GB/s per rank fp%d\n",fpbits);
 
     for(int b=0;b<3;b++){
     for(int r=0;r<3;r++){
@@ -291,15 +308,10 @@ public:
       int N=rhs[r];
       int K=basis[b];
       int BATCH=vol;
-      double p=blas.benchmark<CComplex>(M,N,K,BATCH);
+      Report(M,N,K,BATCH);
-
-      fprintf(FP,"%d, %d, %d, %d, %f\n", M, N, K, BATCH, p);
-      
-      std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<p<<std::endl;
     }}
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"Gflop/s / rank (block project)"<<std::endl;
+    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (block project)"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
     for(int b=0;b<3;b++){
     for(int r=0;r<3;r++){
@@ -307,14 +319,10 @@ public:
       int N=rhs[r];
       int K=blk;
       int BATCH=vol;
-      double p=blas.benchmark<CComplex>(M,N,K,BATCH);
+      Report(M,N,K,BATCH);
-
-      fprintf(FP,"%d, %d, %d, %d, %f\n", M, N, K, BATCH, p);
-      std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<p<<std::endl;
     }}
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
-    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"Gflop/s / rank (block promote)"<<std::endl;
+    std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (block promote)"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
     for(int b=0;b<3;b++){
     for(int r=0;r<3;r++){
@@ -322,12 +330,54 @@ public:
       int N=blk;
       int K=basis[b];
       int BATCH=vol;
-      double p=blas.benchmark<CComplex>(M,N,K,BATCH);
+      Report(M,N,K,BATCH);
-
-      fprintf(FP,"%d, %d, %d, %d, %f\n", M, N, K, BATCH, p);
-      std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << M<<"\t\t"<<N<<"\t\t"<<K<<"\t\t"<<BATCH<<"\t\t"<<p<<std::endl;
     }}
+
+    ///////////////////////////////////////////////////////////////////////////
+    // The coarse stencil apply as the multigrid actually issues it, swept in
+    // the batch count.  One call per stencil point, M = K = nbasis, N = Nrhs,
+    // batch = the LOCAL coarse volume: at 48^3x96 over 288 GCDs with a
+    // {2,2,3,3} block that is 1024 sites, which is small, and the trace shows
+    // the single-precision kernel taking as long as the double one.  The
+    // sweep spans that point and the alternative organisation in which the
+    // stencil legs ride in the batch dimension instead of being separate
+    // calls: 9x and 33x the sites, with the partial results summed after.
+    // Nrhs 1 is the latency-bound case that a single-right-hand-side solve
+    // pays throughout.
+    ///////////////////////////////////////////////////////////////////////////
+    {
+      int  nb[]    = { 60, 64 };
+      int  nrhs_[] = { 1, 12, 24, 32 };
+      // 1024 is the local coarse volume at the production point; the rest are
+      // that times the number of stencil legs folded into the batch.  3 is
+      // the common divisor of the 33-point (PVdagM) and 81-point (HDCG)
+      // stencils, so x3 is the grouping that serves both and leaves only a
+      // three-way linear combination at the end.
+      int  batches[] = { 256, 1024, 3072, 4096, 9216, 33792 };
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (coarse stencil, production shapes)"<<std::endl;
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      for(int b=0;b<2;b++){
+      for(int r=0;r<4;r++){
+      for(int v=0;v<6;v++){
+	Report(nb[b],nrhs_[r],nb[b],batches[v]);
+      }}}
+
+      // The same work with the stencil sum folded along the CONTRACTION
+      // instead of the batch: one GEMM per site with K = npoint*nbasis.  Both
+      // precisions reach their best rates at large K (the block-project
+      // shape, K=256), which this pushes further, at the price of gathering
+      // the neighbour vectors.  33 points is the next-to-nearest stencil,
+      // 81 the full box.
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      std::cout<<GridLogMessage << "  M  "<<"\t\t"<<"N"<<"\t\t\t"<<"K"<<"\t\t"<<"BATCH"<<"\t\t"<<"Gflop/s"<<"\t\t"<<"GB/s / rank (stencil folded along K)"<<std::endl;
+      std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+      for(int b=0;b<2;b++){
+      for(int r=0;r<4;r++){
+	Report(nb[b],nrhs_[r],33*nb[b],1024);
+	Report(nb[b],nrhs_[r],81*nb[b],1024);
+      }}
+    }
     fprintf(FP,"\n\n\n");
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   };

configure.ac

@@ -192,6 +192,18 @@ case ${ac_TRACING} in
 	AC_DEFINE([GRID_TRACING_NONE],[1],[no tracing]);;
 esac
 
+############### distributed dense inverse precision (multigrid coarse-coarse bottom)
+AC_ARG_ENABLE([dense-inverse-precision],
+    [AS_HELP_STRING([--enable-dense-inverse-precision=double|single],[precision of the distributed dense coarse-coarse inversion (default double); independent of the coarse-space precision])],
+    [ac_DENSE_INVERSE=${enable_dense_inverse_precision}], [ac_DENSE_INVERSE=double])
+
+case ${ac_DENSE_INVERSE} in
+    single)
+        AC_DEFINE([GRID_DENSE_INVERSE_SINGLE],[1],[fp32 distributed dense inversion]);;
+    *)
+        AC_DEFINE([GRID_DENSE_INVERSE_DOUBLE],[1],[fp64 distributed dense inversion]);;
+esac
+
 ############### fermions
 AC_ARG_ENABLE([fermion-instantiations],
      [AS_HELP_STRING([--enable-fermion-instantiations=yes|no],[enable fermion instantiations])],

examples/Example_mdagm.cc

@@ -32,50 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 using namespace std;
 using namespace Grid;
 
-// Routes Op/AdjOp -> HermOp so that CoarsenOperator and CreateSubspace
-// both see the HPD operator M†M rather than bare M.
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> &wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
-  void HermOp (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
-  void AdjOp  (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
-  void OpDiag (const Field &in, Field &out)                  { GRID_ASSERT(0); }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) { GRID_ASSERT(0); }
-  void OpDirAll(const Field &in, std::vector<Field> &out)    { GRID_ASSERT(0); }
-  void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
-    wrapped.HermOp(in, out);
-    ComplexD dot = innerProduct(in, out);
-    n1 = real(dot);
-    n2 = norm2(out);
-  }
-};
 
-// Fixed-iteration CG smoother: runs exactly `iters` steps of CG on the
-// shifted operator.  tolerance=0 so CG never exits early.
-template<class Field>
-class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator &_SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator)
-    : _SmootherOperator(SmootherOperator), iters(_iters)
-  {
-    std::cout << GridLogMessage << " CGSmoother order " << iters << std::endl;
-  }
-  void operator()(const Field &in, Field &out)
-  {
-    ConjugateGradient<Field> CG(0.0, iters, false);
-    out = Zero();
-    CG(_SmootherOperator, in, out);
-  }
-};
 
 int main (int argc, char ** argv)
 {

examples/Example_mdagm_cg.cc

@@ -57,46 +57,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 using namespace std;
 using namespace Grid;
 
-// Wraps any LinearOperatorBase so that Op = AdjOp = HermOp.
-// Required when coarsening an HPD operator whose Op != HermOp
-// (e.g. MdagMLinearOperator where Op=M, HermOp=M†M).
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> &wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {}
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
-  void HermOp (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
-  void AdjOp  (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
-  void OpDiag (const Field &in, Field &out)                    { GRID_ASSERT(0); }
-  void OpDir  (const Field &in, Field &out, int dir, int disp) { GRID_ASSERT(0); }
-  void OpDirAll(const Field &in, std::vector<Field> &out)      { GRID_ASSERT(0); }
-  void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
-    wrapped.HermOp(in, out);
-    ComplexD dot = innerProduct(in, out);
-    n1 = real(dot); n2 = norm2(out);
-  }
-};
 
-// Fixed-iteration CG as a smoother (LinearFunction).
-// Used as the IR-shifted smoother: solves (M†M + lo*I) x = b approximately.
-template<class Field>
-class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  LinearOperatorBase<Field> &_op;
-  int iters;
-  CGSmoother(int _iters, LinearOperatorBase<Field> &op) : _op(op), iters(_iters) {
-    std::cout << GridLogMessage << "CGSmoother order " << iters << std::endl;
-  }
-  void operator()(const Field &in, Field &out) {
-    ConjugateGradient<Field> CG(0.0, iters, false);
-    out = Zero();
-    CG(_op, in, out);
-  }
-};
 
 // Two-level V-cycle preconditioner (LinearFunction).
 template<class Fobj, class CComplex, int nbasis>

examples/Example_pvdagm_mrhs.cc

@@ -58,6 +58,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
 using namespace std;
 using namespace Grid;
@@ -183,194 +184,6 @@ public:
   }
 };
 
-//////////////////////////////////////////////////////////////////////
-// Minimal multi-RHS function interface (preconditioner slot)
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-//////////////////////////////////////////////////////////////////////
-// Single-polynomial multi-RHS PGCR (non-Hermitian).
-//
-// Verbatim adaptation of PrecGeneralisedConjugateResidualNonHermitian
-// to std::vector<Field>: every innerProduct / norm2 is SUMMED over the
-// RHS index, so one alpha/beta per step is shared by all RHS -- the
-// single GCR on the enlarged block-diagonal system.
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD   Tolerance;
-  Integer MaxIterations;
-  int mmax;
-  int nstep;
-  int steps;
-  int level;
-  int ZeroGuess = 0; int FirstCycle = 0;  // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-
-  void Level(int lv) { name = "Level " + std::to_string(lv); level=lv; };
-  void Name(std::string n) { name = n; };
-  void SetZeroGuess(int z) { ZeroGuess=z; };
-
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,
-                       LinearOperatorBase<Field> &_Linop,
-                       MrhsLinearFunction<Field> &Prec,
-                       int _mmax,int _nstep)
-    : Tolerance(tol), MaxIterations(maxit), Linop(_Linop), Preconditioner(Prec),
-      mmax(_mmax), nstep(_nstep) { level=1; }
-
-  ///////////////////////////////////////////////////////////////
-  // vector-of-fields linear algebra, reductions summed over rhs
-  ///////////////////////////////////////////////////////////////
-  static RealD vnorm2(std::vector<Field> &x){
-    RealD s=0.0; for(auto &f : x) s+=norm2(f); return s;
-  }
-  static ComplexD vinnerProduct(std::vector<Field> &x, std::vector<Field> &y){
-    ComplexD s(0.0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s;
-  }
-  static void vaxpy(std::vector<Field> &z, ComplexD a, std::vector<Field> &x, std::vector<Field> &y){
-    for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]);
-  }
-  void vOp(std::vector<Field> &in, std::vector<Field> &out){
-    for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]);
-  }
-
-  void operator() (std::vector<Field> &src, std::vector<Field> &psi){
-    RealD cp, ssq, rsq;
-    int nrhs = src.size();
-    GridBase *grid = src[0].Grid();
-
-    ssq=vnorm2(src);
-    rsq=Tolerance*Tolerance*ssq;
-
-    std::vector<Field> r(nrhs,grid);
-
-    GridStopWatch SolverTimer;
-    SolverTimer.Start();
-
-    steps=0;
-    FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-
-      cp=GCRnStep(src,psi,rsq);
-
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-               <<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-
-      if(cp<rsq){
-        SolverTimer.Stop();
-        vOp(psi,r);
-        for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-                 <<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)
-                 <<" true residual "<<std::sqrt(tr/ssq)
-                 <<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-                 <<" MrhsPGCR Time elapsed: Total "<<SolverTimer.Elapsed()<<std::endl;
-        // Per-RHS true residuals: the honest metric under the summed norm
-        for(int rr=0;rr<nrhs;rr++){
-          RealD rn = std::sqrt(norm2(r[rr])/norm2(src[rr]));
-          std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl;
-        }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-
-  RealD GCRnStep(std::vector<Field> &src, std::vector<Field> &psi, RealD rsq){
-
-    RealD cp;
-    ComplexD a, b, rq;
-    RealD zAAz;
-
-    int nrhs = src.size();
-    GridBase *grid = src[0].Grid();
-
-    std::vector<Field> r (nrhs,grid);
-    std::vector<Field> z (nrhs,grid);
-    std::vector<Field> Az(nrhs,grid);
-
-    ////////////////////////////////
-    // history for flexible orthog: [mmax][nrhs]
-    ////////////////////////////////
-    std::vector< std::vector<Field> > q(mmax, std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax, std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-
-    if (ZeroGuess && FirstCycle) {
-      for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; }
-    } else {
-      vOp(psi,Az);
-      for(int rr=0;rr<nrhs;rr++) r[rr] = src[rr]-Az[rr];
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-               <<" MrhsPGCR true residual r = src - A psi   "<<vnorm2(r)<<std::endl;
-    }
-    FirstCycle=0;
-
-    Preconditioner(r,z);
-    vOp(z,Az);
-    zAAz=vnorm2(Az);
-
-    p[0]=z;
-    q[0]=Az;
-    qq[0]=zAAz;
-
-    cp=vnorm2(r);
-
-    for(int k=0;k<nstep;k++){
-
-      steps++;
-
-      int kp     = k+1;
-      int peri_k = k %mmax;
-      int peri_kp= kp%mmax;
-
-      rq = vinnerProduct(q[peri_k],r);
-      a  = rq/qq[peri_k];
-
-      vaxpy(psi,a,p[peri_k],psi);
-      vaxpy(r,-a,q[peri_k],r);
-      cp = vnorm2(r);
-
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
-               <<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-
-      if((k==nstep-1)||(cp<rsq)){
-        return cp;
-      }
-
-      Preconditioner(r,z);
-      vOp(z,Az);
-      zAAz=vnorm2(Az);
-
-      q[peri_kp]=Az;
-      p[peri_kp]=z;
-
-      int northog = ((kp)>(mmax-1))?(mmax-1):(kp);
-      for(int back=0;back<northog;back++){
-        int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-        b = -real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]);
-        vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]);
-      }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); // never reached
-    return cp;
-  }
-};
-
 //////////////////////////////////////////////////////////////////////
 // Trivial multi-RHS preconditioner
 //////////////////////////////////////////////////////////////////////
@@ -382,98 +195,6 @@ public:
   }
 };
 
-//////////////////////////////////////////////////////////////////////
-// MultiRHS two-level V-cycle.
-//
-// Mirrors MGPreconditioner in Example_pvdagm:
-//   out = in (trivial pre)                      [per rhs]
-//   r1  = in - A out                            [per rhs]
-//   batched blockProject -> pack -> ONE mrhs coarse PGCR -> unpack
-//     -> batched blockPromote; out += correction
-//   r2  = in - A out                            [per rhs]
-//   per-RHS fine post-smoother; out += smooth(r2)
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;   // same lattice type on Coarse5d and CoarseMrhs
-
-  LinearOperatorBase<FineField>            &_FineOperator;
-  FineSmoother                             &_PostSmoother;   // single-RHS smoother, looped
-  MultiRHSBlockProject<FineField>          &_Projector;
-  LinearFunction<CoarseVector>             &_CoarseSolve;    // PGCR on the 6D mrhs field
-  GridBase                                 *_CoarseGrid;     // Coarse5d (single rhs)
-  GridBase                                 *_CoarseGridMrhs; // 6D
-
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp,
-                 FineSmoother                  &Post,
-                 MultiRHSBlockProject<FineField> &Projector,
-                 LinearFunction<CoarseVector>  &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp), _PostSmoother(Post), _Projector(Projector),
-      _CoarseSolve(CoarseSolve), _CoarseGrid(CoarseGrid), _CoarseGridMrhs(CoarseGridMrhs) {}
-
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs = in.size();
-    GridBase *fgrid = in[0].Grid();
-    double t;
-
-    std::vector<FineField> vec1(nrhs,fgrid);
-    std::vector<FineField> vec2(nrhs,fgrid);
-
-    // Trivial pre-smoother: out = in (as in Example_pvdagm with simple_fine)
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-
-    // Residual
-    for(int r=0;r<nrhs;r++){
-      _FineOperator.Op(out[r],vec1[r]);
-      sub(vec1[r],in[r],vec1[r]);
-    }
-
-    // Batched fine->coarse, pack rhs into 6D field
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid);
-    std::vector<CoarseVector> Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs);
-    CoarseVector CsolMrhs(_CoarseGridMrhs);
-
-    t=-usecond();
-    _Projector.blockProject(vec1,Csrc_split);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-
-    // ONE coarse solve for all rhs (GEMM coarse mults)
-    t=-usecond();
-    CsolMrhs=Zero();
-    _CoarseSolve(CsrcMrhs,CsolMrhs);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // Unpack, batched coarse->fine, add correction
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-    _Projector.blockPromote(vec1,Csol_split);
-    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-
-    // Residual
-    for(int r=0;r<nrhs;r++){
-      _FineOperator.Op(out[r],vec1[r]);
-      sub(vec1[r],in[r],vec1[r]);
-    }
-
-    // Per-RHS post-smoother (fine level has no batching win; memory-light)
-    t=-usecond();
-    for(int r=0;r<nrhs;r++){
-      vec2[r]=Zero();
-      _PostSmoother(vec1[r],vec2[r]);
-      add(out[r],out[r],vec2[r]);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {

examples/Example_pvdagm_mrhs_3level.cc

@@ -50,6 +50,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
 using namespace std;
 using namespace Grid;
@@ -167,206 +168,6 @@ public:
   void HermOp  (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
 };
 
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from Example_pvdagm_mrhs.cc):
-// reductions summed over rhs -> one alpha/beta per step for the enlarged system.
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;   // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-    p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-      q[peri_kp]=Az; p[peri_kp]=z;
-      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-        b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle: a LinearFunction on the 6D mrhs COARSE field.
-// Mirrors Example_pvdagm_mrhs.cc's MrhsTwoLevelMG one level down, and the
-// single-RHS MGPreconditioner of Example_pvdagm_3level_SVDdefl.cc:
-//   out = in (trivial pre)
-//   r = in - A_coarse out
-//   restrict (unpack 6D coarse -> blockProject -> pack 6D coarse-coarse)
-//   ONE coarse-coarse solve (L3, GEMM)
-//   prolong (unpack -> blockPromote -> pack); out += correction
-//   r = in - A_coarse out
-//   coarse smoother (shifted 6D coarse op); out += smooth(r)
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;          // mrhs coarse op (6D)
-  LinearFunction<CoarseField>              &_CoarseSmoother;    // shifted 6D coarse smoother
-  MultiRHSBlockProject<CoarseField>        &_Projector;         // L2->L3 (vector-based)
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve; // L3 solve (6D cc)
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs; double t;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-
-    // trivial pre-smoother
-    out = in;
-
-    // residual (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
-    std::vector<CoarseField>       csplit(nrhs,_Coarse5d);
-    std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
-    _Projector.blockProject(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
-
-    // L3 solve (6D coarse-coarse, GEMM)
-    t=-usecond();
-    CCsol=Zero();
-    _CoarseCoarseSolve(CCsrc,CCsol);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
-    _Projector.blockPromote(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
-    add(out,out,vec1);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
-
-    // residual + coarse smoother (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-    vec2=Zero();
-    _CoarseSmoother(vec1,vec2);
-    add(out,out,vec2);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle (verbatim from Example_pvdagm_mrhs.cc):
-// per-rhs fine smoother + batched restriction + ONE coarse solve + batched prolong.
-// The coarse solve passed in is now itself three-level (preconditioned by L2->L3).
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    t=-usecond();
-    _Projector.blockProject(vec1,Csrc_split);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-    t=-usecond(); CsolMrhs=Zero(); _CoarseSolve(CsrcMrhs,CsolMrhs); t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-    _Projector.blockPromote(vec1,Csol_split);
-    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    t=-usecond();
-    for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {

examples/Example_pvdagm_mrhs_3level_DenseCoarseMatrix.cc

@@ -50,6 +50,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 #include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
 
 #include <memory>
@@ -173,281 +174,6 @@ public:
   void HermOp  (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
 };
 
-//////////////////////////////////////////////////////////////////////
-// Dense CC solve on the PACKED 6D mrhs coarse-coarse field: drop-in
-// for the L3 PGCR, delegating to the library DenseCoarseMatrix.
-//////////////////////////////////////////////////////////////////////
-template<class DenseType, class CoarseCoarseField>
-class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
-public:
-  DenseType &_Dense;
-  GridBase *_CoarseCoarse5d;
-  int _nrhs;
-  MrhsDenseCCSolve(DenseType &D, GridBase *cc5d, int nrhs)
-    : _Dense(D), _CoarseCoarse5d(cc5d), _nrhs(nrhs) {}
-  using LinearFunction<CoarseCoarseField>::operator();
-  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
-    if ( getenv("DENSE_CC_CHECK") ) {
-      // Audit path: per-rhs 5D unpack so ApplyBatch can run the _Op defect
-      // check per rhs.  ~50ms/call of slice/split overhead -- audit only.
-      CoarseCoarseField tmp(in.Grid());
-      tmp = in;
-      std::vector<CoarseCoarseField> split_in (_nrhs,_CoarseCoarse5d);
-      std::vector<CoarseCoarseField> split_out(_nrhs,_CoarseCoarse5d);
-      for(int r=0;r<_nrhs;r++) ExtractSliceFast(split_in[r], tmp, r, 0);
-      _Dense.ApplyBatch(split_in, split_out);
-      for(int r=0;r<_nrhs;r++) InsertSliceFast(split_out[r], out, r, 0);
-    } else {
-      GRID_TRACE("MrhsDensCCSolve::ApplyBatch6D");
-      _Dense.ApplyBatch6D(in, out, _nrhs);
-    }
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from the
-// frozen Example_pvdagm_mrhs_3level_dense.cc)
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;   // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ GRID_TRACE("GCR-vnorm2"); RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ GRID_TRACE("GCR-vinnerProduct"); ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ GRID_TRACE("GCR-vaxpy"); for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("GCR-vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }  
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    GRID_TRACE((name+"MrhsPGCRNonHermitian").c_str());
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-    p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      {
-        GRID_TRACE("GCR-project");
-        rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-        vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      }      
-      //      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      //      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-
-      {
-        GRID_TRACE("GCR-orthog");
-        q[peri_kp]=Az; p[peri_kp]=z;
-        int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-        for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-          b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-          vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-        qq[peri_kp]=vnorm2(q[peri_kp]);
-      }      
-      //      q[peri_kp]=Az; p[peri_kp]=z;
-      //      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      //      for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-      //        b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-      //        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-      //      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle: LinearFunction on the 6D mrhs COARSE field.
-// The coarse-coarse solve slot takes EITHER the dense mrhs solve
-// (DENSE_CC=1) or the L3 PGCR (DENSE_CC=0).
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;          // mrhs coarse op (6D)
-  LinearFunction<CoarseField>              &_CoarseSmoother;    // shifted 6D coarse smoother
-  MultiRHSBlockProject<CoarseField>        &_Projector;         // L2->L3 (vector-based)
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve; // L3 solve (6D cc)
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs; double t;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-
-    // trivial pre-smoother
-    out = in;
-
-    // residual (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
-    std::vector<CoarseField>       csplit(nrhs,_Coarse5d);
-    std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-
-    t=-usecond();
-    {
-      GRID_TRACE("L2L3-Vcycle - Extract/blockProject/Insert");
-      for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
-      _Projector.blockProject(csplit,ccsplit);
-      for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
-
-    // L3 solve (dense mrhs GEMM, or PGCR)
-    t=-usecond();
-    {
-      GRID_TRACE("L2L3-Vcycle - CoarseCoarseSolve");
-      CCsol=Zero();
-      _CoarseCoarseSolve(CCsrc,CCsol);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
-    t=-usecond();
-    {
-      GRID_TRACE("L2L3-Vcycle - Ext/blockPromote/Ins");
-      for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
-      _Projector.blockPromote(csplit,ccsplit);
-      for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
-      add(out,out,vec1);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
-
-    // residual + coarse smoother (6D coarse)
-    {
-      GRID_TRACE("L2L3-Vcycle - Resid+CoarseSmooth");
-      _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-      vec2=Zero();
-      _CoarseSmoother(vec1,vec2);
-      add(out,out,vec2);
-    }
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle (verbatim from the frozen example)
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
-
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    {
-      GRID_TRACE("L1L2-Vcycle - Resid");
-      for(int r=0;r<nrhs;r++) out[r]=in[r];
-      for(int r=0;r<nrhs;r++){
-	_FineOperator.Op(out[r],vec1[r]);
-	sub(vec1[r],in[r],vec1[r]);
-      }
-    }
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - blockProject");
-      _Projector.blockProject(vec1,Csrc_split);
-      for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - CoarseSolve");
-      CsolMrhs=Zero();
-      _CoarseSolve(CsrcMrhs,CsolMrhs);
-    }
-    t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - ExtractSlice/blockPromote/add delta");
-      for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-      _Projector.blockPromote(vec1,Csol_split);
-      for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-    {
-      GRID_TRACE("L1L2-Vcycle - Residuals");
-      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    }
-    t=-usecond();
-    {
-      GRID_TRACE("L1L2-Vcycle - Smoothers");
-      for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
-    }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -629,7 +355,7 @@ int main (int argc, char ** argv)
     std::cout << GridLogMessage << "**********************************************" << std::endl;
     DenseCC.reset(new DenseCC_t(CoarseCoarse5d));
     DenseCC->Import(LittleDiracOpL2);
-    MrhsDenseCC.reset(new MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector>(*DenseCC, CoarseCoarse5d, nrhs));
+    MrhsDenseCC.reset(new MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector>(*DenseCC, nrhs));
   }
 
   //////////////////////////////////////////////////////////////////////

examples/Example_pvdagm_mrhs_3level_dense.cc

@@ -60,6 +60,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 
 #include <unordered_map>
 #include <memory>
@@ -830,226 +831,6 @@ public:
   }
 };
 
-//////////////////////////////////////////////////////////////////////
-// Dense CC solve on the PACKED 6D mrhs coarse-coarse field: unpack per
-// rhs, ONE batched dense apply, repack.  Drop-in for the L3 PGCR.
-//////////////////////////////////////////////////////////////////////
-template<class CoarseCoarseField>
-class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
-public:
-  DistributedDenseInverse<CoarseCoarseField> &_Dense;
-  GridBase *_CoarseCoarse5d;
-  int _nrhs;
-  MrhsDenseCCSolve(DistributedDenseInverse<CoarseCoarseField> &D, GridBase *cc5d, int nrhs)
-    : _Dense(D), _CoarseCoarse5d(cc5d), _nrhs(nrhs) {}
-  using LinearFunction<CoarseCoarseField>::operator();
-  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
-    if ( getenv("DENSE_CC_CHECK") ) {
-      // Audit path: per-rhs 5D unpack so ApplyBatch can run the _Op defect
-      // check per rhs.  ~50ms/call of slice/split overhead -- audit only.
-      CoarseCoarseField tmp(in.Grid());
-      tmp = in;
-      std::vector<CoarseCoarseField> split_in (_nrhs,_CoarseCoarse5d);
-      std::vector<CoarseCoarseField> split_out(_nrhs,_CoarseCoarse5d);
-      for(int r=0;r<_nrhs;r++) ExtractSliceFast(split_in[r], tmp, r, 0);
-      _Dense.ApplyBatch(split_in, split_out);
-      for(int r=0;r<_nrhs;r++) InsertSliceFast(split_out[r], out, r, 0);
-    } else {
-      _Dense.ApplyBatch6D(in, out, _nrhs);
-    }
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from Example_pvdagm_mrhs.cc)
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;   // caller contract: zero guess => first-cycle r0 = src
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-    p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
-      q[peri_kp]=Az; p[peri_kp]=z;
-      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
-        b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
-        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle: LinearFunction on the 6D mrhs COARSE field.
-// The coarse-coarse solve slot now takes EITHER the dense mrhs solve
-// (DENSE_CC=1) or the L3 PGCR (DENSE_CC=0).
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;          // mrhs coarse op (6D)
-  LinearFunction<CoarseField>              &_CoarseSmoother;    // shifted 6D coarse smoother
-  MultiRHSBlockProject<CoarseField>        &_Projector;         // L2->L3 (vector-based)
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve; // L3 solve (6D cc)
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs; double t;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-
-    // trivial pre-smoother
-    out = in;
-
-    // residual (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
-    std::vector<CoarseField>       csplit(nrhs,_Coarse5d);
-    std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
-    _Projector.blockProject(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
-
-    // L3 solve (dense mrhs GEMM, or PGCR)
-    t=-usecond();
-    CCsol=Zero();
-    _CoarseCoarseSolve(CCsrc,CCsol);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-
-    // prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
-    _Projector.blockPromote(csplit,ccsplit);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
-    add(out,out,vec1);
-    t+=usecond();
-    std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
-
-    // residual + coarse smoother (6D coarse)
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-    vec2=Zero();
-    _CoarseSmoother(vec1,vec2);
-    add(out,out,vec2);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle (verbatim from Example_pvdagm_mrhs.cc)
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    t=-usecond();
-    _Projector.blockProject(vec1,Csrc_split);
-    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
-    t=-usecond(); CsolMrhs=Zero(); _CoarseSolve(CsrcMrhs,CsolMrhs); t+=usecond();
-    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-    t=-usecond();
-    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
-    _Projector.blockPromote(vec1,Csol_split);
-    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
-    for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    t=-usecond();
-    for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
-    t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -1221,13 +1002,13 @@ int main (int argc, char ** argv)
   // its memory peak; needs only the hoisted single-RHS LinOpCC5d).
   //////////////////////////////////////////////////////////////////////
   std::unique_ptr<DistributedDenseInverse<CoarseCoarseVector>> DenseCC;
-  std::unique_ptr<MrhsDenseCCSolve<CoarseCoarseVector>>        MrhsDenseCC;
+  std::unique_ptr<MrhsDenseCCSolve<DistributedDenseInverse<CoarseCoarseVector>,CoarseCoarseVector>>        MrhsDenseCC;
   if (UseDenseCC) {
     std::cout << GridLogMessage << "**********************************************" << std::endl;
     std::cout << GridLogMessage << " Dense CC inverse setup (mrhs bottom)" << std::endl;
     std::cout << GridLogMessage << "**********************************************" << std::endl;
     DenseCC.reset(new DistributedDenseInverse<CoarseCoarseVector>(LinOpCC5d, CoarseCoarse5d, nbasis));
-    MrhsDenseCC.reset(new MrhsDenseCCSolve<CoarseCoarseVector>(*DenseCC, CoarseCoarse5d, nrhs));
+    MrhsDenseCC.reset(new MrhsDenseCCSolve<DistributedDenseInverse<CoarseCoarseVector>,CoarseCoarseVector>(*DenseCC, nrhs));
   }
 
   //////////////////////////////////////////////////////////////////////

examples/Example_pvdagm_multigrid.cc

@@ -33,7 +33,11 @@ Author: Peter Boyle <pboyle@bnl.gov>
 // parameter.  The effective parameters are always printed, so the log
 // describes its own run.
 //
-// Compile-time: -DNBASIS=8 cuts the basis down for laptop runs.
+// Compile-time: NBASIS defaults to 60, the production basis; a subspace file
+// may hold more vectors, the load reads only the first NBASIS.  -DNBASIS=8
+// cuts it down for laptop runs.
+// examples/Makefile.am builds the fp32-coarse and fp32-dense-inversion
+// variants below as their own binaries.
 //
 #include <Grid/Grid.h>
 #include <Grid/lattice/PaddedCell.h>
@@ -48,6 +52,16 @@ using namespace Grid;
 #define NBASIS 60
 #endif
 
+// Precision of the coarse + coarse-coarse sector is a compile-time
+// instantiation: -DCOARSE_SINGLE builds the fp32 coarse space (the dense
+// bottom's apply slab is fp32 either way; its inversion is a configure
+// option).  Both levels carry the same site type iVector<CoarseScalar,NBASIS>.
+#ifdef COARSE_SINGLE
+typedef sTComplexF CoarseScalar_t;
+#else
+typedef sTComplexD CoarseScalar_t;
+#endif
+
 struct PVdagMDriverParams : Serializable {
   GRID_SERIALIZABLE_CLASS_MEMBERS(PVdagMDriverParams,
                                   int,         Ls,
@@ -123,22 +137,46 @@ int main (int argc, char ** argv)
 
   //////////////////////////////////////////////////////////////////////
   // Grids -> coarsening -> dense bottom.  Scope order is lifetime order.
+  //
+  // The fine operator applied during setup is always fp64, on the fp64
+  // basis.  Everything the coarsening produces -- the Galerkin matrix
+  // elements and the transfer operator's store -- takes the coarse precision
+  // (CoarseScalar_t, a compile-time choice).
+  //
+  // There is exactly ONE fine transfer operator.  Its STORE follows the
+  // coarse sector, since the coarse space is what it feeds, while its import
+  // and export accept either fine precision: they are already a layout
+  // transformation, and a scalar conversion inside one is free.  So the fp64
+  // setup and an fp32 V-cycle share the same object and the same basis store.
+  // FinePrecision selects only the fine operator and smoother; the outer
+  // Krylov and its true-residual check stay fp64 throughout.
   //////////////////////////////////////////////////////////////////////
-  typedef PVdagMMultiGridCoarsening<vSpinColourVector,sTComplexD,NBASIS> Coarsening_t;
+  typedef PVdagMMultiGridCoarsening<vSpinColourVector,CoarseScalar_t,NBASIS> Coarsening_t;
+  std::cout << GridLogMessage << "Coarse sector precision (compiled): "
+            << (sizeof(typename GridTypeMapper<CoarseScalar_t>::scalar_type)==sizeof(ComplexF) ? "fp32" : "fp64")
+            << ", nbasis " << NBASIS << std::endl;
 
   MGCoarseGrids CGrids(FGrid, P.MultiGrid.Setup);
-  Coarsening_t  Coarsening(CGrids, P.MultiGrid.Setup);
+  MGFineGridsF  FGridsF(FGrid);
+
+  LatticeGaugeFieldF UmuF(FGridsF.UGridF);
+  precisionChange(UmuF,Umu);
+  MobiusFermionF DdwfF(UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,P.Mass,P.M5,P.MobiusB,P.MobiusC);
+  MobiusFermionF DpvF (UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,1.0,   P.M5,P.MobiusB,P.MobiusC);
+
+  Coarsening_t  Coarsening(CGrids, FGridsF, P.MultiGrid.Setup);
 
   Coarsening.GetSubspace(RNG5, PVdagM);
   Coarsening.Coarsen(PVdagM);
   Coarsening.BuildDenseBottom();
+  Coarsening.CertifyCoarsening(PVdagM);
 
   //////////////////////////////////////////////////////////////////////
   // Solves: mrhs then (optionally) single RHS through the SAME objects.
   //////////////////////////////////////////////////////////////////////
   auto RunSolve = [&](int nr)
   {
-    PVdagMMultiGridSolver<MobiusFermionD,Coarsening_t> Solver(Ddwf,Dpv,Coarsening,P.MultiGrid,nr);
+    PVdagMMultiGridSolver<MobiusFermionD,MobiusFermionF,Coarsening_t> Solver(Ddwf,Dpv,DdwfF,DpvF,Coarsening,P.MultiGrid,nr);
     std::vector<LatticeFermionD> src(nr,FGrid), sol(nr,FGrid);
     for(int r=0;r<nr;r++){ gaussian(RNG5,src[r]); sol[r]=Zero(); }
     Solver.Solve(src,sol);

examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc

@@ -62,6 +62,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #include <Grid/lattice/PaddedCell.h>
 #include <Grid/stencil/GeneralLocalStencil.h>
 #include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
 #include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
 
 #include <memory>
@@ -396,220 +397,6 @@ void PowerIteration(const std::string &name, LinearOperatorBase<Field> &Op, Grid
             << std::endl;
 }
 
-//////////////////////////////////////////////////////////////////////
-// Dense L3 solve on the packed D+1 coarse-coarse field
-//////////////////////////////////////////////////////////////////////
-template<class DenseType, class CoarseCoarseField>
-class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
-public:
-  DenseType &_Dense;
-  int _nrhs;
-  MrhsDenseCCSolve(DenseType &D, int nrhs) : _Dense(D), _nrhs(nrhs) {}
-  using LinearFunction<CoarseCoarseField>::operator();
-  virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
-    _Dense.ApplyBatch6D(in, out, _nrhs);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// mrhs interfaces + single-polynomial mrhs PGCR
-//////////////////////////////////////////////////////////////////////
-template<class Field>
-class MrhsLinearFunction {
-public:
-  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
-};
-
-template<class Field>
-class MrhsPGCRNonHermitian {
-public:
-  RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
-  int ZeroGuess = 0; int FirstCycle = 0;
-  std::string name = "Level 1";
-  LinearOperatorBase<Field> &Linop;
-  MrhsLinearFunction<Field> &Preconditioner;
-  std::function<void(int)> OnStep;      // called with the outer step count after every step (smoother switching)
-  void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
-  void Name(std::string n){ name = n; }
-  void SetZeroGuess(int z){ ZeroGuess=z; }
-  MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
-    : Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
-  static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
-  static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
-  static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
-  void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("MrhsPGCR::vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
-  void operator()(std::vector<Field> &src,std::vector<Field> &psi){
-    RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
-    std::vector<Field> r(nrhs,grid);
-    GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
-    for(int k=0;k<MaxIterations;k++){
-      cp=GCRnStep(src,psi,rsq);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
-      if(cp<rsq){
-        T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
-        RealD tr=vnorm2(r);
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
-                 <<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
-        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
-        return;
-      }
-    }
-    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
-  }
-  RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
-    RealD cp; ComplexD a,b,rq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
-    std::vector<Field> r(nrhs,grid),Az(nrhs,grid);   // Az: restart residual scratch only
-    std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
-    std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
-    std::vector<RealD> qq(mmax);
-    if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
-    else                         { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
-    FirstCycle=0;
-    // p[0]=Prec(r), q[0]=A p[0], produced directly in the history slots (no copies)
-    Preconditioner(r,p[0]); vOp(p[0],q[0]); qq[0]=vnorm2(q[0]); cp=vnorm2(r);
-    for(int k=0;k<nstep;k++){
-      steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
-      if ( OnStep ) OnStep(steps);
-      rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
-      vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
-      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
-      if((k==nstep-1)||(cp<rsq)) return cp;
-      // New direction straight into its history slot: p=Prec(r), q=A p.
-      Preconditioner(r,p[peri_kp]);
-      vOp(p[peri_kp],q[peri_kp]);
-      int northog=((kp)>(mmax-1))?(mmax-1):(kp);
-      MemoryManager::Snapshot(name+" orthog begin step "+std::to_string(steps));
-      {
-	GRID_TRACE("MrhsPGCR orthog");
-        // Classical Gram-Schmidt: all coefficients against the UN-updated new q
-        // (independent, batchable), then apply.  Complex coefficient: the
-        // operator is non-Hermitian, real(<q_j,Aq>) alone left q's non-orthogonal.
-        // Batched per rhs (one fused kernel + one reduction each), the shared
-        // coefficient summed over rhs on the host, ONE GlobalSumVector.
-        std::vector<ComplexD> bcoef(northog,ComplexD(0.0)), part;
-        for(int rr=0;rr<nrhs;rr++){
-          std::vector<const Field*> qwin(northog);
-          for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0); qwin[back]=&q[peri_back][rr]; }
-          rankInnerProductMulti(part,qwin,q[peri_kp][rr]);
-          for(int back=0;back<northog;back++) bcoef[back]+=part[back];
-        }
-        if(northog) grid->GlobalSumVector(&bcoef[0],northog);
-        for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; bcoef[back]=-bcoef[back]/qq[peri_back]; }
-        for(int rr=0;rr<nrhs;rr++){
-          std::vector<const Field*> qwin(northog), pwin(northog);
-          for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; qwin[back]=&q[peri_back][rr]; pwin[back]=&p[peri_back][rr]; }
-          axpyMulti(p[peri_kp][rr],bcoef,pwin);
-          axpyMulti(q[peri_kp][rr],bcoef,qwin);
-        }
-      }
-      qq[peri_kp]=vnorm2(q[peri_kp]);
-      MemoryManager::Snapshot(name+" orthog+vnorm2 end   step "+std::to_string(steps));
-    }
-    GRID_ASSERT(0); return cp;
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L2->L3 mrhs V-cycle on the D+1 coarse field
-//////////////////////////////////////////////////////////////////////
-template<class CoarseField, class CoarseCoarseField>
-class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
-public:
-  LinearOperatorBase<CoarseField>          &_CoarseOp;
-  LinearFunction<CoarseField>              &_CoarseSmoother;
-  MultiRHSBlockProject<CoarseField>        &_Projector;
-  LinearFunction<CoarseCoarseField>        &_CoarseCoarseSolve;
-  GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
-  int _nrhs;
-  MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
-                           LinearFunction<CoarseField> &CoarseSmoother,
-                           MultiRHSBlockProject<CoarseField> &Projector,
-                           LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
-                           GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
-    : _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
-      _CoarseCoarseSolve(CoarseCoarseSolve),
-      _Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
-  using LinearFunction<CoarseField>::operator();
-  virtual void operator()(const CoarseField &in, CoarseField &out) {
-    int nrhs=_nrhs;
-    CoarseField vec1(in.Grid());
-    CoarseField vec2(in.Grid());
-    out = in;
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-
-    // restrict, through the mixed blockProject: D+1 coarse in, D+1 cc out
-    CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
-    CoarseCoarseField CCsol(_CoarseCoarseMrhs);
-    _Projector.blockProject(vec1,CCsrc);
-
-    //    CCsol=Zero(); // Is this necessary?
-    _CoarseCoarseSolve(CCsrc,CCsol);
-
-    _Projector.blockPromote(vec1,CCsol);
-    add(out,out,vec1);
-
-    _CoarseOp.Op(out,vec1);  sub(vec1,in,vec1);
-    //    vec2=Zero(); // Zero guess
-    _CoarseSmoother(vec1,vec2);
-    add(out,out,vec2);
-  }
-};
-
-//////////////////////////////////////////////////////////////////////
-// L1->L2 mrhs V-cycle
-//////////////////////////////////////////////////////////////////////
-template<class FineField, class MrhsCoarseVector, class FineSmoother>
-class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
-public:
-  typedef MrhsCoarseVector CoarseVector;
-  LinearOperatorBase<FineField>   &_FineOperator;
-  FineSmoother                    &_PostSmoother;
-  MultiRHSBlockProject<FineField> &_Projector;
-  LinearFunction<CoarseVector>    &_CoarseSolve;
-  GridBase *_CoarseGrid, *_CoarseGridMrhs;
-  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
-                 MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
-                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
-    : _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
-      _CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
-  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
-    // The whole V-cycle is preconditioner: its fine residuals and the
-    // smoother run with sloppy halos; the caller (the outer Krylov) gets
-    // the exact operator back on exit.
-    GRID_TRACE("MGVcycle");
-    SetFineSloppy(FineSloppyComms);
-    int nrhs=in.size(); GridBase *fgrid=in[0].Grid();
-    std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
-    for(int r=0;r<nrhs;r++) out[r]=in[r];
-    { GRID_TRACE("MGFineResidual");
-      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    }
-    // fine vector -> D+1 coarse, via the mixed blockProject
-    CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
-    { GRID_TRACE("MGProject");
-      _Projector.blockProject(vec1,CsrcMrhs);
-    }
-    CsolMrhs=Zero();
-    { GRID_TRACE("MGCoarseSolve");
-      _CoarseSolve(CsrcMrhs,CsolMrhs);
-    }
-    { GRID_TRACE("MGPromote");
-      _Projector.blockPromote(vec1,CsolMrhs);
-      for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
-    }
-    { GRID_TRACE("MGFineResidual2");
-      for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
-    }
-    { GRID_TRACE("MGPostSmooth");
-      for(int r=0;r<nrhs;r++){
-	//	vec2[r]=Zero();
-	_PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]);
-      }
-    }
-    SetFineSloppy(0);
-  }
-};
 
 int main (int argc, char ** argv)
 {
@@ -1074,6 +861,8 @@ int main (int argc, char ** argv)
     SwitchableSmoother<LatticeFermionD> FineSmootherSlot(SmootherGCR,"Fsmoother GCR");
     MrhsTwoLevelMG<LatticeFermionD,CoarseVector,SwitchableSmoother<LatticeFermionD> >
       ThreeLevelPrecon(PVdagM, FineSmootherSlot, MrhsProjector, L2PGCR, Coarse5d, CMrhs);
+    ThreeLevelPrecon.SetSloppy   = SetFineSloppy;
+    ThreeLevelPrecon.SloppyComms = FineSloppyComms;
 
     MrhsPGCRNonHermitian<LatticeFermionD>
       L1PGCR(OuterTol,1000,PVdagM,ThreeLevelPrecon,OuterMmax,OuterNstep);

examples/Makefile.am

@@ -2,5 +2,23 @@ SUBDIRS = .
 
 include Make.inc
 
+# Precision variants of the two multigrid drivers.  The coarse-sector and
+# dense-inversion precisions are compile-time instantiations, so each is its
+# own binary built from the same source; the fine-level precision is a
+# run-time parameter and needs no variant.  Per-target flags give each
+# variant its own object file.
+bin_PROGRAMS += Example_pvdagm_multigrid_fp32coarse \
+                Example_pvdagm_multigrid_fp32coarse_fp32dense \
+                Example_hdcg_multigrid_fp32coarse
 
+Example_pvdagm_multigrid_fp32coarse_SOURCES  = Example_pvdagm_multigrid.cc
+Example_pvdagm_multigrid_fp32coarse_CPPFLAGS = -DCOARSE_SINGLE
+Example_pvdagm_multigrid_fp32coarse_LDADD    = $(top_builddir)/Grid/libGrid.a
 
+Example_pvdagm_multigrid_fp32coarse_fp32dense_SOURCES  = Example_pvdagm_multigrid.cc
+Example_pvdagm_multigrid_fp32coarse_fp32dense_CPPFLAGS = -DCOARSE_SINGLE -DGRID_DENSE_INVERSE_SINGLE
+Example_pvdagm_multigrid_fp32coarse_fp32dense_LDADD    = $(top_builddir)/Grid/libGrid.a
+
+Example_hdcg_multigrid_fp32coarse_SOURCES  = Example_hdcg_multigrid.cc
+Example_hdcg_multigrid_fp32coarse_CPPFLAGS = -DCOARSE_SINGLE
+Example_hdcg_multigrid_fp32coarse_LDADD    = $(top_builddir)/Grid/libGrid.a

systems/Frontier/pvdagm_multigrid.job

@@ -21,10 +21,14 @@
 # only code path (2D block-cyclic dense inverse, ring-allgather apply,
 # inverseLU big leaves) and are gone from the environment.
 #
-# The binary must be built with -DNBASIS=64 to match the subspace file.
+# The binary is a plain `make` target with the default basis, 60.  The
+# subspace load reads the first 60 vectors of the file.
+# <FinePrecision> selects the arithmetic of the fine level
+# INSIDE the preconditioner (fp64 here; the outer Krylov is fp64 and exact
+# either way).  The precision sweep is in pvdagm_mixed_precision.job.
 ##############################################################################
 
-root=$HOME/PVdagM/Grid/systems/Frontier
+root=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/MGrewrite/Grid/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
 # Paths substituted into the XML below.
@@ -48,6 +52,8 @@ exec numactl -m \$NUMA -N \$NUMA \$*
 EOF
 chmod +x ./select_gpu
 
+# sourceme sets FI_HMEM_ROCR_USE_DMABUF=0 -- the standing avoidance of the CXI
+# NO_TRANSLATION fault at NRHS>=12 (libfabric #12775).
 
 export OMP_NUM_THREADS=7
 ulimit -c 0                             # no 22 GB GPU core dumps
@@ -89,11 +95,13 @@ cat << EOF > params.xml
     <SolveSingleRHS>1</SolveSingleRHS>
     <MultiGrid>
       <Setup>
-        <Block><elem>2</elem><elem>2</elem><elem>3</elem><elem>3</elem></Block>
+        <Block1><elem>2</elem><elem>2</elem><elem>3</elem><elem>3</elem></Block1>
         <Block2><elem>4</elem><elem>4</elem><elem>2</elem><elem>4</elem></Block2>
         <CoarsenBatch>9</CoarsenBatch>
         <SubspaceFile>$SUBSPACE</SubspaceFile>
         <FineSloppyComms>1</FineSloppyComms>
+        <FinePrecision>fp64</FinePrecision>
+        <RetainSubspace>0</RetainSubspace>
       </Setup>
       <FineSmoother>
         <Shift>0.1</Shift><Nstep>6</Nstep><Mmax>4</Mmax>

tests/debug/Test_general_coarse.cc

@@ -40,29 +40,6 @@ using namespace Grid;
 // a second definition here is a duplicate-symbol link error the moment the
 // archive member is pulled in.
 
-///////////////////////
-// Tells little dirac op to use MdagM as the .Op()
-///////////////////////
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void OpDiag (const Field &in, Field &out) {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out){    GRID_ASSERT(0);  };
-  void Op     (const Field &in, Field &out){
-    wrapped.HermOp(in,out);
-  }
-  void AdjOp     (const Field &in, Field &out){
-    wrapped.HermOp(in,out);
-  }
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-  void HermOp(const Field &in, Field &out){
-    wrapped.HermOp(in,out);
-  }
-};
 
 
 int main (int argc, char ** argv)

tests/debug/Test_general_coarse_hdcg.cc

@@ -33,45 +33,6 @@ Author: Peter Boyle <pboyle@bnl.gov>
 using namespace std;
 using namespace Grid;
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
-
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
-
 
 int main (int argc, char ** argv)
 {
@@ -270,15 +231,9 @@ int main (int argc, char ** argv)
   ShiftedHermOpLinearOperator<LatticeFermionD> ShiftedFineHermOp(HermOpEO,MirsShift);
   CGSmoother<LatticeFermionD> CGsmooth(ord,ShiftedFineHermOp) ;
 
-  TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+  MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-    HDCGmrhs(1.0e-8, 500,
+    ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-	     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhs(1.0e-8, 500, FineHermOp, ADEF2, FrbGrid);
-	     CGsmooth,
-	     HPDSolveMrhs,    // Used in M1
-	     HPDSolveMrhs,          // Used in Vstart
-	     MrhsProjector,
-	     MrhsGuesser,
-	     CoarseMrhs);
     
   std::vector<LatticeFermionD> src_mrhs(nrhs,FrbGrid);
   std::vector<LatticeFermionD> res_mrhs(nrhs,FrbGrid);

tests/debug/Test_general_coarse_hdcg_phys.cc

@@ -113,21 +113,6 @@ void LoadBasis(aggregation &Agg, std::string file)
 }
 
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 /*
   void operator() (const Field &in, Field &out) 
   {
@@ -137,25 +122,6 @@ public:
   }
 };
 */
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 int main (int argc, char ** argv)

tests/debug/Test_general_coarse_hdcg_phys48.cc

@@ -155,44 +155,7 @@ void LoadEigenvectors(std::vector<RealD>            &eval,
 #endif
 }
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 int main (int argc, char ** argv)
@@ -408,15 +371,9 @@ int main (int argc, char ** argv)
     MrhsGuesser.ImportEigenBasis(evec,eval);
 
     CGSmoother<LatticeFermionD> CGsmooth(Refineord,ShiftedFineHermOp) ;
-    TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+    MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-      HDCGmrhsRefine(RefineTol, 500,
+    HDCGmrhsRefine_ADEF2(RefineFineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-		     RefineFineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhsRefine(RefineTol, 500, RefineFineHermOp, HDCGmrhsRefine_ADEF2, MrhsProjector.fine_grid);
-		     CGsmooth,
-		     HPDSolveMrhs,    // Used in M1
-		     HPDSolveMrhs,    // Used in Vstart
-		     MrhsProjector,
-		     MrhsGuesser,
-		     CoarseMrhs);
 
     // Reload the first pass aggregates, because we orthogonalised them
     LoadBasis(Aggregates,subspace_file);
@@ -474,15 +431,9 @@ int main (int argc, char ** argv)
   MrhsProjector.Allocate(nbasis,FrbGrid,Coarse5d);
   MrhsProjector.ImportBasis(Aggregates.subspace);
       
-  TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+  MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-    HDCGmrhs(1.0e-8, 500,
+    HDCGmrhs_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-	     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhs(1.0e-8, 500, FineHermOp, HDCGmrhs_ADEF2, MrhsProjector.fine_grid);
-	     CGsmooth,
-	     HPDSolveMrhs,    // Used in M1
-	     HPDSolveMrhs,          // Used in Vstart
-	     MrhsProjector,
-	     MrhsGuesser,
-	     CoarseMrhs);
     
   std::vector<LatticeFermionD> src_mrhs(nrhs,FrbGrid);
   std::vector<LatticeFermionD> res_mrhs(nrhs,FrbGrid);

tests/debug/Test_general_coarse_hdcg_phys48_blockcg.cc

@@ -156,21 +156,6 @@ void LoadEigenvectors(std::vector<RealD>            &eval,
 #endif
 }
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 
 template<class Field> class FixedCGPolynomial : public LinearFunction<Field>
 {
@@ -267,28 +252,6 @@ public:
   }
   
 };
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 
@@ -615,15 +578,9 @@ int main (int argc, char ** argv)
   std::cout << "**************************************"<<std::endl;
   std::cout << "Calling mRHS HDCG"<<std::endl;
   std::cout << "**************************************"<<std::endl;
-  TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+  MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-    HDCGmrhs(1.0e-8, 300,
+    HDCGmrhs_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-	     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhs(1.0e-8, 300, FineHermOp, HDCGmrhs_ADEF2, MrhsProjector.fine_grid);
-	     CGsmooth,
-	     HPDSolveMrhs,    // Used in M1
-	     HPDSolveMrhs,          // Used in Vstart
-	     MrhsProjector,
-	     MrhsGuesser,
-	     CoarseMrhs);
     
   std::vector<LatticeFermionD> src_mrhs(nrhs,FrbGrid);
   std::vector<LatticeFermionD> res_mrhs(nrhs,FrbGrid);
@@ -667,15 +624,9 @@ int main (int argc, char ** argv)
 
   for(auto tol : tols) {
     
-    TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+    MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-      HDCGmrhsSloppy(tol, 500,
+    HDCGmrhsSloppy_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-		     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhsSloppy(tol, 500, FineHermOp, HDCGmrhsSloppy_ADEF2, MrhsProjector.fine_grid);
-		     CGsmooth,
-		     HPDSolveMrhs,    // Used in M1
-		     HPDSolveMrhs,    // Used in Vstart
-		     MrhsProjector,
-		     MrhsGuesser,
-		     CoarseMrhs);
   
     //  Solve again to 10^-5
     for(int r=0;r<nrhs;r++){

tests/debug/Test_general_coarse_hdcg_phys48_lanczos.cc

@@ -156,21 +156,6 @@ void LoadEigenvectors(std::vector<RealD>            &eval,
 #endif
 }
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 
 template<class Field> class FixedCGPolynomial : public LinearFunction<Field>
 {
@@ -267,28 +252,6 @@ public:
   }
   
 };
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 
@@ -614,15 +577,9 @@ int main (int argc, char ** argv)
   std::cout << "**************************************"<<std::endl;
   std::cout << "Calling mRHS HDCG"<<std::endl;
   std::cout << "**************************************"<<std::endl;
-  TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+  MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-    HDCGmrhs(1.0e-8, 300,
+    HDCGmrhs_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-	     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhs(1.0e-8, 300, FineHermOp, HDCGmrhs_ADEF2, MrhsProjector.fine_grid);
-	     CGsmooth,
-	     HPDSolveMrhs,    // Used in M1
-	     HPDSolveMrhs,          // Used in Vstart
-	     MrhsProjector,
-	     MrhsGuesser,
-	     CoarseMrhs);
     
   std::vector<LatticeFermionD> src_mrhs(nrhs,FrbGrid);
   std::vector<LatticeFermionD> res_mrhs(nrhs,FrbGrid);
@@ -647,15 +604,9 @@ int main (int argc, char ** argv)
 
   for(auto tol : tols) {
     
-    TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+    MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-      HDCGmrhsSloppy(tol, 500,
+    HDCGmrhsSloppy_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-		     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhsSloppy(tol, 500, FineHermOp, HDCGmrhsSloppy_ADEF2, MrhsProjector.fine_grid);
-		     CGsmooth,
-		     HPDSolveMrhs,    // Used in M1
-		     HPDSolveMrhs,    // Used in Vstart
-		     MrhsProjector,
-		     MrhsGuesser,
-		     CoarseMrhs);
   
     //  Solve again to 10^-5
     for(int r=0;r<nrhs;r++){

tests/debug/Test_general_coarse_hdcg_phys48_lanczos_subspace.cc

@@ -127,44 +127,7 @@ void LoadEigenvectors(std::vector<RealD>            &eval,
 #endif
 }
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 int main (int argc, char ** argv)

tests/debug/Test_general_coarse_hdcg_phys48_mixed.cc

@@ -100,44 +100,7 @@ void LoadEigenvectors(std::vector<RealD>            &eval,
 #endif
 }
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 int main (int argc, char ** argv)
@@ -348,15 +311,9 @@ int main (int argc, char ** argv)
   MrhsProjector.Allocate(nbasis,FrbGrid,Coarse5d);
   MrhsProjector.ImportBasis(Aggregates.subspace);
       
-  TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+  MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-    HDCGmrhs(1.0e-8, 500,
+    HDCGmrhs_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-	     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhs(1.0e-8, 500, FineHermOp, HDCGmrhs_ADEF2, MrhsProjector.fine_grid);
-	     CGsmooth,
-	     HPDSolveMrhs,    // Used in M1
-	     HPDSolveMrhs,          // Used in Vstart
-	     MrhsProjector,
-	     MrhsGuesser,
-	     CoarseMrhs);
     
   std::vector<LatticeFermionD> src_mrhs(nrhs,FrbGrid);
   std::vector<LatticeFermionD> res_mrhs(nrhs,FrbGrid);

tests/debug/Test_general_coarse_hdcg_phys96_mixed.cc

@@ -100,44 +100,7 @@ void LoadEigenvectors(std::vector<RealD>            &eval,
 #endif
 }
 
-// Want Op in CoarsenOp to call MatPcDagMatPc
-template<class Field>
-class HermOpAdaptor : public LinearOperatorBase<Field>
-{
-  LinearOperatorBase<Field> & wrapped;
-public:
-  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme)  {};
-  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out);  }
-  void HermOp(const Field &in, Field &out)    { wrapped.HermOp(in,out); }
-  void AdjOp     (const Field &in, Field &out){ wrapped.HermOp(in,out);  }
-  void OpDiag (const Field &in, Field &out)                  {    GRID_ASSERT(0);  }
-  void OpDir  (const Field &in, Field &out,int dir,int disp) {    GRID_ASSERT(0);  }
-  void OpDirAll  (const Field &in, std::vector<Field> &out)  {    GRID_ASSERT(0);  };
-  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    GRID_ASSERT(0);  }
-};
 
-template<class Field> class CGSmoother : public LinearFunction<Field>
-{
-public:
-  using LinearFunction<Field>::operator();
-  typedef LinearOperatorBase<Field> FineOperator;
-  FineOperator   & _SmootherOperator;
-  int iters;
-  CGSmoother(int _iters, FineOperator &SmootherOperator) :
-    _SmootherOperator(SmootherOperator),
-    iters(_iters)
-  {
-    std::cout << GridLogMessage<<" Mirs smoother order "<<iters<<std::endl;
-  };
-  void operator() (const Field &in, Field &out) 
-  {
-    ConjugateGradient<Field>  CG(0.0,iters,false); // non-converge is just fine in a smoother
-
-    out=Zero();
-
-    CG(_SmootherOperator,in,out);
-  }
-};
 
 
 int main (int argc, char ** argv)
@@ -356,15 +319,9 @@ int main (int argc, char ** argv)
   CGSmoother<LatticeFermionD> CGsmooth(ord,ShiftedFineHermOp) ;
   
   
-  TwoLevelADEF2mrhs<LatticeFermion,CoarseVector>
+  MrhsADEF2Preconditioner<LatticeFermion,CoarseVector>
-    HDCGmrhs(1.0e-8, 500,
+    HDCGmrhs_ADEF2(FineHermOp, CGsmooth, HPDSolveMrhs, HPDSolveMrhs, MrhsProjector, MrhsGuesser, CoarseMrhs);
-	     FineHermOp,
+  TwoLevelCGmrhs<LatticeFermion> HDCGmrhs(1.0e-8, 500, FineHermOp, HDCGmrhs_ADEF2, MrhsProjector.fine_grid);
-	     CGsmooth,
-	     HPDSolveMrhs,    // Used in M1
-	     HPDSolveMrhs,          // Used in Vstart
-	     MrhsProjector,
-	     MrhsGuesser,
-	     CoarseMrhs);
     
   std::vector<LatticeFermionD> src_mrhs(nrhs,FrbGrid);
   std::vector<LatticeFermionD> res_mrhs(nrhs,FrbGrid);