FOr pvdagm preconditioners if they work

Arm complex instructions on M3/M4 NEON v8.3 and simplify A64FX code paths/broaden.
Grid_vector_types and Simd.h mainly reorg and prep for sComplexD and sComplexF alternat Nsimd=1 types

Grid/algorithms/deflation/MultiRHSBlockProject.h

@@ -61,6 +61,17 @@ public:
   uint64_t coarse_vol;
   uint64_t words;
 
+  ////////////////////////////////////////////////////////////////////////////
+  // Blocking geometry in full local coordinates. Addressing in lSites rather
+  // than (lane,oSite) lets the fine and coarse spaces carry different SIMD
+  // layouts, so an unvectorised coarse space may block a vectorised fine one.
+  ////////////////////////////////////////////////////////////////////////////
+  Coordinate fine_ldimensions;
+  Coordinate coarse_ldimensions;
+  Coordinate block_ldimensions;
+  Coordinate fine_simd;
+  Coordinate coarse_simd;
+
   // Row major layout "C" order:
   // BLAS_V[coarse_vol][nbasis][block_vol][words]
   // BLAS_F[coarse_vol][nrhs][block_vol][words]
@@ -120,9 +131,26 @@ public:
     fine_vol   = fine_grid->lSites();
     coarse_vol = coarse_grid->lSites();
     block_vol = fine_vol/coarse_vol;
-    
+
     words = sizeof(scalar_object)/sizeof(scalar);
 
+    int nd = coarse_grid->_ndimension;
+    GRID_ASSERT(fine_grid->_ndimension == nd);
+
+    fine_ldimensions.resize(nd);
+    coarse_ldimensions.resize(nd);
+    block_ldimensions.resize(nd);
+    fine_simd   = fine_grid->_simd_layout;
+    coarse_simd = coarse_grid->_simd_layout;
+    for(int d=0;d<nd;d++){
+      GRID_ASSERT(fine_grid->_processors[d] == coarse_grid->_processors[d]);
+      fine_ldimensions  [d] = fine_grid->_rdimensions  [d]*fine_grid->_simd_layout  [d];
+      coarse_ldimensions[d] = coarse_grid->_rdimensions[d]*coarse_grid->_simd_layout[d];
+      block_ldimensions [d] = fine_ldimensions[d]/coarse_ldimensions[d];
+      GRID_ASSERT(block_ldimensions[d]*coarse_ldimensions[d] == fine_ldimensions[d]);
+    }
+    GRID_ASSERT(block_vol == fine_vol/coarse_vol);
+
     BLAS_V.resize (fine_vol * words * nbasis );
   }
   void ImportFineGridVectors(std::vector <Field > &vecs, deviceVector<scalar> &blas)
@@ -133,22 +161,16 @@ public:
 
     GRID_ASSERT(vecs[0].Grid()==fine_grid);
 
-    subdivides(coarse_grid,fine_grid); // require they map
-
     int _ndimension = coarse_grid->_ndimension;
-    GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites());
-    
-    Coordinate  block_r      (_ndimension);
-    for(int d=0 ; d<_ndimension;d++){
-      block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
-    }
 
     uint64_t sz = blas.size();
 
     acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
 
     Coordinate fine_rdimensions = fine_grid->_rdimensions;
-    Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate block_l  = block_ldimensions;
+    Coordinate fsimd    = fine_simd;
     int64_t bv= block_vol;
     for(int v=0;v<vecs.size();v++){
 
@@ -162,9 +184,7 @@ public:
 
       // loop over fine sites
       const int Nsimd = vobj::Nsimd();
-      //      std::cout << "sz "<<sz<<std::endl;
-      //      std::cout << "prod "<<Nsimd * coarse_grid->oSites() * block_vol * nvec * words<<std::endl;
-      GRID_ASSERT(sz == Nsimd * coarse_grid->oSites() * block_vol * nvec * words);
+      GRID_ASSERT(sz == coarse_vol * block_vol * nvec * words);
       uint64_t lwords= words; // local variable for copy in to GPU
       accelerator_for(sf,osites,Nsimd,{
 #ifdef GRID_SIMT
@@ -175,27 +195,27 @@ public:
 #endif
 	  // One thread per fine site
 	  Coordinate coor_f(_ndimension);
+	  Coordinate coor_l(_ndimension);
 	  Coordinate coor_b(_ndimension);
 	  Coordinate coor_c(_ndimension);
 
-	  // Fine site to fine coor
+	  // Fine (oSite,lane) to full local coor
 	  Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
+	  Lexicographic::CoorFromIndex(coor_l,lane,fsimd);
+	  for(int d=0;d<_ndimension;d++) coor_f[d] += fine_rdimensions[d]*coor_l[d];
+
+	  for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_l[d];
+	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_l[d];
 
-	  for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
-	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
-	  
 	  int sc;// coarse site
 	  int sb;// block site
-	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
-	  Lexicographic::IndexFromCoor(coor_b,sb,block_r);
+	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
+	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
 
           scalar_object data = extractLane(lane,fineData[sf]);
 
-	  // BLAS layout address calculation
-	  // words * block_vol * nbasis x coarse_vol
-	  // coarse oSite x block vole x lanes
-	  int64_t site = (lane*osites + sc*bv)*nvec
-   	               + v*bv
+	  // BLAS_F[coarse_vol][nvec][block_vol][words]
+	  int64_t site = (sc*nvec + v)*bv
 	               + sb;
 
 	  //	  GRID_ASSERT(site*lwords<sz);
@@ -213,6 +233,318 @@ public:
       //      std::cout << " BlockProjector imported vector"<<v<<std::endl;
     }
   }
+  ////////////////////////////////////////////////////////////////////////////
+  // Import direct from multiRHS fine order, avoiding the unpack to a vector
+  // of single RHS fields.
+  //
+  //   fine mrhs grid  : rhs is dimension 0, undistributed, unvectorised
+  //   Grid  order     : F[fine_vol][nrhs][words]
+  //   BLAS  order     : BLAS_F[lane][coarse_vol][nrhs][block_vol][words]
+  //
+  // The gather of block_vol from fine_vol, and the transpose of nrhs against
+  // block_vol, are the irreducible part: this is not the identity.
+  ////////////////////////////////////////////////////////////////////////////
+  void ImportFineGridMrhsVectors(Field &vec_mrhs, deviceVector<scalar> &blas)
+  {
+    typedef typename Field::vector_object vobj;
+
+    GridBase *fine_mrhs_grid = vec_mrhs.Grid();
+    int _ndimension = coarse_grid->_ndimension;
+
+    GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1);
+    GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1);
+    GRID_ASSERT(fine_mrhs_grid->_processors[0] == 1);
+    for(int d=0;d<_ndimension;d++){
+      GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]);
+      GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]);
+    }
+    int nvec = fine_mrhs_grid->_rdimensions[0];   // nrhs
+
+    uint64_t sz = blas.size();
+    acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
+
+    Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions;
+    Coordinate fine_rdimensions      = fine_grid->_rdimensions;
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate block_l  = block_ldimensions;
+    Coordinate fsimd    = fine_simd;
+    int64_t bv= block_vol;
+
+    autoView( fineData   , vec_mrhs, AcceleratorRead);
+    auto blasData_p  = &blas[0];
+    auto fineData_p  = &fineData[0];
+
+    int64_t osites     = fine_grid->oSites();       // D dimensional
+    int64_t osites_hi  = fine_mrhs_grid->oSites();  // nvec * osites
+
+    const int Nsimd = vobj::Nsimd();
+    GRID_ASSERT(sz == coarse_vol * block_vol * nvec * words);
+    uint64_t lwords= words;
+    int64_t lnvec = nvec;
+
+    accelerator_for(sfr,osites_hi,Nsimd,{
+#ifdef GRID_SIMT
+      {
+	int lane=acceleratorSIMTlane(Nsimd); // buffer lane
+#else
+	for(int lane=0;lane<Nsimd;lane++) {
+#endif
+	  Coordinate coor_hi(_ndimension+1);
+	  Coordinate coor_f(_ndimension);
+	  Coordinate coor_l(_ndimension);
+	  Coordinate coor_b(_ndimension);
+	  Coordinate coor_c(_ndimension);
+
+	  // rhs is dimension 0 of the D+1 grid
+	  Lexicographic::CoorFromIndex(coor_hi,sfr,fine_mrhs_rdimensions);
+	  int v = coor_hi[0];
+	  for(int d=0;d<_ndimension;d++) coor_f[d] = coor_hi[d+1];
+
+	  Lexicographic::CoorFromIndex(coor_l,lane,fsimd);
+	  for(int d=0;d<_ndimension;d++) coor_f[d] += fine_rdimensions[d]*coor_l[d];
+
+	  for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_l[d];
+	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_l[d];
+
+	  int sc;// coarse site
+	  int sb;// block site
+	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
+	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
+
+	  scalar_object data = extractLane(lane,fineData[sfr]);
+
+	  int64_t site = (sc*lnvec + v)*bv
+	               + sb;
+
+	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  *ptr = data;
+#ifdef GRID_SIMT
+      }
+#else
+      }
+#endif
+    });
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Export direct to multiRHS coarse order.
+  //
+  //   Grid  order : C[coarse_vol][nrhs][nbasis]
+  //   BLAS  order : BLAS_C[lane][coarse_vol][nrhs][nbasis]
+  //
+  // At Nsimd()==1 these are the same sequence of addresses.
+  ////////////////////////////////////////////////////////////////////////////
+  template<class vobj>
+  void ExportCoarseGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
+  {
+    typedef typename vobj::scalar_object coarse_scalar_object;
+
+    GridBase *coarse_mrhs_grid = vec_mrhs.Grid();
+    int _ndimension = coarse_grid->_ndimension;
+
+    GRID_ASSERT(coarse_mrhs_grid->_ndimension == _ndimension+1);
+    GRID_ASSERT(coarse_mrhs_grid->_simd_layout[0] == 1);
+    GRID_ASSERT(coarse_mrhs_grid->_processors[0] == 1);
+    for(int d=0;d<_ndimension;d++){
+      GRID_ASSERT(coarse_mrhs_grid->_rdimensions[d+1] == coarse_grid->_rdimensions[d]);
+      GRID_ASSERT(coarse_mrhs_grid->_simd_layout[d+1] == coarse_grid->_simd_layout[d]);
+    }
+    int nvec = coarse_mrhs_grid->_rdimensions[0];   // nrhs
+
+    Coordinate coarse_mrhs_rdimensions = coarse_mrhs_grid->_rdimensions;
+    Coordinate coarse_rdimensions      = coarse_grid->_rdimensions;
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate csimd    = coarse_simd;
+
+    autoView( coarseData   , vec_mrhs, AcceleratorWrite);
+    auto blasData_p    = &blas[0];
+    auto coarseData_p  = &coarseData[0];
+
+    int64_t osites    = coarse_grid->oSites();       // D dimensional
+    int64_t osites_hi = coarse_mrhs_grid->oSites();  // nvec * osites
+
+    const int Nsimd = vobj::Nsimd();
+    uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(cwords==nbasis);
+    int64_t lnvec = nvec;
+
+    accelerator_for(scr,osites_hi,Nsimd,{
+#ifdef GRID_SIMT
+      {
+	int lane=acceleratorSIMTlane(Nsimd); // buffer lane
+#else
+	for(int lane=0;lane<Nsimd;lane++) {
+#endif
+	  Coordinate coor_hi(_ndimension+1);
+	  Coordinate coor_l(_ndimension);
+	  Coordinate coor_c(_ndimension);
+
+	  Lexicographic::CoorFromIndex(coor_hi,scr,coarse_mrhs_rdimensions);
+	  int v = coor_hi[0];
+	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_hi[d+1];
+
+	  Lexicographic::CoorFromIndex(coor_l,lane,csimd);
+	  for(int d=0;d<_ndimension;d++) coor_c[d] += coarse_rdimensions[d]*coor_l[d];
+
+	  int sc;
+	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
+
+	  int64_t blas_site = (sc*lnvec + v)*cwords;
+	  coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	  coarse_scalar_object data = *ptr;
+	  insertLane(lane,coarseData[scr],data);
+#ifdef GRID_SIMT
+      }
+#else
+      }
+#endif
+    });
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Reverse directions: BLAS_F -> fine mrhs field, coarse mrhs field -> BLAS_C
+  ////////////////////////////////////////////////////////////////////////////
+  void ExportFineGridMrhsVectors(Field &vec_mrhs, deviceVector<scalar> &blas)
+  {
+    typedef typename Field::vector_object vobj;
+
+    GridBase *fine_mrhs_grid = vec_mrhs.Grid();
+    int _ndimension = coarse_grid->_ndimension;
+
+    GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1);
+    GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1);
+    for(int d=0;d<_ndimension;d++){
+      GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]);
+      GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]);
+    }
+    int nvec = fine_mrhs_grid->_rdimensions[0];
+
+    Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions;
+    Coordinate fine_rdimensions      = fine_grid->_rdimensions;
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate block_l  = block_ldimensions;
+    Coordinate fsimd    = fine_simd;
+    int64_t bv= block_vol;
+
+    autoView( fineData   , vec_mrhs, AcceleratorWrite);
+    auto blasData_p  = &blas[0];
+    auto fineData_p  = &fineData[0];
+
+    int64_t osites     = fine_grid->oSites();
+    int64_t osites_hi  = fine_mrhs_grid->oSites();
+
+    const int Nsimd = vobj::Nsimd();
+    uint64_t lwords= words;
+    int64_t lnvec = nvec;
+
+    accelerator_for(sfr,osites_hi,Nsimd,{
+#ifdef GRID_SIMT
+      {
+	int lane=acceleratorSIMTlane(Nsimd);
+#else
+	for(int lane=0;lane<Nsimd;lane++) {
+#endif
+	  Coordinate coor_hi(_ndimension+1);
+	  Coordinate coor_f(_ndimension);
+	  Coordinate coor_l(_ndimension);
+	  Coordinate coor_b(_ndimension);
+	  Coordinate coor_c(_ndimension);
+
+	  Lexicographic::CoorFromIndex(coor_hi,sfr,fine_mrhs_rdimensions);
+	  int v = coor_hi[0];
+	  for(int d=0;d<_ndimension;d++) coor_f[d] = coor_hi[d+1];
+
+	  Lexicographic::CoorFromIndex(coor_l,lane,fsimd);
+	  for(int d=0;d<_ndimension;d++) coor_f[d] += fine_rdimensions[d]*coor_l[d];
+
+	  for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_l[d];
+	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_l[d];
+
+	  int sc,sb;
+	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
+	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
+
+	  int64_t site = (sc*lnvec + v)*bv
+	               + sb;
+
+	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
+	  scalar_object data = *ptr;
+	  insertLane(lane,fineData[sfr],data);
+#ifdef GRID_SIMT
+      }
+#else
+      }
+#endif
+    });
+  }
+
+  template<class vobj>
+  void ImportCoarseGridMrhsVectors(Lattice<vobj> &vec_mrhs, deviceVector<scalar> &blas)
+  {
+    typedef typename vobj::scalar_object coarse_scalar_object;
+
+    GridBase *coarse_mrhs_grid = vec_mrhs.Grid();
+    int _ndimension = coarse_grid->_ndimension;
+
+    GRID_ASSERT(coarse_mrhs_grid->_ndimension == _ndimension+1);
+    GRID_ASSERT(coarse_mrhs_grid->_simd_layout[0] == 1);
+    for(int d=0;d<_ndimension;d++){
+      GRID_ASSERT(coarse_mrhs_grid->_rdimensions[d+1] == coarse_grid->_rdimensions[d]);
+      GRID_ASSERT(coarse_mrhs_grid->_simd_layout[d+1] == coarse_grid->_simd_layout[d]);
+    }
+    int nvec = coarse_mrhs_grid->_rdimensions[0];
+
+    Coordinate coarse_mrhs_rdimensions = coarse_mrhs_grid->_rdimensions;
+    Coordinate coarse_rdimensions      = coarse_grid->_rdimensions;
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate csimd    = coarse_simd;
+
+    autoView( coarseData   , vec_mrhs, AcceleratorRead);
+    auto blasData_p    = &blas[0];
+    auto coarseData_p  = &coarseData[0];
+
+    int64_t osites    = coarse_grid->oSites();
+    int64_t osites_hi = coarse_mrhs_grid->oSites();
+
+    const int Nsimd = vobj::Nsimd();
+    uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(cwords==nbasis);
+    int64_t lnvec = nvec;
+
+    accelerator_for(scr,osites_hi,Nsimd,{
+#ifdef GRID_SIMT
+      {
+	int lane=acceleratorSIMTlane(Nsimd);
+#else
+	for(int lane=0;lane<Nsimd;lane++) {
+#endif
+	  Coordinate coor_hi(_ndimension+1);
+	  Coordinate coor_l(_ndimension);
+	  Coordinate coor_c(_ndimension);
+
+	  Lexicographic::CoorFromIndex(coor_hi,scr,coarse_mrhs_rdimensions);
+	  int v = coor_hi[0];
+	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_hi[d+1];
+
+	  Lexicographic::CoorFromIndex(coor_l,lane,csimd);
+	  for(int d=0;d<_ndimension;d++) coor_c[d] += coarse_rdimensions[d]*coor_l[d];
+
+	  int sc;
+	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
+
+	  coarse_scalar_object data = extractLane(lane,coarseData[scr]);
+
+	  int64_t blas_site = (sc*lnvec + v)*cwords;
+	  coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
+	  *ptr = data;
+#ifdef GRID_SIMT
+      }
+#else
+      }
+#endif
+    });
+  }
+
   void ExportFineGridVectors(std::vector <Field> &vecs, deviceVector<scalar> &blas)
   {
     typedef typename Field::vector_object vobj;
@@ -221,17 +553,12 @@ public:
 
     GRID_ASSERT(vecs[0].Grid()==fine_grid);
 
-    subdivides(coarse_grid,fine_grid); // require they map
-
     int _ndimension = coarse_grid->_ndimension;
-    GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites());
-    
-    Coordinate  block_r      (_ndimension);
-    for(int d=0 ; d<_ndimension;d++){
-      block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
-    }
+
     Coordinate fine_rdimensions = fine_grid->_rdimensions;
-    Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate block_l  = block_ldimensions;
+    Coordinate fsimd    = fine_simd;
 
     //    std::cout << " export fine Blas norm "<<blasNorm2(blas)<<std::endl;
 
@@ -259,23 +586,24 @@ public:
 #endif
 	  // One thread per fine site
 	  Coordinate coor_f(_ndimension);
+	  Coordinate coor_l(_ndimension);
 	  Coordinate coor_b(_ndimension);
 	  Coordinate coor_c(_ndimension);
 
 	  Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
+	  Lexicographic::CoorFromIndex(coor_l,lane,fsimd);
+	  for(int d=0;d<_ndimension;d++) coor_f[d] += fine_rdimensions[d]*coor_l[d];
+
+	  for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_l[d];
+	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_l[d];
 
-	  for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
-	  for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
-	  
 	  int sc;
 	  int sb;
-	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
-	  Lexicographic::IndexFromCoor(coor_b,sb,block_r);
+	  Lexicographic::IndexFromCoor(coor_c,sc,coarse_l);
+	  Lexicographic::IndexFromCoor(coor_b,sb,block_l);
 
-	  // BLAS layout address calculation
-	  // words * block_vol * nbasis x coarse_vol 	  
-	  int64_t site = (lane*osites + sc*bv)*nvec
-   	               + v*bv
+	  // BLAS_F[coarse_vol][nvec][block_vol][words]
+	  int64_t site = (sc*nvec + v)*bv
 	               + sb;
 
 	  scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
@@ -306,7 +634,9 @@ public:
     uint64_t sz = blas.size();
 
     Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
-    
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate csimd    = coarse_simd;
+
     for(int v=0;v<vecs.size();v++){
 
       //      std::cout << " BlockProjector importing coarse vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
@@ -330,8 +660,17 @@ public:
 	  for(int lane=0;lane<Nsimd;lane++) {
 #endif
            // C_br per site
-	    int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
-	    
+	    Coordinate coor_c(_ndimension);
+	    Coordinate coor_l(_ndimension);
+	    Lexicographic::CoorFromIndex(coor_c,sc,coarse_rdimensions);
+	    Lexicographic::CoorFromIndex(coor_l,lane,csimd);
+	    for(int d=0;d<_ndimension;d++) coor_c[d] += coarse_rdimensions[d]*coor_l[d];
+
+	    int scl;
+	    Lexicographic::IndexFromCoor(coor_c,scl,coarse_l);
+
+	    int64_t blas_site = (scl*nvec + v)*cwords;
+
 	    coarse_scalar_object data = extractLane(lane,coarseData[sc]);
 
 	    coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
@@ -360,7 +699,9 @@ public:
     uint64_t sz = blas.size();
 
     Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
-    
+    Coordinate coarse_l = coarse_ldimensions;
+    Coordinate csimd    = coarse_simd;
+
     //    std::cout << " export coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
     for(int v=0;v<vecs.size();v++){
 
@@ -385,7 +726,16 @@ public:
 #else
 	  for(int lane=0;lane<Nsimd;lane++) {
 #endif
-	    int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
+	    Coordinate coor_c(_ndimension);
+	    Coordinate coor_l(_ndimension);
+	    Lexicographic::CoorFromIndex(coor_c,sc,coarse_rdimensions);
+	    Lexicographic::CoorFromIndex(coor_l,lane,csimd);
+	    for(int d=0;d<_ndimension;d++) coor_c[d] += coarse_rdimensions[d]*coor_l[d];
+
+	    int scl;
+	    Lexicographic::IndexFromCoor(coor_c,scl,coarse_l);
+
+	    int64_t blas_site = (scl*nvec + v)*cwords;
 	    coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
 	    coarse_scalar_object data = *ptr;
 	    insertLane(lane,coarseData[sc],data);
@@ -508,6 +858,168 @@ public:
     ExportFineGridVectors(fine, BLAS_F);
     //    std::cout << " exported "<<std::endl;
   }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // multiRHS ordered interfaces.  The GEMM is identical; only the import and
+  // export differ, so those are the whole of the layout question.
+  ////////////////////////////////////////////////////////////////////////////
+  template<class cobj>
+  void blockProject(Field &fine_mrhs,Lattice<cobj> &coarse_mrhs)
+  {
+    int nrhs = fine_mrhs.Grid()->_rdimensions[0];
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(nbasis==_nbasis);
+    GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
+
+    BLAS_F.resize (fine_vol * words * nrhs );
+    BLAS_C.resize (coarse_vol * nbasis * nrhs );
+
+    ImportFineGridMrhsVectors(fine_mrhs,BLAS_F);
+    ProjectBLAS(nrhs);
+    ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
+  }
+
+  template<class cobj>
+  void blockPromote(Field &fine_mrhs,Lattice<cobj> &coarse_mrhs)
+  {
+    int nrhs = fine_mrhs.Grid()->_rdimensions[0];
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(nbasis==_nbasis);
+    GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
+
+    BLAS_F.resize (fine_vol * words * nrhs );
+    BLAS_C.resize (coarse_vol * nbasis * nrhs );
+
+    ImportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
+    PromoteBLAS(nrhs);
+    ExportFineGridMrhsVectors(fine_mrhs,BLAS_F);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Mixed orderings. A single RHS fine operator produces a vector of fine
+  // fields with no packing; the coarse side is still wanted in mrhs order.
+  ////////////////////////////////////////////////////////////////////////////
+  template<class cobj>
+  void blockProject(std::vector<Field> &fine,Lattice<cobj> &coarse_mrhs)
+  {
+    int nrhs = fine.size();
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(nbasis==_nbasis);
+    GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
+
+    BLAS_F.resize (fine_vol * words * nrhs );
+    BLAS_C.resize (coarse_vol * nbasis * nrhs );
+
+    ImportFineGridVectors(fine,BLAS_F);
+    ProjectBLAS(nrhs);
+    ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
+  }
+
+  template<class cobj>
+  void blockProject(Field &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
+  {
+    int nrhs = fine_mrhs.Grid()->_rdimensions[0];
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(nbasis==_nbasis);
+    GRID_ASSERT(coarse.size()==nrhs);
+
+    BLAS_F.resize (fine_vol * words * nrhs );
+    BLAS_C.resize (coarse_vol * nbasis * nrhs );
+
+    ImportFineGridMrhsVectors(fine_mrhs,BLAS_F);
+    ProjectBLAS(nrhs);
+    ExportCoarseGridVectors(coarse,BLAS_C);
+  }
+
+  template<class cobj>
+  void blockPromote(std::vector<Field> &fine,Lattice<cobj> &coarse_mrhs)
+  {
+    int nrhs = fine.size();
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(nbasis==_nbasis);
+    GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs);
+
+    BLAS_F.resize (fine_vol * words * nrhs );
+    BLAS_C.resize (coarse_vol * nbasis * nrhs );
+
+    ImportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C);
+    PromoteBLAS(nrhs);
+    ExportFineGridVectors(fine,BLAS_F);
+  }
+
+  template<class cobj>
+  void blockPromote(Field &fine_mrhs,std::vector< Lattice<cobj> > &coarse)
+  {
+    int nrhs = fine_mrhs.Grid()->_rdimensions[0];
+    int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
+    GRID_ASSERT(nbasis==_nbasis);
+    GRID_ASSERT(coarse.size()==nrhs);
+
+    BLAS_F.resize (fine_vol * words * nrhs );
+    BLAS_C.resize (coarse_vol * nbasis * nrhs );
+
+    ImportCoarseGridVectors(coarse,BLAS_C);
+    PromoteBLAS(nrhs);
+    ExportFineGridMrhsVectors(fine_mrhs,BLAS_F);
+  }
+
+  ////////////////////////////////////////////////////////////////////////////
+  // Pointer tables and the batched GEMM, shared by both orderings
+  ////////////////////////////////////////////////////////////////////////////
+  void BLASPointers(int nrhs,
+		    deviceVector<scalar *> &Vd,
+		    deviceVector<scalar *> &Fd,
+		    deviceVector<scalar *> &Cd)
+  {
+    for(int c=0;c<coarse_vol;c++){
+      scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
+      scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
+      scalar * Ch = & BLAS_C[c*nrhs*nbasis];
+      acceleratorPut(Vd[c],Vh);
+      acceleratorPut(Fd[c],Fh);
+      acceleratorPut(Cd[c],Ch);
+    }
+  }
+
+  // C_br = V^dag F
+  void ProjectBLAS(int nrhs)
+  {
+    deviceVector<scalar *> Vd(coarse_vol);
+    deviceVector<scalar *> Fd(coarse_vol);
+    deviceVector<scalar *> Cd(coarse_vol);
+    BLASPointers(nrhs,Vd,Fd,Cd);
+
+    GridBLAS BLAS;
+    int64_t vw = block_vol * words;
+    BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
+		     nbasis,nrhs,vw,
+		     scalar(1.0),
+		     Vd,
+		     Fd,
+		     scalar(0.0),  // wipe out C
+		     Cd);
+    BLAS.synchronise();
+  }
+
+  // F_xr = Vxb Cbr
+  void PromoteBLAS(int nrhs)
+  {
+    deviceVector<scalar *> Vd(coarse_vol);
+    deviceVector<scalar *> Fd(coarse_vol);
+    deviceVector<scalar *> Cd(coarse_vol);
+    BLASPointers(nrhs,Vd,Fd,Cd);
+
+    GridBLAS BLAS;
+    int64_t vw = block_vol * words;
+    BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
+		     vw,nrhs,nbasis,
+		     scalar(1.0),
+		     Vd,
+		     Cd,
+		     scalar(0.0),  // wipe out F
+		     Fd);
+    BLAS.synchronise();
+  }
 };
 
 NAMESPACE_END(Grid);

Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHSV2.h

@@ -0,0 +1,851 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/GeneralCoarsenedMatrixMultiRHS.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+
+NAMESPACE_BEGIN(Grid);
+
+
+// Fine Object == (per site) type of fine field
+// nbasis      == number of deflation vectors
+template<class Fobj,class CComplex,int nbasis>
+class MultiGeneralCoarsenedOperatorV2 : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > >  {
+public:
+  typedef typename CComplex::scalar_object SComplex;
+  typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
+  typedef MultiGeneralCoarsenedOperatorV2<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
+
+  typedef iVector<CComplex,nbasis >           siteVector;
+  typedef iMatrix<CComplex,nbasis >           siteMatrix;
+  typedef iVector<SComplex,nbasis >           calcVector;
+  typedef iMatrix<SComplex,nbasis >           calcMatrix;
+  typedef Lattice<iScalar<CComplex> >         CoarseComplexField;
+  typedef Lattice<siteVector>                 CoarseVector;
+  typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
+  typedef iMatrix<CComplex,nbasis >  Cobj;
+  typedef iVector<CComplex,nbasis >  Cvec;
+  typedef Lattice< CComplex >   CoarseScalar; // used for inner products on fine field
+  typedef Lattice<Fobj >        FineField;
+  typedef CoarseVector Field;
+
+  // Block operations on the fine vectors carry the fine layout, which need
+  // not be the coarse one
+  typedef decltype(innerProduct(Fobj(),Fobj())) FineInner;
+  typedef Lattice<FineInner>                    FineComplexField;
+  typedef Lattice<FineInner>                    BlockComplexField;
+
+  ////////////////////
+  // Data members
+  //
+  // Nrhs independent: the D dimensional coarse grid, the geometry, the padded
+  // cell that supplies the stencil grid, the stencil, and the matrix elements.
+  //
+  // Nrhs dependent: the D+1 grid, its padded cell, and the BLAS B/C buffers
+  // with their pointer tables. Owned by SetNRHS().
+  ////////////////////
+  GridCartesian *       _CoarseGrid;        // D dimensional
+  NonLocalStencilGeometry geom;
+  NonLocalStencilGeometry geom_srhs;
+  PaddedCell CellD;                         // D dimensional, supplies stencil grid
+  GeneralLocalStencil Stencil;              // D dimensional
+
+  int                   _Nrhs;
+  GridCartesian *       _CoarseGridMulti;   // D+1 dimensional, SetNRHS
+  PaddedCell *          CellMulti;          // D+1 dimensional, SetNRHS
+
+  deviceVector<calcVector> BLAS_B;
+  deviceVector<calcVector> BLAS_C;
+  std::vector<deviceVector<calcMatrix> > BLAS_A;
+
+  std::vector<deviceVector<ComplexD *> > BLAS_AP;
+  std::vector<deviceVector<ComplexD *> > BLAS_BP;
+  deviceVector<ComplexD *>               BLAS_CP;
+
+  ///////////////////////
+  // Interface
+  ///////////////////////
+  GridBase      * Grid(void)           { CheckGridSet(); return _CoarseGridMulti; };
+  GridCartesian * CoarseGrid(void)     { CheckGridSet(); return _CoarseGridMulti; };
+  GridCartesian * CoarseGridD(void)    { return _CoarseGrid; };        // lower dimensional grid
+  int             Nrhs(void)           { CheckGridSet(); return _Nrhs; };
+
+  void CheckGridSet(void)
+  {
+    if ( _CoarseGridMulti == nullptr ) {
+      std::cout << GridLogError
+		<< "MultiGeneralCoarsenedOperatorV2: the multiRHS grid has not been set."
+		<< std::endl;
+      std::cout << GridLogError
+		<< "  Call SetGrid(CoarseGridMulti) with the D+1 dimensional grid your"
+		<< std::endl;
+      std::cout << GridLogError
+		<< "  coarse vectors live on, before Grid(), Nrhs() or M()."
+		<< std::endl;
+      GRID_ASSERT(_CoarseGridMulti != nullptr);
+    }
+  }
+
+  // Can be used to do I/O on the operator matrices externally
+  void SetMatrix (int p,CoarseMatrix & A)
+  {
+    GRID_ASSERT(A.size()==geom_srhs.npoint);
+    GridtoBLAS(A[p],BLAS_A[p]);
+  }
+  void GetMatrix (int p,CoarseMatrix & A)
+  {
+    GRID_ASSERT(A.size()==geom_srhs.npoint);
+    BLAStoGrid(A[p],BLAS_A[p]);
+  }
+  void CopyMatrix (GeneralCoarseOp &_Op)
+  {
+    for(int p=0;p<geom.npoint;p++){
+      auto Aup = _Op.Cell.Extract(_Op._A[p]);
+      //Unpadded
+      GridtoBLAS(Aup,BLAS_A[p]);
+    }
+  }
+  /*
+  void CheckMatrix (GeneralCoarseOp &_Op)
+  {
+    std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
+    for(int p=0;p<geom.npoint;p++){
+      //Unpadded
+      auto Aup = _Op.Cell.Extract(_Op._A[p]);
+      auto Ack = Aup;
+      BLAStoGrid(Ack,BLAS_A[p]);
+      std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
+      std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
+    }
+    std::cout <<"************* "<<std::endl;
+  }
+  */
+  
+  ///////////////////////////////////////////////////////////////////////////
+  // Constructor takes the D dimensional coarse grid. Everything built here
+  // is independent of Nrhs, in particular the matrix elements, which must
+  // survive a change of Nrhs untouched.
+  ///////////////////////////////////////////////////////////////////////////
+  MultiGeneralCoarsenedOperatorV2(NonLocalStencilGeometry &_geom,GridCartesian *CoarseGrid) :
+    _CoarseGrid(CoarseGrid),
+    geom_srhs(_geom),
+    geom(CoarseGrid,_geom.hops,_geom.skip),
+    CellD(geom.Depth(),CoarseGrid),
+    Stencil(CellD.grids.back(),geom.shifts), // D dimensional padded cell stencil
+    _Nrhs(-1),
+    _CoarseGridMulti(nullptr),
+    CellMulti(nullptr)
+  {
+    int32_t unpadded_sites = _CoarseGrid->lSites();
+
+    /////////////////////////////////////////////////
+    // Matrix elements and their pointer table
+    /////////////////////////////////////////////////
+    BLAS_A.resize(geom.npoint);
+    BLAS_AP.resize(geom.npoint);
+    for(int p=0;p<geom.npoint;p++){
+      BLAS_A[p].resize (unpadded_sites); // no ghost zone, npoint elements
+      BLAS_AP[p].resize(unpadded_sites);
+    }
+
+    // Site identity mapping for A
+    for(int p=0;p<geom.npoint;p++){
+      for(int ss=0;ss<unpadded_sites;ss++){
+	ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
+	acceleratorPut(BLAS_AP[p][ss],ptr);
+      }
+    }
+  }
+
+  virtual ~MultiGeneralCoarsenedOperatorV2()
+  {
+    ReleaseGrid();
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Free everything SetGrid allocated. The D+1 grid is borrowed from the
+  // caller and is never deleted here. Safe to call repeatedly and before
+  // the destructor.
+  ///////////////////////////////////////////////////////////////////////////
+  void ReleaseGrid(void)
+  {
+    if ( CellMulti != nullptr ) { delete CellMulti; CellMulti = nullptr; }
+
+    _CoarseGridMulti = nullptr;  // borrowed, not owned
+    _Nrhs            = -1;
+
+    BLAS_B.resize(0);
+    BLAS_C.resize(0);
+    for(int p=0;p<BLAS_BP.size();p++){
+      BLAS_BP[p].resize(0);
+    }
+    BLAS_BP.resize(0);
+    BLAS_CP.resize(0);
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Everything that depends on Nrhs. Idempotent; lazy called on demand.
+  //
+  // The stencil lives on the D dimensional padded grid. PaddedCell only pads
+  // a dimension when it is distributed, and the rhs direction never is, so
+  // the D+1 padded grid is exactly Nrhs copies of the D dimensional padded
+  // grid with rhs innermost. The neighbour offset therefore carries an Nrhs
+  // factor, in the same way the Nsimd factor is carried.
+  ///////////////////////////////////////////////////////////////////////////
+  void SetGrid(GridCartesian *CoarseGridMulti)
+  {
+    GRID_ASSERT(CoarseGridMulti != nullptr);
+
+    if ( CoarseGridMulti == _CoarseGridMulti ) return; // idempotent on identity
+
+    ReleaseGrid();
+
+    /////////////////////////////////////////////////
+    // The D+1 grid is supplied and owned by the caller. Two operators over
+    // the same coarse space must share one grid object or their fields will
+    // not conform, so this is never manufactured internally.
+    /////////////////////////////////////////////////
+    int nd = _CoarseGrid->_ndimension;
+
+    GRID_ASSERT(CoarseGridMulti->_ndimension == nd+1);
+    GRID_ASSERT(CoarseGridMulti->_processors[0] == 1);   // rhs is not distributed
+    for(int d=0;d<nd;d++){
+      GRID_ASSERT(CoarseGridMulti->_fdimensions[d+1] == _CoarseGrid->_fdimensions[d]);
+      GRID_ASSERT(CoarseGridMulti->_processors [d+1] == _CoarseGrid->_processors [d]);
+      GRID_ASSERT(CoarseGridMulti->_simd_layout[d+1] == _CoarseGrid->_simd_layout[d]);
+    }
+
+    _CoarseGridMulti = CoarseGridMulti;
+    _Nrhs            = CoarseGridMulti->_fdimensions[0];
+    GRID_ASSERT(_Nrhs>=1);
+
+    int nrhs = _Nrhs;
+
+    CellMulti = new PaddedCell(geom.Depth(),_CoarseGridMulti);
+
+    int32_t padded_sites   = CellD.grids.back()->lSites(); // D dimensional
+    int32_t unpadded_sites = _CoarseGrid->lSites();        // D dimensional
+
+    // The neighbour offset multiplication by nrhs is exact only if the D+1
+    // padded volume is nrhs copies of the D dimensional one. Check it.
+    GRID_ASSERT(CellMulti->grids.back()->lSites() == nrhs*padded_sites);
+    GRID_ASSERT(_CoarseGridMulti->lSites()        == nrhs*unpadded_sites);
+
+    /////////////////////////////////////////////////
+    // Device data vector storage
+    /////////////////////////////////////////////////
+    BLAS_B.resize(nrhs *padded_sites);   // includes ghost zone
+    BLAS_C.resize(nrhs *unpadded_sites); // no ghost zone
+    BLAS_BP.resize(geom.npoint);
+    for(int p=0;p<geom.npoint;p++){
+      BLAS_BP[p].resize(unpadded_sites);
+    }
+    BLAS_CP.resize(unpadded_sites);
+
+    // Site identity mapping for C
+    for(int ss=0;ss<unpadded_sites;ss++){
+      ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
+      acceleratorPut(BLAS_CP[ss],ptr);
+    }
+
+    // Neighbour table is more complicated
+    int32_t j=0; // Interior point counter (unpadded)
+    for(int32_t s=0;s<padded_sites;s++){ // D volume, padded
+      int ghost_zone=0;
+      for(int32_t point = 0 ; point < geom.npoint; point++){
+	int i=s*geom.npoint+point;
+	if( Stencil._entries[i]._wrap ) { // stencil is indexed by the oSite of the D dim grid
+	  ghost_zone=1; // If general stencil wrapped in any direction, wrap=1
+	}
+      }
+
+      if( ghost_zone==0) {
+	for(int32_t point = 0 ; point < geom.npoint; point++){
+	  int i=s*geom.npoint+point;
+ 	  int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite, D dim
+	  nbr = nbr*nrhs;                                              // D -> D+1, rhs innermost
+	  GRID_ASSERT(nbr<BLAS_B.size());
+	  ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
+	  acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
+	}
+	j++;
+      }
+    }
+    GRID_ASSERT(j==unpadded_sites);
+  }
+  template<class vobj> void GridtoBLAS(const Lattice<vobj> &from,deviceVector<typename vobj::scalar_object> &to)
+  {
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  GridBase *Fg = from.Grid();
+  GRID_ASSERT(!Fg->_isCheckerBoarded);
+  int nd = Fg->_ndimension;
+
+  to.resize(Fg->lSites());
+
+  Coordinate LocalLatt = Fg->LocalDimensions();
+  size_t nsite = 1;
+  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
+
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  // do the index calc on the GPU
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  Coordinate f_ostride = Fg->_ostride;
+  Coordinate f_istride = Fg->_istride;
+  Coordinate f_rdimensions = Fg->_rdimensions;
+
+  autoView(from_v,from,AcceleratorRead);
+  auto to_v = &to[0];
+
+  const int words=sizeof(vobj)/sizeof(vector_type);
+  accelerator_for(idx,nsite,1,{
+      
+      Coordinate from_coor, base;
+      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
+      for(int i=0;i<nd;i++){
+	from_coor[i] = base[i];
+      }
+      int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*(from_coor[d]%f_rdimensions[d]);
+      int from_lane = 0; for(int d=0;d<nd;d++) from_lane+=f_istride[d]*(from_coor[d]/f_rdimensions[d]);
+
+      const vector_type* from = (const vector_type *)&from_v[from_oidx];
+      scalar_type* to = (scalar_type *)&to_v[idx];
+      
+      scalar_type stmp;
+      for(int w=0;w<words;w++){
+	stmp = getlane(from[w], from_lane);
+	to[w] = stmp;
+      }
+    });
+  }    
+  template<class vobj> void BLAStoGrid(Lattice<vobj> &grid,deviceVector<typename vobj::scalar_object> &in)
+  {
+  typedef typename vobj::scalar_object sobj;
+  typedef typename vobj::scalar_type scalar_type;
+  typedef typename vobj::vector_type vector_type;
+
+  GridBase *Tg = grid.Grid();
+  GRID_ASSERT(!Tg->_isCheckerBoarded);
+  int nd = Tg->_ndimension;
+  
+  GRID_ASSERT(in.size()==Tg->lSites());
+
+  Coordinate LocalLatt = Tg->LocalDimensions();
+  size_t nsite = 1;
+  for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
+
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  // do the index calc on the GPU
+  ////////////////////////////////////////////////////////////////////////////////////////////////
+  Coordinate t_ostride = Tg->_ostride;
+  Coordinate t_istride = Tg->_istride;
+  Coordinate t_rdimensions = Tg->_rdimensions;
+
+  autoView(to_v,grid,AcceleratorWrite);
+  auto from_v = &in[0];
+
+  const int words=sizeof(vobj)/sizeof(vector_type);
+  accelerator_for(idx,nsite,1,{
+      
+      Coordinate to_coor, base;
+      Lexicographic::CoorFromIndex(base,idx,LocalLatt);
+      for(int i=0;i<nd;i++){
+	to_coor[i] = base[i];
+      }
+      int to_oidx = 0; for(int d=0;d<nd;d++) to_oidx+=t_ostride[d]*(to_coor[d]%t_rdimensions[d]);
+      int to_lane = 0; for(int d=0;d<nd;d++) to_lane+=t_istride[d]*(to_coor[d]/t_rdimensions[d]);
+
+      vector_type* to = (vector_type *)&to_v[to_oidx];
+      scalar_type* from = (scalar_type *)&from_v[idx];
+      
+      scalar_type stmp;
+      for(int w=0;w<words;w++){
+	stmp=from[w];
+	putlane(to[w], stmp, to_lane);
+      }
+    });
+  }
+  ///////////////////////////////////////////////////////////////////////////
+  // Shared by both CoarsenOperator variants
+  //
+  //     conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
+  //       =  \sum_{l in ball}  e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
+  //       =  \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
+  //       = M_{kl} A_ji^{b.b+l}
+  //
+  //     Where q_k = delta_k . (2*M_PI/global_nb[mu])
+  //     Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenFourierMatrix(GridBase *CoarseGrid,Eigen::MatrixXcd &invMkl)
+  {
+    const int npoint = geom_srhs.npoint;
+    Coordinate clatt = CoarseGrid->GlobalDimensions();
+    int Nd = CoarseGrid->Nd();
+
+    Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
+    ComplexD ci(0.0,1.0);
+    for(int k=0;k<npoint;k++){ // Loop over momenta
+      for(int l=0;l<npoint;l++){ // Loop over nbr relative
+	ComplexD phase(0.0,0.0);
+	for(int mu=0;mu<Nd;mu++){
+	  RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
+	  phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
+	}
+	phase=exp(phase*ci);
+	Mkl(k,l) = phase;
+      }
+    }
+    invMkl = Mkl.inverse();
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // blockOrthogonalise and blockZAXPY are block operations on the fine
+  // vectors, using a coarse shaped field only as an index set. They need a
+  // grid carrying the fine SIMD layout, which the coarse space no longer
+  // does. Constructed local to the caller so it cannot be mistaken for the
+  // coarse grid.
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenBlockGridLayout(GridBase *grid,GridBase *CoarseGrid,
+			      Coordinate &latt,Coordinate &simd,Coordinate &mpi)
+  {
+    int nd = CoarseGrid->_ndimension;
+    latt.resize(nd); simd.resize(nd); mpi.resize(nd);
+    for(int d=0;d<nd;d++){
+      latt[d] = CoarseGrid->_fdimensions[d];
+      simd[d] = grid->_simd_layout[d];
+      mpi [d] = CoarseGrid->_processors[d];
+    }
+  }
+
+  // D+1 coarse grid holding the batch, rhs innermost and unvectorised
+  void CoarsenBatchGridLayout(GridBase *CoarseGrid,int batch,
+			      Coordinate &latt,Coordinate &simd,Coordinate &mpi)
+  {
+    latt.resize(1,batch); simd.resize(1,1); mpi.resize(1,1);
+    latt[0]=batch; simd[0]=1; mpi[0]=1;
+    for(int d=0;d<CoarseGrid->_ndimension;d++){
+      latt.push_back(CoarseGrid->_fdimensions[d]);
+      simd.push_back(CoarseGrid->_simd_layout[d]);
+      mpi .push_back(CoarseGrid->_processors[d]);
+    }
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // The Fourier inverse needs the phase in the coarse layout and the basis
+  // phasing needs it in the fine layout; each is built from its own
+  // coordinates rather than transferred.
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenPhases(GridBase *grid,GridBase *CoarseGrid,GridCartesian *BlockGrid,
+		     std::vector<CoarseComplexField> &pha,
+		     std::vector<FineComplexField> &phaF)
+  {
+    const int npoint = geom_srhs.npoint;
+    Coordinate clatt = CoarseGrid->GlobalDimensions();
+    int Nd = CoarseGrid->Nd();
+    ComplexD ci(0.0,1.0);
+
+    typedef typename CComplex::scalar_type SComplex;
+    FineComplexField one(grid); one=SComplex(1.0);
+    FineComplexField zz(grid);  zz = Zero();
+    BlockComplexField pha_blk (BlockGrid);
+    BlockComplexField blk_coor(BlockGrid);
+
+    for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
+      CoarseComplexField coor(CoarseGrid);
+      pha[p] =Zero();
+      pha_blk=Zero();
+      for(int mu=0;mu<Nd;mu++){
+	RealD TwoPiL =  M_PI * 2.0/ clatt[mu];
+	LatticeCoordinate(coor,mu);
+	pha[p]  = pha[p]  + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
+	LatticeCoordinate(blk_coor,mu);
+	pha_blk = pha_blk + (TwoPiL * geom_srhs.shifts[p][mu]) * blk_coor;
+      }
+      pha[p] =exp(pha[p] *ci);
+      pha_blk=exp(pha_blk*ci);
+
+      blockZAXPY(phaF[p],pha_blk,one,zz);
+    }
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Remove the bulk phase from the batch of coarse projections and
+  // accumulate the Fourier inverse into A. Both variants reach here with
+  // TmpProj in the same batch coarse order, so this is shared verbatim.
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenAccumulate(int p,int i0,int nbv,int batch,
+			 Eigen::MatrixXcd &invMkl,
+			 std::vector<CoarseComplexField> &pha,
+			 CoarseComplexField &phaB,
+			 CoarseVector &TmpProj,
+			 std::vector<CoarseMatrix> &_A,
+			 GridBase *CoarseGrid)
+  {
+    typedef typename CComplex::scalar_type SComplex;
+    const int npoint = geom_srhs.npoint;
+
+    for(int b=0;b<batch;b++) InsertSliceFast(pha[p],phaB,b,0);
+    TmpProj = conjugate(phaB)*TmpProj;
+
+    int osites=CoarseGrid->oSites();
+    for(int k=0;k<npoint;k++){
+      SComplex sc(invMkl(p,k).real(),invMkl(p,k).imag());
+      CComplex coef(sc);
+      autoView( A_v  , _A[k], AcceleratorWrite);
+      autoView( TP_v , TmpProj, AcceleratorRead);
+      accelerator_for(sss, osites, 1, {
+	  for(int b=0;b<nbv;b++){
+	    for(int j=0;j<nbasis;j++){
+	      A_v[sss](i0+b,j) = A_v[sss](i0+b,j) + coef*TP_v[b+batch*sss](j);
+	    }
+	  }
+      });
+    }
+  }
+
+  void CoarsenReport(RealD tphase,RealD tphaseBZ,RealD tslice,
+		     RealD tmat,RealD tproj,RealD tinv)
+  {
+    std::cout << GridLogMessage<<"CoarsenOperator phase  "<<tphase<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator slice  "<<tslice <<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator mat    "<<tmat <<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator proj   "<<tproj<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"CoarsenOperator inv    "<<tinv<<" us"<<std::endl;
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Coarsen a NATIVELY multiRHS fine operator.
+  //
+  // linop acts on the D+1 dimensional fine grid FineGridMulti, with the batch
+  // of phased basis vectors carried in the rhs direction. A single RHS
+  // operator can be promoted with MrhsPromotedOperator, but that pays an
+  // ExtractSlice/InsertSlice pair per rhs; prefer the single RHS variant
+  // below in that case.
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
+		       GridCartesian *FineGridMulti,
+		       std::vector<FineField> &Subspace,
+		       GridBase *CoarseGrid)
+  {
+    RealD tproj=0.0, tmat=0.0, tphase=0.0, tphaseBZ=0.0, tslice=0.0, tinv=0.0;
+
+    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs (multiRHS fine operator)"<< std::endl;
+
+    GRID_ASSERT(Subspace.size()==nbasis);
+    GridBase *grid = Subspace[0].Grid();
+
+    GRID_ASSERT(FineGridMulti->_ndimension == grid->_ndimension+1);
+    GRID_ASSERT(FineGridMulti->_processors[0] == 1);
+    for(int d=0;d<grid->_ndimension;d++){
+      GRID_ASSERT(FineGridMulti->_fdimensions[d+1] == grid->_fdimensions[d]);
+      GRID_ASSERT(FineGridMulti->_processors [d+1] == grid->_processors [d]);
+    }
+    int batch = FineGridMulti->_fdimensions[0];
+
+    Coordinate blatt,bsimd,bmpi;
+    CoarsenBlockGridLayout(grid,CoarseGrid,blatt,bsimd,bmpi);
+    GridCartesian BlockGrid(blatt,bsimd,bmpi);
+
+    BlockComplexField InnerProd(&BlockGrid);
+    blockOrthogonalise(InnerProd,Subspace);
+
+    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
+    Projector.Allocate(nbasis,grid,CoarseGrid);
+    Projector.ImportBasis(Subspace);
+
+    const int npoint = geom_srhs.npoint;
+
+    Eigen::MatrixXcd invMkl;
+    CoarsenFourierMatrix(CoarseGrid,invMkl);
+
+    FineField phaV(grid);
+    std::vector<FineComplexField>   phaF(npoint,grid);
+    std::vector<CoarseComplexField> pha (npoint,CoarseGrid);
+
+    tphase=-usecond();
+    CoarsenPhases(grid,CoarseGrid,&BlockGrid,pha,phaF);
+    tphase+=usecond();
+
+    std::vector<CoarseMatrix> _A;
+    _A.resize(npoint,CoarseGrid);
+    for(int k=0;k<npoint;k++) _A[k] = Zero();
+
+    Coordinate cmlatt,cmsimd,cmmpi;
+    CoarsenBatchGridLayout(CoarseGrid,batch,cmlatt,cmsimd,cmmpi);
+    GridCartesian CoarseBatchGrid(cmlatt,cmsimd,cmmpi);
+
+    CoarseVector       TmpProj(&CoarseBatchGrid);
+    CoarseComplexField phaB(&CoarseBatchGrid);
+
+    FineField hi_in (FineGridMulti);
+    FineField hi_out(FineGridMulti);
+    FineField zzF(grid); zzF = Zero();
+
+    for(int i0=0;i0<nbasis;i0+=batch){ // Loop over batches of basis vectors
+
+      int nbv = MIN(batch,nbasis-i0);
+      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i0<<"/"<<nbasis<< std::endl;
+
+      for(int p=0;p<npoint;p++){ // Loop over momenta
+
+	// One phase, applied to the whole batch. Tail slices are zeroed so
+	// the operator never sees undefined data.
+	for(int b=0;b<nbv;b++){
+	  tphaseBZ-=usecond();
+	  phaV = phaF[p]*Subspace[i0+b];
+	  tphaseBZ+=usecond();
+	  tslice-=usecond();
+	  InsertSliceFast(phaV,hi_in,b,0);
+	  tslice+=usecond();
+	}
+	tslice-=usecond();
+	for(int b=nbv;b<batch;b++){
+	  InsertSliceFast(zzF,hi_in,b,0);
+	}
+	tslice+=usecond();
+
+	tmat-=usecond();
+	linop.Op(hi_in,hi_out);
+	tmat+=usecond();
+
+	tproj-=usecond();
+	Projector.blockProject(hi_out,TmpProj);
+	tproj+=usecond();
+
+	tinv-=usecond();
+	CoarsenAccumulate(p,i0,nbv,batch,invMkl,pha,phaB,TmpProj,_A,CoarseGrid);
+	tinv+=usecond();
+      }
+    }
+
+    for(int p=0;p<npoint;p++){
+      GridtoBLAS(_A[p],BLAS_A[p]);
+    }
+    CoarsenReport(tphase,tphaseBZ,tslice,tmat,tproj,tinv);
+  }
+
+  ///////////////////////////////////////////////////////////////////////////
+  // Coarsen a SINGLE RHS fine operator.
+  //
+  // No multiRHS packing: the operator is applied once per phased basis
+  // vector and the batch is assembled on the coarse side by the mixed
+  // blockProject, which takes a vector of fine fields and writes the batch
+  // coarse field the accumulate expects. Only nbv applications per momentum,
+  // so a batch that does not divide nbasis wastes nothing, and the live fine
+  // storage is batch fields rather than two D+1 fields of extent batch.
+  ///////////////////////////////////////////////////////////////////////////
+  void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
+		       std::vector<FineField> &Subspace,
+		       GridBase *CoarseGrid,
+		       int batch)
+  {
+    RealD tproj=0.0, tmat=0.0, tphase=0.0, tphaseBZ=0.0, tslice=0.0, tinv=0.0;
+
+    std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs (single RHS fine operator)"<< std::endl;
+
+    GRID_ASSERT(Subspace.size()==nbasis);
+    GRID_ASSERT(batch>=1);
+    GridBase *grid = Subspace[0].Grid();
+
+    Coordinate blatt,bsimd,bmpi;
+    CoarsenBlockGridLayout(grid,CoarseGrid,blatt,bsimd,bmpi);
+    GridCartesian BlockGrid(blatt,bsimd,bmpi);
+
+    BlockComplexField InnerProd(&BlockGrid);
+    blockOrthogonalise(InnerProd,Subspace);
+
+    MultiRHSBlockProject<Lattice<Fobj> >    Projector;
+    Projector.Allocate(nbasis,grid,CoarseGrid);
+    Projector.ImportBasis(Subspace);
+
+    const int npoint = geom_srhs.npoint;
+
+    Eigen::MatrixXcd invMkl;
+    CoarsenFourierMatrix(CoarseGrid,invMkl);
+
+    FineField phaV(grid);
+    std::vector<FineComplexField>   phaF(npoint,grid);
+    std::vector<CoarseComplexField> pha (npoint,CoarseGrid);
+
+    tphase=-usecond();
+    CoarsenPhases(grid,CoarseGrid,&BlockGrid,pha,phaF);
+    tphase+=usecond();
+
+    std::vector<CoarseMatrix> _A;
+    _A.resize(npoint,CoarseGrid);
+    for(int k=0;k<npoint;k++) _A[k] = Zero();
+
+    Coordinate cmlatt,cmsimd,cmmpi;
+    CoarsenBatchGridLayout(CoarseGrid,batch,cmlatt,cmsimd,cmmpi);
+    GridCartesian CoarseBatchGrid(cmlatt,cmsimd,cmmpi);
+
+    CoarseVector       TmpProj(&CoarseBatchGrid);
+    CoarseComplexField phaB(&CoarseBatchGrid);
+
+    std::vector<FineField> MphaV(batch,grid);
+
+    for(int i0=0;i0<nbasis;i0+=batch){ // Loop over batches of basis vectors
+
+      int nbv = MIN(batch,nbasis-i0);
+      std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i0<<"/"<<nbasis<< std::endl;
+
+      for(int p=0;p<npoint;p++){ // Loop over momenta
+
+	for(int b=0;b<nbv;b++){
+	  tphaseBZ-=usecond();
+	  phaV = phaF[p]*Subspace[i0+b];
+	  tphaseBZ+=usecond();
+	  tmat-=usecond();
+	  linop.Op(phaV,MphaV[b]);
+	  tmat+=usecond();
+	}
+	// The accumulate reads only the first nbv slices, but the projector
+	// sees the whole vector, so the tail must not be undefined.
+	for(int b=nbv;b<batch;b++) MphaV[b] = Zero();
+
+	tproj-=usecond();
+	Projector.blockProject(MphaV,TmpProj);
+	tproj+=usecond();
+
+	tinv-=usecond();
+	CoarsenAccumulate(p,i0,nbv,batch,invMkl,pha,phaB,TmpProj,_A,CoarseGrid);
+	tinv+=usecond();
+      }
+    }
+
+    for(int p=0;p<npoint;p++){
+      GridtoBLAS(_A[p],BLAS_A[p]);
+    }
+    CoarsenReport(tphase,tphaseBZ,tslice,tmat,tproj,tinv);
+  }
+  void Mdag(const CoarseVector &in, CoarseVector &out)
+  {
+    this->M(in,out);
+  }
+  void M (const CoarseVector &in, CoarseVector &out)
+  {
+    //    std::cout << GridLogMessage << "New Mrhs coarse"<<std::endl;
+    conformable(CoarseGrid(),in.Grid());
+    conformable(in.Grid(),out.Grid());
+    out.Checkerboard() = in.Checkerboard();
+
+    RealD t_tot;
+    RealD t_exch;
+    RealD t_GtoB;
+    RealD t_BtoG;
+    RealD t_mult;
+
+    CheckGridSet();
+    if ( in.Grid() != _CoarseGridMulti ) {
+      std::cout << GridLogError
+		<< "MultiGeneralCoarsenedOperatorV2::M called with a field on a"
+		<< std::endl;
+      std::cout << GridLogError
+		<< "  different grid object from the one given to SetGrid(). Two"
+		<< std::endl;
+      std::cout << GridLogError
+		<< "  grids of identical shape do not conform; share one object."
+		<< std::endl;
+      GRID_ASSERT(in.Grid() == _CoarseGridMulti);
+    }
+
+    t_tot=-usecond();
+    CoarseVector tin=in;
+    t_exch=-usecond();
+    CoarseVector pin = CellMulti->ExchangePeriodic(tin); //padded input
+    t_exch+=usecond();
+
+    CoarseVector pout(pin.Grid());
+
+    int npoint = geom.npoint;
+    typedef calcMatrix* Aview;
+    typedef LatticeView<Cvec> Vview;
+      
+    const int Nsimd = CComplex::Nsimd();
+
+    int64_t nrhs  =pin.Grid()->GlobalDimensions()[0];
+    GRID_ASSERT(nrhs>=1);
+
+    RealD flops,bytes;
+    int64_t osites=in.Grid()->oSites(); // unpadded
+    int64_t unpadded_vol = CoarseGrid()->lSites()/nrhs;
+    
+    flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
+    bytes = 1.0*osites*sizeof(siteMatrix)*npoint/pin.Grid()->GlobalDimensions()[0]
+          + 2.0*osites*sizeof(siteVector)*npoint;
+    
+
+    t_GtoB=-usecond();
+    GridtoBLAS(pin,BLAS_B);
+    t_GtoB+=usecond();
+
+    GridBLAS BLAS;
+
+    t_mult=-usecond();
+    for(int p=0;p<geom.npoint;p++){
+      RealD c = 1.0;
+      if (p==0) c = 0.0;
+      ComplexD beta(c);
+
+      BLAS.gemmBatched(nbasis,nrhs,nbasis,
+		       ComplexD(1.0),
+		       BLAS_AP[p], 
+		       BLAS_BP[p], 
+		       ComplexD(c), 
+		       BLAS_CP);
+    }
+    BLAS.synchronise();
+    t_mult+=usecond();
+
+    t_BtoG=-usecond();
+    BLAStoGrid(out,BLAS_C);
+    t_BtoG+=usecond();
+    t_tot+=usecond();
+    /*
+    std::cout << GridLogMessage << "New Mrhs coarse DONE "<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult exch "<<t_exch<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult mult "<<t_mult<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult GtoB  "<<t_GtoB<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult BtoG  "<<t_BtoG<<" us"<<std::endl;
+    std::cout << GridLogMessage<<"Coarse Mult tot  "<<t_tot<<" us"<<std::endl;
+    */
+    //    std::cout << GridLogMessage<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse Kernel flops "<< flops<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse Kernel flop/s "<< flops/t_mult<<" mflop/s"<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse Kernel bytes/s "<< bytes/t_mult/1000<<" GB/s"<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse overall flops/s "<< flops/t_tot<<" mflop/s"<<std::endl;
+    //    std::cout << GridLogMessage<<"Coarse total bytes   "<< bytes/1e6<<" MB"<<std::endl;
+  };
+  virtual  void Mdiag    (const Field &in, Field &out){ GRID_ASSERT(0);};
+  virtual  void Mdir     (const Field &in, Field &out,int dir, int disp){assert(0);};
+  virtual  void MdirAll  (const Field &in, std::vector<Field> &out){assert(0);};
+};
+  
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/MrhsPromotedOperator.h

@@ -0,0 +1,161 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./lib/algorithms/multigrid/MrhsPromotedOperator.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////////////////////////////////////////////
+// Present a D dimensional operator as a D+1 dimensional operator with Nrhs in
+// dimension 0.  Field type is unchanged; only the Grid differs, so this is a
+// LinearOperatorBase and callers need no template parameter: a native mrhs
+// operator derives from the same base and substitutes without a call site
+// change.
+//
+// Slices in and out around the wrapped operator.  No arithmetic beyond the
+// wrapped call, but one ExtractSliceFast/InsertSliceFast pair per right hand
+// side: data motion, not work.
+//
+// AdjOp is carried so that A^dag may be coarsened as a separate coarse
+// operator when needed, rather than doubling coarse storage.
+///////////////////////////////////////////////////////////////////////////////
+template<class Field>
+class MrhsPromotedOperator : public LinearOperatorBase<Field>
+{
+private:
+
+  LinearOperatorBase<Field> &_LinOp;
+  GridBase                  *_LowGrid;
+  int                        _Nrhs;
+
+public:
+
+  MrhsPromotedOperator(LinearOperatorBase<Field> &LinOp,GridBase *LowGrid,int Nrhs)
+    : _LinOp(LinOp), _LowGrid(LowGrid), _Nrhs(Nrhs)
+  {
+    GRID_ASSERT(_Nrhs>=1);
+  }
+
+  GridBase *LowGrid(void) { return _LowGrid; }
+  int       Nrhs(void)    { return _Nrhs;    }
+
+  // Reset on each call; retrieve and accumulate in the caller
+  RealD tslice;
+  RealD top;
+
+  void OpDiag (const Field &in, Field &out)
+  {
+    SliceLoop(in,out,[&](Field &i,Field &o){ _LinOp.OpDiag(i,o); });
+  }
+
+  void Op     (const Field &in, Field &out)
+  {
+    SliceLoop(in,out,[&](Field &i,Field &o){ _LinOp.Op(i,o); });
+  }
+
+  void AdjOp  (const Field &in, Field &out)
+  {
+    SliceLoop(in,out,[&](Field &i,Field &o){ _LinOp.AdjOp(i,o); });
+  }
+
+  void HermOp (const Field &in, Field &out)
+  {
+    SliceLoop(in,out,[&](Field &i,Field &o){ _LinOp.HermOp(i,o); });
+  }
+
+  void OpDir  (const Field &in, Field &out,int dir,int disp)
+  {
+    SliceLoop(in,out,[&](Field &i,Field &o){ _LinOp.OpDir(i,o,dir,disp); });
+  }
+
+  //////////////////////////////////////////////////////////////////
+  // Norms of the D+1 field are the sums over slices
+  //////////////////////////////////////////////////////////////////
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)
+  {
+    Conformable(in,out);
+    Field lo_in (_LowGrid);
+    Field lo_out(_LowGrid);
+    n1=0.0;
+    n2=0.0;
+    for(int r=0;r<_Nrhs;r++){
+      RealD r1,r2;
+      ExtractSliceFast(lo_in,in,r,0);
+      _LinOp.HermOpAndNorm(lo_in,lo_out,r1,r2);
+      InsertSliceFast(lo_out,out,r,0);
+      n1=n1+r1;
+      n2=n2+r2;
+    }
+  }
+
+  void OpDirAll(const Field &in, std::vector<Field> &out)
+  {
+    int npoint = out.size();
+    Field lo_in(_LowGrid);
+    std::vector<Field> lo_out(npoint,_LowGrid);
+    for(int r=0;r<_Nrhs;r++){
+      ExtractSliceFast(lo_in,in,r,0);
+      _LinOp.OpDirAll(lo_in,lo_out);
+      for(int p=0;p<npoint;p++){
+        InsertSliceFast(lo_out[p],out[p],r,0);
+      }
+    }
+  }
+
+private:
+
+  void Conformable(const Field &in,const Field &out)
+  {
+    conformable(in.Grid(),out.Grid());
+    GRID_ASSERT(in.Grid()->_ndimension == _LowGrid->_ndimension+1);
+    GRID_ASSERT(in.Grid()->_fdimensions[0] == _Nrhs);
+  }
+
+  template<class Kernel>
+  void SliceLoop(const Field &in,Field &out,Kernel K)
+  {
+    Conformable(in,out);
+    Field lo_in (_LowGrid);
+    Field lo_out(_LowGrid);
+    tslice=0.0;
+    top=0.0;
+    for(int r=0;r<_Nrhs;r++){
+      tslice-=usecond();
+      ExtractSliceFast(lo_in,in,r,0);
+      tslice+=usecond();
+      top-=usecond();
+      K(lo_in,lo_out);
+      top+=usecond();
+      tslice-=usecond();
+      InsertSliceFast(lo_out,out,r,0);
+      tslice+=usecond();
+    }
+  }
+
+};
+
+NAMESPACE_END(Grid);

Grid/algorithms/multigrid/MultiGrid.h

@@ -32,3 +32,5 @@ Author: Peter Boyle <pboyle@bnl.gov>
 #include <Grid/algorithms/multigrid/CoarsenedMatrix.h>
 #include <Grid/algorithms/multigrid/GeneralCoarsenedMatrix.h>
 #include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHS.h>
+#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHSV2.h>
+#include <Grid/algorithms/multigrid/MrhsPromotedOperator.h>

Grid/algorithms/multigrid/RecursiveSchurInverse.h

@@ -36,34 +36,20 @@ NAMESPACE_BEGIN(Grid);
 // RecursiveSchurInverse: distributed dense inversion by recursive Schur
 // complement over a binary rank-range tree.
 //
-// CONTRACT: the caller presents an N x N matrix in RANK-MAJOR row ordering,
-// distributed by rows -- rank r owns global rows [rowStart[r], rowStart[r+1])
-// -- and receives its rows of the INVERSE in the same layout.  This class
-// knows nothing of lattices or coarse operators; it consumes a GridBase for
-// world collectives, GridBLAS for GEMMs and GridBLASInverse for the leaf
-// inversions (all of which have Eigen reference backends, so the whole
-// algorithm unit-tests on a CPU-only laptop build under mpirun).
+// Contract: rank r owns global rows [rowStart[r], rowStart[r+1]) of an
+// N x N matrix in rank-major ordering, and receives its rows of the
+// inverse in the same layout.  Consumes GridBase collectives, GridBLAS
+// and GridBLASInverse only; Eigen reference backends permit CPU unit
+// testing under mpirun (Test_schur_inverse).
 //
-// PRECISION (decision 2026-08-14, superseding the fp32-merge design): the
-// ENTIRE inversion runs in fp64 (ComplexD).  The apply-side fp32 gain is
-// taken where it matters -- inside the iterative process -- by rounding the
-// finished inverse ONCE when the caller stores it in the fp32 apply slab.
-// Consequences: merge-growth error accumulates in eps64 and the terminal
-// rounding gives representation-only ~eps32 accuracy independent of growth;
-// the Newton-Schulz refinement and the fp32 escalation ladder are DELETED
-// (resurrectable from git history if a future scale forces reduced-precision
-// merges).  Setup cost: ~2x panel-gather bytes and ~2x transient memory,
-// once per setup; fp64 GEMM runs at fp32 rate on CDNA2/PVC.
+// Arithmetic is fp64 throughout; the caller rounds once into fp32 storage.
 //
-// EXECUTION MODEL: SPMD full-tree walk.  Every rank executes the identical
-// recursion call sequence; participation in DATA is ownership-gated, and
-// every collective is a world-communicator zero-fill GlobalSumVector.  No
-// sub-communicators exist, so no deadlock surface exists.
+// Execution: SPMD full-tree walk.  Every rank makes the identical call
+// sequence; data participation is ownership-gated; all collectives are
+// world-wide, so no deadlock surface exists.
 //
-// STORAGE CONVENTION (pinned by unit test T1b, Test_schur_inverse.cc):
-// BlockRows is COLUMN-MAJOR with ld = rows, matching the BLAS world:
-// element (i,j) lives at data[ i + j*ld ]; a column window [col0, col0+w)
-// is the contiguous slice starting at data[ col0*ld ].
+// Storage: BlockRows is column-major, ld = rows; element (i,j) at
+// data[i + j*ld]; a column window is the contiguous slice at data[col0*ld].
 ///////////////////////////////////////////////////////////////////////////////
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -113,14 +99,12 @@ public:
   GridBLAS              BLAS;
   GridBLASInverse       INV;
 
-  // Growth telemetry (diagnostic, not load-bearing at fp64): one entry per
-  // merge node, walk order
+  // Growth telemetry: one entry per merge node, walk order
   std::vector<double>   telNormB;      // ||B||_F = ||A11inv A12||_F
   std::vector<double>   telSratio;     // ||S||_F / ||A22||_F
   double                telLeafMaxInv; // max |(leaf inverse)_ij| over leaves
 
-  // Phase timers/counters (this rank), accumulated across the whole walk:
-  // where does the setup wall actually go?  Reported by ReportTelemetry.
+  // Phase timers/counters, reported by ReportTelemetry
   double                tMemset;       // device panel zero-fill
   double                tDeposit;      // owner rows -> panel (device kernel)
   double                tAllreduce;    // GlobalSumVector on device panels
@@ -132,17 +116,8 @@ public:
   uint64_t              nAllreduce;    // panel collectives
   uint64_t              nGatherGemm;   // GatherGemm calls
 
-  // PERSISTENT DEVICE panel, grow-only across the whole walk.
-  // Persistent: fresh per-call allocations defeat the MPI registration
-  // cache (measured 13% at N=138k).  DEVICE-RESIDENT: the collective runs
-  // on the device pointer via GPU-aware MPI, and panel assembly (zero-fill
-  // + owner deposit) is device-side -- the entire host round-trip (stage
-  // D2H, 92s memset, deposit, H2D at N=138k) is deleted.  Measured
-  // motivation (MPI_benchmark/gather_mpi, Frontier, 16-128 ranks):
-  // host-buffer allreduce 0.8-1.3 GB/s vs device-buffer 5.9-6.8 GB/s;
-  // device allgatherv/ring 20-24 GB/s is the planned stage-2 pattern.
-  // DEVICE BUILDS REQUIRE GPU-aware MPI (MPICH_GPU_SUPPORT_ENABLED=1,
-  // standard in production); CPU builds are unaffected (device == host).
+  // Persistent grow-only device panel; assembly and collectives are
+  // device-resident.  Device builds require GPU-aware MPI.
   deviceVector<ComplexD> dPanelBuf;
 
   ///////////////////////////////////////////////////////////////////////////
@@ -207,7 +182,7 @@ public:
   }
 
   ///////////////////////////////////////////////////////////////////////////
-  // THE communication primitive (plan 3.4 / 4B.3).
+  // The communication primitive.
   //
   //   C(:, colC : colC+widthB) <-   beta  * C(:, colC : colC+widthB)
   //                               + alpha * A(:, colA : colA+widthA) * Bsub
@@ -215,15 +190,12 @@ public:
   // Bsub is the widthA x widthB sub-block of a row-distributed operand
   // owned by ranks [rB0, rB1): owner r contributes its rows of
   // B(:, colB : colB+widthB) at sub-block row offset
-  // rowStart[r] - rowStart[rB0].  The sub-block is gathered in panelBytes
-  // row-chunks, DEVICE-NATIVE: device zero-fill + device deposit kernel +
-  // world GlobalSumVector on the device pointer (GPU-aware MPI on device
-  // builds; see the dPanelBuf comment for the measured motivation).
+  // rowStart[r] - rowStart[rB0], gathered in panelBytes row-chunks by
+  // device zero-fill + deposit kernel + GlobalSumVector.
   //
-  // SPMD rules: EVERY rank calls (the collectives are world-wide);
-  // non-owners of B add zeros; ranks with A.rows == 0 skip all local
-  // compute but still make every collective call.  Column offsets are
-  // LOCAL buffer offsets -- non-participants pass 0.
+  // Every rank calls; non-owners of B add zeros; ranks with A.rows == 0
+  // skip local compute but make every collective call.  Column offsets
+  // are local buffer offsets -- non-participants pass 0.
   ///////////////////////////////////////////////////////////////////////////
   void GatherGemm(ComplexD alpha,
                   BlockRows &A, int64_t colA, int64_t widthA,
@@ -280,6 +252,8 @@ public:
     {
       int64_t kchunk = std::min(kc, k-k0);
 
+      // Zeroing must complete before MPI reads the panel; it is the sole
+      // producer on non-owner ranks.
       tMemset -= usecond();
       acceleratorMemSet(&dPanel[0], 0, (uint64_t)kchunk*n*sizeof(ComplexD));
       tMemset += usecond();
@@ -289,8 +263,7 @@ public:
         int64_t i1 = std::min(k0+kchunk, myOff+B.rows);
         if ( i1 > i0 )
         {
-          // Device-side deposit straight from the B window: strided
-          // block copy as a flat kernel (len rows of n columns).
+          // Deposit my rows: strided block copy, len rows x n columns
           int64_t   len   = i1-i0;
           int64_t   brows = B.rows;
           int64_t   dof   = i0-k0;
@@ -380,11 +353,8 @@ public:
   }
 
   ///////////////////////////////////////////////////////////////////////////
-  // Leaf inversion.  Purely LOCAL -- the calling rank owns the whole
-  // width x width leaf (width == my row count); no collectives, so the
-  // SPMD walk stays uniform with other ranks doing nothing.  The window
-  // is contiguous (ld == rows == width): invert IN PLACE via
-  // GridBLASInverse.  Everything is already fp64; no promote/demote.
+  // Leaf inversion: local, in place on the contiguous diagonal window.
+  // No collectives.
   ///////////////////////////////////////////////////////////////////////////
   void LeafInvert(int64_t col0, int64_t width, BlockRows &Arows)
   {

Grid/json/json.hpp

@@ -21895,7 +21895,7 @@ inline void swap(nlohmann::NLOHMANN_BASIC_JSON_TPL& j1, nlohmann::NLOHMANN_BASIC
 /// @brief user-defined string literal for JSON values
 /// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json/
 JSON_HEDLEY_NON_NULL(1)
-inline nlohmann::json operator "" _json(const char* s, std::size_t n)
+inline nlohmann::json operator ""_json(const char* s, std::size_t n)
 {
     return nlohmann::json::parse(s, s + n);
 }
@@ -21903,7 +21903,7 @@ inline nlohmann::json operator "" _json(const char* s, std::size_t n)
 /// @brief user-defined string literal for JSON pointer
 /// @sa https://json.nlohmann.me/api/basic_json/operator_literal_json_pointer/
 JSON_HEDLEY_NON_NULL(1)
-inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n)
+inline nlohmann::json::json_pointer operator ""_json_pointer(const char* s, std::size_t n)
 {
     return nlohmann::json::json_pointer(std::string(s, n));
 }

Grid/lattice/Lattice_ET.h

@@ -74,7 +74,7 @@ accelerator_inline vobj predicatedWhere(const iobj &predicate,
 
   extract(iftrue, truevals);
   extract(iffalse, falsevals);
-  extract<vInteger, Integer>(TensorRemove(predicate), mask);
+  extract(TensorRemove(predicate), mask);
 
   for (int s = 0; s < Nsimd; s++) {
     if (mask[s]) falsevals[s] = truevals[s];

Grid/lattice/Lattice_comparison.h

@@ -40,16 +40,16 @@ NAMESPACE_BEGIN(Grid);
 //Query supporting logical &&, ||, 
 //////////////////////////////////////////////////////////////////////////
 
-typedef iScalar<vInteger> vPredicate ;
+template<class vobj> using vPredicate = iScalar<IntegerPredicate<vobj> > ;
 
 //////////////////////////////////////////////////////////////////////////
 // compare lattice to lattice
 //////////////////////////////////////////////////////////////////////////
 
 template<class vfunctor,class lobj,class robj>  
-inline Lattice<vPredicate> LLComparison(vfunctor op,const Lattice<lobj> &lhs,const Lattice<robj> &rhs)
+inline Lattice<vPredicate<lobj> > LLComparison(vfunctor op,const Lattice<lobj> &lhs,const Lattice<robj> &rhs)
 {
-  Lattice<vPredicate> ret(rhs.Grid());
+  Lattice<vPredicate<lobj> > ret(rhs.Grid());
   autoView( lhs_v, lhs, CpuRead);
   autoView( rhs_v, rhs, CpuRead);
   autoView( ret_v, ret, CpuWrite);
@@ -62,9 +62,9 @@ inline Lattice<vPredicate> LLComparison(vfunctor op,const Lattice<lobj> &lhs,con
 // compare lattice to scalar
 //////////////////////////////////////////////////////////////////////////
 template<class vfunctor,class lobj,class robj> 
-inline Lattice<vPredicate> LSComparison(vfunctor op,const Lattice<lobj> &lhs,const robj &rhs)
+inline Lattice<vPredicate<lobj> > LSComparison(vfunctor op,const Lattice<lobj> &lhs,const robj &rhs)
 {
-  Lattice<vPredicate> ret(lhs.Grid());
+  Lattice<vPredicate<lobj> > ret(lhs.Grid());
   autoView( lhs_v, lhs, CpuRead);
   autoView( ret_v, ret, CpuWrite);
   thread_for( ss, lhs_v.size(), {
@@ -76,9 +76,9 @@ inline Lattice<vPredicate> LSComparison(vfunctor op,const Lattice<lobj> &lhs,con
 // compare scalar to lattice
 //////////////////////////////////////////////////////////////////////////
 template<class vfunctor,class lobj,class robj> 
-inline Lattice<vPredicate> SLComparison(vfunctor op,const lobj &lhs,const Lattice<robj> &rhs)
+inline Lattice<vPredicate<robj> > SLComparison(vfunctor op,const lobj &lhs,const Lattice<robj> &rhs)
 {
-  Lattice<vPredicate> ret(rhs.Grid());
+  Lattice<vPredicate<robj> > ret(rhs.Grid());
   autoView( rhs_v, rhs, CpuRead);
   autoView( ret_v, ret, CpuWrite);
   thread_for( ss, rhs_v.size(), {
@@ -92,87 +92,87 @@ inline Lattice<vPredicate> SLComparison(vfunctor op,const lobj &lhs,const Lattic
 //////////////////////////////////////////////////////////////////////////
 // Less than
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator < (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<lobj> > operator < (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
   return LLComparison(vlt<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator < (const Lattice<lobj> & lhs, const robj & rhs) {
+inline Lattice<vPredicate<lobj> > operator < (const Lattice<lobj> & lhs, const robj & rhs) {
   return LSComparison(vlt<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator < (const lobj & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<robj> > operator < (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(vlt<lobj,robj>(),lhs,rhs);
 }
   
 // Less than equal
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator <= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<lobj> > operator <= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
   return LLComparison(vle<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator <= (const Lattice<lobj> & lhs, const robj & rhs) {
+inline Lattice<vPredicate<lobj> > operator <= (const Lattice<lobj> & lhs, const robj & rhs) {
   return LSComparison(vle<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator <= (const lobj & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<robj> > operator <= (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(vle<lobj,robj>(),lhs,rhs);
 }
   
 // Greater than 
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator > (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<lobj> > operator > (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
   return LLComparison(vgt<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator > (const Lattice<lobj> & lhs, const robj & rhs) {
+inline Lattice<vPredicate<lobj> > operator > (const Lattice<lobj> & lhs, const robj & rhs) {
   return LSComparison(vgt<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator > (const lobj & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<robj> > operator > (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(vgt<lobj,robj>(),lhs,rhs);
 }
   
   
 // Greater than equal
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator >= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<lobj> > operator >= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
   return LLComparison(vge<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator >= (const Lattice<lobj> & lhs, const robj & rhs) {
+inline Lattice<vPredicate<lobj> > operator >= (const Lattice<lobj> & lhs, const robj & rhs) {
   return LSComparison(vge<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator >= (const lobj & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<robj> > operator >= (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(vge<lobj,robj>(),lhs,rhs);
 }
    
 // equal
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator == (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<lobj> > operator == (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
   return LLComparison(veq<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator == (const Lattice<lobj> & lhs, const robj & rhs) {
+inline Lattice<vPredicate<lobj> > operator == (const Lattice<lobj> & lhs, const robj & rhs) {
   return LSComparison(veq<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator == (const lobj & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<robj> > operator == (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(veq<lobj,robj>(),lhs,rhs);
 }
    
    
 // not equal
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator != (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<lobj> > operator != (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
   return LLComparison(vne<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator != (const Lattice<lobj> & lhs, const robj & rhs) {
+inline Lattice<vPredicate<lobj> > operator != (const Lattice<lobj> & lhs, const robj & rhs) {
   return LSComparison(vne<lobj,robj>(),lhs,rhs);
 }
 template<class lobj,class robj>
-inline Lattice<vPredicate> operator != (const lobj & lhs, const Lattice<robj> & rhs) {
+inline Lattice<vPredicate<robj> > operator != (const lobj & lhs, const Lattice<robj> & rhs) {
   return SLComparison(vne<lobj,robj>(),lhs,rhs);
 }
 NAMESPACE_END(Grid);

Grid/lattice/Lattice_comparison_utils.h

@@ -39,47 +39,53 @@ NAMESPACE_BEGIN(Grid);
   //
   // Should guard this with isGridTensor<> enable if?
   /////////////////////////////////////////
+  //
+  // Predicate type matching the operand's vectorisation
+  //
+  template<class obj> using IntegerPredicate =
+    typename GridTypeMapper<typename GridTypeMapper<obj>::vector_type>::Integerified;
+
   //
   // Generic list of functors
   //
   template<class lobj,class robj> class veq {
   public:
-    accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
+    accelerator IntegerPredicate<lobj> operator()(const lobj &lhs, const robj &rhs)
     { 
       return (lhs) == (rhs);
     }
   };
   template<class lobj,class robj> class vne {
   public:
-    accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
+    accelerator IntegerPredicate<lobj> operator()(const lobj &lhs, const robj &rhs)
     { 
       return (lhs) != (rhs);
     }
   };
   template<class lobj,class robj> class vlt {
   public:
-    accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
+    accelerator IntegerPredicate<lobj> operator()(const lobj &lhs, const robj &rhs)
     { 
       return (lhs) < (rhs);
     }
   };
   template<class lobj,class robj> class vle {
   public:
-    accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
+    accelerator IntegerPredicate<lobj> operator()(const lobj &lhs, const robj &rhs)
     { 
       return (lhs) <= (rhs);
     }
   };
   template<class lobj,class robj> class vgt {
   public:
-    accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
+    accelerator IntegerPredicate<lobj> operator()(const lobj &lhs, const robj &rhs)
     { 
       return (lhs) > (rhs);
     }
   };
   template<class lobj,class robj> class vge {
     public:
-    accelerator vInteger operator()(const lobj &lhs, const robj &rhs)
+    accelerator IntegerPredicate<lobj> operator()(const lobj &lhs, const robj &rhs)
     { 
       return (lhs) >= (rhs);
     }
@@ -133,78 +139,78 @@ NAMESPACE_BEGIN(Grid);
   // Integer and real get extra relational functions.
   //////////////////////////////////////////////////////////////////////////////////////////////////////
   template<class sfunctor, class vsimd,IfNotComplex<vsimd> = 0> 
-    accelerator_inline vInteger Comparison(sfunctor sop,const vsimd & lhs, const vsimd & rhs)
+    accelerator_inline IntegerPredicate<vsimd> Comparison(sfunctor sop,const vsimd & lhs, const vsimd & rhs)
     {
       typedef typename vsimd::scalar_type scalar;
       ExtractBuffer<scalar> vlhs(vsimd::Nsimd());   // Use functors to reduce this to single implementation
       ExtractBuffer<scalar> vrhs(vsimd::Nsimd());
       ExtractBuffer<Integer> vpred(vsimd::Nsimd());
-      vInteger ret;
+      IntegerPredicate<vsimd> ret;
       extract<vsimd,scalar>(lhs,vlhs);
       extract<vsimd,scalar>(rhs,vrhs);
       for(int s=0;s<vsimd::Nsimd();s++){
 	vpred[s] = sop(vlhs[s],vrhs[s]);
       }
-      merge<vInteger,Integer>(ret,vpred);
+      merge<IntegerPredicate<vsimd>,Integer>(ret,vpred);
       return ret;
     }
 
   template<class sfunctor, class vsimd,IfNotComplex<vsimd> = 0> 
-    accelerator_inline vInteger Comparison(sfunctor sop,const vsimd & lhs, const typename vsimd::scalar_type & rhs)
+    accelerator_inline IntegerPredicate<vsimd> Comparison(sfunctor sop,const vsimd & lhs, const typename vsimd::scalar_type & rhs)
     {
       typedef typename vsimd::scalar_type scalar;
       ExtractBuffer<scalar> vlhs(vsimd::Nsimd());   // Use functors to reduce this to single implementation
       ExtractBuffer<Integer> vpred(vsimd::Nsimd());
-      vInteger ret;
+      IntegerPredicate<vsimd> ret;
       extract<vsimd,scalar>(lhs,vlhs);
       for(int s=0;s<vsimd::Nsimd();s++){
 	vpred[s] = sop(vlhs[s],rhs);
       }
-      merge<vInteger,Integer>(ret,vpred);
+      merge<IntegerPredicate<vsimd>,Integer>(ret,vpred);
       return ret;
     }
 
   template<class sfunctor, class vsimd,IfNotComplex<vsimd> = 0> 
-    accelerator_inline vInteger Comparison(sfunctor sop,const typename vsimd::scalar_type & lhs, const vsimd & rhs)
+    accelerator_inline IntegerPredicate<vsimd> Comparison(sfunctor sop,const typename vsimd::scalar_type & lhs, const vsimd & rhs)
     {
       typedef typename vsimd::scalar_type scalar;
       ExtractBuffer<scalar> vrhs(vsimd::Nsimd());   // Use functors to reduce this to single implementation
       ExtractBuffer<Integer> vpred(vsimd::Nsimd());
-      vInteger ret;
+      IntegerPredicate<vsimd> ret;
       extract<vsimd,scalar>(rhs,vrhs);
       for(int s=0;s<vsimd::Nsimd();s++){
 	vpred[s] = sop(lhs,vrhs[s]);
       }
-      merge<vInteger,Integer>(ret,vpred);
+      merge<IntegerPredicate<vsimd>,Integer>(ret,vpred);
       return ret;
     }
 
 #define DECLARE_RELATIONAL_EQ(op,functor) \
   template<class vsimd,IfSimd<vsimd> = 0>\
-    accelerator_inline vInteger operator op (const vsimd & lhs, const vsimd & rhs)\
+    accelerator_inline IntegerPredicate<vsimd> operator op (const vsimd & lhs, const vsimd & rhs)\
     {\
       typedef typename vsimd::scalar_type scalar;\
       return Comparison(functor<scalar,scalar>(),lhs,rhs);\
     }\
   template<class vsimd,IfSimd<vsimd> = 0>\
-    accelerator_inline vInteger operator op (const vsimd & lhs, const typename vsimd::scalar_type & rhs) \
+    accelerator_inline IntegerPredicate<vsimd> operator op (const vsimd & lhs, const typename vsimd::scalar_type & rhs) \
     {\
       typedef typename vsimd::scalar_type scalar;\
       return Comparison(functor<scalar,scalar>(),lhs,rhs);\
     }\
   template<class vsimd,IfSimd<vsimd> = 0>\
-    accelerator_inline vInteger operator op (const typename vsimd::scalar_type & lhs, const vsimd & rhs) \
+    accelerator_inline IntegerPredicate<vsimd> operator op (const typename vsimd::scalar_type & lhs, const vsimd & rhs) \
     {\
       typedef typename vsimd::scalar_type scalar;\
       return Comparison(functor<scalar,scalar>(),lhs,rhs);\
     }\
   template<class vsimd>\
-    accelerator_inline vInteger operator op(const iScalar<vsimd> &lhs,const typename vsimd::scalar_type &rhs) \
+    accelerator_inline IntegerPredicate<vsimd> operator op(const iScalar<vsimd> &lhs,const typename vsimd::scalar_type &rhs) \
     {									\
       return lhs._internal op rhs;					\
     }									\
   template<class vsimd>\
-    accelerator_inline vInteger operator op(const typename vsimd::scalar_type &lhs,const iScalar<vsimd> &rhs) \
+    accelerator_inline IntegerPredicate<vsimd> operator op(const typename vsimd::scalar_type &lhs,const iScalar<vsimd> &rhs) \
     {									\
       return lhs op rhs._internal;					\
     }									\
@@ -212,7 +218,7 @@ NAMESPACE_BEGIN(Grid);
 #define DECLARE_RELATIONAL(op,functor) \
   DECLARE_RELATIONAL_EQ(op,functor)    \
   template<class vsimd>\
-    accelerator_inline vInteger operator op(const iScalar<vsimd> &lhs,const iScalar<vsimd> &rhs)\
+    accelerator_inline IntegerPredicate<vsimd> operator op(const iScalar<vsimd> &lhs,const iScalar<vsimd> &rhs)\
     {									\
       return lhs._internal op rhs._internal;				\
     }									

Grid/parallelIO/BinaryIO.h

@@ -262,6 +262,16 @@ class BinaryIO {
   static const int BINARYIO_READ          = 0x02;
   static const int BINARYIO_WRITE         = 0x01;
 
+  // Single point of control for the aggregated path. Setting
+  // GRID_BINARYIO_NOAGGREGATE falls back to plain lexicographic I/O.
+  static int DefaultControl(void)
+  {
+    static int ctrl = getenv("GRID_BINARYIO_NOAGGREGATE")
+                    ? BINARYIO_LEXICOGRAPHIC
+                    : BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE;
+    return ctrl;
+  }
+
 #ifdef USE_MPI_IO
   /////////////////////////////////////////////////////////////////////////////
   // Aggregation: self controlled transposition onto an I/O friendly layout.
@@ -989,7 +999,7 @@ class BinaryIO {
 				       uint32_t &nersc_csum,
 				       uint32_t &scidac_csuma,
 				       uint32_t &scidac_csumb,
-				       int control=BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE
+				       int control=DefaultControl()
 				       )
   {
     typedef typename vobj::scalar_object sobj;
@@ -1028,7 +1038,7 @@ class BinaryIO {
 					  uint32_t &nersc_csum,
 					  uint32_t &scidac_csuma,
 					  uint32_t &scidac_csumb,
-					  int control=BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE)
+					  int control=DefaultControl())
   {
     typedef typename vobj::scalar_object sobj;
     typedef typename vobj::Realified::scalar_type word;    word w=0;
@@ -1115,7 +1125,7 @@ class BinaryIO {
     std::cout << GridLogMessage << "RNG read I/O on file " << file << std::endl;
 
     std::vector<RNGstate> iodata(lsites);
-    IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE,
+    IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|DefaultControl(),
 	     nersc_csum,scidac_csuma,scidac_csumb);
 
     timer.Start();
@@ -1194,7 +1204,7 @@ class BinaryIO {
     });
     timer.Stop();
 
-    IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC|BINARYIO_AGGREGATE,
+    IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|DefaultControl(),
 	     nersc_csum,scidac_csuma,scidac_csumb);
     iodata.resize(1);
     {

Grid/parallelIO/MetaData.h

@@ -180,7 +180,7 @@ template<class Impl>
 class GaugeStatistics
 {
 public:
-  void operator()(Lattice<vLorentzColourMatrixD> & data,FieldMetaData &header)
+  void operator()(typename Impl::Field & data,FieldMetaData &header)
   {
     header.link_trace = WilsonLoops<Impl>::linkTrace(data);
     header.plaquette  = WilsonLoops<Impl>::avgPlaquette(data);
@@ -188,13 +188,24 @@ public:
 };
 typedef GaugeStatistics<PeriodicGimplD> PeriodicGaugeStatistics;
 typedef GaugeStatistics<ConjugateGimplD> ConjugateGaugeStatistics;
+typedef GaugeStatistics<lexPeriodicGimplD> lexPeriodicGaugeStatistics;
+typedef GaugeStatistics<lexConjugateGimplD> lexConjugateGaugeStatistics;
 template<> inline void PrepareMetaData<vLorentzColourMatrixD>(Lattice<vLorentzColourMatrixD> & field, FieldMetaData &header)
 {
   GridBase *grid = field.Grid();
   std::string format = getFormatString<vLorentzColourMatrixD>();
   header.floating_point = format;
   header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac
-  GridMetaData(grid,header); 
+  GridMetaData(grid,header);
+  MachineCharacteristics(header);
+}
+template<> inline void PrepareMetaData<sLorentzColourMatrixD>(Lattice<sLorentzColourMatrixD> & field, FieldMetaData &header)
+{
+  GridBase *grid = field.Grid();
+  std::string format = getFormatString<sLorentzColourMatrixD>();
+  header.floating_point = format;
+  header.checksum = 0x0; // Nersc checksum unused in ILDG, Scidac
+  GridMetaData(grid,header);
   MachineCharacteristics(header);
 }
 

Grid/parallelIO/NerscIO.h

@@ -136,12 +136,14 @@ public:
   // Now the meat: the object readers
   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-  template<class GaugeStats=PeriodicGaugeStatistics>
-  static inline void readConfiguration(GaugeField &Umu,
-				       FieldMetaData& header,
-				       std::string file,
-				       GaugeStats GaugeStatisticsCalculator=GaugeStats())
+  template<class GaugeFieldType,class GaugeStats>
+  static inline void readConfigurationInternal(GaugeFieldType &Umu,
+					       FieldMetaData& header,
+					       std::string file,
+					       GaugeStats GaugeStatisticsCalculator)
   {
+    typedef typename GaugeFieldType::vector_object vobj;
+    typedef typename GaugeFieldType::scalar_object sobj;
 
     GridBase *grid = Umu.Grid();
     uint64_t offset = readHeader(file,Umu.Grid(),header);
@@ -162,24 +164,24 @@ public:
     const std::string stNC = std::to_string( Nc ) ;
     if ( header.data_type == std::string("4D_SU"+stNC+"_GAUGE") ) {
       if ( ieee32 || ieee32big ) {
-	BinaryIO::readLatticeObject<vLorentzColourMatrixD, LorentzColour2x3F> 
-	  (Umu,file,Gauge3x2munger<LorentzColour2x3F,LorentzColourMatrix>(), offset,format,
+	BinaryIO::readLatticeObject<vobj, LorentzColour2x3F> 
+	  (Umu,file,Gauge3x2munger<LorentzColour2x3F,sobj>(), offset,format,
 	   nersc_csum,scidac_csuma,scidac_csumb);
       }
       if ( ieee64 || ieee64big ) {
-	BinaryIO::readLatticeObject<vLorentzColourMatrixD, LorentzColour2x3D> 
-	  (Umu,file,Gauge3x2munger<LorentzColour2x3D,LorentzColourMatrix>(),offset,format,
+	BinaryIO::readLatticeObject<vobj, LorentzColour2x3D> 
+	  (Umu,file,Gauge3x2munger<LorentzColour2x3D,sobj>(),offset,format,
 	   nersc_csum,scidac_csuma,scidac_csumb);
       }
     } else if ( header.data_type == std::string("4D_SU"+stNC+"_GAUGE_"+stNC+"x"+stNC) ) {
       if ( ieee32 || ieee32big ) {
-	BinaryIO::readLatticeObject<vLorentzColourMatrixD,LorentzColourMatrixF>
-	  (Umu,file,GaugeSimpleMunger<LorentzColourMatrixF,LorentzColourMatrix>(),offset,format,
+	BinaryIO::readLatticeObject<vobj,LorentzColourMatrixF>
+	  (Umu,file,GaugeSimpleMunger<LorentzColourMatrixF,sobj>(),offset,format,
 	   nersc_csum,scidac_csuma,scidac_csumb);
       }
       if ( ieee64 || ieee64big ) {
-	BinaryIO::readLatticeObject<vLorentzColourMatrixD,LorentzColourMatrixD>
-	  (Umu,file,GaugeSimpleMunger<LorentzColourMatrixD,LorentzColourMatrix>(),offset,format,
+	BinaryIO::readLatticeObject<vobj,LorentzColourMatrixD>
+	  (Umu,file,GaugeSimpleMunger<LorentzColourMatrixD,sobj>(),offset,format,
 	   nersc_csum,scidac_csuma,scidac_csumb);
       }
     } else {
@@ -198,12 +200,11 @@ public:
     if ( fabs(clone.plaquette -header.plaquette ) >=  1.0e-5 ) { 
       std::cout << " Plaquette mismatch "<<std::endl;
     }
-    if ( nersc_csum != header.checksum ) { 
+    if ( nersc_csum != header.checksum ) {
       std::cerr << " checksum mismatch " << std::endl;
       std::cerr << " plaqs " << clone.plaquette << " " << header.plaquette << std::endl;
       std::cerr << " trace " << clone.link_trace<< " " << header.link_trace<< std::endl;
       std::cerr << " nersc_csum  " <<std::hex<< nersc_csum << " " << header.checksum<< std::dec<< std::endl;
-      exit(0);
     }
     if(exitOnReadPlaquetteMismatch()) GRID_ASSERT(fabs(clone.plaquette -header.plaquette ) < 1.0e-5 );
     GRID_ASSERT(fabs(clone.link_trace-header.link_trace) < 1.0e-6 );
@@ -212,6 +213,26 @@ public:
     std::cout<<GridLogMessage <<"NERSC Configuration "<<file<< " and plaquette, link trace, and checksum agree"<<std::endl;
   }
 
+  //////////////////////////////////////////////////////////////////////////
+  // Constrained frontends: vectorised and lexicographic gauge fields
+  //////////////////////////////////////////////////////////////////////////
+  template<class GaugeStats=PeriodicGaugeStatistics>
+  static inline void readConfiguration(Lattice<vLorentzColourMatrixD> &Umu,
+				       FieldMetaData& header,
+				       std::string file,
+				       GaugeStats GaugeStatisticsCalculator=GaugeStats())
+  {
+    readConfigurationInternal(Umu,header,file,GaugeStatisticsCalculator);
+  }
+  template<class GaugeStats=lexPeriodicGaugeStatistics>
+  static inline void readConfiguration(Lattice<sLorentzColourMatrixD> &Umu,
+				       FieldMetaData& header,
+				       std::string file,
+				       GaugeStats GaugeStatisticsCalculator=GaugeStats())
+  {
+    readConfigurationInternal(Umu,header,file,GaugeStatisticsCalculator);
+  }
+
   // Preferred interface
   template<class GaugeStats=PeriodicGaugeStatistics>
   static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
@@ -220,7 +241,16 @@ public:
 					std::string ens_id = std::string("UKQCD"),
 					unsigned int sequence_number = 1)
   {
-    writeConfiguration(Umu,file,0,1,ens_label,ens_id,sequence_number);
+    writeConfigurationInternal<Lattice<vLorentzColourMatrixD>,GaugeStats>(Umu,file,0,1,ens_label,ens_id,sequence_number);
+  }
+  template<class GaugeStats=lexPeriodicGaugeStatistics>
+  static inline void writeConfiguration(Lattice<sLorentzColourMatrixD > &Umu,
+					std::string file, 
+					std::string ens_label = std::string("DWF"),
+					std::string ens_id = std::string("UKQCD"),
+					unsigned int sequence_number = 1)
+  {
+    writeConfigurationInternal<Lattice<sLorentzColourMatrixD>,GaugeStats>(Umu,file,0,1,ens_label,ens_id,sequence_number);
   }
   template<class GaugeStats=PeriodicGaugeStatistics>
   static inline void writeConfiguration(Lattice<vLorentzColourMatrixD > &Umu,
@@ -231,7 +261,29 @@ public:
 					std::string ens_id = std::string("UKQCD"),
 					unsigned int sequence_number = 1)
   {
-    typedef vLorentzColourMatrixD vobj;
+    writeConfigurationInternal<Lattice<vLorentzColourMatrixD>,GaugeStats>(Umu,file,two_row,bits32,ens_label,ens_id,sequence_number);
+  }
+  template<class GaugeStats=lexPeriodicGaugeStatistics>
+  static inline void writeConfiguration(Lattice<sLorentzColourMatrixD > &Umu,
+					std::string file, 
+					int two_row,
+					int bits32,
+					std::string ens_label = std::string("DWF"),
+					std::string ens_id = std::string("UKQCD"),
+					unsigned int sequence_number = 1)
+  {
+    writeConfigurationInternal<Lattice<sLorentzColourMatrixD>,GaugeStats>(Umu,file,two_row,bits32,ens_label,ens_id,sequence_number);
+  }
+  template<class GaugeFieldType,class GaugeStats>
+  static inline void writeConfigurationInternal(GaugeFieldType &Umu,
+					std::string file, 
+					int two_row,
+					int bits32,
+					std::string ens_label = std::string("DWF"),
+					std::string ens_id = std::string("UKQCD"),
+					unsigned int sequence_number = 1)
+  {
+    typedef typename GaugeFieldType::vector_object vobj;
     typedef typename vobj::scalar_object sobj;
 
     FieldMetaData header;
@@ -277,9 +329,12 @@ public:
 						nersc_csum,scidac_csuma,scidac_csumb);
     }
     header.checksum = nersc_csum;
-    if ( grid->IsBoss() ) { 
+    if ( grid->IsBoss() ) {
       writeHeader(header,file);
     }
+    // The header is written twice; the checksum is only known after the data.
+    // Other ranks must not proceed to read it until the rewrite has landed.
+    grid->Barrier();
 
     std::cout<<GridLogMessage <<"Written NERSC Configuration on "<< file << " checksum "
 	     <<std::hex<<header.checksum
@@ -330,11 +385,14 @@ public:
     uint32_t nersc_csum,scidac_csuma,scidac_csumb;
     BinaryIO::writeRNG(serial,parallel,file,offset,nersc_csum,scidac_csuma,scidac_csumb);
     header.checksum = nersc_csum;
-	if ( grid->IsBoss() ) { 
+	if ( grid->IsBoss() ) {
     offset = writeHeader(header,file);
 	}
+    // As for the gauge field: the checksum is only known after the data, so
+    // the header is rewritten. Hold the other ranks until that has landed.
+    grid->Barrier();
 
-    std::cout<<GridLogMessage 
+    std::cout<<GridLogMessage
 	     <<"Written NERSC RNG STATE "<<file<< " checksum "
 	     <<std::hex<<header.checksum
 	     <<std::dec<<std::endl;
@@ -372,9 +430,8 @@ public:
     uint32_t nersc_csum,scidac_csuma,scidac_csumb;
     BinaryIO::readRNG(serial,parallel,file,offset,nersc_csum,scidac_csuma,scidac_csumb);
 
-    if ( nersc_csum != header.checksum ) { 
+    if ( nersc_csum != header.checksum ) {
       std::cerr << "checksum mismatch "<<std::hex<< nersc_csum <<" "<<header.checksum<<std::dec<<std::endl;
-      exit(0);
     }
     GRID_ASSERT(nersc_csum == header.checksum );
 

Grid/qcd/QCD.h

@@ -127,7 +127,6 @@ typedef iSpinMatrix<ComplexD >          SpinMatrixD;
 typedef iSpinMatrix<vComplex >          vSpinMatrix;
 typedef iSpinMatrix<vComplexF>          vSpinMatrixF;
 typedef iSpinMatrix<vComplexD>          vSpinMatrixD;
-typedef iSpinMatrix<vComplexD2>         vSpinMatrixD2;
 
 // Colour Matrix
 typedef iColourMatrix<Complex  >        ColourMatrix;
@@ -137,7 +136,6 @@ typedef iColourMatrix<ComplexD >        ColourMatrixD;
 typedef iColourMatrix<vComplex >        vColourMatrix;
 typedef iColourMatrix<vComplexF>        vColourMatrixF;
 typedef iColourMatrix<vComplexD>        vColourMatrixD;
-typedef iColourMatrix<vComplexD2>       vColourMatrixD2;
 
 // SpinColour matrix
 typedef iSpinColourMatrix<Complex  >    SpinColourMatrix;
@@ -147,7 +145,6 @@ typedef iSpinColourMatrix<ComplexD >    SpinColourMatrixD;
 typedef iSpinColourMatrix<vComplex >    vSpinColourMatrix;
 typedef iSpinColourMatrix<vComplexF>    vSpinColourMatrixF;
 typedef iSpinColourMatrix<vComplexD>    vSpinColourMatrixD;
-typedef iSpinColourMatrix<vComplexD2>   vSpinColourMatrixD2;
 
 // SpinColourSpinColour matrix
 typedef iSpinColourSpinColourMatrix<Complex  >    SpinColourSpinColourMatrix;
@@ -157,17 +154,6 @@ typedef iSpinColourSpinColourMatrix<ComplexD >    SpinColourSpinColourMatrixD;
 typedef iSpinColourSpinColourMatrix<vComplex >    vSpinColourSpinColourMatrix;
 typedef iSpinColourSpinColourMatrix<vComplexF>    vSpinColourSpinColourMatrixF;
 typedef iSpinColourSpinColourMatrix<vComplexD>    vSpinColourSpinColourMatrixD;
-typedef iSpinColourSpinColourMatrix<vComplexD2>   vSpinColourSpinColourMatrixD2;
-
-// SpinColourSpinColour matrix
-typedef iSpinColourSpinColourMatrix<Complex  >    SpinColourSpinColourMatrix;
-typedef iSpinColourSpinColourMatrix<ComplexF >    SpinColourSpinColourMatrixF;
-typedef iSpinColourSpinColourMatrix<ComplexD >    SpinColourSpinColourMatrixD;
-
-typedef iSpinColourSpinColourMatrix<vComplex >    vSpinColourSpinColourMatrix;
-typedef iSpinColourSpinColourMatrix<vComplexF>    vSpinColourSpinColourMatrixF;
-typedef iSpinColourSpinColourMatrix<vComplexD>    vSpinColourSpinColourMatrixD;
-typedef iSpinColourSpinColourMatrix<vComplexD2>   vSpinColourSpinColourMatrixD2;
 
 // LorentzColour
 typedef iLorentzColourMatrix<Complex  > LorentzColourMatrix;
@@ -177,7 +163,6 @@ typedef iLorentzColourMatrix<ComplexD > LorentzColourMatrixD;
 typedef iLorentzColourMatrix<vComplex >  vLorentzColourMatrix;
 typedef iLorentzColourMatrix<vComplexF>  vLorentzColourMatrixF;
 typedef iLorentzColourMatrix<vComplexD>  vLorentzColourMatrixD;
-typedef iLorentzColourMatrix<vComplexD2> vLorentzColourMatrixD2;
 
 // LorentzComplex
 typedef iLorentzComplex<Complex  > LorentzComplex;
@@ -196,7 +181,6 @@ typedef iDoubleStoredColourMatrix<ComplexD > DoubleStoredColourMatrixD;
 typedef iDoubleStoredColourMatrix<vComplex >  vDoubleStoredColourMatrix;
 typedef iDoubleStoredColourMatrix<vComplexF>  vDoubleStoredColourMatrixF;
 typedef iDoubleStoredColourMatrix<vComplexD>  vDoubleStoredColourMatrixD;
-typedef iDoubleStoredColourMatrix<vComplexD2> vDoubleStoredColourMatrixD2;
 
 //G-parity flavour matrix
 typedef iGparityFlavourMatrix<Complex> GparityFlavourMatrix;
@@ -206,7 +190,6 @@ typedef iGparityFlavourMatrix<ComplexD> GparityFlavourMatrixD;
 typedef iGparityFlavourMatrix<vComplex>   vGparityFlavourMatrix;
 typedef iGparityFlavourMatrix<vComplexF>  vGparityFlavourMatrixF;
 typedef iGparityFlavourMatrix<vComplexD>  vGparityFlavourMatrixD;
-typedef iGparityFlavourMatrix<vComplexD2> vGparityFlavourMatrixD2;
 
 
 // Spin vector
@@ -217,7 +200,6 @@ typedef iSpinVector<ComplexD>           SpinVectorD;
 typedef iSpinVector<vComplex >           vSpinVector;
 typedef iSpinVector<vComplexF>           vSpinVectorF;
 typedef iSpinVector<vComplexD>           vSpinVectorD;
-typedef iSpinVector<vComplexD2>          vSpinVectorD2;
 
 // Colour vector
 typedef iColourVector<Complex >         ColourVector;
@@ -227,7 +209,6 @@ typedef iColourVector<ComplexD>         ColourVectorD;
 typedef iColourVector<vComplex >         vColourVector;
 typedef iColourVector<vComplexF>         vColourVectorF;
 typedef iColourVector<vComplexD>         vColourVectorD;
-typedef iColourVector<vComplexD2>        vColourVectorD2;
 
 // SpinColourVector
 typedef iSpinColourVector<Complex >     SpinColourVector;
@@ -237,7 +218,6 @@ typedef iSpinColourVector<ComplexD>     SpinColourVectorD;
 typedef iSpinColourVector<vComplex >     vSpinColourVector;
 typedef iSpinColourVector<vComplexF>     vSpinColourVectorF;
 typedef iSpinColourVector<vComplexD>     vSpinColourVectorD;
-typedef iSpinColourVector<vComplexD2>    vSpinColourVectorD2;
 
 // HalfSpin vector
 typedef iHalfSpinVector<Complex >       HalfSpinVector;
@@ -247,7 +227,6 @@ typedef iHalfSpinVector<ComplexD>       HalfSpinVectorD;
 typedef iHalfSpinVector<vComplex >       vHalfSpinVector;
 typedef iHalfSpinVector<vComplexF>       vHalfSpinVectorF;
 typedef iHalfSpinVector<vComplexD>       vHalfSpinVectorD;
-typedef iHalfSpinVector<vComplexD2>      vHalfSpinVectorD2;
 
 // HalfSpinColour vector
 typedef iHalfSpinColourVector<Complex > HalfSpinColourVector;
@@ -257,7 +236,6 @@ typedef iHalfSpinColourVector<ComplexD> HalfSpinColourVectorD;
 typedef iHalfSpinColourVector<vComplex >  vHalfSpinColourVector;
 typedef iHalfSpinColourVector<vComplexF>  vHalfSpinColourVectorF;
 typedef iHalfSpinColourVector<vComplexD>  vHalfSpinColourVectorD;
-typedef iHalfSpinColourVector<vComplexD2> vHalfSpinColourVectorD2;
 
 //G-parity flavour vector
 typedef iGparityFlavourVector<Complex >         GparityFlavourVector;
@@ -267,7 +245,6 @@ typedef iGparityFlavourVector<ComplexD>         GparityFlavourVectorD;
 typedef iGparityFlavourVector<vComplex >         vGparityFlavourVector;
 typedef iGparityFlavourVector<vComplexF>         vGparityFlavourVectorF;
 typedef iGparityFlavourVector<vComplexD>         vGparityFlavourVectorD;
-typedef iGparityFlavourVector<vComplexD2>        vGparityFlavourVectorD2;
     
 // singlets
 typedef iSinglet<Complex >         TComplex;     // FIXME This is painful. Tensor singlet complex type.
@@ -277,7 +254,6 @@ typedef iSinglet<ComplexD>         TComplexD;    // FIXME This is painful. Tenso
 typedef iSinglet<vComplex >        vTComplex ;   // what if we don't know the tensor structure
 typedef iSinglet<vComplexF>        vTComplexF;   // what if we don't know the tensor structure
 typedef iSinglet<vComplexD>        vTComplexD;   // what if we don't know the tensor structure
-typedef iSinglet<vComplexD2>       vTComplexD2;   // what if we don't know the tensor structure
 
 typedef iSinglet<Real >            TReal;        // Shouldn't need these; can I make it work without?
 typedef iSinglet<RealF>            TRealF;       // Shouldn't need these; can I make it work without?
@@ -295,27 +271,22 @@ typedef iSinglet<Integer >         TInteger;
 typedef Lattice<vColourMatrix>          LatticeColourMatrix;
 typedef Lattice<vColourMatrixF>         LatticeColourMatrixF;
 typedef Lattice<vColourMatrixD>         LatticeColourMatrixD;
-typedef Lattice<vColourMatrixD2>        LatticeColourMatrixD2;
 
 typedef Lattice<vSpinMatrix>            LatticeSpinMatrix;
 typedef Lattice<vSpinMatrixF>           LatticeSpinMatrixF;
 typedef Lattice<vSpinMatrixD>           LatticeSpinMatrixD;
-typedef Lattice<vSpinMatrixD2>          LatticeSpinMatrixD2;
 
 typedef Lattice<vSpinColourMatrix>      LatticeSpinColourMatrix;
 typedef Lattice<vSpinColourMatrixF>     LatticeSpinColourMatrixF;
 typedef Lattice<vSpinColourMatrixD>     LatticeSpinColourMatrixD;
-typedef Lattice<vSpinColourMatrixD2>    LatticeSpinColourMatrixD2;
 
 typedef Lattice<vSpinColourSpinColourMatrix>      LatticeSpinColourSpinColourMatrix;
 typedef Lattice<vSpinColourSpinColourMatrixF>     LatticeSpinColourSpinColourMatrixF;
 typedef Lattice<vSpinColourSpinColourMatrixD>     LatticeSpinColourSpinColourMatrixD;
-typedef Lattice<vSpinColourSpinColourMatrixD2>    LatticeSpinColourSpinColourMatrixD2;
 
 typedef Lattice<vLorentzColourMatrix>   LatticeLorentzColourMatrix;
 typedef Lattice<vLorentzColourMatrixF>  LatticeLorentzColourMatrixF;
 typedef Lattice<vLorentzColourMatrixD>  LatticeLorentzColourMatrixD;
-typedef Lattice<vLorentzColourMatrixD2> LatticeLorentzColourMatrixD2;
 
 typedef Lattice<vLorentzComplex>  LatticeLorentzComplex;
 typedef Lattice<vLorentzComplexF> LatticeLorentzComplexF;
@@ -325,32 +296,26 @@ typedef Lattice<vLorentzComplexD> LatticeLorentzComplexD;
 typedef Lattice<vDoubleStoredColourMatrix>   LatticeDoubleStoredColourMatrix;
 typedef Lattice<vDoubleStoredColourMatrixF>  LatticeDoubleStoredColourMatrixF;
 typedef Lattice<vDoubleStoredColourMatrixD>  LatticeDoubleStoredColourMatrixD;
-typedef Lattice<vDoubleStoredColourMatrixD2> LatticeDoubleStoredColourMatrixD2;
 
 typedef Lattice<vSpinVector>            LatticeSpinVector;
 typedef Lattice<vSpinVectorF>           LatticeSpinVectorF;
 typedef Lattice<vSpinVectorD>           LatticeSpinVectorD;
-typedef Lattice<vSpinVectorD2>          LatticeSpinVectorD2;
 
 typedef Lattice<vColourVector>          LatticeColourVector;
 typedef Lattice<vColourVectorF>         LatticeColourVectorF;
 typedef Lattice<vColourVectorD>         LatticeColourVectorD;
-typedef Lattice<vColourVectorD2>        LatticeColourVectorD2;
 
 typedef Lattice<vSpinColourVector>      LatticeSpinColourVector;
 typedef Lattice<vSpinColourVectorF>     LatticeSpinColourVectorF;
 typedef Lattice<vSpinColourVectorD>     LatticeSpinColourVectorD;
-typedef Lattice<vSpinColourVectorD2>    LatticeSpinColourVectorD2;
 
 typedef Lattice<vHalfSpinVector>        LatticeHalfSpinVector;
 typedef Lattice<vHalfSpinVectorF>       LatticeHalfSpinVectorF;
 typedef Lattice<vHalfSpinVectorD>       LatticeHalfSpinVectorD;
-typedef Lattice<vHalfSpinVectorD2>      LatticeHalfSpinVectorD2;
 
 typedef Lattice<vHalfSpinColourVector>   LatticeHalfSpinColourVector;
 typedef Lattice<vHalfSpinColourVectorF>  LatticeHalfSpinColourVectorF;
 typedef Lattice<vHalfSpinColourVectorD>  LatticeHalfSpinColourVectorD;
-typedef Lattice<vHalfSpinColourVectorD2> LatticeHalfSpinColourVectorD2;
 
 typedef Lattice<vTReal>            LatticeReal;
 typedef Lattice<vTRealF>           LatticeRealF;
@@ -359,7 +324,6 @@ typedef Lattice<vTRealD>           LatticeRealD;
 typedef Lattice<vTComplex>         LatticeComplex;
 typedef Lattice<vTComplexF>        LatticeComplexF;
 typedef Lattice<vTComplexD>        LatticeComplexD;
-typedef Lattice<vTComplexD2>       LatticeComplexD2;
 
 typedef Lattice<vTInteger>         LatticeInteger; // Predicates for "where"
 
@@ -370,39 +334,124 @@ typedef Lattice<vTInteger>         LatticeInteger; // Predicates for "where"
 typedef LatticeHalfSpinColourVector   LatticeHalfFermion;
 typedef LatticeHalfSpinColourVectorF  LatticeHalfFermionF;
 typedef LatticeHalfSpinColourVectorD  LatticeHalfFermionD;
-typedef LatticeHalfSpinColourVectorD2 LatticeHalfFermionD2;
 
 typedef LatticeSpinColourVector      LatticeFermion;
 typedef LatticeSpinColourVectorF     LatticeFermionF;
 typedef LatticeSpinColourVectorD     LatticeFermionD;
-typedef LatticeSpinColourVectorD2    LatticeFermionD2;
 
 typedef LatticeSpinColourMatrix                LatticePropagator;
 typedef LatticeSpinColourMatrixF               LatticePropagatorF;
 typedef LatticeSpinColourMatrixD               LatticePropagatorD;
-typedef LatticeSpinColourMatrixD2              LatticePropagatorD2;
 
 typedef LatticeLorentzColourMatrix             LatticeGaugeField;
 typedef LatticeLorentzColourMatrixF            LatticeGaugeFieldF;
 typedef LatticeLorentzColourMatrixD            LatticeGaugeFieldD;
-typedef LatticeLorentzColourMatrixD2           LatticeGaugeFieldD2;
 
 typedef LatticeDoubleStoredColourMatrix        LatticeDoubledGaugeField;
 typedef LatticeDoubleStoredColourMatrixF       LatticeDoubledGaugeFieldF;
 typedef LatticeDoubleStoredColourMatrixD       LatticeDoubledGaugeFieldD;
-typedef LatticeDoubleStoredColourMatrixD2      LatticeDoubledGaugeFieldD2;
 
 template<class GF> using LorentzScalar = Lattice<iScalar<typename GF::vector_object::element> >;
 
 typedef Lattice<vColourVector>          LatticeStaggeredFermion;    
 typedef Lattice<vColourVectorF>         LatticeStaggeredFermionF;    
 typedef Lattice<vColourVectorD>         LatticeStaggeredFermionD;    
-typedef Lattice<vColourVectorD2>        LatticeStaggeredFermionD2;    
 
 typedef Lattice<vColourMatrix>          LatticeStaggeredPropagator; 
 typedef Lattice<vColourMatrixF>         LatticeStaggeredPropagatorF; 
 typedef Lattice<vColourMatrixD>         LatticeStaggeredPropagatorD; 
-typedef Lattice<vColourMatrixD2>        LatticeStaggeredPropagatorD2; 
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Lexicographic lattice types: Nsimd()==1 containers of the same tensors.
+// Single and double precision only; no default precision alias.
+/////////////////////////////////////////////////////////////////////////////
+
+// Site objects over the width one simd types
+typedef iSpinMatrix<sComplexF>                        sSpinMatrixF;
+typedef iSpinMatrix<sComplexD>                        sSpinMatrixD;
+typedef iColourMatrix<sComplexF>                      sColourMatrixF;
+typedef iColourMatrix<sComplexD>                      sColourMatrixD;
+typedef iSpinColourMatrix<sComplexF>                  sSpinColourMatrixF;
+typedef iSpinColourMatrix<sComplexD>                  sSpinColourMatrixD;
+typedef iSpinColourSpinColourMatrix<sComplexF>        sSpinColourSpinColourMatrixF;
+typedef iSpinColourSpinColourMatrix<sComplexD>        sSpinColourSpinColourMatrixD;
+typedef iLorentzColourMatrix<sComplexF>               sLorentzColourMatrixF;
+typedef iLorentzColourMatrix<sComplexD>               sLorentzColourMatrixD;
+typedef iLorentzComplex<sComplexF>                    sLorentzComplexF;
+typedef iLorentzComplex<sComplexD>                    sLorentzComplexD;
+typedef iDoubleStoredColourMatrix<sComplexF>          sDoubleStoredColourMatrixF;
+typedef iDoubleStoredColourMatrix<sComplexD>          sDoubleStoredColourMatrixD;
+typedef iGparityFlavourMatrix<sComplexF>              sGparityFlavourMatrixF;
+typedef iGparityFlavourMatrix<sComplexD>              sGparityFlavourMatrixD;
+typedef iSpinVector<sComplexF>                        sSpinVectorF;
+typedef iSpinVector<sComplexD>                        sSpinVectorD;
+typedef iColourVector<sComplexF>                      sColourVectorF;
+typedef iColourVector<sComplexD>                      sColourVectorD;
+typedef iSpinColourVector<sComplexF>                  sSpinColourVectorF;
+typedef iSpinColourVector<sComplexD>                  sSpinColourVectorD;
+typedef iHalfSpinVector<sComplexF>                    sHalfSpinVectorF;
+typedef iHalfSpinVector<sComplexD>                    sHalfSpinVectorD;
+typedef iHalfSpinColourVector<sComplexF>              sHalfSpinColourVectorF;
+typedef iHalfSpinColourVector<sComplexD>              sHalfSpinColourVectorD;
+typedef iGparityFlavourVector<sComplexF>              sGparityFlavourVectorF;
+typedef iGparityFlavourVector<sComplexD>              sGparityFlavourVectorD;
+typedef iSinglet<sComplexF>                           sTComplexF;
+typedef iSinglet<sComplexD>                           sTComplexD;
+typedef iSinglet<sRealF>                              sTRealF;
+typedef iSinglet<sRealD>                              sTRealD;
+typedef iSinglet<sInteger>                            sTInteger;
+
+// Lattices of these
+typedef Lattice<sColourMatrixF>                       lexLatticeColourMatrixF;
+typedef Lattice<sColourMatrixD>                       lexLatticeColourMatrixD;
+typedef Lattice<sSpinMatrixF>                         lexLatticeSpinMatrixF;
+typedef Lattice<sSpinMatrixD>                         lexLatticeSpinMatrixD;
+typedef Lattice<sSpinColourMatrixF>                   lexLatticeSpinColourMatrixF;
+typedef Lattice<sSpinColourMatrixD>                   lexLatticeSpinColourMatrixD;
+typedef Lattice<sSpinColourSpinColourMatrixF>         lexLatticeSpinColourSpinColourMatrixF;
+typedef Lattice<sSpinColourSpinColourMatrixD>         lexLatticeSpinColourSpinColourMatrixD;
+typedef Lattice<sLorentzColourMatrixF>                lexLatticeLorentzColourMatrixF;
+typedef Lattice<sLorentzColourMatrixD>                lexLatticeLorentzColourMatrixD;
+typedef Lattice<sLorentzComplexF>                     lexLatticeLorentzComplexF;
+typedef Lattice<sLorentzComplexD>                     lexLatticeLorentzComplexD;
+typedef Lattice<sDoubleStoredColourMatrixF>           lexLatticeDoubleStoredColourMatrixF;
+typedef Lattice<sDoubleStoredColourMatrixD>           lexLatticeDoubleStoredColourMatrixD;
+typedef Lattice<sSpinVectorF>                         lexLatticeSpinVectorF;
+typedef Lattice<sSpinVectorD>                         lexLatticeSpinVectorD;
+typedef Lattice<sColourVectorF>                       lexLatticeColourVectorF;
+typedef Lattice<sColourVectorD>                       lexLatticeColourVectorD;
+typedef Lattice<sSpinColourVectorF>                   lexLatticeSpinColourVectorF;
+typedef Lattice<sSpinColourVectorD>                   lexLatticeSpinColourVectorD;
+typedef Lattice<sHalfSpinVectorF>                     lexLatticeHalfSpinVectorF;
+typedef Lattice<sHalfSpinVectorD>                     lexLatticeHalfSpinVectorD;
+typedef Lattice<sHalfSpinColourVectorF>               lexLatticeHalfSpinColourVectorF;
+typedef Lattice<sHalfSpinColourVectorD>               lexLatticeHalfSpinColourVectorD;
+typedef Lattice<sTRealF>                              lexLatticeRealF;
+typedef Lattice<sTRealD>                              lexLatticeRealD;
+typedef Lattice<sTComplexF>                           lexLatticeComplexF;
+typedef Lattice<sTComplexD>                           lexLatticeComplexD;
+typedef Lattice<sTInteger>                            lexLatticeInteger;
+typedef Lattice<sColourVectorF>                       lexLatticeStaggeredFermionF;
+typedef Lattice<sColourVectorD>                       lexLatticeStaggeredFermionD;
+typedef Lattice<sColourMatrixF>                       lexLatticeStaggeredPropagatorF;
+typedef Lattice<sColourMatrixD>                       lexLatticeStaggeredPropagatorD;
+
+// Physical names for things
+typedef lexLatticeHalfSpinColourVectorF               lexLatticeHalfFermionF;
+typedef lexLatticeHalfSpinColourVectorD               lexLatticeHalfFermionD;
+typedef lexLatticeSpinColourVectorF                   lexLatticeFermionF;
+typedef lexLatticeSpinColourVectorD                   lexLatticeFermionD;
+typedef lexLatticeSpinColourMatrixF                   lexLatticePropagatorF;
+typedef lexLatticeSpinColourMatrixD                   lexLatticePropagatorD;
+typedef lexLatticeLorentzColourMatrixF                lexLatticeGaugeFieldF;
+typedef lexLatticeLorentzColourMatrixD                lexLatticeGaugeFieldD;
+typedef lexLatticeDoubleStoredColourMatrixF           lexLatticeDoubledGaugeFieldF;
+typedef lexLatticeDoubleStoredColourMatrixD           lexLatticeDoubledGaugeFieldD;
+
+/////////////////////////////////////////////////////////////////////////////
+// End lexicographic lattice types
+/////////////////////////////////////////////////////////////////////////////
 
 //////////////////////////////////////////////////////////////////////////////
 // Peek and Poke named after physics attributes

Grid/qcd/action/fermion/CloverHelpers.h

@@ -143,6 +143,8 @@ public:
   static int getNMAX(Lattice<iImplClover<vComplexD2>> &t, RealD R) {return getNMAX(1e-12,R);}
   static int getNMAX(Lattice<iImplClover<vComplexD>> &t, RealD R) {return getNMAX(1e-12,R);}
   static int getNMAX(Lattice<iImplClover<vComplexF>> &t, RealD R) {return getNMAX(1e-6,R);}
+  static int getNMAX(Lattice<iImplClover<sComplexD>> &t, RealD R) {return getNMAX(1e-12,R);}
+  static int getNMAX(Lattice<iImplClover<sComplexF>> &t, RealD R) {return getNMAX(1e-6,R);}
 
   static void Instantiate(CloverField& Clover, CloverField& CloverInv, RealD csw_t, RealD diag_mass) {
     GridBase* grid = Clover.Grid();
@@ -268,6 +270,8 @@ public:
 
   static int getNMAX(Lattice<iImplClover<vComplexD>> &t, RealD R) {return getNMAX(1e-12,R);}
   static int getNMAX(Lattice<iImplClover<vComplexF>> &t, RealD R) {return getNMAX(1e-6,R);}
+  static int getNMAX(Lattice<iImplClover<sComplexD>> &t, RealD R) {return getNMAX(1e-12,R);}
+  static int getNMAX(Lattice<iImplClover<sComplexF>> &t, RealD R) {return getNMAX(1e-6,R);}
 
   static void InstantiateClover(CloverField& Clover, CloverField& CloverInv, RealD csw_t, RealD diag_mass) {
 

Grid/qcd/action/fermion/DomainWallVec5dImpl.h

@@ -40,15 +40,12 @@ public:
   static const int Dimension = Representation::Dimension;
   static const bool isFundamental = Representation::isFundamental;
   static const bool LsVectorised=true;
-  static const int Nhcs = Options::Nhcs;
       
   typedef typename Options::_Coeff_t Coeff_t;      
-  typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
   
   template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
   template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Dimension>, Ns> >;
   template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
-  template <typename vtype> using iImplHalfCommSpinor    = iScalar<iVector<iVector<vtype, Dimension>, Nhcs> >;
   template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
   template <typename vtype> using iImplGaugeField        = iVector<iScalar<iMatrix<vtype, Dimension> >, Nd>;
   template <typename vtype> using iImplGaugeLink         = iScalar<iScalar<iMatrix<vtype, Dimension> > >;
@@ -56,7 +53,6 @@ public:
   typedef iImplSpinor<Simd>            SiteSpinor;
   typedef iImplPropagator<Simd>        SitePropagator;
   typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
-  typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
   typedef Lattice<SiteSpinor>          FermionField;
   typedef Lattice<SitePropagator>      PropagatorField;
 
@@ -68,7 +64,7 @@ public:
   typedef iImplGaugeLink<typename Simd::scalar_type>          SiteScalarGaugeLink;  // scalar
   typedef Lattice<SiteDoubledGaugeField>                      DoubledGaugeField;
       
-  typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
+  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
   typedef WilsonImplParams ImplParams;
   typedef WilsonStencil<SiteSpinor, SiteHalfSpinor,ImplParams> StencilImpl;
   typedef typename StencilImpl::View_type StencilView;
@@ -201,16 +197,8 @@ typedef DomainWallVec5dImpl<vComplex ,FundamentalRepresentation, CoeffReal> Doma
 typedef DomainWallVec5dImpl<vComplexF,FundamentalRepresentation, CoeffReal> DomainWallVec5dImplF; // Float
 typedef DomainWallVec5dImpl<vComplexD,FundamentalRepresentation, CoeffReal> DomainWallVec5dImplD; // Double
  
-typedef DomainWallVec5dImpl<vComplex ,FundamentalRepresentation, CoeffRealHalfComms> DomainWallVec5dImplRL; // Real.. whichever prec
-typedef DomainWallVec5dImpl<vComplexF,FundamentalRepresentation, CoeffRealHalfComms> DomainWallVec5dImplFH; // Float
-typedef DomainWallVec5dImpl<vComplexD,FundamentalRepresentation, CoeffRealHalfComms> DomainWallVec5dImplDF; // Double
- 
 typedef DomainWallVec5dImpl<vComplex ,FundamentalRepresentation,CoeffComplex> ZDomainWallVec5dImplR; // Real.. whichever prec
 typedef DomainWallVec5dImpl<vComplexF,FundamentalRepresentation,CoeffComplex> ZDomainWallVec5dImplF; // Float
 typedef DomainWallVec5dImpl<vComplexD,FundamentalRepresentation,CoeffComplex> ZDomainWallVec5dImplD; // Double
- 
-typedef DomainWallVec5dImpl<vComplex ,FundamentalRepresentation,CoeffComplexHalfComms> ZDomainWallVec5dImplRL; // Real.. whichever prec
-typedef DomainWallVec5dImpl<vComplexF,FundamentalRepresentation,CoeffComplexHalfComms> ZDomainWallVec5dImplFH; // Float
-typedef DomainWallVec5dImpl<vComplexD,FundamentalRepresentation,CoeffComplexHalfComms> ZDomainWallVec5dImplDF; // Double
 
 NAMESPACE_END(Grid);

Grid/qcd/action/fermion/Fermion.h

@@ -114,7 +114,7 @@ NAMESPACE_CHECK(DWFutils);
 // Cayley 5d
 NAMESPACE_BEGIN(Grid);
 
-typedef WilsonFermion<WilsonImplD2> WilsonFermionD2;
+//typedef WilsonFermion<WilsonImplD2> WilsonFermionD2;
 typedef WilsonFermion<WilsonImplF> WilsonFermionF;
 typedef WilsonFermion<WilsonImplD> WilsonFermionD;
 
@@ -138,7 +138,7 @@ typedef WilsonFermion<SpWilsonTwoIndexSymmetricImplF> SpWilsonTwoIndexSymmetricF
 typedef WilsonFermion<SpWilsonTwoIndexSymmetricImplD> SpWilsonTwoIndexSymmetricFermionD;
 
 // Twisted mass fermion
-typedef WilsonTMFermion<WilsonImplD2> WilsonTMFermionD2;
+//typedef WilsonTMFermion<WilsonImplD2> WilsonTMFermionD2;
 typedef WilsonTMFermion<WilsonImplF> WilsonTMFermionF;
 typedef WilsonTMFermion<WilsonImplD> WilsonTMFermionD;
 
@@ -146,11 +146,11 @@ typedef WilsonTMFermion<WilsonImplD> WilsonTMFermionD;
 template <typename WImpl> using WilsonClover = WilsonCloverFermion<WImpl, CloverHelpers<WImpl>>;
 template <typename WImpl> using WilsonExpClover = WilsonCloverFermion<WImpl, ExpCloverHelpers<WImpl>>;
 
-typedef WilsonClover<WilsonImplD2> WilsonCloverFermionD2;
+//typedef WilsonClover<WilsonImplD2> WilsonCloverFermionD2;
 typedef WilsonClover<WilsonImplF> WilsonCloverFermionF;
 typedef WilsonClover<WilsonImplD> WilsonCloverFermionD;
 
-typedef WilsonExpClover<WilsonImplD2> WilsonExpCloverFermionD2;
+//typedef WilsonExpClover<WilsonImplD2> WilsonExpCloverFermionD2;
 typedef WilsonExpClover<WilsonImplF> WilsonExpCloverFermionF;
 typedef WilsonExpClover<WilsonImplD> WilsonExpCloverFermionD;
 
@@ -168,15 +168,15 @@ template <typename WImpl> using CompactWilsonClover = CompactWilsonCloverFermion
 template <typename WImpl> using CompactWilsonClover5D = CompactWilsonCloverFermion5D<WImpl, CompactCloverHelpers<WImpl>>;
 template <typename WImpl> using CompactWilsonExpClover = CompactWilsonCloverFermion<WImpl, CompactExpCloverHelpers<WImpl>>;
 
-typedef CompactWilsonClover<WilsonImplD2> CompactWilsonCloverFermionD2;
+//typedef CompactWilsonClover<WilsonImplD2> CompactWilsonCloverFermionD2;
 typedef CompactWilsonClover<WilsonImplF> CompactWilsonCloverFermionF;
 typedef CompactWilsonClover<WilsonImplD> CompactWilsonCloverFermionD;
 
-typedef CompactWilsonClover5D<WilsonImplD2> CompactWilsonCloverFermion5DD2;
+//typedef CompactWilsonClover5D<WilsonImplD2> CompactWilsonCloverFermion5DD2;
 typedef CompactWilsonClover5D<WilsonImplF> CompactWilsonCloverFermion5DF;
 typedef CompactWilsonClover5D<WilsonImplD> CompactWilsonCloverFermion5DD;
 
-typedef CompactWilsonExpClover<WilsonImplD2> CompactWilsonExpCloverFermionD2;
+//typedef CompactWilsonExpClover<WilsonImplD2> CompactWilsonExpCloverFermionD2;
 typedef CompactWilsonExpClover<WilsonImplF> CompactWilsonExpCloverFermionF;
 typedef CompactWilsonExpClover<WilsonImplD> CompactWilsonExpCloverFermionD;
 
@@ -192,56 +192,56 @@ typedef CompactWilsonClover<WilsonTwoIndexAntiSymmetricImplD> CompactWilsonClove
 // Domain Wall fermions
 typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;
 typedef DomainWallFermion<WilsonImplD> DomainWallFermionD;
-typedef DomainWallFermion<WilsonImplD2> DomainWallFermionD2;
+//typedef DomainWallFermion<WilsonImplD2> DomainWallFermionD2;
 
-typedef DomainWallEOFAFermion<WilsonImplD2> DomainWallEOFAFermionD2;
+//typedef DomainWallEOFAFermion<WilsonImplD2> DomainWallEOFAFermionD2;
 typedef DomainWallEOFAFermion<WilsonImplF> DomainWallEOFAFermionF;
 typedef DomainWallEOFAFermion<WilsonImplD> DomainWallEOFAFermionD;
 
-typedef MobiusFermion<WilsonImplD2> MobiusFermionD2;
+//typedef MobiusFermion<WilsonImplD2> MobiusFermionD2;
 typedef MobiusFermion<WilsonImplF> MobiusFermionF;
 typedef MobiusFermion<WilsonImplD> MobiusFermionD;
 
-typedef MobiusEOFAFermion<WilsonImplD2> MobiusEOFAFermionD2;
+//typedef MobiusEOFAFermion<WilsonImplD2> MobiusEOFAFermionD2;
 typedef MobiusEOFAFermion<WilsonImplF> MobiusEOFAFermionF;
 typedef MobiusEOFAFermion<WilsonImplD> MobiusEOFAFermionD;
 
-typedef ZMobiusFermion<ZWilsonImplD2> ZMobiusFermionD2;
+//typedef ZMobiusFermion<ZWilsonImplD2> ZMobiusFermionD2;
 typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
 typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
 
-typedef ScaledShamirFermion<WilsonImplD2> ScaledShamirFermionD2;
+//typedef ScaledShamirFermion<WilsonImplD2> ScaledShamirFermionD2;
 typedef ScaledShamirFermion<WilsonImplF> ScaledShamirFermionF;
 typedef ScaledShamirFermion<WilsonImplD> ScaledShamirFermionD;
 
-typedef MobiusZolotarevFermion<WilsonImplD2> MobiusZolotarevFermionD2;
+//typedef MobiusZolotarevFermion<WilsonImplD2> MobiusZolotarevFermionD2;
 typedef MobiusZolotarevFermion<WilsonImplF> MobiusZolotarevFermionF;
 typedef MobiusZolotarevFermion<WilsonImplD> MobiusZolotarevFermionD;
-typedef ShamirZolotarevFermion<WilsonImplD2> ShamirZolotarevFermionD2;
+//typedef ShamirZolotarevFermion<WilsonImplD2> ShamirZolotarevFermionD2;
 typedef ShamirZolotarevFermion<WilsonImplF> ShamirZolotarevFermionF;
 typedef ShamirZolotarevFermion<WilsonImplD> ShamirZolotarevFermionD;
 
-typedef OverlapWilsonCayleyTanhFermion<WilsonImplD2> OverlapWilsonCayleyTanhFermionD2;
+//typedef OverlapWilsonCayleyTanhFermion<WilsonImplD2> OverlapWilsonCayleyTanhFermionD2;
 typedef OverlapWilsonCayleyTanhFermion<WilsonImplF> OverlapWilsonCayleyTanhFermionF;
 typedef OverlapWilsonCayleyTanhFermion<WilsonImplD> OverlapWilsonCayleyTanhFermionD;
-typedef OverlapWilsonCayleyZolotarevFermion<WilsonImplD2> OverlapWilsonCayleyZolotarevFermionD2;
+//typedef OverlapWilsonCayleyZolotarevFermion<WilsonImplD2> OverlapWilsonCayleyZolotarevFermionD2;
 typedef OverlapWilsonCayleyZolotarevFermion<WilsonImplF> OverlapWilsonCayleyZolotarevFermionF;
 typedef OverlapWilsonCayleyZolotarevFermion<WilsonImplD> OverlapWilsonCayleyZolotarevFermionD;
 
 // Continued fraction
-typedef OverlapWilsonContFracTanhFermion<WilsonImplD2> OverlapWilsonContFracTanhFermionD2;
+//typedef OverlapWilsonContFracTanhFermion<WilsonImplD2> OverlapWilsonContFracTanhFermionD2;
 typedef OverlapWilsonContFracTanhFermion<WilsonImplF> OverlapWilsonContFracTanhFermionF;
 typedef OverlapWilsonContFracTanhFermion<WilsonImplD> OverlapWilsonContFracTanhFermionD;
-typedef OverlapWilsonContFracZolotarevFermion<WilsonImplD2> OverlapWilsonContFracZolotarevFermionD2;
+//typedef OverlapWilsonContFracZolotarevFermion<WilsonImplD2> OverlapWilsonContFracZolotarevFermionD2;
 typedef OverlapWilsonContFracZolotarevFermion<WilsonImplF> OverlapWilsonContFracZolotarevFermionF;
 typedef OverlapWilsonContFracZolotarevFermion<WilsonImplD> OverlapWilsonContFracZolotarevFermionD;
 
 // Partial fraction
-typedef OverlapWilsonPartialFractionTanhFermion<WilsonImplD2> OverlapWilsonPartialFractionTanhFermionD2;
+//typedef OverlapWilsonPartialFractionTanhFermion<WilsonImplD2> OverlapWilsonPartialFractionTanhFermionD2;
 typedef OverlapWilsonPartialFractionTanhFermion<WilsonImplF> OverlapWilsonPartialFractionTanhFermionF;
 typedef OverlapWilsonPartialFractionTanhFermion<WilsonImplD> OverlapWilsonPartialFractionTanhFermionD;
 
-typedef OverlapWilsonPartialFractionZolotarevFermion<WilsonImplD2> OverlapWilsonPartialFractionZolotarevFermionD2;
+//typedef OverlapWilsonPartialFractionZolotarevFermion<WilsonImplD2> OverlapWilsonPartialFractionZolotarevFermionD2;
 typedef OverlapWilsonPartialFractionZolotarevFermion<WilsonImplF> OverlapWilsonPartialFractionZolotarevFermionF;
 typedef OverlapWilsonPartialFractionZolotarevFermion<WilsonImplD> OverlapWilsonPartialFractionZolotarevFermionD;
 
@@ -252,19 +252,19 @@ typedef WilsonFermion<GparityWilsonImplD>     GparityWilsonFermionD;
 typedef DomainWallFermion<GparityWilsonImplF> GparityDomainWallFermionF;
 typedef DomainWallFermion<GparityWilsonImplD> GparityDomainWallFermionD;
 
-typedef DomainWallEOFAFermion<GparityWilsonImplR> GparityDomainWallEOFAFermionD2;
+//typedef DomainWallEOFAFermion<GparityWilsonImplR> GparityDomainWallEOFAFermionD2;
 typedef DomainWallEOFAFermion<GparityWilsonImplF> GparityDomainWallEOFAFermionF;
 typedef DomainWallEOFAFermion<GparityWilsonImplD> GparityDomainWallEOFAFermionD;
 
-typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionD2;
+//typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionD2;
 typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
 typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
 
-typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionD2;
+//typedef MobiusFermion<GparityWilsonImplR> GparityMobiusFermionD2;
 typedef MobiusFermion<GparityWilsonImplF> GparityMobiusFermionF;
 typedef MobiusFermion<GparityWilsonImplD> GparityMobiusFermionD;
 
-typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusEOFAFermionD2;
+//typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusEOFAFermionD2;
 typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusEOFAFermionF;
 typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusEOFAFermionD;
 

Grid/qcd/action/fermion/FermionOperatorImpl.h

@@ -88,51 +88,14 @@ NAMESPACE_BEGIN(Grid);
 //  }
 //////////////////////////////////////////////
 
-template <class T> struct SamePrecisionMapper {
-  typedef T HigherPrecVector ;
-  typedef T LowerPrecVector ;
-};
-template <class T> struct LowerPrecisionMapper {  };
-template <> struct LowerPrecisionMapper<vRealF> {
-  typedef vRealF HigherPrecVector ;
-  typedef vRealH LowerPrecVector ;
-};
-template <> struct LowerPrecisionMapper<vRealD> {
-  typedef vRealD HigherPrecVector ;
-  typedef vRealF LowerPrecVector ;
-};
-template <> struct LowerPrecisionMapper<vComplexF> {
-  typedef vComplexF HigherPrecVector ;
-  typedef vComplexH LowerPrecVector ;
-};
-template <> struct LowerPrecisionMapper<vComplexD> {
-  typedef vComplexD HigherPrecVector ;
-  typedef vComplexF LowerPrecVector ;
-};
 
 struct CoeffReal {
 public:
   typedef RealD _Coeff_t;
-  static const int Nhcs = 2;
-  template<class Simd> using PrecisionMapper = SamePrecisionMapper<Simd>;
-};
-struct CoeffRealHalfComms {
-public:
-  typedef RealD _Coeff_t;
-  static const int Nhcs = 1;
-  template<class Simd> using PrecisionMapper = LowerPrecisionMapper<Simd>;
 };
 struct CoeffComplex {
 public:
   typedef ComplexD _Coeff_t;
-  static const int Nhcs = 2;
-  template<class Simd> using PrecisionMapper = SamePrecisionMapper<Simd>;
-};
-struct CoeffComplexHalfComms {
-public:
-  typedef ComplexD _Coeff_t;
-  static const int Nhcs = 1;
-  template<class Simd> using PrecisionMapper = LowerPrecisionMapper<Simd>;
 };
 
 ////////////////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/GparityWilsonImpl.h

@@ -48,7 +48,6 @@ public:
 
  static const int Dimension = Representation::Dimension;
  static const bool isFundamental = Representation::isFundamental;
- static const int Nhcs = Options::Nhcs;
  static const bool LsVectorised=false;
  static const bool isGparity=true;
 
@@ -56,18 +55,15 @@ public:
  INHERIT_GIMPL_TYPES(Gimpl);
  
  typedef typename Options::_Coeff_t Coeff_t;
- typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
       
  template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Dimension>, Ns>,   Ngp>;
  template <typename vtype> using iImplPropagator        = iMatrix<iMatrix<iMatrix<vtype, Dimension>, Ns>,   Ngp>;
  template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Dimension>, Nhs>,  Ngp>;
- template <typename vtype> using iImplHalfCommSpinor    = iVector<iVector<iVector<vtype, Dimension>, Nhcs>, Ngp>;
  template <typename vtype> using iImplDoubledGaugeField = iVector<iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>, Ngp>;
 
   typedef iImplSpinor<Simd>            SiteSpinor;
   typedef iImplPropagator<Simd>        SitePropagator;
   typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
-  typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
   typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
 
   typedef Lattice<SiteSpinor> FermionField;
@@ -75,7 +71,7 @@ public:
   typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
  
   typedef GparityWilsonImplParams ImplParams;
-  typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
+  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
   typedef WilsonStencil<SiteSpinor, SiteHalfSpinor, ImplParams> StencilImpl;
   typedef typename StencilImpl::View_type StencilView;
       
@@ -408,9 +404,5 @@ public:
 typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffReal> GparityWilsonImplR;  // Real.. whichever prec
 typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffReal> GparityWilsonImplF;  // Float
 typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffReal> GparityWilsonImplD;  // Double
- 
-//typedef GparityWilsonImpl<vComplex , FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplRL;  // Real.. whichever prec
-//typedef GparityWilsonImpl<vComplexF, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplFH;  // Float
-//typedef GparityWilsonImpl<vComplexD, FundamentalRepresentation,CoeffRealHalfComms> GparityWilsonImplDF;  // Double
 
 NAMESPACE_END(Grid);

Grid/qcd/action/fermion/StaggeredImpl.h

@@ -176,4 +176,13 @@ typedef StaggeredImpl<vComplex,  FundamentalRepresentation > StaggeredImplR;   /
 typedef StaggeredImpl<vComplexF, FundamentalRepresentation > StaggeredImplF;  // Float
 typedef StaggeredImpl<vComplexD, FundamentalRepresentation > StaggeredImplD;  // Double
 
+/////////////////////////////////////////////////////////////////////////////
+// Lexicographic staggered implementations
+/////////////////////////////////////////////////////////////////////////////
+typedef StaggeredImpl<sComplexF, FundamentalRepresentation > lexStaggeredImplF;
+typedef StaggeredImpl<sComplexD, FundamentalRepresentation > lexStaggeredImplD;
+/////////////////////////////////////////////////////////////////////////////
+// End lexicographic staggered implementations
+/////////////////////////////////////////////////////////////////////////////
+
 NAMESPACE_END(Grid);

Grid/qcd/action/fermion/WilsonCompressor.h

@@ -38,7 +38,7 @@ NAMESPACE_BEGIN(Grid);
 
 
 //Could make FaceGather a template param, but then behaviour is runtime not compile time
-template<class _HCspinor,class _Hspinor,class _Spinor, class projector>
+template<class _Hspinor,class _Spinor, class projector>
 class WilsonCompressorTemplate : public FaceGatherSimple
 {
 public:
@@ -53,13 +53,11 @@ public:
 
   typedef _Spinor         SiteSpinor;
   typedef _Hspinor     SiteHalfSpinor;
-  typedef _HCspinor SiteHalfCommSpinor;
-  typedef typename SiteHalfCommSpinor::vector_type vComplexLow;
-  typedef typename SiteHalfSpinor::vector_type     vComplexHigh;
-  constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vComplexHigh);
+  typedef typename SiteHalfSpinor::vector_type vector_type;    
+  constexpr static int Nw=sizeof(SiteHalfSpinor)/sizeof(vector_type);
 
   accelerator_inline int CommDatumSize(void) const {
-    return sizeof(SiteHalfCommSpinor);
+    return sizeof(SiteHalfSpinor);
   }
 
   /*****************************************************/
@@ -159,7 +157,7 @@ public:
     spProj(result,in);							\
   }									\
   };									\
-  template<typename HCS,typename HS,typename S> using Compressor = WilsonCompressorTemplate<HCS,HS,S,Projector>;
+  template<typename HS,typename S> using Compressor = WilsonCompressorTemplate<HS,S,Projector>;
 
 DECLARE_PROJ(WilsonXpProjector,WilsonXpCompressor,spProjXp);
 DECLARE_PROJ(WilsonYpProjector,WilsonYpCompressor,spProjYp);
@@ -188,7 +186,7 @@ public:
     }
   }
 };
-template<typename HCS,typename HS,typename S> using WilsonCompressor = WilsonCompressorTemplate<HCS,HS,S,WilsonProjector>;
+template<typename HS,typename S> using WilsonCompressor = WilsonCompressorTemplate<HS,S,WilsonProjector>;
 
 // Fast comms buffer manipulation which should inline right through (avoid direction
 // dependent logic that prevents inlining
@@ -237,7 +235,6 @@ public:
     // Use types to select the write direction by directon compressor
     typedef typename compressor::SiteSpinor         SiteSpinor;
     typedef typename compressor::SiteHalfSpinor     SiteHalfSpinor;
-    typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
 
     this->_grid->StencilBarrier();
 
@@ -245,14 +242,14 @@ public:
     
     this->u_comm_offset=0;
       
-    WilsonXpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> XpCompress; 
-    WilsonYpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> YpCompress; 
-    WilsonZpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> ZpCompress; 
-    WilsonTpCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> TpCompress;
-    WilsonXmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> XmCompress; 
-    WilsonYmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> YmCompress; 
-    WilsonZmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> ZmCompress; 
-    WilsonTmCompressor<SiteHalfCommSpinor,SiteHalfSpinor,SiteSpinor> TmCompress;
+    WilsonXpCompressor<SiteHalfSpinor,SiteSpinor> XpCompress; 
+    WilsonYpCompressor<SiteHalfSpinor,SiteSpinor> YpCompress; 
+    WilsonZpCompressor<SiteHalfSpinor,SiteSpinor> ZpCompress; 
+    WilsonTpCompressor<SiteHalfSpinor,SiteSpinor> TpCompress;
+    WilsonXmCompressor<SiteHalfSpinor,SiteSpinor> XmCompress; 
+    WilsonYmCompressor<SiteHalfSpinor,SiteSpinor> YmCompress; 
+    WilsonZmCompressor<SiteHalfSpinor,SiteSpinor> ZmCompress; 
+    WilsonTmCompressor<SiteHalfSpinor,SiteSpinor> TmCompress;
 
     int dag = compress.dag;
     int face_idx=0;

Grid/qcd/action/fermion/WilsonImpl.h

@@ -42,7 +42,6 @@ public:
   static const bool isFundamental = Representation::isFundamental;
   static const bool LsVectorised=false;
   static const bool isGparity=false;
-  static const int Nhcs = Options::Nhcs;
 
   typedef PeriodicGaugeImpl<GaugeImplTypes<S, Dimension > > Gimpl;
   INHERIT_GIMPL_TYPES(Gimpl);
@@ -51,25 +50,22 @@ public:
   constexpr bool is_fundamental() const{return Dimension == Nc ? 1 : 0;}
     
   typedef typename Options::_Coeff_t Coeff_t;
-  typedef typename Options::template PrecisionMapper<Simd>::LowerPrecVector SimdL;
       
   template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
   template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Dimension>, Ns> >;
   template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
-  template <typename vtype> using iImplHalfCommSpinor    = iScalar<iVector<iVector<vtype, Dimension>, Nhcs> >;
   template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
     
   typedef iImplSpinor<Simd>            SiteSpinor;
   typedef iImplPropagator<Simd>        SitePropagator;
   typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
-  typedef iImplHalfCommSpinor<SimdL>   SiteHalfCommSpinor;
   typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
     
   typedef Lattice<SiteSpinor>            FermionField;
   typedef Lattice<SitePropagator>        PropagatorField;
   typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
     
-  typedef WilsonCompressor<SiteHalfCommSpinor,SiteHalfSpinor, SiteSpinor> Compressor;
+  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
   typedef WilsonImplParams ImplParams;
   typedef WilsonStencil<SiteSpinor, SiteHalfSpinor,ImplParams> StencilImpl;
   typedef const typename StencilImpl::View_type StencilView;
@@ -242,12 +238,19 @@ public:
 typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffReal > WilsonImplR;  // Real.. whichever prec
 typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffReal > WilsonImplF;  // Float
 typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffReal > WilsonImplD;  // Double
-typedef WilsonImpl<vComplexD2, FundamentalRepresentation, CoeffReal > WilsonImplD2;  // Double
+
+/////////////////////////////////////////////////////////////////////////////
+// Lexicographic Wilson implementations
+/////////////////////////////////////////////////////////////////////////////
+typedef WilsonImpl<sComplexF, FundamentalRepresentation, CoeffReal > lexWilsonImplF;
+typedef WilsonImpl<sComplexD, FundamentalRepresentation, CoeffReal > lexWilsonImplD;
+/////////////////////////////////////////////////////////////////////////////
+// End lexicographic Wilson implementations
+/////////////////////////////////////////////////////////////////////////////
 
 typedef WilsonImpl<vComplex,  FundamentalRepresentation, CoeffComplex > ZWilsonImplR; // Real.. whichever prec
 typedef WilsonImpl<vComplexF, FundamentalRepresentation, CoeffComplex > ZWilsonImplF; // Float
 typedef WilsonImpl<vComplexD, FundamentalRepresentation, CoeffComplex > ZWilsonImplD; // Double
-typedef WilsonImpl<vComplexD2, FundamentalRepresentation, CoeffComplex > ZWilsonImplD2; // Double
 
 typedef WilsonImpl<vComplex,  AdjointRepresentation, CoeffReal > WilsonAdjImplR;   // Real.. whichever prec
 typedef WilsonImpl<vComplexF, AdjointRepresentation, CoeffReal > WilsonAdjImplF;  // Float

Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h

@@ -684,7 +684,7 @@ void StaggeredKernels<Impl>::DhopSiteAsm(StencilView &st,
 #ifdef STAG_VEC5D
   // This is the single precision 5th direction vectorised kernel
 #include <Grid/simd/Intel512single.h>
-template <> void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView &st,
+template <> inline void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView &st,
 								    DoubledGaugeFieldView &U,
 								    DoubledGaugeFieldView &UUU,
 								    SiteSpinor *buf, int sF,
@@ -739,7 +739,7 @@ template <> void StaggeredKernels<StaggeredVec5dImplF>::DhopSiteAsm(StencilView
 }
 
 #include <Grid/simd/Intel512double.h>
-template <> void StaggeredKernels<StaggeredVec5dImplD>::DhopSiteAsm(StencilView &st, 
+template <> inline void StaggeredKernels<StaggeredVec5dImplD>::DhopSiteAsm(StencilView &st, 
 								    DoubledGaugeFieldView &U,
 								    DoubledGaugeFieldView &UUU,
 								    SiteSpinor *buf, int sF,
@@ -825,7 +825,7 @@ template <> void StaggeredKernels<StaggeredVec5dImplD>::DhopSiteAsm(StencilView
   // This is the single precision 5th direction vectorised kernel
 
 #include <Grid/simd/Intel512single.h>
-template <> void StaggeredKernels<StaggeredImplF>::DhopSiteAsm(StencilView &st, 
+template <> inline void StaggeredKernels<StaggeredImplF>::DhopSiteAsm(StencilView &st, 
 							       DoubledGaugeFieldView &U,
 							       DoubledGaugeFieldView &UUU,
 							       SiteSpinor *buf, int sF,
@@ -894,7 +894,7 @@ template <> void StaggeredKernels<StaggeredImplF>::DhopSiteAsm(StencilView &st,
 }
 
 #include <Grid/simd/Intel512double.h>
-template <> void StaggeredKernels<StaggeredImplD>::DhopSiteAsm(StencilView &st, 
+template <> inline void StaggeredKernels<StaggeredImplD>::DhopSiteAsm(StencilView &st, 
 							       DoubledGaugeFieldView &U,
 							       DoubledGaugeFieldView &UUU,
 							       SiteSpinor *buf, int sF,

Grid/qcd/action/fermion/instantiation/StaggeredKernelsInstantiation.cc.master

@@ -29,6 +29,7 @@ directory
 #include <Grid/qcd/action/fermion/FermionCore.h>
 #include <Grid/qcd/action/fermion/implementation/StaggeredKernelsImplementation.h>
 #include <Grid/qcd/action/fermion/implementation/StaggeredKernelsHand.h>
+#include <Grid/qcd/action/fermion/implementation/StaggeredKernelsAsm.h>
 
 NAMESPACE_BEGIN(Grid);
 

Grid/qcd/action/fermion/instantiation/generate_instantiations.sh

@@ -2,14 +2,15 @@
 
 STAG_IMPL_LIST=" \
 	   StaggeredImplF \
-	   StaggeredImplD  "
-
-STAG5_IMPL_LIST=""
+	   StaggeredImplD \
+	   lexStaggeredImplF \
+	   lexStaggeredImplD  "
 
 WILSON_IMPL_LIST=" \
 	   WilsonImplF \
 	   WilsonImplD \
-	   WilsonImplD2 \
+	   lexWilsonImplF \
+	   lexWilsonImplD \
 	   SpWilsonImplF \
 	   SpWilsonImplD \
 	   WilsonAdjImplF \
@@ -32,9 +33,10 @@ COMPACT_WILSON_IMPL_LIST=" \
 DWF_IMPL_LIST=" \
 	   WilsonImplF \
 	   WilsonImplD \
-	   WilsonImplD2 \
+	   lexWilsonImplF \
+	   lexWilsonImplD \
 	   ZWilsonImplF \
-	   ZWilsonImplD2 "
+	   ZWilsonImplD "
 
 GDWF_IMPL_LIST=" \
 	   GparityWilsonImplF \
@@ -109,8 +111,3 @@ do
   ln -f -s ../$f.cc.master $impl/$f$impl.cc
 done
 done
-
-CC_LIST=" \
-  ImprovedStaggeredFermion5DInstantiation \
-  StaggeredKernelsInstantiation "
-

Grid/qcd/action/fermion/instantiation/lexStaggeredImplD/ImprovedStaggeredFermion5DInstantiationlexStaggeredImplD.cc

@@ -0,0 +1 @@
+../ImprovedStaggeredFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplD/ImprovedStaggeredFermionInstantiationlexStaggeredImplD.cc

@@ -0,0 +1 @@
+../ImprovedStaggeredFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplD/NaiveStaggeredFermionInstantiationlexStaggeredImplD.cc

@@ -0,0 +1 @@
+../NaiveStaggeredFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplD/StaggeredKernelsInstantiationlexStaggeredImplD.cc

@@ -0,0 +1 @@
+../StaggeredKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplD/impl.h

@@ -0,0 +1 @@
+#define IMPLEMENTATION lexStaggeredImplD

Grid/qcd/action/fermion/instantiation/lexStaggeredImplF/ImprovedStaggeredFermion5DInstantiationlexStaggeredImplF.cc

@@ -0,0 +1 @@
+../ImprovedStaggeredFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplF/ImprovedStaggeredFermionInstantiationlexStaggeredImplF.cc

@@ -0,0 +1 @@
+../ImprovedStaggeredFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplF/NaiveStaggeredFermionInstantiationlexStaggeredImplF.cc

@@ -0,0 +1 @@
+../NaiveStaggeredFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplF/StaggeredKernelsInstantiationlexStaggeredImplF.cc

@@ -0,0 +1 @@
+../StaggeredKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexStaggeredImplF/impl.h

@@ -0,0 +1 @@
+#define IMPLEMENTATION lexStaggeredImplF

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/CayleyFermion5DInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../CayleyFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/ContinuedFractionFermion5DInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../ContinuedFractionFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/DomainWallEOFAFermionInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../DomainWallEOFAFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/MobiusEOFAFermionInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../MobiusEOFAFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/PartialFractionFermion5DInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../PartialFractionFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/WilsonCloverFermionInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonCloverFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/WilsonFermion5DInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/WilsonFermionInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/WilsonKernelsInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/WilsonTMFermionInstantiationlexWilsonImplD.cc

@@ -0,0 +1 @@
+../WilsonTMFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplD/impl.h

@@ -0,0 +1 @@
+#define IMPLEMENTATION lexWilsonImplD

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/CayleyFermion5DInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../CayleyFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/ContinuedFractionFermion5DInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../ContinuedFractionFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/DomainWallEOFAFermionInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../DomainWallEOFAFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/MobiusEOFAFermionInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../MobiusEOFAFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/PartialFractionFermion5DInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../PartialFractionFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/WilsonCloverFermionInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonCloverFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/WilsonFermion5DInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonFermion5DInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/WilsonFermionInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/WilsonKernelsInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonKernelsInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/WilsonTMFermionInstantiationlexWilsonImplF.cc

@@ -0,0 +1 @@
+../WilsonTMFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/lexWilsonImplF/impl.h

@@ -0,0 +1 @@
+#define IMPLEMENTATION lexWilsonImplF

Grid/qcd/action/gauge/GaugeImplTypes.h

@@ -60,8 +60,10 @@ NAMESPACE_BEGIN(Grid);
   typedef typename Impl::SiteField SiteField;	    \
   typedef typename Impl::Field Field;
 
-// hardcodes the exponential approximation in the template
-template <class S, int Nrepresentation = Nc, int Nexp = 12, class Group = SU<Nc> > class GaugeImplTypes {
+// Order of the exponential approximation
+const int NexpDefault = 12;
+
+template <class S, int Nrepresentation = Nc, int Nexp = NexpDefault, class Group = SU<Nc> > class GaugeImplTypes {
 public:
   typedef S Simd;
   typedef typename Simd::scalar_type scalar_type;
@@ -188,14 +190,29 @@ typedef GaugeImplTypes<vComplex, Nc> GimplTypesR;
 typedef GaugeImplTypes<vComplexF, Nc> GimplTypesF;
 typedef GaugeImplTypes<vComplexD, Nc> GimplTypesD;
 
-typedef GaugeImplTypes<vComplex, Nc, 12, Sp<Nc> > SpGimplTypesR;
-typedef GaugeImplTypes<vComplexF, Nc, 12, Sp<Nc> > SpGimplTypesF;
-typedef GaugeImplTypes<vComplexD, Nc, 12, Sp<Nc> > SpGimplTypesD;
+typedef GaugeImplTypes<vComplex, Nc, NexpDefault, Sp<Nc> > SpGimplTypesR;
+typedef GaugeImplTypes<vComplexF, Nc, NexpDefault, Sp<Nc> > SpGimplTypesF;
+typedef GaugeImplTypes<vComplexD, Nc, NexpDefault, Sp<Nc> > SpGimplTypesD;
 
 typedef GaugeImplTypes<vComplex, SU<Nc>::AdjointDimension> GimplAdjointTypesR;
 typedef GaugeImplTypes<vComplexF, SU<Nc>::AdjointDimension> GimplAdjointTypesF;
 typedef GaugeImplTypes<vComplexD, SU<Nc>::AdjointDimension> GimplAdjointTypesD;
 
+/////////////////////////////////////////////////////////////////////////////
+// Lexicographic gauge implementation types
+/////////////////////////////////////////////////////////////////////////////
+typedef GaugeImplTypes<sComplexF, Nc> lexGimplTypesF;
+typedef GaugeImplTypes<sComplexD, Nc> lexGimplTypesD;
+
+typedef GaugeImplTypes<sComplexF, Nc, NexpDefault, Sp<Nc> > lexSpGimplTypesF;
+typedef GaugeImplTypes<sComplexD, Nc, NexpDefault, Sp<Nc> > lexSpGimplTypesD;
+
+typedef GaugeImplTypes<sComplexF, SU<Nc>::AdjointDimension> lexGimplAdjointTypesF;
+typedef GaugeImplTypes<sComplexD, SU<Nc>::AdjointDimension> lexGimplAdjointTypesD;
+/////////////////////////////////////////////////////////////////////////////
+// End lexicographic gauge implementation types
+/////////////////////////////////////////////////////////////////////////////
+
 
 
 

Grid/qcd/action/gauge/GaugeImplementations.h

@@ -197,6 +197,24 @@ typedef PeriodicGaugeImpl<SpGimplTypesR> SpPeriodicGimplR; // Real.. whichever p
 typedef PeriodicGaugeImpl<SpGimplTypesF> SpPeriodicGimplF; // Float
 typedef PeriodicGaugeImpl<SpGimplTypesD> SpPeriodicGimplD; // Double
 
+/////////////////////////////////////////////////////////////////////////////
+// Lexicographic gauge implementations
+/////////////////////////////////////////////////////////////////////////////
+typedef PeriodicGaugeImpl<lexGimplTypesF> lexPeriodicGimplF;
+typedef PeriodicGaugeImpl<lexGimplTypesD> lexPeriodicGimplD;
+
+typedef PeriodicGaugeImpl<lexGimplAdjointTypesF> lexPeriodicGimplAdjF;
+typedef PeriodicGaugeImpl<lexGimplAdjointTypesD> lexPeriodicGimplAdjD;
+
+typedef ConjugateGaugeImpl<lexGimplTypesF> lexConjugateGimplF;
+typedef ConjugateGaugeImpl<lexGimplTypesD> lexConjugateGimplD;
+
+typedef PeriodicGaugeImpl<lexSpGimplTypesF> lexSpPeriodicGimplF;
+typedef PeriodicGaugeImpl<lexSpGimplTypesD> lexSpPeriodicGimplD;
+/////////////////////////////////////////////////////////////////////////////
+// End lexicographic gauge implementations
+/////////////////////////////////////////////////////////////////////////////
+
 
 NAMESPACE_END(Grid);
 

Grid/qcd/action/pseudofermion/TwoFlavourPVdagMPseudoFermion.h

@@ -0,0 +1,183 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./lib/qcd/action/pseudofermion/TwoFlavourPVdagMPseudoFermion.h
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////////////////////////////////////////////
+// Two flavour pseudofermion on the COMPOSITE operator  F = PVdag M :
+//
+//    S1 = phi^dag (Fdag F)^-1 phi
+//
+//    integral  ==>  det( Mdag PV PVdag M ) = |det M|^2 |det PV|^2
+//
+// i.e. the target two-flavour |det M|^2 TIMES an excess |det PV|^2, to be
+// cancelled by compensator monomials (two TwoFlavourBosonPseudoFermionAction
+// instances on PV, each contributing |det PV|^-2, net |det PV|^-4; together
+// with this action's |det PV|^2 the ensemble carries |det M|^2/|det PV|^2 --
+// the standard DWF quotient).
+//
+// Why this shape: F = PVdag M is exactly the operator the non-Hermitian
+// multigrid coarsens, so its cycles precondition (Fdag F) natively.  The
+// outer solve is CG on a Hermitian positive definite system -- the
+// non-normality is quarantined inside the preconditioner.  One solve per
+// force evaluation; all other force ingredients are matrix multiplies.
+//
+// Heatbath is exact by OPERATOR ALGEBRA (no wall/projection identity):
+//    refresh:  phi = Fdag eta   ==>   S1 = eta^dag F (Fdag F)^-1 Fdag eta
+//                                        = |eta|^2       (to solver tol)
+//
+// Hasenbusch: nothing here requires PVOp to have mass one.  Any
+// (heavier,lighter) pair F(m1,m2) = D^dag(m1) D(m2) works, each rung
+// coarsenable by the same machinery; compensate intermediate-mass excess
+// dets with boson monomials on the heavier operator.
+//
+// Solver slots map  b -> (Fdag F)^-1 b  (full grid, zero guess imposed
+// internally).  The class is agnostic to the implementation: plain CG on
+// the normal equations for testing; sequential MG solves of Fdag and F, or
+// preconditioned CG with a frozen-cycle G Gdag preconditioner in production.
+///////////////////////////////////////////////////////////////////////////////
+template<class Impl>
+class TwoFlavourPVdagMPseudoFermionAction : public Action<typename Impl::GaugeField> {
+public:
+  INHERIT_IMPL_TYPES(Impl);
+
+private:
+  FermionOperator<Impl> & PVOp; // the heavier / Pauli-Villars operator
+  FermionOperator<Impl> & MOp;  // the lighter operator
+
+  LinearFunction<FermionField> &DerivSolver;  // b -> (FdagF)^-1 b, MD tolerance
+  LinearFunction<FermionField> &ActionSolver; // b -> (FdagF)^-1 b, accept/reject tolerance
+
+  FermionField Phi; // the pseudo fermion field for this trajectory
+
+  ////////////////////////////////////////////////////////////////////
+  // F = PVdag M  and  Fdag = Mdag PV
+  ////////////////////////////////////////////////////////////////////
+  void Fapply(const FermionField &in, FermionField &out) {
+    FermionField tmp(MOp.FermionGrid());
+    MOp.M(in,tmp);
+    PVOp.Mdag(tmp,out);
+  }
+  void FdagApply(const FermionField &in, FermionField &out) {
+    FermionField tmp(MOp.FermionGrid());
+    PVOp.M(in,tmp);
+    MOp.Mdag(tmp,out);
+  }
+
+public:
+  TwoFlavourPVdagMPseudoFermionAction(FermionOperator<Impl>  &_PVOp,
+				      FermionOperator<Impl>  &_MOp,
+				      LinearFunction<FermionField> & DS,
+				      LinearFunction<FermionField> & AS
+				      ) : PVOp(_PVOp),
+					  MOp(_MOp),
+					  DerivSolver(DS),
+					  ActionSolver(AS),
+					  Phi(_MOp.FermionGrid())
+  {};
+
+  virtual std::string action_name(){return "TwoFlavourPVdagMPseudoFermionAction";}
+
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+    return sstream.str();
+  }
+
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
+    // P(phi) = e^{- phi^dag (FdagF)^-1 phi} ; phi = Fdag eta ; P(eta) = e^{-eta^dag eta}
+    // e^{-x^2/2 sig^2} => sig^2 = 0.5 ; eta enters with width 1/sqrt(2).
+    RealD scale = std::sqrt(0.5);
+    FermionField eta(MOp.FermionGrid());
+    gaussian(pRNG,eta);
+    eta = eta * scale;
+    refresh(U,eta);
+  }
+
+  // Deterministic-noise variant (test hook, TwoFlavourEvenOddRatio idiom):
+  // after this,  S(U) == norm2(eta)  exactly (to solver tolerance).
+  void refresh(const GaugeField &U, const FermionField &eta) {
+    PVOp.ImportGauge(U);
+    MOp.ImportGauge(U);
+    FdagApply(eta,Phi);            // NO solve: heatbath is two matmuls
+    std::cout << GridLogMessage << action_name() << " refresh |Phi|^2 = "<< norm2(Phi)<<std::endl;
+  }
+
+  //////////////////////////////////////////////////////
+  // S1 = phi^dag (FdagF)^-1 phi
+  //////////////////////////////////////////////////////
+  virtual RealD S(const GaugeField &U) {
+    PVOp.ImportGauge(U);
+    MOp.ImportGauge(U);
+
+    FermionField X(MOp.FermionGrid());
+    X = Zero();
+    ActionSolver(Phi,X);                       // X = (FdagF)^-1 phi
+
+    RealD action = real(innerProduct(Phi,X));  // Hermitian positive kernel
+    return action;
+  }
+
+  //////////////////////////////////////////////////////
+  // dS1 = - X^dag [ dFdag F + Fdag dF ] X ,   X = (FdagF)^-1 phi,  Y = F X
+  //
+  // dF    = dPVdag M + PVdag dM   ==>  with A = M X,  B = PV Y :
+  //
+  // dS1 = - X^dag dMdag B - B^dag dM X - A^dag dPV Y - Y^dag dPVdag A
+  //
+  // ONE solve; A,Y,B by matrix multiply (Y = PVdag A reuses A).
+  //////////////////////////////////////////////////////
+  virtual void deriv(const GaugeField &U,GaugeField & dSdU) {
+    PVOp.ImportGauge(U);
+    MOp.ImportGauge(U);
+
+    FermionField X(MOp.FermionGrid());
+    FermionField Y(MOp.FermionGrid());
+    FermionField A(MOp.FermionGrid());
+    FermionField B(MOp.FermionGrid());
+
+    GaugeField force(MOp.GaugeGrid());
+
+    X = Zero();
+    DerivSolver(Phi,X);            // X = (FdagF)^-1 phi
+    MOp.M(X,A);                    // A = M X
+    PVOp.Mdag(A,Y);                // Y = PVdag A = F X
+    PVOp.M(Y,B);                   // B = PV Y
+
+    // dS1 = -( X^dag dMdag B + B^dag dM X + A^dag dPV Y + Y^dag dPVdag A )
+    MOp.MDeriv (force, X, B, DaggerYes); dSdU =      -force;
+    MOp.MDeriv (force, B, X, DaggerNo ); dSdU = dSdU -force;
+    PVOp.MDeriv(force, A, Y, DaggerNo ); dSdU = dSdU -force;
+    PVOp.MDeriv(force, Y, A, DaggerYes); dSdU = dSdU -force;
+
+    dSdU *= -1.0;  // Grid action sign convention (cf TwoFlavourRatio.h)
+  };
+};
+
+NAMESPACE_END(Grid);

Grid/qcd/utils/GaugeGroup.h

@@ -36,7 +36,7 @@ directory
 // This is the only reason why the second template parameter has a name.
 #define ONLY_IF_SU                                                       \
   typename dummy_name = group_name,                                      \
-           typename named_dummy = std::enable_if_t <                                 \
+          typename named_dummy = std::enable_if_t <                                 \
                           std::is_same<dummy_name, group_name>::value && \
                       is_su<dummy_name>::value >
 
@@ -89,10 +89,8 @@ template <int ncolour, class group_name>
 class GaugeGroup {
  public:
   static const int Dimension = ncolour;
-  static const int AdjointDimension =
-      compute_adjoint_dimension<group_name>(ncolour);
-  static const int AlgebraDimension =
-      compute_adjoint_dimension<group_name>(ncolour);
+  static const int AdjointDimension = compute_adjoint_dimension<group_name>(ncolour);
+  static const int AlgebraDimension = compute_adjoint_dimension<group_name>(ncolour);
 
   template <typename vtype>
   using iSU2Matrix = iScalar<iScalar<iMatrix<vtype, 2> > >;
@@ -101,38 +99,37 @@ class GaugeGroup {
   template <typename vtype>
   using iAlgebraVector = iScalar<iScalar<iVector<vtype, AdjointDimension> > >;
   template <typename vtype>
-  using iSUnAlgebraMatrix =
-    iScalar<iScalar<iMatrix<vtype, AdjointDimension> > >;
+  using iSUnAlgebraMatrix = iScalar<iScalar<iMatrix<vtype, AdjointDimension> > >;
   static int su2subgroups(void) { return su2subgroups(group_name()); }
 
   //////////////////////////////////////////////////////////////////////////////////////////////////
   // Types can be accessed as SU<2>::Matrix , SU<2>::vSUnMatrix,
   // SU<2>::LatticeMatrix etc...
   //////////////////////////////////////////////////////////////////////////////////////////////////
-  typedef iGroupMatrix<Complex> Matrix;
+  typedef iGroupMatrix<Complex>  Matrix;
   typedef iGroupMatrix<ComplexF> MatrixF;
   typedef iGroupMatrix<ComplexD> MatrixD;
 
-  typedef iGroupMatrix<vComplex> vMatrix;
+  typedef iGroupMatrix<vComplex>  vMatrix;
   typedef iGroupMatrix<vComplexF> vMatrixF;
   typedef iGroupMatrix<vComplexD> vMatrixD;
 
   // For the projectors to the algebra
   // these should be real...
   // keeping complex for consistency with the SIMD vector types
-  typedef iAlgebraVector<Complex> AlgebraVector;
+  typedef iAlgebraVector<Complex>  AlgebraVector;
   typedef iAlgebraVector<ComplexF> AlgebraVectorF;
   typedef iAlgebraVector<ComplexD> AlgebraVectorD;
 
-  typedef iAlgebraVector<vComplex> vAlgebraVector;
+  typedef iAlgebraVector<vComplex>  vAlgebraVector;
   typedef iAlgebraVector<vComplexF> vAlgebraVectorF;
   typedef iAlgebraVector<vComplexD> vAlgebraVectorD;
 
-  typedef Lattice<vMatrix> LatticeMatrix;
+  typedef Lattice<vMatrix>  LatticeMatrix;
   typedef Lattice<vMatrixF> LatticeMatrixF;
   typedef Lattice<vMatrixD> LatticeMatrixD;
   
-  typedef Lattice<vAlgebraVector> LatticeAlgebraVector;
+  typedef Lattice<vAlgebraVector>  LatticeAlgebraVector;
   typedef Lattice<vAlgebraVectorF> LatticeAlgebraVectorF;
   typedef Lattice<vAlgebraVectorD> LatticeAlgebraVectorD;
    
@@ -145,15 +142,15 @@ class GaugeGroup {
   typedef Lattice<vAlgebraMatrixD> LatticeAlgebraMatrixD;
   
 
-  typedef iSU2Matrix<Complex> SU2Matrix;
+  typedef iSU2Matrix<Complex>  SU2Matrix;
   typedef iSU2Matrix<ComplexF> SU2MatrixF;
   typedef iSU2Matrix<ComplexD> SU2MatrixD;
 
-  typedef iSU2Matrix<vComplex> vSU2Matrix;
+  typedef iSU2Matrix<vComplex>  vSU2Matrix;
   typedef iSU2Matrix<vComplexF> vSU2MatrixF;
   typedef iSU2Matrix<vComplexD> vSU2MatrixD;
 
-  typedef Lattice<vSU2Matrix> LatticeSU2Matrix;
+  typedef Lattice<vSU2Matrix>  LatticeSU2Matrix;
   typedef Lattice<vSU2MatrixF> LatticeSU2MatrixF;
   typedef Lattice<vSU2MatrixD> LatticeSU2MatrixD;
 
@@ -296,16 +293,18 @@ class GaugeGroup {
       Umu = ProjectOnGeneralGroup(Umu);
     }
   }
-       
-
   
   template <int N,class vComplex_t>
-  static Lattice<iScalar<iScalar<iMatrix<vComplex_t, N> > > > ProjectOnGeneralGroup(const Lattice<iScalar<iScalar<iMatrix<vComplex_t, N> > > > &Umu) {
+  static Lattice<iScalar<iScalar<iMatrix<vComplex_t, N> > > >
+  ProjectOnGeneralGroup(const Lattice<iScalar<iScalar<iMatrix<vComplex_t, N> > > > &Umu)
+  {
     return ProjectOnGeneralGroup(Umu, group_name());
   }
 
-  template <int N,class vComplex_t>       // Projects on SU(N), Sp(2N), with unit determinant, by first projecting on general group and then enforcing unit determinant
-  static void ProjectOnSpecialGroup(Lattice<iScalar<iScalar<iMatrix<vComplex_t, N> > > > &Umu) {
+  // Projects on SU(N), Sp(2N), with unit determinant, by first projecting on general group and then enforcing unit determinant
+  template <int N,class vComplex_t>  
+  static void ProjectOnSpecialGroup(Lattice<iScalar<iScalar<iMatrix<vComplex_t, N> > > > &Umu)
+  {
        Umu = ProjectOnGeneralGroup(Umu);
        auto det = Determinant(Umu);
 
@@ -319,14 +318,15 @@ class GaugeGroup {
    }
 
   template <int N,class vComplex_t>    // reunitarise, resimplectify... previously ProjectSUn
-    static void ProjectOnSpecialGroup(Lattice<iVector<iScalar<iMatrix<vComplex_t, N> >, Nd> > &U) {
+  static void ProjectOnSpecialGroup(Lattice<iVector<iScalar<iMatrix<vComplex_t, N> >, Nd> > &U)
+  {
       // Reunitarise
       for (int mu = 0; mu < Nd; mu++) {
         auto Umu = PeekIndex<LorentzIndex>(U, mu);
         ProjectOnSpecialGroup(Umu);
         PokeIndex<LorentzIndex>(U, Umu, mu);
       }
-    }
+  }
     
   template <typename GaugeField>
   static void HotConfiguration(GridParallelRNG &pRNG, GaugeField &out) {

Grid/simd/Grid_a64fx-2.h

@@ -933,6 +933,8 @@ typedef Optimization::Div            DivSIMD;
 typedef Optimization::Mult           MultSIMD;
 typedef Optimization::MultComplex    MultComplexSIMD;
 typedef Optimization::MultAddComplex MultAddComplexSIMD;
+// Asserts MultAddComplexSIMD and a three argument MultSIMD operator()
+#define GRID_ARCH_HAS_COMPLEX_MULT_ADD 1
 typedef Optimization::MultRealPart   MultRealPartSIMD;
 typedef Optimization::MaddRealPart   MaddRealPartSIMD;
 typedef Optimization::Conj           ConjSIMD;

Grid/simd/Grid_a64fx-fixedsize.h

@@ -760,6 +760,8 @@ typedef Optimization::Div            DivSIMD;
 typedef Optimization::Mult           MultSIMD;
 typedef Optimization::MultComplex    MultComplexSIMD;
 typedef Optimization::MultAddComplex MultAddComplexSIMD;
+// Asserts MultAddComplexSIMD and a three argument MultSIMD operator()
+#define GRID_ARCH_HAS_COMPLEX_MULT_ADD 1
 typedef Optimization::MultRealPart   MultRealPartSIMD;
 typedef Optimization::MaddRealPart   MaddRealPartSIMD;
 typedef Optimization::Conj           ConjSIMD;

Grid/simd/Grid_doubled_vector.h

@@ -339,54 +339,6 @@ typedef Grid_simd2<complex<double>  , vComplexD>  vComplexD2;
 typedef Grid_simd2<double           , vRealD>     vRealD2;
 
 
-
-/////////////////////////////////////////
-// Some traits to recognise the types
-/////////////////////////////////////////
-template <typename T>
-struct is_simd : public std::false_type {};
-template <> struct is_simd<vRealF>     : public std::true_type {};
-template <> struct is_simd<vRealD>     : public std::true_type {};
-template <> struct is_simd<vRealH>     : public std::true_type {};
-template <> struct is_simd<vComplexF>  : public std::true_type {};
-template <> struct is_simd<vComplexD>  : public std::true_type {};
-template <> struct is_simd<vComplexH>  : public std::true_type {};
-template <> struct is_simd<vInteger>   : public std::true_type {};
-template <> struct is_simd<vRealD2>    : public std::true_type {};
-template <> struct is_simd<vComplexD2> : public std::true_type {};
-
-template <typename T> using IfSimd    = Invoke<std::enable_if<is_simd<T>::value, int> >;
-template <typename T> using IfNotSimd = Invoke<std::enable_if<!is_simd<T>::value, unsigned> >;
-
-///////////////////////////////////////////////
-// insert / extract with complex support
-///////////////////////////////////////////////
-template <class S, class V>
-accelerator_inline S getlane(const Grid_simd<S, V> &in,int lane) {
-  return in.getlane(lane);
-}
-template <class S, class V>
-accelerator_inline void putlane(Grid_simd<S, V> &vec,const S &_S, int lane){
-  vec.putlane(_S,lane);
-}
-template <class S,IfNotSimd<S> = 0 >
-accelerator_inline S getlane(const S &in,int lane) {
-  return in;
-}
-template <class S,IfNotSimd<S> = 0 >
-accelerator_inline void putlane(S &vec,const S &_S, int lane){
-  vec = _S;
-}
-template <class S, class V>
-accelerator_inline S getlane(const Grid_simd2<S, V> &in,int lane) {
-  return in.getlane(lane);
-}
-template <class S, class V>
-accelerator_inline void putlane(Grid_simd2<S, V> &vec,const S &_S, int lane){
-  vec.putlane(_S,lane);
-}
-
-
 ////////////////////////////////////////////////////////////////////
 // General rotate
 ////////////////////////////////////////////////////////////////////

Grid/simd/Grid_neon.h

@@ -42,6 +42,16 @@
 #define GEN_SIMD_WIDTH 16u
 #endif
 
+//////////////////////////////////////////////////////////////////////////////
+// ARMv8.3 complex assist (FCMLA). Kept separate from the ACLE
+// __ARM_FEATURE_COMPLEX: the instructions are also present on A64FX, where
+// their throughput is 1:4 and this path would be a regression. Revisit the
+// rule once measured per target.
+//////////////////////////////////////////////////////////////////////////////
+#ifdef __ARM_FEATURE_COMPLEX
+#define GRID_ARM_FEATURE_COMPLEX 1
+#endif
+
 #include "Grid_generic_types.h"
 #include <arm_neon.h>
 
@@ -226,6 +236,22 @@ struct Div{
 };
 
 struct MultComplex{
+#ifdef GRID_ARM_FEATURE_COMPLEX
+  // fcmla #0  : r += (ar,ar)*(br,bi) = ( ar*br, ar*bi)
+  // fcmla #90 : r += (-ai,ai)*(bi,br) = (-ai*bi, ai*br)
+  // Complex float: two products, two instructions
+  inline float32x4_t operator()(float32x4_t a, float32x4_t b){
+    float32x4_t r = vdupq_n_f32(0.0f);
+    r = vcmlaq_f32(r, a, b);
+    return vcmlaq_rot90_f32(r, a, b);
+  }
+  // Complex double
+  inline float64x2_t operator()(float64x2_t a, float64x2_t b){
+    float64x2_t r = vdupq_n_f64(0.0);
+    r = vcmlaq_f64(r, a, b);
+    return vcmlaq_rot90_f64(r, a, b);
+  }
+#else
   // Complex float
   inline float32x4_t operator()(float32x4_t a, float32x4_t b){
 
@@ -273,8 +299,23 @@ struct MultComplex{
     // r5 = vmulq_f64(r0, a);
     // return vaddq_f64(r4, r5);
   }
+#endif
 };
 
+#ifdef GRID_ARM_FEATURE_COMPLEX
+struct MultAddComplex{
+  // Complex a*b+c
+  inline float32x4_t operator()(float32x4_t a, float32x4_t b, float32x4_t c){
+    c = vcmlaq_f32(c, a, b);
+    return vcmlaq_rot90_f32(c, a, b);
+  }
+  inline float64x2_t operator()(float64x2_t a, float64x2_t b, float64x2_t c){
+    c = vcmlaq_f64(c, a, b);
+    return vcmlaq_rot90_f64(c, a, b);
+  }
+};
+#endif
+
 struct Mult{
   // Real float
   inline float32x4_t mac(float32x4_t a, float32x4_t b, float32x4_t c){
@@ -285,6 +326,13 @@ struct Mult{
     //return vaddq_f64(vmulq_f64(b,c),a);
     return vfmaq_f64(a, b, c);
   }
+  // Real a*b+c
+  inline float32x4_t operator()(float32x4_t a, float32x4_t b, float32x4_t c){
+    return vfmaq_f32(c, a, b);
+  }
+  inline float64x2_t operator()(float64x2_t a, float64x2_t b, float64x2_t c){
+    return vfmaq_f64(c, a, b);
+  }
   inline float32x4_t operator()(float32x4_t a, float32x4_t b){
     return vmulq_f32(a,b);
   }
@@ -585,6 +633,11 @@ typedef Optimization::Sub         SubSIMD;
 typedef Optimization::Div         DivSIMD;
 typedef Optimization::Mult        MultSIMD;
 typedef Optimization::MultComplex MultComplexSIMD;
+#ifdef GRID_ARM_FEATURE_COMPLEX
+typedef Optimization::MultAddComplex MultAddComplexSIMD;
+// Asserts MultAddComplexSIMD and a three argument MultSIMD operator()
+#define GRID_ARCH_HAS_COMPLEX_MULT_ADD 1
+#endif
 typedef Optimization::MultRealPart MultRealPartSIMD;
 typedef Optimization::MaddRealPart MaddRealPartSIMD;
 typedef Optimization::Conj        ConjSIMD;

Grid/simd/Grid_scalar_support.h

@@ -0,0 +1,204 @@
+#pragma once
+
+#if defined(GRID_CUDA) || defined(GRID_HIP)
+#include <thrust/complex.h>
+#endif
+
+////////////////////////////////////////////////////////////////////////
+// Define scalar and vector floating point types
+//
+// Scalar:   RealF, RealD, ComplexF, ComplexD
+//
+// Vector:  vRealF, vRealD, vComplexF, vComplexD
+//
+// Vector types are arch dependent
+////////////////////////////////////////////////////////////////////////
+
+#define _MM_SELECT_FOUR_FOUR(A,B,C,D) ((A<<6)|(B<<4)|(C<<2)|(D))
+#define _MM_SELECT_FOUR_FOUR_STRING(A,B,C,D) "((" #A "<<6)|(" #B "<<4)|(" #C "<<2)|(" #D "))"
+#define _MM_SELECT_EIGHT_TWO(A,B,C,D,E,F,G,H) ((A<<7)|(B<<6)|(C<<5)|(D<<4)|(E<<3)|(F<<2)|(G<<4)|(H))
+#define _MM_SELECT_FOUR_TWO (A,B,C,D) _MM_SELECT_EIGHT_TWO(0,0,0,0,A,B,C,D)
+#define _MM_SELECT_TWO_TWO  (A,B)     _MM_SELECT_FOUR_TWO(0,0,A,B)
+
+#define RotateBit (0x100)
+
+NAMESPACE_BEGIN(Grid);
+
+typedef uint32_t Integer;
+
+typedef  float  RealF;
+typedef  double RealD;
+#ifdef GRID_DEFAULT_PRECISION_DOUBLE
+typedef RealD   Real;
+#else
+typedef RealF  Real;
+#endif
+
+#if defined(GRID_CUDA) || defined(GRID_HIP)
+typedef thrust::complex<RealF> ComplexF;
+typedef thrust::complex<RealD> ComplexD;
+typedef thrust::complex<Real>  Complex;
+typedef thrust::complex<uint16_t>  ComplexH;
+template<class T> using complex = thrust::complex<T>;
+
+accelerator_inline ComplexD pow(const ComplexD& r,RealD y){ return(thrust::pow(r,(double)y)); }
+accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(thrust::pow(r,(float)y)); }
+#else 
+typedef std::complex<RealF> ComplexF;
+typedef std::complex<RealD> ComplexD;
+typedef std::complex<Real>  Complex;
+typedef std::complex<uint16_t>  ComplexH; // Hack
+template<class T> using complex = std::complex<T>;
+
+accelerator_inline ComplexD pow(const ComplexD& r,RealD y){ return(std::pow(r,y)); }
+accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(std::pow(r,y)); }
+#endif
+
+//accelerator_inline RealD pow(const RealD& r,RealD y){ return(std::pow(r,y)); }
+//accelerator_inline RealD sqrt(const RealD  & r){ return std::sqrt(r); }
+
+// This comes from ::pow already from math.h and CUDA
+// Calls either Grid::pow for complex, or std::pow for real
+// Problem is CUDA math_functions is exposing ::pow, and I can't define
+
+using std::abs;
+using std::pow;
+using std::sqrt;
+using std::log;
+using std::exp;
+using std::sin;
+using std::cos;
+using std::asin;
+using std::acos;
+
+
+accelerator_inline RealF    conjugate(const RealF  & r){ return r; }
+accelerator_inline RealD    conjugate(const RealD  & r){ return r; }
+accelerator_inline ComplexD conjugate(const ComplexD& r){ return(conj(r)); }
+accelerator_inline ComplexF conjugate(const ComplexF& r ){ return(conj(r)); }
+
+accelerator_inline RealF    adj(const RealF  & r){ return r; }
+accelerator_inline RealD    adj(const RealD  & r){ return r; }
+accelerator_inline ComplexD adj(const ComplexD& r){ return(conjugate(r)); }
+accelerator_inline ComplexF adj(const ComplexF& r ){ return(conjugate(r)); }
+
+#if defined(GRID_CUDA) || defined(GRID_HIP)
+//Provide for convenience
+inline std::complex<double> conjugate(const std::complex<double>& r){ return(conj(r)); }
+inline std::complex<float>  conjugate(const std::complex<float>& r) { return(conj(r)); }
+inline std::complex<double> adj(const std::complex<double>& r)      { return(conj(r)); }
+inline std::complex<float>  adj(const std::complex<float>& r)       { return(conj(r)); }
+#endif
+
+accelerator_inline RealF real(const RealF  & r){ return r; }
+accelerator_inline RealD real(const RealD  & r){ return r; }
+accelerator_inline RealF real(const ComplexF  & r){ return r.real(); }
+accelerator_inline RealD real(const ComplexD  & r){ return r.real(); }
+
+accelerator_inline RealF imag(const ComplexF  & r){ return r.imag(); }
+accelerator_inline RealD imag(const ComplexD  & r){ return r.imag(); }
+
+accelerator_inline ComplexD innerProduct(const ComplexD & l, const ComplexD & r) { return conjugate(l)*r; }
+accelerator_inline ComplexF innerProduct(const ComplexF & l, const ComplexF & r) { return conjugate(l)*r; }
+accelerator_inline RealD innerProduct(const RealD & l, const RealD & r) { return l*r; }
+accelerator_inline RealF innerProduct(const RealF & l, const RealF & r) { return l*r; }
+
+accelerator_inline ComplexD Reduce(const ComplexD& r){ return r; }
+accelerator_inline ComplexF Reduce(const ComplexF& r){ return r; }
+accelerator_inline RealD Reduce(const RealD& r){ return r; }
+accelerator_inline RealF Reduce(const RealF& r){ return r; }
+
+accelerator_inline RealD toReal(const ComplexD& r){ return r.real(); }
+accelerator_inline RealF toReal(const ComplexF& r){ return r.real(); }
+accelerator_inline RealD toReal(const RealD& r){ return r; }
+accelerator_inline RealF toReal(const RealF& r){ return r; }
+  
+////////////////////////////////////////////////////////////////////////////////
+//Provide support functions for basic real and complex data types required by Grid
+//Single and double precision versions. Should be able to template this once only.
+////////////////////////////////////////////////////////////////////////////////
+accelerator_inline void mac (ComplexD * __restrict__ y,const ComplexD * __restrict__ a,const ComplexD *__restrict__ x){ *y = (*a) * (*x)+(*y); };
+accelerator_inline void mult(ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) * (*r);}
+accelerator_inline void sub (ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) - (*r);}
+accelerator_inline void add (ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) + (*r);}
+// conjugate already supported for complex
+  
+accelerator_inline void mac (ComplexF * __restrict__ y,const ComplexF * __restrict__ a,const ComplexF *__restrict__ x){ *y = (*a) * (*x)+(*y); }
+accelerator_inline void mult(ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) * (*r); }
+accelerator_inline void sub (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) - (*r); }
+accelerator_inline void add (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) + (*r); }
+  
+//conjugate already supported for complex
+accelerator_inline ComplexF timesI(const ComplexF &r)     { return(ComplexF(-r.imag(),r.real()));}
+accelerator_inline ComplexD timesI(const ComplexD &r)     { return(ComplexD(-r.imag(),r.real()));}
+accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(ComplexF(r.imag(),-r.real()));}
+accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(ComplexD(r.imag(),-r.real()));}
+//accelerator_inline ComplexF timesI(const ComplexF &r)     { return(r*ComplexF(0.0,1.0));}
+//accelerator_inline ComplexD timesI(const ComplexD &r)     { return(r*ComplexD(0.0,1.0));}
+//accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(r*ComplexF(0.0,-1.0));}
+//accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(r*ComplexD(0.0,-1.0));}
+
+// define projections to real and imaginay parts
+accelerator_inline ComplexF projReal(const ComplexF &r){return( ComplexF(r.real(), 0.0));}
+accelerator_inline ComplexD projReal(const ComplexD &r){return( ComplexD(r.real(), 0.0));}
+accelerator_inline ComplexF projImag(const ComplexF &r){return (ComplexF(r.imag(), 0.0 ));}
+accelerator_inline ComplexD projImag(const ComplexD &r){return (ComplexD(r.imag(), 0.0));}
+
+// define auxiliary functions for complex computations
+accelerator_inline void timesI(ComplexF &ret,const ComplexF &r)     { ret = timesI(r);}
+accelerator_inline void timesI(ComplexD &ret,const ComplexD &r)     { ret = timesI(r);}
+accelerator_inline void timesMinusI(ComplexF &ret,const ComplexF &r){ ret = timesMinusI(r);}
+accelerator_inline void timesMinusI(ComplexD &ret,const ComplexD &r){ ret = timesMinusI(r);}
+  
+accelerator_inline void mac (RealD * __restrict__ y,const RealD * __restrict__ a,const RealD *__restrict__ x){ *y = (*a) * (*x)+(*y);}
+accelerator_inline void mult(RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) * (*r);}
+accelerator_inline void sub (RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) - (*r);}
+accelerator_inline void add (RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) + (*r);}
+  
+accelerator_inline void mac (RealF * __restrict__ y,const RealF * __restrict__ a,const RealF *__restrict__ x){  *y = (*a) * (*x)+(*y); }
+accelerator_inline void mult(RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) * (*r); }
+accelerator_inline void sub (RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) - (*r); }
+accelerator_inline void add (RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) + (*r); }
+  
+accelerator_inline void vstream(ComplexF &l, const ComplexF &r){ l=r;}
+accelerator_inline void vstream(ComplexD &l, const ComplexD &r){ l=r;}
+accelerator_inline void vstream(RealF &l, const RealF &r){ l=r;}
+accelerator_inline void vstream(RealD &l, const RealD &r){ l=r;}
+  
+accelerator_inline ComplexD toComplex(const RealD &in) { return ComplexD(in);}
+accelerator_inline ComplexF toComplex(const RealF &in) { return ComplexF(in);}
+  
+class Zero{};
+//static Zero Zero();
+template<class itype> accelerator_inline void zeroit(itype &arg)   { arg=Zero();};
+template<>            accelerator_inline void zeroit(ComplexF &arg){ arg=0; };
+template<>            accelerator_inline void zeroit(ComplexD &arg){ arg=0; };
+template<>            accelerator_inline void zeroit(RealF &arg)   { arg=0; };
+template<>            accelerator_inline void zeroit(RealD &arg)   { arg=0; };
+
+// More limited Integer support  
+accelerator_inline Integer Reduce(const Integer& r){ return r; }
+accelerator_inline void mac (Integer * __restrict__ y,const Integer * __restrict__ a,const Integer *__restrict__ x){  *y = (*a) * (*x)+(*y); }
+accelerator_inline void mult(Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) * (*r); }
+accelerator_inline void sub (Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) - (*r); }
+accelerator_inline void add (Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) + (*r); }
+accelerator_inline void vstream(Integer &l, const RealD &r){ l=r;}
+template<>            accelerator_inline void zeroit(Integer &arg)   { arg=0; };
+
+accelerator_inline Integer mod (Integer a,Integer y) { return a%y;}
+accelerator_inline Integer div (Integer a,Integer y) { return a/y;}
+//accelerator_inline Integer abs (Integer &a) { return a%y;}
+
+//////////////////////////////////////////////////////////
+// Permute
+// Permute 0 every ABCDEFGH -> BA DC FE HG
+// Permute 1 every ABCDEFGH -> CD AB GH EF
+// Permute 2 every ABCDEFGH -> EFGH ABCD
+// Permute 3 possible on longer iVector lengths (512bit = 8 double = 16 single)
+// Permute 4 possible on half precision @512bit vectors.
+//
+// Defined inside SIMD specialization files
+//////////////////////////////////////////////////////////
+template<class VectorSIMD> accelerator_inline void Gpermute(VectorSIMD &y,const VectorSIMD &b,int perm);
+
+NAMESPACE_END(Grid);

Grid/simd/Grid_scalar_types.h

@@ -0,0 +1,568 @@
+#pragma once
+NAMESPACE_BEGIN(Grid);
+template <class Scalar_type>
+class Grid_simd1 {
+public:
+  typedef typename RealPart<Scalar_type>::type Real;
+  typedef Scalar_type vector_type;
+  typedef Scalar_type scalar_type;
+
+  vector_type v;
+
+  static accelerator_inline constexpr int Nsimd(void) {
+    return 1;
+  }
+  accelerator_inline Grid_simd1 &operator=(const Grid_simd1 &&rhs) {
+    v = rhs.v;
+    return *this;
+  };
+  accelerator_inline Grid_simd1 &operator=(const Grid_simd1 &rhs) {
+    v = rhs.v;
+    return *this;
+  };  // faster than not declaring it and leaving to the compiler
+
+  accelerator Grid_simd1() = default;
+  accelerator_inline Grid_simd1(const Grid_simd1 &rhs)  : v(rhs.v){};  // compiles in movaps
+  accelerator_inline Grid_simd1(const Grid_simd1 &&rhs) : v(rhs.v){};
+
+  accelerator_inline Grid_simd1(const Real a) { v=Scalar_type(a); };
+
+  template <typename S = Scalar_type> accelerator_inline
+  Grid_simd1(const typename std::enable_if<is_complex<S>::value, S>::type a) {
+    v=Scalar_type(a);
+  };
+
+  /////////////////////////////
+  // Constructors
+  /////////////////////////////
+  accelerator_inline Grid_simd1 &  operator=(const Zero &z) {
+    v=scalar_type(0);
+    return *this;
+  }
+
+  ///////////////////////////////////////////////
+  // mac, mult, sub, add, adj
+  ///////////////////////////////////////////////
+
+  friend accelerator_inline void mac(Grid_simd1 *__restrict__ y,
+				     const Grid_simd1 *__restrict__ a,
+				     const Grid_simd1 *__restrict__ x) {
+    *y = (*a) * (*x) + (*y);
+  };
+
+  friend accelerator_inline void mult(Grid_simd1 *__restrict__ y,
+				      const Grid_simd1 *__restrict__ l,
+				      const Grid_simd1 *__restrict__ r) {
+    *y = (*l) * (*r);
+  }
+
+  friend accelerator_inline void sub(Grid_simd1 *__restrict__ y,
+				     const Grid_simd1 *__restrict__ l,
+				     const Grid_simd1 *__restrict__ r) {
+    *y = (*l) - (*r);
+  }
+  friend accelerator_inline void add(Grid_simd1 *__restrict__ y,
+				     const Grid_simd1 *__restrict__ l,
+				     const Grid_simd1 *__restrict__ r) {
+    *y = (*l) + (*r);
+  }
+  friend accelerator_inline void mac(Grid_simd1 *__restrict__ y,
+				     const Scalar_type *__restrict__ a,
+				     const Grid_simd1 *__restrict__ x) {
+    *y = (*a) * (*x) + (*y);
+  };
+  friend accelerator_inline void mult(Grid_simd1 *__restrict__ y,
+				      const Scalar_type *__restrict__ l,
+				      const Grid_simd1 *__restrict__ r) {
+    *y = (*l) * (*r);
+  }
+  friend accelerator_inline void sub(Grid_simd1 *__restrict__ y,
+				     const Scalar_type *__restrict__ l,
+				     const Grid_simd1 *__restrict__ r) {
+    *y = (*l) - (*r);
+  }
+  friend accelerator_inline void add(Grid_simd1 *__restrict__ y,
+				     const Scalar_type *__restrict__ l,
+				     const Grid_simd1 *__restrict__ r) {
+    *y = (*l) + (*r);
+  }
+
+  friend accelerator_inline void mac(Grid_simd1 *__restrict__ y,
+				     const Grid_simd1 *__restrict__ a,
+				     const Scalar_type *__restrict__ x) {
+    *y = (*a) * (*x) + (*y);
+  };
+  friend accelerator_inline void mult(Grid_simd1 *__restrict__ y,
+				      const Grid_simd1 *__restrict__ l,
+				      const Scalar_type *__restrict__ r) {
+    *y = (*l) * (*r);
+  }
+  friend accelerator_inline void sub(Grid_simd1 *__restrict__ y,
+				     const Grid_simd1 *__restrict__ l,
+				     const Scalar_type *__restrict__ r) {
+    *y = (*l) - (*r);
+  }
+  friend accelerator_inline void add(Grid_simd1 *__restrict__ y,
+				     const Grid_simd1 *__restrict__ l,
+				     const Scalar_type *__restrict__ r) {
+    *y = (*l) + (*r);
+  }
+
+  ////////////////////////////////////////////////////////////////////////
+  // FIXME:  gonna remove these load/store, get, set, prefetch
+  ////////////////////////////////////////////////////////////////////////
+  friend accelerator_inline void vset(Grid_simd1 &ret, Scalar_type *a) {
+    ret.v = *a;
+  }
+
+  ///////////////////////
+  // Vstore
+  ///////////////////////
+  friend accelerator_inline void vstore(const Grid_simd1 &ret, Scalar_type *a) {
+    *a=ret.v;
+  }
+
+  ///////////////////////
+  // Vprefetch
+  ///////////////////////
+  friend accelerator_inline void vprefetch(const Grid_simd1 &v) { }
+
+  ///////////////////////
+  // Reduce
+  ///////////////////////
+  friend accelerator_inline Scalar_type Reduce(const Grid_simd1 &in) {
+    return in.v;
+  }
+  ////////////////////////////
+  // operator scalar * simd
+  ////////////////////////////
+  friend accelerator_inline Grid_simd1 operator*(const Scalar_type &a, Grid_simd1 b) {
+    Grid_simd1 va;
+    va.v=a;
+    return va * b;
+  }
+  friend accelerator_inline Grid_simd1 operator*(Grid_simd1 b, const Scalar_type &a) {
+    return a * b;
+  }
+
+  //////////////////////////////////
+  // Divides
+  //////////////////////////////////
+  friend accelerator_inline Grid_simd1 operator/(const Scalar_type &a, Grid_simd1 b) {
+    Grid_simd1 va;
+    va.v = a;
+    return va / b;
+  }
+  friend accelerator_inline Grid_simd1 operator/(Grid_simd1 b, const Scalar_type &a) {
+    Grid_simd1 va;
+    va.v=a;
+    return b / va;
+  }
+  ///////////////////////
+  // Unary negation
+  ///////////////////////
+  friend accelerator_inline Grid_simd1 operator-(const Grid_simd1 &r) {
+    Grid_simd1 ret;
+    ret.v = scalar_type(0);
+    ret = ret - r;
+    return ret;
+  }
+  // *=,+=,-= operators
+  accelerator_inline Grid_simd1 &operator*=(const Grid_simd1 &r) {
+    *this = (*this) * r;
+    return *this;
+  }
+  accelerator_inline Grid_simd1 &operator+=(const Grid_simd1 &r) {
+    *this = *this + r;
+    return *this;
+  }
+  accelerator_inline Grid_simd1 &operator-=(const Grid_simd1 &r) {
+    *this = *this - r;
+    return *this;
+  }
+
+  ///////////////////////////////////////
+  // Not all functions are supported
+  // through SIMD and must breakout to
+  // scalar type and back again. This
+  // provides support
+  ///////////////////////////////////////
+
+  template <class functor>
+  friend accelerator_inline Grid_simd1 SimdApply(const functor &func, const Grid_simd1 &v) {
+    Grid_simd1 ret;
+    ret.v = func(v.v);
+    return ret;
+  }
+  template <class functor>
+  friend accelerator_inline Grid_simd1 SimdApplyBinop(const functor &func,
+                                         const Grid_simd1 &x,
+                                         const Grid_simd1 &y) {
+    Grid_simd1 ret;
+    ret.v = func(x.v,y.v);
+    return ret;
+  }
+  ///////////////////////
+  // Exchange
+  // Al Ah , Bl Bh -> Al Bl Ah,Bh
+  ///////////////////////
+  friend accelerator_inline void exchange(Grid_simd1 &out1,Grid_simd1 &out2,Grid_simd1 in1,Grid_simd1 in2,int n)
+  {
+    assert(0);
+  }
+  friend accelerator_inline void exchange0(Grid_simd1 &out1,Grid_simd1 &out2,Grid_simd1 in1,Grid_simd1 in2){
+    assert(0);
+  }
+  friend accelerator_inline void exchange1(Grid_simd1 &out1,Grid_simd1 &out2,Grid_simd1 in1,Grid_simd1 in2){
+    assert(0);
+  }
+  friend accelerator_inline void exchange2(Grid_simd1 &out1,Grid_simd1 &out2,Grid_simd1 in1,Grid_simd1 in2){
+    assert(0);
+  }
+  friend accelerator_inline void exchange3(Grid_simd1 &out1,Grid_simd1 &out2,Grid_simd1 in1,Grid_simd1 in2){
+    assert(0);
+  }
+  ////////////////////////////////////////////////////////////////////
+  // Permute: unreachable at Nsimd=1
+  ////////////////////////////////////////////////////////////////////
+  friend accelerator_inline void permute0(Grid_simd1 &y, Grid_simd1 b) {
+    assert(0);
+  }
+  friend accelerator_inline void permute1(Grid_simd1 &y, Grid_simd1 b) {
+    assert(0);
+  }
+  friend accelerator_inline void permute2(Grid_simd1 &y, Grid_simd1 b) {
+    assert(0);
+  }
+  friend accelerator_inline void permute3(Grid_simd1 &y, Grid_simd1 b) {
+    assert(0);
+  }
+  friend accelerator_inline void permute(Grid_simd1 &y, Grid_simd1 b, int perm) {
+    assert(0);
+  }
+
+  ///////////////////////////////
+  // Getting single lanes
+  ///////////////////////////////
+  accelerator_inline Scalar_type getlane(int lane) const {
+    return v;
+  }
+  accelerator_inline void putlane(const Scalar_type &S, int lane){
+    v=S;
+  }
+
+};
+template <class Scalar>
+inline std::ostream& operator<< (std::ostream& stream, const Grid_simd1<Scalar> &o){
+  stream<<"<"<<o.v<<">";
+  return stream;
+}
+
+typedef Grid_simd1<RealF> sRealF;
+typedef Grid_simd1<RealD> sRealD;
+typedef Grid_simd1<ComplexF> sComplexF;
+typedef Grid_simd1<ComplexD> sComplexD;
+typedef Grid_simd1<Integer>  sInteger;
+
+/////////////////////////////////////////
+// Permute
+/////////////////////////////////////////
+
+//accelerator_inline void permute(sComplexD &y,sComplexD b, int perm) {  y=b; }
+//accelerator_inline void permute(sComplexF &y,sComplexF b, int perm) {  y=b; }
+//accelerator_inline void permute(sRealD &y,sRealD b, int perm) {  y=b; }
+//accelerator_inline void permute(sRealF &y,sRealF b, int perm) {  y=b; }
+
+////////////////////////////////////////////////////////////////////
+// General rotate
+////////////////////////////////////////////////////////////////////
+template <class S, IfNotComplex<S> = 0>
+accelerator_inline Grid_simd1<S> rotate(Grid_simd1<S> b, int nrot) { return b; }
+template <class S, IfComplex<S> = 0>
+accelerator_inline Grid_simd1<S> rotate(Grid_simd1<S> b, int nrot) {  return b; }
+template <class S, IfNotComplex<S> =0>
+accelerator_inline void rotate( Grid_simd1<S> &ret,Grid_simd1<S> b,int nrot)
+{
+  ret = b;
+}
+template <class S, IfComplex<S> =0>
+accelerator_inline void rotate(Grid_simd1<S> &ret,Grid_simd1<S> b,int nrot)
+{
+  ret = b;
+}
+
+template <class S>
+accelerator_inline void vbroadcast(Grid_simd1<S> &ret,const Grid_simd1<S> &src,int lane){
+  ret = src;
+}
+template <class S,  IfComplex<S> =0>
+accelerator_inline void rbroadcast(Grid_simd1<S> &ret,const Grid_simd1<S> &src,int lane){
+  ret.v = real(src.v);
+}
+
+///////////////////////
+// Splat
+///////////////////////
+
+// this is only for the complex version
+template <class S,  IfComplex<S> = 0, class ABtype>
+accelerator_inline void vsplat(Grid_simd1<S> &ret, ABtype a, ABtype b) {
+  ret.v = S(a,b);
+}
+
+// overload if complex
+template <class S>
+accelerator_inline void vsplat(Grid_simd1<S> &ret, EnableIf<is_complex<S>, S> c) {
+  vsplat(ret, real(c), imag(c));
+}
+template <class S>
+accelerator_inline void rsplat(Grid_simd1<S> &ret, EnableIf<is_complex<S>, S> c) {
+  vsplat(ret, real(c), real(c));
+}
+// if real fill with a, if complex fill with a in the real part (first function
+// above)
+template <class S>
+accelerator_inline void vsplat(Grid_simd1<S> &ret, NotEnableIf<is_complex<S>, S> a) {
+  ret.v = a;
+}
+//////////////////////////
+
+
+///////////////////////////////////////////////
+// Initialise to 1,0,i for the correct types
+///////////////////////////////////////////////
+// For complex types
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void vone(Grid_simd1<S> &ret) {
+  vsplat(ret, S(1.0, 0.0));
+}
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void vzero(Grid_simd1<S> &ret) {
+  vsplat(ret, S(0.0, 0.0));
+}  // use xor?
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void vcomplex_i(Grid_simd1<S> &ret) {
+  vsplat(ret, S(0.0, 1.0));
+}
+
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void visign(Grid_simd1<S> &ret) {
+  vsplat(ret, S(1.0, -1.0));
+}
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void vrsign(Grid_simd1<S> &ret) {
+  vsplat(ret, S(-1.0, 1.0));
+}
+
+// if not complex overload here
+template <class S,  IfReal<S> = 0>
+accelerator_inline void vone(Grid_simd1<S> &ret) {
+  vsplat(ret, S(1.0));
+}
+template <class S,  IfReal<S> = 0>
+accelerator_inline void vzero(Grid_simd1<S> &ret) {
+  vsplat(ret, S(0.0));
+}
+
+// For integral types
+template <class S,  IfInteger<S> = 0>
+accelerator_inline void vone(Grid_simd1<S> &ret) {
+  vsplat(ret, 1);
+}
+template <class S,  IfInteger<S> = 0>
+accelerator_inline void vzero(Grid_simd1<S> &ret) {
+  vsplat(ret, 0);
+}
+template <class S,  IfInteger<S> = 0>
+accelerator_inline void vtrue(Grid_simd1<S> &ret) {
+  vsplat(ret, 0xFFFFFFFF);
+}
+template <class S,  IfInteger<S> = 0>
+accelerator_inline void vfalse(Grid_simd1<S> &ret) {
+  vsplat(ret, 0);
+}
+template <class S>
+accelerator_inline void zeroit(Grid_simd1<S> &z) {
+  vzero(z);
+}
+
+///////////////////////
+// Vstream
+///////////////////////
+template <class S,  IfReal<S> = 0>
+accelerator_inline void vstream(Grid_simd1<S> &out, const Grid_simd1<S> &in) {
+  out = in;
+}
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void vstream(Grid_simd1<S> &out, const Grid_simd1<S> &in) {
+  out = in;
+}
+template <class S,  IfInteger<S> = 0>
+accelerator_inline void vstream(Grid_simd1<S> &out, const Grid_simd1<S> &in) {
+  out = in;
+}
+
+////////////////////////////////////
+// Arithmetic operator overloads +,-,*
+////////////////////////////////////
+template <class S>
+accelerator_inline Grid_simd1<S> operator+(Grid_simd1<S> a, Grid_simd1<S> b) {
+  Grid_simd1<S> ret;
+  ret.v = a.v+b.v;
+  return ret;
+};
+
+template <class S>
+accelerator_inline Grid_simd1<S> operator-(Grid_simd1<S> a, Grid_simd1<S> b) {
+  Grid_simd1<S> ret;
+  ret.v = a.v-b.v;
+  return ret;
+};
+
+// Distinguish between complex types and others
+template <class S,  IfComplex<S> = 0>
+accelerator_inline Grid_simd1<S> real_mult(Grid_simd1<S> a, Grid_simd1<S> b) {
+  Grid_simd1<S> ret;
+  ret.v = S(real(a.v)*real(b.v),real(a.v)*imag(b.v));
+  return ret;
+};
+template <class S,  IfComplex<S> = 0>
+accelerator_inline Grid_simd1<S> real_madd(Grid_simd1<S> a, Grid_simd1<S> b, Grid_simd1<S> c) {
+  Grid_simd1<S> ret;
+  ret = real_mult(a,b) + c;
+  return ret;
+};
+
+
+// Distinguish between complex types and others
+template <class S>
+accelerator_inline Grid_simd1<S> operator*(Grid_simd1<S> a, Grid_simd1<S> b) {
+  Grid_simd1<S> ret;
+#ifndef STRICT_COMPLEX_MUL
+  // Direct product, matching the vector types. std::complex adds an inf/NaN
+  // recovery branch. Define STRICT_COMPLEX_MUL to restore std::complex.
+  if constexpr ( is_complex<S>::value ) {
+    ret.v = S(real(a.v)*real(b.v) - imag(a.v)*imag(b.v),
+	      real(a.v)*imag(b.v) + imag(a.v)*real(b.v));
+  } else {
+    ret.v = a.v*b.v;
+  }
+#else
+  ret.v = a.v*b.v;
+#endif
+  return ret;
+};
+///////////////////////
+// Conjugate
+///////////////////////
+template <class S,  IfComplex<S> = 0>
+accelerator_inline Grid_simd1<S> conjugate(const Grid_simd1<S> &in) {
+  Grid_simd1<S> ret;
+  ret.v = S(real(in.v),-imag(in.v));
+  return ret;
+}
+template <class S,  IfNotComplex<S> = 0>
+accelerator_inline Grid_simd1<S> conjugate(const Grid_simd1<S> &in) {
+  return in;  // for real objects
+}
+// Suppress adj for integer types... // odd; why conjugate above but not adj??
+template <class S,  IfNotInteger<S> = 0>
+accelerator_inline Grid_simd1<S> adj(const Grid_simd1<S> &in) {
+  return conjugate(in);
+}
+
+///////////////////////
+// timesMinusI
+///////////////////////
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void timesMinusI(Grid_simd1<S> &ret, const Grid_simd1<S> &in) {
+  ret.v = S(imag(in.v),-real(in.v));
+}
+template <class S,  IfComplex<S> = 0>
+accelerator_inline Grid_simd1<S> timesMinusI(const Grid_simd1<S> &in) {
+  Grid_simd1<S> ret;
+  timesMinusI(ret,in);
+  return ret;
+}
+template <class S,  IfNotComplex<S> = 0>
+accelerator_inline Grid_simd1<S> timesMinusI(const Grid_simd1<S> &in) {
+  return in;
+}
+///////////////////////
+// timesI
+///////////////////////
+template <class S,  IfComplex<S> = 0>
+accelerator_inline void timesI(Grid_simd1<S> &ret, const Grid_simd1<S> &in) {
+  ret.v = S(-imag(in.v),real(in.v));
+}
+template <class S,  IfComplex<S> = 0>
+accelerator_inline Grid_simd1<S> timesI(const Grid_simd1<S> &in) {
+  Grid_simd1<S> ret;
+  timesI(ret,in);
+  return ret;
+}
+template <class S,  IfNotComplex<S> = 0>
+accelerator_inline Grid_simd1<S> timesI(const Grid_simd1<S> &in) {
+  return in;
+}
+
+// Distinguish between complex types and others
+template <class S>
+accelerator_inline Grid_simd1<S> operator/(Grid_simd1<S> a, Grid_simd1<S> b) {
+  Grid_simd1<S> ret;
+  ret.v = a.v/b.v;
+  return ret;
+};
+
+
+/////////////////////
+// Inner, outer
+/////////////////////
+template <class S>
+accelerator_inline Grid_simd1<S> innerProduct(const Grid_simd1<S> &l,const Grid_simd1<S> &r) {
+  return conjugate(l) * r;
+}
+template <class S>
+accelerator_inline Grid_simd1<S> outerProduct(const Grid_simd1<S> &l,const Grid_simd1<S> &r) {
+  return l * conjugate(r);
+}
+
+template <class S>
+accelerator_inline Grid_simd1<S> trace(const Grid_simd1<S> &arg) {
+  return arg;
+}
+////////////////////////////////////////////////////////////
+// copy/splat complex real parts into real;
+// insert real into complex and zero imag;
+////////////////////////////////////////////////////////////
+accelerator_inline sRealF toReal(const sComplexF &in) {
+  sRealF ret;
+  ret.v=real(in.v);
+  return ret;
+}
+accelerator_inline sRealD toReal(const sComplexD &in) {
+  sRealD ret;
+  ret.v=real(in.v);
+  return ret;
+}
+accelerator_inline sComplexF toComplex(sRealF &in)
+{
+  sComplexF ret;
+  ret.v = in.v;
+  return ret;
+}
+accelerator_inline sComplexD toComplex(sRealD &in)
+{
+  sComplexD ret;
+  ret.v = in.v;
+  return ret;
+}
+
+accelerator_inline void precisionChange(sRealF *out,const sRealD *in,int nvec){ assert(nvec==1); out->v = in->v;}
+accelerator_inline void precisionChange(sRealD *out,const sRealF *in,int nvec){ assert(nvec==1); out->v = in->v;}
+accelerator_inline void precisionChange(sComplexF *out,const sComplexD *in,int nvec){ assert(nvec==1); out->v = in->v;}
+accelerator_inline void precisionChange(sComplexD *out,const sComplexF *in,int nvec){ assert(nvec==1); out->v = in->v;}
+
+
+NAMESPACE_END(Grid);
+

Grid/simd/Grid_scalar_unops.h

@@ -0,0 +1,62 @@
+#pragma once
+NAMESPACE_BEGIN(Grid);
+/////////////
+// Unary operations
+/////////////
+template <class S>
+accelerator_inline Grid_simd1<S> real(const Grid_simd1<S> &r) {
+  return SimdApply(RealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> imag(const Grid_simd1<S> &r) {
+  return SimdApply(ImagFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> sqrt(const Grid_simd1<S> &r) {
+  return SimdApply(SqrtRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> cos(const Grid_simd1<S> &r) {
+  return SimdApply(CosRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> sin(const Grid_simd1<S> &r) {
+  return SimdApply(SinRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> acos(const Grid_simd1<S> &r) {
+  return SimdApply(AcosRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> asin(const Grid_simd1<S> &r) {
+  return SimdApply(AsinRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> log(const Grid_simd1<S> &r) {
+  return SimdApply(LogRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> abs(const Grid_simd1<S> &r) {
+  return SimdApply(AbsRealFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> exp(const Grid_simd1<S> &r) {
+  return SimdApply(ExpFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> Not(const Grid_simd1<S> &r) {
+  return SimdApply(NotFunctor<S>(), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> pow(const Grid_simd1<S> &r, double y) {
+  return SimdApply(PowRealFunctor<S>(y), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> mod(const Grid_simd1<S> &r, Integer y) {
+  return SimdApply(ModIntFunctor<S>(y), r);
+}
+template <class S>
+accelerator_inline Grid_simd1<S> div(const Grid_simd1<S> &r, Integer y) {
+  return SimdApply(DivIntFunctor<S>(y), r);
+}
+NAMESPACE_END(Grid);

Grid/simd/Grid_vector_types.h

@@ -36,8 +36,7 @@ Author: Michael Marshall <michael.marshall@ed.ac.au>
 */
 // Time-stamp: <2015-07-10 17:45:33 neo>
 //---------------------------------------------------------------------------
-#ifndef GRID_VECTOR_TYPES
-#define GRID_VECTOR_TYPES
+#pragma once
 
 // PAB - Lifted and adapted from Eigen, which is GPL V2
 struct Grid_half {
@@ -169,78 +168,6 @@ accelerator_inline Grid_half sfw_float_to_half(float ff) {
 
 NAMESPACE_BEGIN(Grid);
 
-
-//////////////////////////////////////
-// To take the floating point type of real/complex type
-//////////////////////////////////////
-template <typename T>
-struct RealPart {
-  typedef T type;
-};
-template <typename T>
-struct RealPart<complex<T> > {
-  typedef T type;
-};
-
-#include <type_traits>
-
-//////////////////////////////////////
-// demote a vector to real type
-//////////////////////////////////////
-// type alias used to simplify the syntax of std::enable_if
-template <typename T> using Invoke = typename T::type;
-template <typename Condition, typename ReturnType = void> using EnableIf    = Invoke<std::enable_if<Condition::value, ReturnType> >;
-template <typename Condition, typename ReturnType = void> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
-
-////////////////////////////////////////////////////////
-// Check for complexity with type traits
-template <typename T> struct is_complex : public std::false_type {};
-template <> struct is_complex<ComplexD> : public std::true_type {};
-template <> struct is_complex<ComplexF> : public std::true_type {};
-
-template <typename T> struct is_ComplexD : public std::false_type {};
-template <> struct is_ComplexD<ComplexD> : public std::true_type {};
-
-template <typename T> struct is_ComplexF : public std::false_type {};
-template <> struct is_ComplexF<ComplexF> : public std::true_type {};
-
-template<typename T, typename V=void> struct is_real : public std::false_type {};
-template<typename T> struct is_real<T, typename std::enable_if<std::is_floating_point<T>::value,
-  void>::type> : public std::true_type {};
-
-template<typename T, typename V=void> struct is_integer : public std::false_type {};
-template<typename T> struct is_integer<T, typename std::enable_if<std::is_integral<T>::value,
-  void>::type> : public std::true_type {};
-
-template <typename T>              using IfReal    = Invoke<std::enable_if<is_real<T>::value, int> >;
-template <typename T>              using IfComplex = Invoke<std::enable_if<is_complex<T>::value, int> >;
-template <typename T>              using IfInteger = Invoke<std::enable_if<is_integer<T>::value, int> >;
-template <typename T1,typename T2> using IfSame    = Invoke<std::enable_if<std::is_same<T1,T2>::value, int> >;
-
-template <typename T>              using IfNotReal    = Invoke<std::enable_if<!is_real<T>::value, int> >;
-template <typename T>              using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
-template <typename T>              using IfNotInteger = Invoke<std::enable_if<!is_integer<T>::value, int> >;
-template <typename T1,typename T2> using IfNotSame    = Invoke<std::enable_if<!std::is_same<T1,T2>::value, int> >;
-
-////////////////////////////////////////////////////////
-// Define the operation templates functors
-// general forms to allow for vsplat syntax
-// need explicit declaration of types when used since
-// clang cannot automatically determine the output type sometimes
-template <class Out, class Input1, class Input2, class Input3, class Operation>
-Out accelerator_inline  trinary(Input1 src_1, Input2 src_2, Input3 src_3, Operation op) {
-  return op(src_1, src_2, src_3);
-}
-template <class Out, class Input1, class Input2, class Operation>
-Out accelerator_inline binary(Input1 src_1, Input2 src_2, Operation op) {
-  return op(src_1, src_2);
-}
-template <class Out, class Input, class Operation>
-Out accelerator_inline  unary(Input src, Operation op) {
-  return op(src);
-}
-///////////////////////////////////////////////
-
 /*
   @brief Grid_simd class for the SIMD vector type operations
 */
@@ -378,11 +305,18 @@ public:
 
   // FIXME -- alias this to an accelerator_inline MAC struct.
 
-  #if defined(A64FX) || defined(A64FXFIXEDSIZE)
+  #ifdef GRID_ARCH_HAS_COMPLEX_MULT_ADD
+  // Architecture has a fused complex multiply-add (FCMLA and friends).
+  // Defining GRID_ARCH_HAS_COMPLEX_MULT_ADD asserts BOTH MultAddComplexSIMD
+  // and a three argument MultSIMD operator() computing a*b+c.
   friend accelerator_inline void mac(Grid_simd *__restrict__ y,
 				     const Grid_simd *__restrict__ a,
 				     const Grid_simd *__restrict__ x) {
-    *y = fxmac((*a), (*x), (*y));
+    if constexpr ( is_complex<Scalar_type>::value ) {
+      y->v = trinary<Vector_type>(a->v, x->v, y->v, MultAddComplexSIMD());
+    } else {
+      y->v = trinary<Vector_type>(a->v, x->v, y->v, MultSIMD());
+    }
   };
   #else
   friend accelerator_inline void mac(Grid_simd *__restrict__ y,
@@ -670,7 +604,6 @@ typedef Grid_simd<complex<uint16_t>, SIMD_Htype> vComplexH;
 typedef Grid_simd<complex<float>   , SIMD_Ftype> vComplexF;
 typedef Grid_simd<complex<double>  , SIMD_Dtype> vComplexD;
 #endif
-
 /////////////////////////////////////////
 // Pointer type to use on extractLane
 /////////////////////////////////////////
@@ -885,25 +818,7 @@ accelerator_inline Grid_simd<S, V> operator*(Grid_simd<S, V> a, Grid_simd<S, V>
   return ret;
 };
 
-// ---------------- A64FX MAC -------------------
-// Distinguish between complex types and others
-#if defined(A64FX) || defined(A64FXFIXEDSIZE)
-template <class S, class V, IfComplex<S> = 0>
-accelerator_inline Grid_simd<S, V> fxmac(Grid_simd<S, V> a, Grid_simd<S, V> b, Grid_simd<S, V> c) {
-  Grid_simd<S, V> ret;
-  ret.v = trinary<V>(a.v, b.v, c.v, MultAddComplexSIMD());
-  return ret;
-};
-
-// Real/Integer types
-template <class S, class V, IfNotComplex<S> = 0>
-accelerator_inline Grid_simd<S, V> fxmac(Grid_simd<S, V> a, Grid_simd<S, V> b, Grid_simd<S, V> c) {
-  Grid_simd<S, V> ret;
-  ret.v = trinary<V>(a.v, b.v, c.v, MultSIMD());
-  return ret;
-};
-#endif
-// ----------------------------------------------
+// mac() selects MultAddComplexSIMD directly where the architecture has it.
 
 
 ///////////////////////
@@ -1141,13 +1056,3 @@ template<class vobj> void gpermute(vobj & inout,int perm){
 
 NAMESPACE_END(Grid);
 
-#ifdef GRID_SYCL
-template<> struct sycl::is_device_copyable<Grid::vComplexF> : public std::true_type {};
-template<> struct sycl::is_device_copyable<Grid::vComplexD> : public std::true_type {};
-template<> struct sycl::is_device_copyable<Grid::vRealF   > : public std::true_type {};
-template<> struct sycl::is_device_copyable<Grid::vRealD   > : public std::true_type {};
-template<> struct sycl::is_device_copyable<Grid::vInteger > : public std::true_type {};
-#endif
-
-
-#endif

Grid/simd/Simd.h

@@ -28,216 +28,163 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#ifndef GRID_SIMD_H
-#define GRID_SIMD_H
+#pragma once
 
-#if defined(GRID_CUDA) || defined(GRID_HIP)
-#include <thrust/complex.h>
-#endif
+#include <type_traits>
 
-////////////////////////////////////////////////////////////////////////
-// Define scalar and vector floating point types
-//
-// Scalar:   RealF, RealD, ComplexF, ComplexD
-//
-// Vector:  vRealF, vRealD, vComplexF, vComplexD
-//
-// Vector types are arch dependent
-////////////////////////////////////////////////////////////////////////
-
-#define _MM_SELECT_FOUR_FOUR(A,B,C,D) ((A<<6)|(B<<4)|(C<<2)|(D))
-#define _MM_SELECT_FOUR_FOUR_STRING(A,B,C,D) "((" #A "<<6)|(" #B "<<4)|(" #C "<<2)|(" #D "))"
-#define _MM_SELECT_EIGHT_TWO(A,B,C,D,E,F,G,H) ((A<<7)|(B<<6)|(C<<5)|(D<<4)|(E<<3)|(F<<2)|(G<<4)|(H))
-#define _MM_SELECT_FOUR_TWO (A,B,C,D) _MM_SELECT_EIGHT_TWO(0,0,0,0,A,B,C,D)
-#define _MM_SELECT_TWO_TWO  (A,B)     _MM_SELECT_FOUR_TWO(0,0,A,B)
-
-#define RotateBit (0x100)
+#include <Grid/simd/Grid_scalar_support.h>
 
 NAMESPACE_BEGIN(Grid);
-
-typedef uint32_t Integer;
-
-typedef  float  RealF;
-typedef  double RealD;
-#ifdef GRID_DEFAULT_PRECISION_DOUBLE
-typedef RealD   Real;
-#else
-typedef RealF  Real;
-#endif
-
-#if defined(GRID_CUDA) || defined(GRID_HIP)
-typedef thrust::complex<RealF> ComplexF;
-typedef thrust::complex<RealD> ComplexD;
-typedef thrust::complex<Real>  Complex;
-typedef thrust::complex<uint16_t>  ComplexH;
-template<class T> using complex = thrust::complex<T>;
-
-accelerator_inline ComplexD pow(const ComplexD& r,RealD y){ return(thrust::pow(r,(double)y)); }
-accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(thrust::pow(r,(float)y)); }
-#else 
-typedef std::complex<RealF> ComplexF;
-typedef std::complex<RealD> ComplexD;
-typedef std::complex<Real>  Complex;
-typedef std::complex<uint16_t>  ComplexH; // Hack
-template<class T> using complex = std::complex<T>;
-
-accelerator_inline ComplexD pow(const ComplexD& r,RealD y){ return(std::pow(r,y)); }
-accelerator_inline ComplexF pow(const ComplexF& r,RealF y){ return(std::pow(r,y)); }
-#endif
-
-//accelerator_inline RealD pow(const RealD& r,RealD y){ return(std::pow(r,y)); }
-//accelerator_inline RealD sqrt(const RealD  & r){ return std::sqrt(r); }
-
-// This comes from ::pow already from math.h and CUDA
-// Calls either Grid::pow for complex, or std::pow for real
-// Problem is CUDA math_functions is exposing ::pow, and I can't define
-
-using std::abs;
-using std::pow;
-using std::sqrt;
-using std::log;
-using std::exp;
-using std::sin;
-using std::cos;
-using std::asin;
-using std::acos;
+//////////////////////////////////////
+// To take the floating point type of real/complex type
+//////////////////////////////////////
+template <typename T>
+struct RealPart {
+  typedef T type;
+};
+template <typename T>
+struct RealPart<std::complex<T> > {
+  typedef T type;
+};
 
 
-accelerator_inline RealF    conjugate(const RealF  & r){ return r; }
-accelerator_inline RealD    conjugate(const RealD  & r){ return r; }
-accelerator_inline ComplexD conjugate(const ComplexD& r){ return(conj(r)); }
-accelerator_inline ComplexF conjugate(const ComplexF& r ){ return(conj(r)); }
+// type alias used to simplify the syntax of std::enable_if
+template <typename T> using Invoke = typename T::type;
+template <typename Condition, typename ReturnType = void> using EnableIf    = Invoke<std::enable_if<Condition::value, ReturnType> >;
+template <typename Condition, typename ReturnType = void> using NotEnableIf = Invoke<std::enable_if<!Condition::value, ReturnType> >;
 
-accelerator_inline RealF    adj(const RealF  & r){ return r; }
-accelerator_inline RealD    adj(const RealD  & r){ return r; }
-accelerator_inline ComplexD adj(const ComplexD& r){ return(conjugate(r)); }
-accelerator_inline ComplexF adj(const ComplexF& r ){ return(conjugate(r)); }
+////////////////////////////////////////////////////////
+// Check for complexity with type traits
+template <typename T> struct is_complex : public std::false_type {};
+template <> struct is_complex<ComplexD> : public std::true_type {};
+template <> struct is_complex<ComplexF> : public std::true_type {};
 
-#if defined(GRID_CUDA) || defined(GRID_HIP)
-//Provide for convenience
-inline std::complex<double> conjugate(const std::complex<double>& r){ return(conj(r)); }
-inline std::complex<float>  conjugate(const std::complex<float>& r) { return(conj(r)); }
-inline std::complex<double> adj(const std::complex<double>& r)      { return(conj(r)); }
-inline std::complex<float>  adj(const std::complex<float>& r)       { return(conj(r)); }
-#endif
+template <typename T> struct is_ComplexD : public std::false_type {};
+template <> struct is_ComplexD<ComplexD> : public std::true_type {};
 
-accelerator_inline RealF real(const RealF  & r){ return r; }
-accelerator_inline RealD real(const RealD  & r){ return r; }
-accelerator_inline RealF real(const ComplexF  & r){ return r.real(); }
-accelerator_inline RealD real(const ComplexD  & r){ return r.real(); }
+template <typename T> struct is_ComplexF : public std::false_type {};
+template <> struct is_ComplexF<ComplexF> : public std::true_type {};
 
-accelerator_inline RealF imag(const ComplexF  & r){ return r.imag(); }
-accelerator_inline RealD imag(const ComplexD  & r){ return r.imag(); }
+template<typename T, typename V=void> struct is_real : public std::false_type {};
+template<typename T> struct is_real<T, typename std::enable_if<std::is_floating_point<T>::value,
+  void>::type> : public std::true_type {};
 
-accelerator_inline ComplexD innerProduct(const ComplexD & l, const ComplexD & r) { return conjugate(l)*r; }
-accelerator_inline ComplexF innerProduct(const ComplexF & l, const ComplexF & r) { return conjugate(l)*r; }
-accelerator_inline RealD innerProduct(const RealD & l, const RealD & r) { return l*r; }
-accelerator_inline RealF innerProduct(const RealF & l, const RealF & r) { return l*r; }
+template<typename T, typename V=void> struct is_integer : public std::false_type {};
+template<typename T> struct is_integer<T, typename std::enable_if<std::is_integral<T>::value,
+  void>::type> : public std::true_type {};
 
-accelerator_inline ComplexD Reduce(const ComplexD& r){ return r; }
-accelerator_inline ComplexF Reduce(const ComplexF& r){ return r; }
-accelerator_inline RealD Reduce(const RealD& r){ return r; }
-accelerator_inline RealF Reduce(const RealF& r){ return r; }
+template <typename T>              using IfReal    = Invoke<std::enable_if<is_real<T>::value, int> >;
+template <typename T>              using IfComplex = Invoke<std::enable_if<is_complex<T>::value, int> >;
+template <typename T>              using IfInteger = Invoke<std::enable_if<is_integer<T>::value, int> >;
+template <typename T1,typename T2> using IfSame    = Invoke<std::enable_if<std::is_same<T1,T2>::value, int> >;
 
-accelerator_inline RealD toReal(const ComplexD& r){ return r.real(); }
-accelerator_inline RealF toReal(const ComplexF& r){ return r.real(); }
-accelerator_inline RealD toReal(const RealD& r){ return r; }
-accelerator_inline RealF toReal(const RealF& r){ return r; }
-  
-////////////////////////////////////////////////////////////////////////////////
-//Provide support functions for basic real and complex data types required by Grid
-//Single and double precision versions. Should be able to template this once only.
-////////////////////////////////////////////////////////////////////////////////
-accelerator_inline void mac (ComplexD * __restrict__ y,const ComplexD * __restrict__ a,const ComplexD *__restrict__ x){ *y = (*a) * (*x)+(*y); };
-accelerator_inline void mult(ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) * (*r);}
-accelerator_inline void sub (ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) - (*r);}
-accelerator_inline void add (ComplexD * __restrict__ y,const ComplexD * __restrict__ l,const ComplexD *__restrict__ r){ *y = (*l) + (*r);}
-// conjugate already supported for complex
-  
-accelerator_inline void mac (ComplexF * __restrict__ y,const ComplexF * __restrict__ a,const ComplexF *__restrict__ x){ *y = (*a) * (*x)+(*y); }
-accelerator_inline void mult(ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) * (*r); }
-accelerator_inline void sub (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) - (*r); }
-accelerator_inline void add (ComplexF * __restrict__ y,const ComplexF * __restrict__ l,const ComplexF *__restrict__ r){ *y = (*l) + (*r); }
-  
-//conjugate already supported for complex
-accelerator_inline ComplexF timesI(const ComplexF &r)     { return(ComplexF(-r.imag(),r.real()));}
-accelerator_inline ComplexD timesI(const ComplexD &r)     { return(ComplexD(-r.imag(),r.real()));}
-accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(ComplexF(r.imag(),-r.real()));}
-accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(ComplexD(r.imag(),-r.real()));}
-//accelerator_inline ComplexF timesI(const ComplexF &r)     { return(r*ComplexF(0.0,1.0));}
-//accelerator_inline ComplexD timesI(const ComplexD &r)     { return(r*ComplexD(0.0,1.0));}
-//accelerator_inline ComplexF timesMinusI(const ComplexF &r){ return(r*ComplexF(0.0,-1.0));}
-//accelerator_inline ComplexD timesMinusI(const ComplexD &r){ return(r*ComplexD(0.0,-1.0));}
-
-// define projections to real and imaginay parts
-accelerator_inline ComplexF projReal(const ComplexF &r){return( ComplexF(r.real(), 0.0));}
-accelerator_inline ComplexD projReal(const ComplexD &r){return( ComplexD(r.real(), 0.0));}
-accelerator_inline ComplexF projImag(const ComplexF &r){return (ComplexF(r.imag(), 0.0 ));}
-accelerator_inline ComplexD projImag(const ComplexD &r){return (ComplexD(r.imag(), 0.0));}
-
-// define auxiliary functions for complex computations
-accelerator_inline void timesI(ComplexF &ret,const ComplexF &r)     { ret = timesI(r);}
-accelerator_inline void timesI(ComplexD &ret,const ComplexD &r)     { ret = timesI(r);}
-accelerator_inline void timesMinusI(ComplexF &ret,const ComplexF &r){ ret = timesMinusI(r);}
-accelerator_inline void timesMinusI(ComplexD &ret,const ComplexD &r){ ret = timesMinusI(r);}
-  
-accelerator_inline void mac (RealD * __restrict__ y,const RealD * __restrict__ a,const RealD *__restrict__ x){ *y = (*a) * (*x)+(*y);}
-accelerator_inline void mult(RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) * (*r);}
-accelerator_inline void sub (RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) - (*r);}
-accelerator_inline void add (RealD * __restrict__ y,const RealD * __restrict__ l,const RealD *__restrict__ r){ *y = (*l) + (*r);}
-  
-accelerator_inline void mac (RealF * __restrict__ y,const RealF * __restrict__ a,const RealF *__restrict__ x){  *y = (*a) * (*x)+(*y); }
-accelerator_inline void mult(RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) * (*r); }
-accelerator_inline void sub (RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) - (*r); }
-accelerator_inline void add (RealF * __restrict__ y,const RealF * __restrict__ l,const RealF *__restrict__ r){ *y = (*l) + (*r); }
-  
-accelerator_inline void vstream(ComplexF &l, const ComplexF &r){ l=r;}
-accelerator_inline void vstream(ComplexD &l, const ComplexD &r){ l=r;}
-accelerator_inline void vstream(RealF &l, const RealF &r){ l=r;}
-accelerator_inline void vstream(RealD &l, const RealD &r){ l=r;}
-  
-accelerator_inline ComplexD toComplex(const RealD &in) { return ComplexD(in);}
-accelerator_inline ComplexF toComplex(const RealF &in) { return ComplexF(in);}
-  
-class Zero{};
-//static Zero Zero();
-template<class itype> accelerator_inline void zeroit(itype &arg)   { arg=Zero();};
-template<>            accelerator_inline void zeroit(ComplexF &arg){ arg=0; };
-template<>            accelerator_inline void zeroit(ComplexD &arg){ arg=0; };
-template<>            accelerator_inline void zeroit(RealF &arg)   { arg=0; };
-template<>            accelerator_inline void zeroit(RealD &arg)   { arg=0; };
-
-// More limited Integer support  
-accelerator_inline Integer Reduce(const Integer& r){ return r; }
-accelerator_inline void mac (Integer * __restrict__ y,const Integer * __restrict__ a,const Integer *__restrict__ x){  *y = (*a) * (*x)+(*y); }
-accelerator_inline void mult(Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) * (*r); }
-accelerator_inline void sub (Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) - (*r); }
-accelerator_inline void add (Integer * __restrict__ y,const Integer * __restrict__ l,const Integer *__restrict__ r){ *y = (*l) + (*r); }
-accelerator_inline void vstream(Integer &l, const RealD &r){ l=r;}
-template<>            accelerator_inline void zeroit(Integer &arg)   { arg=0; };
-
-accelerator_inline Integer mod (Integer a,Integer y) { return a%y;}
-accelerator_inline Integer div (Integer a,Integer y) { return a/y;}
-//accelerator_inline Integer abs (Integer &a) { return a%y;}
-
-//////////////////////////////////////////////////////////
-// Permute
-// Permute 0 every ABCDEFGH -> BA DC FE HG
-// Permute 1 every ABCDEFGH -> CD AB GH EF
-// Permute 2 every ABCDEFGH -> EFGH ABCD
-// Permute 3 possible on longer iVector lengths (512bit = 8 double = 16 single)
-// Permute 4 possible on half precision @512bit vectors.
-//
-// Defined inside SIMD specialization files
-//////////////////////////////////////////////////////////
-template<class VectorSIMD>
-accelerator_inline void Gpermute(VectorSIMD &y,const VectorSIMD &b,int perm);
+template <typename T>              using IfNotReal    = Invoke<std::enable_if<!is_real<T>::value, int> >;
+template <typename T>              using IfNotComplex = Invoke<std::enable_if<!is_complex<T>::value, int> >;
+template <typename T>              using IfNotInteger = Invoke<std::enable_if<!is_integer<T>::value, int> >;
+template <typename T1,typename T2> using IfNotSame    = Invoke<std::enable_if<!std::is_same<T1,T2>::value, int> >;
 
+////////////////////////////////////////////////////////
+// Define the operation templates functors
+// general forms to allow for vsplat syntax
+// need explicit declaration of types when used since
+// clang cannot automatically determine the output type sometimes
+template <class Out, class Input1, class Input2, class Input3, class Operation>
+Out accelerator_inline  trinary(Input1 src_1, Input2 src_2, Input3 src_3, Operation op) {
+  return op(src_1, src_2, src_3);
+}
+template <class Out, class Input1, class Input2, class Operation>
+Out accelerator_inline binary(Input1 src_1, Input2 src_2, Operation op) {
+  return op(src_1, src_2);
+}
+template <class Out, class Input, class Operation>
+Out accelerator_inline  unary(Input src, Operation op) {
+  return op(src);
+}
 NAMESPACE_END(Grid);
+///////////////////////////////////////////////
 
 #include <Grid/simd/Grid_vector_types.h>
 #include <Grid/simd/Grid_doubled_vector.h>
+#include <Grid/simd/Grid_scalar_types.h>
+
+#ifdef GRID_SYCL
+template<> struct sycl::is_device_copyable<Grid::vComplexF> : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::vComplexD> : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::vRealF   > : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::vRealD   > : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::vInteger > : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::sComplexF> : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::sComplexD> : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::sRealF   > : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::sRealD   > : public std::true_type {};
+template<> struct sycl::is_device_copyable<Grid::sInteger > : public std::true_type {};
+#endif
+
+/////////////////////////////////////////
+// Detect vector types
+/////////////////////////////////////////
+NAMESPACE_BEGIN(Grid);
+template <typename T>
+struct is_simd : public std::false_type {};
+template <> struct is_simd<vRealF>     : public std::true_type {};
+template <> struct is_simd<vRealD>     : public std::true_type {};
+template <> struct is_simd<vRealH>     : public std::true_type {};
+template <> struct is_simd<vComplexF>  : public std::true_type {};
+template <> struct is_simd<vComplexD>  : public std::true_type {};
+template <> struct is_simd<vComplexH>  : public std::true_type {};
+template <> struct is_simd<vInteger>   : public std::true_type {};
+template <> struct is_simd<sRealF>     : public std::true_type {};
+template <> struct is_simd<sRealD>     : public std::true_type {};
+template <> struct is_simd<sComplexF>  : public std::true_type {};
+template <> struct is_simd<sComplexD>  : public std::true_type {};
+template <> struct is_simd<sInteger>   : public std::true_type {};
+
+template <typename T> using IfSimd    = Invoke<std::enable_if<is_simd<T>::value, int> >;
+template <typename T> using IfNotSimd = Invoke<std::enable_if<!is_simd<T>::value, unsigned> >;
+
+
+///////////////////////////////////////////////
+// insert / extract with complex support
+///////////////////////////////////////////////
+template <class S, class V>
+accelerator_inline S getlane(const Grid_simd<S, V> &in,int lane) {
+  return in.getlane(lane);
+}
+template <class S, class V>
+accelerator_inline void putlane(Grid_simd<S, V> &vec,const S &_S, int lane){
+  vec.putlane(_S,lane);
+}
+template <class S,IfNotSimd<S> = 0 >
+accelerator_inline S getlane(const S &in,int lane) {
+  return in;
+}
+template <class S,IfNotSimd<S> = 0 >
+accelerator_inline void putlane(S &vec,const S &_S, int lane){
+  vec = _S;
+}
+template <class S, class V>
+accelerator_inline S getlane(const Grid_simd2<S, V> &in,int lane) {
+  return in.getlane(lane);
+}
+template <class S, class V>
+accelerator_inline void putlane(Grid_simd2<S, V> &vec,const S &_S, int lane){
+  vec.putlane(_S,lane);
+}
+template <class S>
+accelerator_inline S getlane(const Grid_simd1<S> &in,int lane) {
+  return in.getlane(lane);
+}
+template <class S>
+accelerator_inline void putlane(Grid_simd1<S> &vec,const S &_S, int lane){
+  vec.putlane(_S,lane);
+}
+
+NAMESPACE_END(Grid);
+
+
+
 #include <Grid/simd/Grid_vector_unops.h>
+#include <Grid/simd/Grid_scalar_unops.h>
 
 NAMESPACE_BEGIN(Grid);
 
@@ -317,4 +264,4 @@ inline std::ostream& operator<< (std::ostream& stream, const vInteger &o){
 }
 
 NAMESPACE_END(Grid)
-#endif
+

Grid/tensors/Tensor_inner.h

@@ -155,12 +155,27 @@ accelerator_inline vComplexD innerProductD(const vComplexF &l,const vComplexF &r
   return innerProduct(la,ra) + innerProduct(lb,rb); 
 }
 accelerator_inline vRealD innerProductD(const vRealF &l,const vRealF &r)
-{  
+{
   vRealD la,lb;
   vRealD ra,rb;
   Optimization::PrecisionChange::StoD(l.v,la.v,lb.v);
   Optimization::PrecisionChange::StoD(r.v,ra.v,rb.v);
-  return innerProduct(la,ra) + innerProduct(lb,rb); 
+  return innerProduct(la,ra) + innerProduct(lb,rb);
+}
+
+accelerator_inline sComplexD innerProductD(const sComplexD &l,const sComplexD &r){  return innerProduct(l,r); }
+accelerator_inline sRealD    innerProductD(const sRealD    &l,const sRealD    &r){  return innerProduct(l,r); }
+accelerator_inline sComplexD innerProductD(const sComplexF &l,const sComplexF &r)
+{
+  sComplexD ret;
+  ret.v = innerProduct(ComplexD(l.v),ComplexD(r.v));
+  return ret;
+}
+accelerator_inline sRealD innerProductD(const sRealF &l,const sRealF &r)
+{
+  sRealD ret;
+  ret.v = innerProduct(RealD(l.v),RealD(r.v));
+  return ret;
 }
 
 // Now do it for vector, matrix, scalar
@@ -212,6 +227,27 @@ accelerator_inline RealD    innerProductD2(const RealF    &l,const RealF    &r){
 accelerator_inline vComplexD innerProductD2(const vComplexD &l,const vComplexD &r){  return innerProduct(l,r); }
 accelerator_inline vRealD    innerProductD2(const vRealD    &l,const vRealD    &r){  return innerProduct(l,r); }
 
+// Lex chart. D2 means double precision carried at the F type's Nsimd, so that
+// both fit the same Grid; sComplexF and sComplexD are both Nsimd 1, so
+// sComplexD2 would be sComplexD and no separate type is needed.
+accelerator_inline sComplexD innerProductD2(const sComplexD &l,const sComplexD &r){  return innerProduct(l,r); }
+accelerator_inline sRealD    innerProductD2(const sRealD    &l,const sRealD    &r){  return innerProduct(l,r); }
+
+accelerator_inline sComplexD innerProductD2(const sComplexF &l,const sComplexF &r)
+{
+  sComplexD dl,dr;
+  precisionChange(&dl,&l,1);
+  precisionChange(&dr,&r,1);
+  return innerProduct(dl,dr);
+}
+accelerator_inline sRealD innerProductD2(const sRealF &l,const sRealF &r)
+{
+  sRealD dl,dr;
+  precisionChange(&dl,&l,1);
+  precisionChange(&dr,&r,1);
+  return innerProduct(dl,dr);
+}
+
 accelerator_inline vComplexD2 innerProductD2(const vComplexF &l,const vComplexF &r)
 {  
   vComplexD2 dl,dr;

Grid/tensors/Tensor_traits.h

@@ -56,6 +56,10 @@ NAMESPACE_BEGIN(Grid);
   template<>               struct isGridFundamental<ComplexD>      : public std::true_type  { static constexpr bool notvalue = false; };
   template<>               struct isGridFundamental<RealF>         : public std::true_type  { static constexpr bool notvalue = false; };
   template<>               struct isGridFundamental<RealD>         : public std::true_type  { static constexpr bool notvalue = false; };
+  template<>               struct isGridFundamental<sComplexF>     : public std::true_type  { static constexpr bool notvalue = false; };
+  template<>               struct isGridFundamental<sComplexD>     : public std::true_type  { static constexpr bool notvalue = false; };
+  template<>               struct isGridFundamental<sRealF>        : public std::true_type  { static constexpr bool notvalue = false; };
+  template<>               struct isGridFundamental<sRealD>        : public std::true_type  { static constexpr bool notvalue = false; };
   template<>               struct isGridFundamental<vComplexD2>    : public std::true_type  { static constexpr bool notvalue = false; };
   template<>               struct isGridFundamental<vRealD2>       : public std::true_type  { static constexpr bool notvalue = false; };
 
@@ -103,6 +107,7 @@ NAMESPACE_BEGIN(Grid);
     typedef RealD scalar_objectD;
     typedef ComplexF Complexified;
     typedef RealF Realified;
+    typedef Integer Integerified;
     typedef RealD DoublePrecision;
     typedef RealD DoublePrecision2;
   };
@@ -116,6 +121,7 @@ NAMESPACE_BEGIN(Grid);
     typedef RealD scalar_objectD;
     typedef ComplexD Complexified;
     typedef RealD Realified;
+    typedef Integer Integerified;
     typedef RealD DoublePrecision;
     typedef RealD DoublePrecision2;
   };
@@ -129,6 +135,7 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD scalar_objectD;
     typedef ComplexF Complexified;
     typedef RealF Realified;
+    typedef Integer Integerified;
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
   };
@@ -142,6 +149,7 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD scalar_objectD;
     typedef ComplexD Complexified;
     typedef RealD Realified;
+    typedef Integer Integerified;
     typedef ComplexD DoublePrecision;
     typedef ComplexD DoublePrecision2;
   };
@@ -157,6 +165,7 @@ NAMESPACE_BEGIN(Grid);
     typedef scalar_typeD scalar_objectD;
     typedef scalar_type Complexified;
     typedef RealF Realified;
+    typedef Integer Integerified;
     typedef scalar_typeD DoublePrecision;
     typedef scalar_typeD DoublePrecision2;
   };
@@ -170,6 +179,7 @@ NAMESPACE_BEGIN(Grid);
     typedef scalar_typeD scalar_objectD;
     typedef scalar_type Complexified;
     typedef RealD Realified;
+    typedef Integer Integerified;
     typedef scalar_typeD DoublePrecision;
     typedef scalar_typeD DoublePrecision2;
   };
@@ -185,6 +195,7 @@ NAMESPACE_BEGIN(Grid);
     typedef Integer scalar_objectD;
     typedef void Complexified;
     typedef void Realified;
+    typedef Integer Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
   };
@@ -199,6 +210,7 @@ NAMESPACE_BEGIN(Grid);
     typedef RealD  scalar_objectD;
     typedef vComplexF Complexified;
     typedef vRealF Realified;
+    typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD2 DoublePrecision2;
   };
@@ -212,6 +224,7 @@ NAMESPACE_BEGIN(Grid);
     typedef RealD  scalar_objectD;
     typedef vComplexD Complexified;
     typedef vRealD Realified;
+    typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD DoublePrecision2;
   };
@@ -225,6 +238,7 @@ NAMESPACE_BEGIN(Grid);
     typedef RealD  scalar_objectD;
     typedef vComplexD2 Complexified;
     typedef vRealD2 Realified;
+    typedef vInteger Integerified;
     typedef vRealD2 DoublePrecision;
     typedef vRealD2 DoublePrecision2;
   };
@@ -239,6 +253,7 @@ NAMESPACE_BEGIN(Grid);
     typedef RealD  scalar_objectD;
     typedef vComplexH Complexified;
     typedef vRealH Realified;
+    typedef vInteger Integerified;
     typedef vRealD DoublePrecision;
     typedef vRealD DoublePrecision2;
   };
@@ -253,6 +268,7 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD  scalar_objectD;
     typedef vComplexH Complexified;
     typedef vRealH Realified;
+    typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD DoublePrecision2;
   };
@@ -266,6 +282,7 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD  scalar_objectD;
     typedef vComplexF Complexified;
     typedef vRealF Realified;
+    typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD2 DoublePrecision2;
   };
@@ -279,6 +296,7 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD  scalar_objectD;
     typedef vComplexD Complexified;
     typedef vRealD Realified;
+    typedef vInteger Integerified;
     typedef vComplexD DoublePrecision;
     typedef vComplexD DoublePrecision2;
   };
@@ -292,6 +310,7 @@ NAMESPACE_BEGIN(Grid);
     typedef ComplexD  scalar_objectD;
     typedef vComplexD2 Complexified;
     typedef vRealD2 Realified;
+    typedef vInteger Integerified;
     typedef vComplexD2 DoublePrecision;
     typedef vComplexD2 DoublePrecision2;
   };
@@ -305,6 +324,77 @@ NAMESPACE_BEGIN(Grid);
     typedef  Integer scalar_objectD;
     typedef void Complexified;
     typedef void Realified;
+    typedef vInteger Integerified;
+    typedef void DoublePrecision;
+    typedef void DoublePrecision2;
+  };
+  template<> struct GridTypeMapper<sRealF> : public GridTypeMapper_Base {
+    typedef RealF  scalar_type;
+    typedef RealD  scalar_typeD;
+    typedef sRealF vector_type;
+    typedef sRealD vector_typeD;
+    typedef sRealF tensor_reduced;
+    typedef RealF  scalar_object;
+    typedef RealD  scalar_objectD;
+    typedef sComplexF Complexified;
+    typedef sRealF Realified;
+    typedef sInteger Integerified;
+    typedef sRealD DoublePrecision;
+    typedef sRealD DoublePrecision2;
+  };
+  template<> struct GridTypeMapper<sRealD> : public GridTypeMapper_Base {
+    typedef RealD  scalar_type;
+    typedef RealD  scalar_typeD;
+    typedef sRealD vector_type;
+    typedef sRealD vector_typeD;
+    typedef sRealD tensor_reduced;
+    typedef RealD  scalar_object;
+    typedef RealD  scalar_objectD;
+    typedef sComplexD Complexified;
+    typedef sRealD Realified;
+    typedef sInteger Integerified;
+    typedef sRealD DoublePrecision;
+    typedef sRealD DoublePrecision2;
+  };
+  template<> struct GridTypeMapper<sComplexF> : public GridTypeMapper_Base {
+    typedef ComplexF  scalar_type;
+    typedef ComplexD  scalar_typeD;
+    typedef sComplexF vector_type;
+    typedef sComplexD vector_typeD;
+    typedef sComplexF tensor_reduced;
+    typedef ComplexF  scalar_object;
+    typedef ComplexD  scalar_objectD;
+    typedef sComplexF Complexified;
+    typedef sRealF Realified;
+    typedef sInteger Integerified;
+    typedef sComplexD DoublePrecision;
+    typedef sComplexD DoublePrecision2;
+  };
+  template<> struct GridTypeMapper<sComplexD> : public GridTypeMapper_Base {
+    typedef ComplexD  scalar_type;
+    typedef ComplexD  scalar_typeD;
+    typedef sComplexD vector_type;
+    typedef sComplexD vector_typeD;
+    typedef sComplexD tensor_reduced;
+    typedef ComplexD  scalar_object;
+    typedef ComplexD  scalar_objectD;
+    typedef sComplexD Complexified;
+    typedef sRealD Realified;
+    typedef sInteger Integerified;
+    typedef sComplexD DoublePrecision;
+    typedef sComplexD DoublePrecision2;
+  };
+  template<> struct GridTypeMapper<sInteger> : public GridTypeMapper_Base {
+    typedef Integer  scalar_type;
+    typedef Integer  scalar_typeD;
+    typedef sInteger vector_type;
+    typedef sInteger vector_typeD;
+    typedef sInteger tensor_reduced;
+    typedef Integer  scalar_object;
+    typedef Integer  scalar_objectD;
+    typedef void Complexified;
+    typedef void Realified;
+    typedef sInteger Integerified;
     typedef void DoublePrecision;
     typedef void DoublePrecision2;
   };
@@ -324,6 +414,7 @@ NAMESPACE_BEGIN(Grid);
     using scalar_objectD  = iScalar<typename BaseTraits::scalar_objectD>;
     using Complexified    = iScalar<typename BaseTraits::Complexified>;
     using Realified       = iScalar<typename BaseTraits::Realified>;
+    using Integerified    = iScalar<typename BaseTraits::Integerified>;
     using DoublePrecision = iScalar<typename BaseTraits::DoublePrecision>;
     using DoublePrecision2= iScalar<typename BaseTraits::DoublePrecision2>;
     static constexpr int Rank = BaseTraits::Rank + 1;
@@ -339,6 +430,7 @@ NAMESPACE_BEGIN(Grid);
     using scalar_objectD  = iVector<typename BaseTraits::scalar_objectD,  N>;
     using Complexified    = iVector<typename BaseTraits::Complexified,    N>;
     using Realified       = iVector<typename BaseTraits::Realified,       N>;
+    using Integerified    = iVector<typename BaseTraits::Integerified,    N>;
     using DoublePrecision = iVector<typename BaseTraits::DoublePrecision, N>;
     using DoublePrecision2= iVector<typename BaseTraits::DoublePrecision2, N>;
     static constexpr int Rank = BaseTraits::Rank + 1;
@@ -354,6 +446,7 @@ NAMESPACE_BEGIN(Grid);
     using scalar_objectD  = iMatrix<typename BaseTraits::scalar_objectD,  N>;
     using Complexified    = iMatrix<typename BaseTraits::Complexified,    N>;
     using Realified       = iMatrix<typename BaseTraits::Realified,       N>;
+    using Integerified    = iMatrix<typename BaseTraits::Integerified,    N>;
     using DoublePrecision = iMatrix<typename BaseTraits::DoublePrecision, N>;
     using DoublePrecision2= iMatrix<typename BaseTraits::DoublePrecision2, N>;
     static constexpr int Rank = BaseTraits::Rank + 2;

Grid/util/FlightRecorder.cc

@@ -208,7 +208,7 @@ bool FlightRecorder::CsumLog(uint64_t hex)
         std::cerr<<"FlightRecorder::CsumLog Oops, I did it again "<< CsumLoggingCounter
 		 <<std::hex<<" "<<hex<<" "<<hexref<<std::dec<<std::endl;
 
-	fprintf(stderr,"%s:%d Oops, I did it again! Reproduce failure for csum %d %lx expect %lx\n",
+	fprintf(stderr,"%s:%d Oops, I did it again! Reproduce failure for csum %d %llx expect %llx\n",
 		GridHostname(),
 		GlobalSharedMemory::WorldShmRank,
 		CsumLoggingCounter,hex, hexref);

HMC/Mobius2p1f_DD_EOFA_96I_mixed.cc

@@ -167,10 +167,6 @@ int main(int argc, char **argv) {
   typedef typename FermionAction::FermionField FermionField;
   typedef typename FermionActionF::FermionField FermionFieldF;
 
-  typedef WilsonImplD2 FermionImplPolicyD2;
-  typedef MobiusFermionD2 FermionActionD2;
-  typedef MobiusEOFAFermionD2 FermionEOFAActionD2;
-  typedef typename FermionActionD2::FermionField FermionFieldD2;
   
   typedef Grid::XmlReader       Serialiser;
 

benchmarks/Benchmark_dwf_lex.cc

@@ -0,0 +1,149 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./benchmarks/Benchmark_dwf_lex.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// Single core domain wall Dhop: vectorised chart against lexicographic
+// (Nsimd()==1), identical gauge field and source, in one process.
+//
+// Run at both precisions.  Where the vectorised chart also has Nsimd()==1 the
+// difference is instruction quality and register pressure alone; where it has
+// lanes, the excess over that is the layout.
+//
+#include <Grid/Grid.h>
+
+using namespace Grid;
+
+template<class vobjOut,class vobjIn>
+void transfer(Lattice<vobjOut> &out,const Lattice<vobjIn> &in)
+{
+  typedef typename vobjIn::scalar_object sobj;
+  std::vector<sobj> buf;
+  unvectorizeToLexOrdArray(buf,in);
+  vectorizeFromLexOrdArray(buf,out);
+}
+
+template<class Op,class Field>
+double TimeDhop(Op &D,Field &src,Field &res,int ncall)
+{
+  D.Dhop(src,res,DaggerNo);
+  double t0 = usecond();
+  for(int i=0;i<ncall;i++){
+    D.Dhop(src,res,DaggerNo);
+  }
+  double t1 = usecond();
+  return (t1-t0)/ncall;
+}
+
+template<class vImpl,class lexImpl>
+void Sweep(std::string name,int Ls,Coordinate mpi)
+{
+  typedef DomainWallFermion<vImpl>   vDwf;
+  typedef DomainWallFermion<lexImpl> lexDwf;
+  typedef typename vImpl::Simd       vSimd;
+
+  const long unsigned int single_site_flops = 8*Nc*(7+16*Nc);
+
+  Coordinate vsimd = GridDefaultSimd(Nd,vSimd::Nsimd());
+  Coordinate lsimd({1,1,1,1});
+
+  std::cout << GridLogMessage << "==== " << name
+	    << "  Nsimd(simd chart) = " << vSimd::Nsimd()
+	    << "  Nsimd(lex chart) = 1  Ls = " << Ls << std::endl;
+  std::cout << GridLogMessage
+	    << "  L    localvol    MB/field    simd us     lex us    simd Gflop/s  lex Gflop/s  ratio"
+	    << std::endl;
+
+  // L divisible by 4: red-black needs an even reduced dimension, and the simd
+  // layout can take 2 in one direction.
+  for(auto L : std::vector<int>({4,8,12,16,20})){
+
+    Coordinate latt({L,L,L,L});
+
+    GridCartesian         *vU  = new GridCartesian(latt,vsimd,mpi);
+    GridRedBlackCartesian *vUr = SpaceTimeGrid::makeFourDimRedBlackGrid(vU);
+    GridCartesian         *vF  = SpaceTimeGrid::makeFiveDimGrid(Ls,vU);
+    GridRedBlackCartesian *vFr = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,vU);
+
+    GridCartesian         *lU  = new GridCartesian(latt,lsimd,mpi);
+    GridRedBlackCartesian *lUr = SpaceTimeGrid::makeFourDimRedBlackGrid(lU);
+    GridCartesian         *lF  = SpaceTimeGrid::makeFiveDimGrid(Ls,lU);
+    GridRedBlackCartesian *lFr = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,lU);
+
+    GridParallelRNG pRNG4(vU); pRNG4.SeedFixedIntegers(std::vector<int>({1,2,3,4}));
+    GridParallelRNG pRNG5(vF); pRNG5.SeedFixedIntegers(std::vector<int>({5,6,7,8}));
+
+    typename vDwf::GaugeField   Umu(vU);  SU<Nc>::HotConfiguration(pRNG4,Umu);
+    typename lexDwf::GaugeField Ulex(lU); transfer(Ulex,Umu);
+
+    typename vDwf::FermionField   vsrc(vF),vres(vF);  random(pRNG5,vsrc);
+    typename lexDwf::FermionField lsrc(lF),lres(lF);  transfer(lsrc,vsrc);
+
+    vDwf   Dv(Umu ,*vF,*vFr,*vU,*vUr,0.05,1.8);
+    lexDwf Dl(Ulex,*lF,*lFr,*lU,*lUr,0.05,1.8);
+
+    double volume=Ls; for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
+
+    int ncall = (L<=8) ? 200 : 50;
+    double tv=1e30,tl=1e30;
+    for(int rep=0;rep<3;rep++){
+      tv = std::min(tv,TimeDhop(Dv,vsrc,vres,ncall));
+      tl = std::min(tl,TimeDhop(Dl,lsrc,lres,ncall));
+    }
+
+    double gf_v = single_site_flops*volume/tv/1.0e3;
+    double gf_l = single_site_flops*volume/tl/1.0e3;
+    double mb   = volume*sizeof(typename vDwf::FermionField::scalar_object)/1024./1024.;
+
+    std::cout << GridLogMessage
+	      << std::setw(3) << L
+	      << std::setw(12) << (long)volume
+	      << std::setw(11) << std::fixed << std::setprecision(2) << mb
+	      << std::setw(11) << std::setprecision(1) << tv
+	      << std::setw(11) << tl
+	      << std::setw(13) << std::setprecision(3) << gf_v
+	      << std::setw(13) << gf_l
+	      << std::setw(8)  << std::setprecision(2) << gf_v/gf_l
+	      << std::endl;
+
+    delete vFr; delete vF; delete vUr; delete vU;
+    delete lFr; delete lF; delete lUr; delete lU;
+  }
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  const int Ls = 8;
+  Coordinate mpi = GridDefaultMpi();
+
+  Sweep<WilsonImplD,lexWilsonImplD>("DOUBLE",Ls,mpi);
+  Sweep<WilsonImplF,lexWilsonImplF>("SINGLE",Ls,mpi);
+
+  Grid_finalize();
+}

benchmarks/Benchmark_prec_change.cc

@@ -33,6 +33,7 @@ using namespace Grid;
 
 int main (int argc, char ** argv)
 {
+  
   Grid_init(&argc,&argv);
 
   int Ls = 12;
@@ -61,9 +62,6 @@ int main (int argc, char ** argv)
   LatticeFermionD field_d(FGridD), tmp_d(FGridD);
   random(RNG5,field_d); tmp_d = field_d;
 
-  LatticeFermionD2 field_d2(FGridF), tmp_d2(FGridF);
-  precisionChange(field_d2, field_d); tmp_d2 = field_d2;
-
   LatticeFermionF field_f(FGridF), tmp_f(FGridF);
   precisionChange(field_f, field_d); tmp_f = field_f;
 
@@ -141,49 +139,5 @@ int main (int argc, char ** argv)
   }
   std::cout << "d->s " << time_ds/N << "us" << " s->d " << time_sd/N << "us" << std::endl;
 
-
-  std::cout<<GridLogMessage << "Benchmarking single<->double2 (fields initially device-resident)" << std::endl;
-  time_sd = time_ds = 0;
-  for(int i=0;i<N;i++){
-    field_d2 = tmp_d2;
-    field_f = tmp_f;
-
-    double start=usecond();
-    precisionChangeFast(field_d2,field_f);
-    double stop=usecond();
-    time_sd += stop - start;
-
-    field_d2 = tmp_d2;
-    field_f = tmp_f;
-
-    start=usecond();
-    precisionChangeFast(field_f,field_d2);
-    stop=usecond();
-    time_ds += stop - start;
-  }
-  std::cout << "d->s " << time_ds/N << "us" << " s->d " << time_sd/N << "us" << std::endl;
-
-
-  std::cout<<GridLogMessage << "Benchmarking single<->double2 through standard precisionChange call(fields initially device-resident) [NB: perf should be the same as the previous test!]" << std::endl;
-  time_sd = time_ds = 0;
-  for(int i=0;i<N;i++){
-    field_d2 = tmp_d2;
-    field_f = tmp_f;
-
-    double start=usecond();
-    precisionChange(field_d2,field_f);
-    double stop=usecond();
-    time_sd += stop - start;
-
-    field_d2 = tmp_d2;
-    field_f = tmp_f;
-
-    start=usecond();
-    precisionChange(field_f,field_d2);
-    stop=usecond();
-    time_ds += stop - start;
-  }
-  std::cout << "d->s " << time_ds/N << "us" << " s->d " << time_sd/N << "us" << std::endl;
-
   Grid_finalize();
 }

configure.ac

@@ -574,6 +574,9 @@ case ${ax_cv_cxx_compiler_vendor} in
       NEONv8)
         AC_DEFINE([NEONV8],[1],[ARMv8 NEON])
         SIMD_FLAGS='-march=armv8-a';;
+      NEONv8.3)
+        AC_DEFINE([NEONV8],[1],[ARMv8.3 NEON])
+        SIMD_FLAGS='-march=armv8.3-a';;
       QPX|BGQ)
         AC_DEFINE([QPX],[1],[QPX intrinsics for BG/Q])
         SIMD_FLAGS='';;
@@ -907,6 +910,7 @@ AC_CONFIG_FILES(tests/Makefile)
 AC_CONFIG_FILES(tests/IO/Makefile)
 AC_CONFIG_FILES(tests/core/Makefile)
 AC_CONFIG_FILES(tests/debug/Makefile)
+AC_CONFIG_FILES(tests/lexLattice/Makefile)
 AC_CONFIG_FILES(tests/forces/Makefile)
 AC_CONFIG_FILES(tests/hmc/Makefile)
 AC_CONFIG_FILES(tests/solver/Makefile)

examples/Example_mdagm.cc

@@ -0,0 +1,240 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_mdagm.cc
+
+    Copyright (C) 2023
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+
+using namespace std;
+using namespace Grid;
+
+// Routes Op/AdjOp -> HermOp so that CoarsenOperator and CreateSubspace
+// both see the HPD operator M†M rather than bare M.
+template<class Field>
+class HermOpAdaptor : public LinearOperatorBase<Field>
+{
+  LinearOperatorBase<Field> &wrapped;
+public:
+  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {};
+  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
+  void HermOp (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
+  void AdjOp  (const Field &in, Field &out)   { wrapped.HermOp(in,out); }
+  void OpDiag (const Field &in, Field &out)                  { GRID_ASSERT(0); }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) { GRID_ASSERT(0); }
+  void OpDirAll(const Field &in, std::vector<Field> &out)    { GRID_ASSERT(0); }
+  void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
+    wrapped.HermOp(in, out);
+    ComplexD dot = innerProduct(in, out);
+    n1 = real(dot);
+    n2 = norm2(out);
+  }
+};
+
+// Fixed-iteration CG smoother: runs exactly `iters` steps of CG on the
+// shifted operator.  tolerance=0 so CG never exits early.
+template<class Field>
+class CGSmoother : public LinearFunction<Field>
+{
+public:
+  using LinearFunction<Field>::operator();
+  typedef LinearOperatorBase<Field> FineOperator;
+  FineOperator &_SmootherOperator;
+  int iters;
+  CGSmoother(int _iters, FineOperator &SmootherOperator)
+    : _SmootherOperator(SmootherOperator), iters(_iters)
+  {
+    std::cout << GridLogMessage << " CGSmoother order " << iters << std::endl;
+  }
+  void operator()(const Field &in, Field &out)
+  {
+    ConjugateGradient<Field> CG(0.0, iters, false);
+    out = Zero();
+    CG(_SmootherOperator, in, out);
+  }
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  const int Ls    = 24;
+  const int nbasis = 60;
+  const int cb    = 0;
+  RealD M5  = 1.8;
+  RealD b   = 1.5;
+  RealD c   = 0.5;
+
+  RealD mass = 0.00078;
+  { const char *e = getenv("MASS"); if (e && *e) mass = atof(e); }
+
+  std::cout << GridLogMessage << "Mass: " << mass
+            << "  Ls: " << Ls << "  b=" << b << " c=" << c << std::endl;
+  std::cout << GridLogMessage << "nbasis: " << nbasis << std::endl;
+
+  // ── Grids ──────────────────────────────────────────────────────────────
+  std::vector<int> lat_size{48,48,48,96};
+
+  GridCartesian         *UGrid   = SpaceTimeGrid::makeFourDimGrid(lat_size,
+                                     GridDefaultSimd(Nd,vComplex::Nsimd()),
+                                     GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         *FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  Coordinate Block({4,4,3,4});
+  Coordinate clatt = lat_size;
+  for (int d = 0; d < (int)clatt.size(); d++) clatt[d] /= Block[d];
+
+  GridCartesian *Coarse4d = SpaceTimeGrid::makeFourDimGrid(clatt,
+                              GridDefaultSimd(Nd,vComplex::Nsimd()),
+                              GridDefaultMpi());
+  GridCartesian *Coarse5d = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
+
+  // ── RNGs ───────────────────────────────────────────────────────────────
+  GridParallelRNG RNG5(FGrid);   RNG5.SeedFixedIntegers({5,6,7,8});
+  GridParallelRNG RNG4(UGrid);   RNG4.SeedFixedIntegers({1,2,3,4});
+
+  // ── Gauge field ────────────────────────────────────────────────────────
+  LatticeGaugeField Umu(UGrid);
+  FieldMetaData header;
+  NerscIO::readConfiguration(Umu, header, std::string("/ccs/home/poare/ckpoint_lat.1000"));
+
+  // ── Fermion operator ───────────────────────────────────────────────────
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
+
+  MdagMLinearOperator<MobiusFermionD,LatticeFermionD> MdagMOp(Ddwf);
+  HermOpAdaptor<LatticeFermionD> HermFineOp(MdagMOp);
+
+  // ── Coarse geometry ────────────────────────────────────────────────────
+  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+  typedef LittleDiracOperator::CoarseVector CoarseVector;
+  typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
+
+  NextToNearestStencilGeometry5D geom(Coarse5d);
+
+  // ── Power method: estimate upper end of M†M spectrum ───────────────────
+  LatticeFermionD pm_src(FGrid); random(RNG5, pm_src);
+  PowerMethod<LatticeFermionD> PM;
+  RealD hi = PM(HermFineOp, pm_src);
+  std::cout << GridLogMessage << "Power method: hi = " << hi << std::endl;
+
+  // ── Smoother: fixed-iteration CG on (M†M + lo) ─────────────────────────
+  // lo/hi ~ 2/95 matches the HDCG ratio; tune empirically.
+  RealD lo  = hi / 40.0;
+  int   ord = 12;
+  std::cout << GridLogMessage << "Smoother shift lo = " << lo
+            << "  order = " << ord << std::endl;
+  ShiftedHermOpLinearOperator<LatticeFermionD> ShiftedFineOp(HermFineOp, lo);
+  CGSmoother<LatticeFermionD> Smoother(ord, ShiftedFineOp);
+
+  // ── Subspace via CG inverse iteration ──────────────────────────────────
+  Subspace Aggregates(Coarse5d, FGrid, cb);
+  Aggregates.CreateSubspace(RNG5, HermFineOp, nbasis);
+
+  // ── Cheap coarse deflation: diagonalise W BEFORE block-GS ──────────────
+  // Orthogonalise() applies block-local Gram-Schmidt which rotates subspace[i]
+  // into orthonormal block-local combinations, destroying the near-null
+  // property of individual vectors.  We must extract the coarse zero-mode
+  // combinations χₖ = Σᵢ V[i,k] ψᵢ from the pre-GS near-null vectors first.
+  std::vector<LatticeFermionD> chi(nbasis, FGrid);
+  {
+    std::vector<LatticeFermionD> &psi = Aggregates.subspace;
+    LatticeFermionD tmp(FGrid);
+    Eigen::MatrixXcd W = Eigen::MatrixXcd::Zero(nbasis, nbasis);
+    for (int i = 0; i < nbasis; i++) {
+      HermFineOp.Op(psi[i], tmp);
+      for (int j = 0; j < nbasis; j++)
+        W(j, i) = TensorRemove(innerProduct(psi[j], tmp));
+    }
+    Eigen::SelfAdjointEigenSolver<Eigen::MatrixXcd> esolver(W);
+    for (int k = 0; k < nbasis; k++) {
+      chi[k] = Zero();
+      for (int i = 0; i < nbasis; i++)
+        chi[k] += ComplexD(esolver.eigenvectors()(i, k)) * psi[i];
+    }
+  }
+
+  Aggregates.Orthogonalise();
+
+  // ── Coarse operator ────────────────────────────────────────────────────
+  LittleDiracOperator LittleDiracOp(geom, FGrid, Coarse5d);
+  LittleDiracOp.CoarsenOperator(HermFineOp, Aggregates);
+
+  // ── Coarse linear operator ─────────────────────────────────────────────
+  HermitianLinearOperator<LittleDiracOperator,CoarseVector> LinOpCoarse(LittleDiracOp);
+
+  // ── Project χₖ to coarse grid; Rayleigh quotients give deflation evals ─
+  // ProjectToSubspace uses the post-GS basis, correctly mapping the pre-GS
+  // near-null combinations to coarse vectors via the block-local U†(x_c).
+  std::vector<CoarseVector> coarse_deflation_vecs(nbasis, Coarse5d);
+  std::vector<RealD>        coarse_deflation_evals(nbasis);
+  {
+    CoarseVector Ac(Coarse5d);
+    for (int k = 0; k < nbasis; k++) {
+      Aggregates.ProjectToSubspace(coarse_deflation_vecs[k], chi[k]);
+      RealD n = norm2(coarse_deflation_vecs[k]);
+      coarse_deflation_vecs[k] *= 1.0 / std::sqrt(n);
+      LinOpCoarse.HermOp(coarse_deflation_vecs[k], Ac);
+      coarse_deflation_evals[k] = real(TensorRemove(innerProduct(coarse_deflation_vecs[k], Ac)));
+      std::cout << GridLogMessage << "Coarse deflation eval[" << k << "] = "
+                << coarse_deflation_evals[k] << std::endl;
+    }
+  }
+
+  // ── Coarse solve: CG + deflation guesser ──────────────────────────────
+  ConjugateGradient<CoarseVector>  coarseCG(5.0e-2, 10000, false);
+  DeflatedGuesser<CoarseVector>    coarseGuess(coarse_deflation_vecs, coarse_deflation_evals);
+  HPDSolver<CoarseVector> CoarseSolve(LinOpCoarse, coarseCG, coarseGuess);
+
+  // ── ADEF2 outer solve ──────────────────────────────────────────────────
+  LatticeFermionD src(FGrid);    random(RNG5, src);
+  LatticeFermionD result(FGrid); result = Zero();
+
+  TwoLevelADEF2<LatticeFermionD, CoarseVector, Subspace>
+    HDCG(1.0e-8, 1000,
+         HermFineOp,
+         Smoother,
+         CoarseSolve,   // used in PcgM1
+         CoarseSolve,   // used in Vstart
+         Aggregates);
+
+  HDCG(src, result);
+
+  // ── Reference RBCG ─────────────────────────────────────────────────────
+#if 0
+  {
+    SchurDiagMooeeOperator<MobiusFermionD,LatticeFermion> HermOpEO(Ddwf);
+    LatticeFermionD rb_src(FrbGrid);    random(RNG5, rb_src);
+    LatticeFermionD rb_res(FrbGrid);    rb_res = Zero();
+    ConjugateGradient<LatticeFermionD> CG(1.0e-8, 30000, false);
+    CG(HermOpEO, rb_src, rb_res);
+  }
+#endif
+
+  Grid_finalize();
+  return 0;
+}

examples/Example_mdagm_cg.cc

@@ -0,0 +1,355 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_mdagm_cg.cc
+
+    Copyright (C) 2024
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+// Two-level multigrid preconditioned CG for M†M (full Möbius DWF operator).
+// Single RHS, targeting HMC use case.
+//
+// Operator hierarchy:
+//   Fine:   M†M, acted on via HermOpAdaptor so Op = HermOp = M†M
+//   Coarse: 33-point GeneralCoarsenedMatrix (NextToNearest, 2-hop M†M)
+//
+// Setup:
+//   1. Chebyshev filter (T_600 x T_2500) on M†M to build near-null subspace {ψᵢ}
+//   2. Before block-local Gram-Schmidt, compute W_ij = <ψᵢ|M†M|ψⱼ> (nbasis×nbasis)
+//      and diagonalize it. The eigenvectors of W are approximate zero modes of the
+//      coarse operator; projecting them onto the coarse grid gives coarse deflation
+//      vectors. Block GS destroys the near-null property of individual basis vectors
+//      (they become local linear combinations), so this step must precede Orthogonalise.
+//   3. Block-local Gram-Schmidt (Orthogonalise), then CoarsenOperator.
+//   4. Project W eigenvectors χₖ = Σᵢ V[i,k] ψᵢ to coarse grid → deflation vectors.
+//
+// Solvers:
+//   Smoother:    CG(nIter) on (M†M + lo*I)
+//   Coarse:      DeflatedGuesser (nbasis W-eigenvectors) + CG to tolerance 5e-2
+//   Outer:       PrecGeneralisedConjugateResidual with two-level V-cycle preconditioner
+
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h>
+#include <Grid/algorithms/iterative/NormalEquations.h>
+
+using namespace std;
+using namespace Grid;
+
+// Wraps any LinearOperatorBase so that Op = AdjOp = HermOp.
+// Required when coarsening an HPD operator whose Op != HermOp
+// (e.g. MdagMLinearOperator where Op=M, HermOp=M†M).
+template<class Field>
+class HermOpAdaptor : public LinearOperatorBase<Field>
+{
+  LinearOperatorBase<Field> &wrapped;
+public:
+  HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {}
+  void Op     (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
+  void HermOp (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
+  void AdjOp  (const Field &in, Field &out)   { wrapped.HermOp(in, out); }
+  void OpDiag (const Field &in, Field &out)                    { GRID_ASSERT(0); }
+  void OpDir  (const Field &in, Field &out, int dir, int disp) { GRID_ASSERT(0); }
+  void OpDirAll(const Field &in, std::vector<Field> &out)      { GRID_ASSERT(0); }
+  void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
+    wrapped.HermOp(in, out);
+    ComplexD dot = innerProduct(in, out);
+    n1 = real(dot); n2 = norm2(out);
+  }
+};
+
+// Fixed-iteration CG as a smoother (LinearFunction).
+// Used as the IR-shifted smoother: solves (M†M + lo*I) x = b approximately.
+template<class Field>
+class CGSmoother : public LinearFunction<Field>
+{
+public:
+  using LinearFunction<Field>::operator();
+  LinearOperatorBase<Field> &_op;
+  int iters;
+  CGSmoother(int _iters, LinearOperatorBase<Field> &op) : _op(op), iters(_iters) {
+    std::cout << GridLogMessage << "CGSmoother order " << iters << std::endl;
+  }
+  void operator()(const Field &in, Field &out) {
+    ConjugateGradient<Field> CG(0.0, iters, false);
+    out = Zero();
+    CG(_op, in, out);
+  }
+};
+
+// Two-level V-cycle preconditioner (LinearFunction).
+template<class Fobj, class CComplex, int nbasis>
+class MGPreconditioner : public LinearFunction<Lattice<Fobj>>
+{
+public:
+  using LinearFunction<Lattice<Fobj>>::operator();
+  typedef Aggregation<Fobj, CComplex, nbasis>  Aggregates;
+  typedef typename Aggregates::FineField        FineField;
+  typedef typename Aggregates::CoarseVector     CoarseVector;
+  typedef LinearOperatorBase<FineField>         FineOperator;
+  typedef LinearFunction<FineField>             FineSmoother;
+  typedef LinearOperatorBase<CoarseVector>      CoarseOperator;
+  typedef LinearFunction<CoarseVector>          CoarseSolver;
+
+  Aggregates    &_Aggregates;
+  FineOperator  &_FineOperator;
+  FineSmoother  &_PreSmoother;
+  FineSmoother  &_PostSmoother;
+  CoarseOperator &_CoarseOperator;
+  CoarseSolver  &_CoarseSolve;
+
+  MGPreconditioner(Aggregates     &Agg,
+                   FineOperator   &Fine,
+                   FineSmoother   &Pre,
+                   FineSmoother   &Post,
+                   CoarseOperator &CoarseOp,
+                   CoarseSolver   &CoarseSolve)
+    : _Aggregates(Agg), _FineOperator(Fine),
+      _PreSmoother(Pre), _PostSmoother(Post),
+      _CoarseOperator(CoarseOp), _CoarseSolve(CoarseSolve) {}
+
+  virtual void operator()(const FineField &in, FineField &out)
+  {
+    GridBase *CoarseGrid = _Aggregates.CoarseGrid;
+    CoarseVector Csrc(CoarseGrid);
+    CoarseVector Csol(CoarseGrid);
+    FineField vec1(in.Grid());
+    FineField vec2(in.Grid());
+
+    double t;
+
+    // Pre-smooth
+    out = Zero();
+    t = -usecond();
+    _PreSmoother(in, out);
+    t += usecond();
+    std::cout << GridLogMessage << "PreSmoother " << t/1e3 << " ms" << std::endl;
+
+    // Residual -> coarse projection
+    _FineOperator.Op(out, vec1);
+    sub(vec1, in, vec1);
+    t = -usecond();
+    _Aggregates.ProjectToSubspace(Csrc, vec1);
+    t += usecond();
+    std::cout << GridLogMessage << "ProjectToSubspace " << t/1e3 << " ms" << std::endl;
+
+    // Coarse solve
+    Csol = Zero();
+    t = -usecond();
+    _CoarseSolve(Csrc, Csol);
+    t += usecond();
+    std::cout << GridLogMessage << "CoarseSolve " << t/1e3 << " ms" << std::endl;
+
+    // Promote correction and accumulate
+    t = -usecond();
+    _Aggregates.PromoteFromSubspace(Csol, vec1);
+    t += usecond();
+    std::cout << GridLogMessage << "PromoteFromSubspace " << t/1e3 << " ms" << std::endl;
+    add(out, out, vec1);
+
+    // Post-smooth on updated residual
+    _FineOperator.Op(out, vec1);
+    sub(vec1, in, vec1);
+    vec2 = Zero();
+    t = -usecond();
+    _PostSmoother(vec1, vec2);
+    t += usecond();
+    std::cout << GridLogMessage << "PostSmoother " << t/1e3 << " ms" << std::endl;
+    add(out, out, vec2);
+  }
+};
+
+int main(int argc, char **argv)
+{
+  Grid_init(&argc, &argv);
+
+  // Physical parameters matching the 48^3 x 96 test ensemble.
+  // Target HMC ensemble is 128^3 x 288, Ls=12 at a^{-1}=3.5 GeV.
+  const int Ls    = 24;
+  const int nbasis = 62;
+  RealD mass = 0.00078;
+  RealD M5   = 1.8;
+  RealD b    = 1.5;
+  RealD c    = 0.5;
+  { const char *e = getenv("MASS"); if (e && *e) mass = atof(e); }
+
+  std::cout << GridLogMessage << "MdagM multigrid CG: mass=" << mass
+            << " Ls=" << Ls << " b=" << b << " c=" << c << std::endl;
+
+  GridCartesian         *UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(),
+                                     GridDefaultSimd(Nd, vComplex::Nsimd()),
+                                     GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         *FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Coarse grid: divide by 2 in each dimension, matching Example_mdagm.cc.
+  // Coarse dims {24,24,24,48} -> local {8,8,6,6} with MPI 3.3.4.8: all even,
+  // compatible with any power-of-2 SIMD layout.
+  Coordinate clatt = GridDefaultLatt();
+  for (int d = 0; d < (int)clatt.size(); d++) clatt[d] /= 2;
+
+  GridCartesian *Coarse4d = SpaceTimeGrid::makeFourDimGrid(clatt,
+                              GridDefaultSimd(Nd, vComplex::Nsimd()),
+                              GridDefaultMpi());
+  GridCartesian *Coarse5d = SpaceTimeGrid::makeFiveDimGrid(1, Coarse4d);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4);
+
+  LatticeGaugeField Umu(UGrid);
+  FieldMetaData header;
+  std::string file("ckpoint_lat.1000");
+  NerscIO::readConfiguration(Umu, header, file);
+
+  ///////////////////////////////////////////////////////////
+  // Fine operator: M†M on the full 5D lattice
+  // MdagMLinearOperator::Op = M (not M†M), HermOp = M†M.
+  // HermOpAdaptor makes Op = HermOp = M†M everywhere.
+  ///////////////////////////////////////////////////////////
+  MobiusFermionD Ddwf(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mass, M5, b, c);
+  MdagMLinearOperator<MobiusFermionD, LatticeFermionD> MdagMOp(Ddwf);
+  HermOpAdaptor<LatticeFermionD> HermFineOp(MdagMOp);
+
+  ///////////////////////////////////////////////////////////
+  // Coarse operator: 33-point stencil for 2-hop M†M
+  ///////////////////////////////////////////////////////////
+  typedef GeneralCoarsenedMatrix<vSpinColourVector, vTComplex, nbasis> LittleDiracOperator;
+  typedef LittleDiracOperator::CoarseVector CoarseVector;
+
+  NextToNearestStencilGeometry5D geom(Coarse5d);
+  LittleDiracOperator LittleDiracOp(geom, FGrid, Coarse5d);
+
+  ///////////////////////////////////////////////////////////
+  // Near-null subspace via Chebyshev filter on M†M.
+  // After CreateSubspaceChebyshevNew, subspace[i] = ψᵢ are the
+  // original near-null vectors. Block-local Gram-Schmidt (Orthogonalise)
+  // will rotate them into orthonormal block basis, destroying the near-null
+  // property of individual vectors while preserving the subspace span.
+  // We must compute the cheap coarse deflation BEFORE Orthogonalise.
+  ///////////////////////////////////////////////////////////
+  typedef Aggregation<vSpinColourVector, vTComplex, nbasis> Subspace;
+  Subspace Aggregates(Coarse5d, FGrid, 0);
+
+  RealD hi = 92.0;
+  Aggregates.CreateSubspaceChebyshevNew(RNG5, HermFineOp, hi);
+
+  // Compute W_ij = <ψᵢ|M†M|ψⱼ> and diagonalise BEFORE Orthogonalise.
+  // Orthogonalise() applies block-local Gram-Schmidt which rotates subspace[i]
+  // into an orthonormal block basis, destroying the near-null property of
+  // individual vectors. The original span is preserved, but we need the
+  // pre-GS ψᵢ to form χₖ = Σᵢ V[i,k] ψᵢ (approximate coarse zero modes).
+  std::vector<LatticeFermionD> chi(nbasis, FGrid);
+  {
+    std::vector<LatticeFermionD> &psi = Aggregates.subspace; // pre-GS near-null vecs
+    LatticeFermionD tmp(FGrid);
+    Eigen::MatrixXcd W = Eigen::MatrixXcd::Zero(nbasis, nbasis);
+    for (int i = 0; i < nbasis; i++) {
+      HermFineOp.Op(psi[i], tmp);
+      for (int j = 0; j < nbasis; j++)
+        W(j, i) = TensorRemove(innerProduct(psi[j], tmp));
+    }
+    Eigen::SelfAdjointEigenSolver<Eigen::MatrixXcd> esolver(W);
+    for (int k = 0; k < nbasis; k++) {
+      chi[k] = Zero();
+      for (int i = 0; i < nbasis; i++)
+        chi[k] += ComplexD(esolver.eigenvectors()(i, k)) * psi[i];
+    }
+  }
+
+  Aggregates.Orthogonalise();
+
+  ///////////////////////////////////////////////////////////
+  // Build coarse operator from subspace (post-GS basis)
+  ///////////////////////////////////////////////////////////
+  std::cout << GridLogMessage << "CoarsenOperator" << std::endl;
+  LittleDiracOp.CoarsenOperator(HermFineOp, Aggregates);
+
+  ///////////////////////////////////////////////////////////
+  // Coarse linear operator (HPD: Op = HermOp = coarse M†M)
+  ///////////////////////////////////////////////////////////
+  HermitianLinearOperator<LittleDiracOperator, CoarseVector> LinOpCoarse(LittleDiracOp);
+
+  ///////////////////////////////////////////////////////////
+  // Project χₖ onto the coarse grid using the post-GS subspace.
+  // ProjectToSubspace implicitly applies U†(x_c) at each block,
+  // correctly mapping the pre-GS near-null combinations to coarse vectors.
+  // Rayleigh quotients on the coarse operator give accurate eigenvalues.
+  ///////////////////////////////////////////////////////////
+  std::vector<CoarseVector> coarse_deflation_vecs(nbasis, Coarse5d);
+  std::vector<RealD>        coarse_deflation_evals(nbasis);
+  {
+    CoarseVector Ac(Coarse5d);
+    for (int k = 0; k < nbasis; k++) {
+      Aggregates.ProjectToSubspace(coarse_deflation_vecs[k], chi[k]);
+      RealD n = norm2(coarse_deflation_vecs[k]);
+      coarse_deflation_vecs[k] *= 1.0 / std::sqrt(n);
+      LinOpCoarse.HermOp(coarse_deflation_vecs[k], Ac);
+      coarse_deflation_evals[k] = real(TensorRemove(innerProduct(coarse_deflation_vecs[k], Ac)));
+      std::cout << GridLogMessage << "Coarse deflation eval[" << k << "] = "
+                << coarse_deflation_evals[k] << std::endl;
+    }
+  }
+
+  ///////////////////////////////////////////////////////////
+  // Coarse CG solver with deflation guesser
+  ///////////////////////////////////////////////////////////
+  ConjugateGradient<CoarseVector> coarseCG(5.0e-2, 10000, false);
+  DeflatedGuesser<CoarseVector>   coarseGuess(coarse_deflation_vecs, coarse_deflation_evals);
+  HPDSolver<CoarseVector> CoarseSolve(LinOpCoarse, coarseCG, coarseGuess);
+
+  ///////////////////////////////////////////////////////////
+  // Fine-grid smoother: fixed-iteration CG on (M†M + lo*I)
+  // lo=2.0 is the IR-shift high-pass parameter from mrhs-HDCG.
+  ///////////////////////////////////////////////////////////
+  RealD lo           = 2.0;
+  int   smootherIters = 8;
+  ShiftedHermOpLinearOperator<LatticeFermionD> ShiftedOp(HermFineOp, lo);
+  CGSmoother<LatticeFermionD> Smoother(smootherIters, ShiftedOp);
+
+  ///////////////////////////////////////////////////////////
+  // Two-level V-cycle multigrid preconditioner
+  ///////////////////////////////////////////////////////////
+  MGPreconditioner<vSpinColourVector, vTComplex, nbasis>
+    TwoLevelPrecon(Aggregates, HermFineOp, Smoother, Smoother, LinOpCoarse, CoarseSolve);
+
+  ///////////////////////////////////////////////////////////
+  // Outer solver: preconditioned GCR on M†M
+  ///////////////////////////////////////////////////////////
+  LatticeFermionD src(FGrid);    random(RNG5, src);
+  LatticeFermionD result(FGrid); result = Zero();
+
+  PrecGeneralisedConjugateResidual<LatticeFermionD>
+    PGCR(1.0e-8, 10000, HermFineOp, TwoLevelPrecon, 16, 16);
+
+  double t = -usecond();
+  PGCR(src, result);
+  t += usecond();
+  std::cout << GridLogMessage << "Total solve time: " << t/1e6 << " s" << std::endl;
+
+  Grid_finalize();
+  return 0;
+}

examples/Example_pvdagm_mrhs.cc

@@ -0,0 +1,662 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_pvdagm_mrhs.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+
+// MultiRHS (valence) two-level multigrid for PVdagM.
+//
+// Philosophy: NO block Krylov, NO per-RHS lockstep coefficients. A single
+// GCR polynomial for the enlarged block-diagonal system diag(A,...,A):
+// all inner products are summed over the RHS index, giving one alpha/beta
+// per step shared by every RHS.
+//
+// Level structure mirrors Example_pvdagm:
+//   outer:  MrhsPGCRNonHermitian on PVdagM over std::vector<LatticeFermionD>
+//   precon: V-cycle -- per-RHS fine post-smoother (16-step shifted GCR),
+//           batched restriction (MultiRHSBlockProject / GEMM),
+//           ONE coarse PGCR on the 6D mrhs coarse operator
+//           (MultiGeneralCoarsenedMatrix, GEMM mults -- the ~10x win),
+//           batched prolongation.
+//
+// The coarse operator is coarsened once with the standard single-RHS
+// machinery (subspace cache reused) and imported via CopyMatrix.
+//
+// Memory note: outer restart history is 2*mmax*nrhs fine fields
+// (49GB/field global at 48^3x96,Ls=24). Default mmax=8, nrhs=12 needs
+// ~10TB for history; use OuterMmax / NRHS to fit the partition.
+//
+// Env vars: MASS, SUBSPACE_FILE, BLOCK (dotted e.g. 4.4.3.4),
+//           NRHS (default 12, multiple of Nsimd),
+//           FineSmootherShift, FineSmootherOrder,
+//           CoarseSolverTol, CoarseSolverOrder,
+//           OuterMmax, OuterNstep, OuterTol
+
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+
+using namespace std;
+using namespace Grid;
+
+RealD FineSmootherShift = 0.1;
+int   FineSmootherOrder = 16;
+RealD CoarseSolverTol   = 0.03;
+int   CoarseSolverOrder = 200;
+RealD OuterTol          = 1.0e-8;
+int   OuterMmax         = 8;
+int   OuterNstep        = 8;
+int   Nrhs              = 12;
+RealD mass              = 0.00078;
+
+void ParseEnvironment(void)
+{
+  if(getenv("MASS"))              mass              = atof(getenv("MASS"));
+  if(getenv("FineSmootherShift")) FineSmootherShift = atof(getenv("FineSmootherShift"));
+  if(getenv("FineSmootherOrder")) FineSmootherOrder = atoi(getenv("FineSmootherOrder"));
+  if(getenv("CoarseSolverTol"))   CoarseSolverTol   = atof(getenv("CoarseSolverTol"));
+  if(getenv("CoarseSolverOrder")) CoarseSolverOrder = atoi(getenv("CoarseSolverOrder"));
+  if(getenv("OuterTol"))          OuterTol          = atof(getenv("OuterTol"));
+  if(getenv("OuterMmax"))         OuterMmax         = atoi(getenv("OuterMmax"));
+  if(getenv("OuterNstep"))        OuterNstep        = atoi(getenv("OuterNstep"));
+  if(getenv("NRHS"))              Nrhs              = atoi(getenv("NRHS"));
+
+  std::cout << GridLogMessage << "PARAM: MASS              " << mass              << std::endl;
+  std::cout << GridLogMessage << "PARAM: NRHS              " << Nrhs              << std::endl;
+  std::cout << GridLogMessage << "PARAM: FineSmootherShift " << FineSmootherShift << std::endl;
+  std::cout << GridLogMessage << "PARAM: FineSmootherOrder " << FineSmootherOrder << std::endl;
+  std::cout << GridLogMessage << "PARAM: CoarseSolverTol   " << CoarseSolverTol   << std::endl;
+  std::cout << GridLogMessage << "PARAM: CoarseSolverOrder " << CoarseSolverOrder << std::endl;
+  std::cout << GridLogMessage << "PARAM: OuterTol          " << OuterTol          << std::endl;
+  std::cout << GridLogMessage << "PARAM: OuterMmax         " << OuterMmax         << std::endl;
+  std::cout << GridLogMessage << "PARAM: OuterNstep        " << OuterNstep        << std::endl;
+}
+
+template <class Field>
+void saveSubspace(std::vector<Field> &subspace, std::string const fname){
+#ifdef HAVE_LIME
+  std::cout << Grid::GridLogMessage << "Saving subspace (" << subspace.size() << " vectors) to: " << fname << std::endl;
+  Grid::emptyUserRecord record;
+  Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
+  SW.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++)
+    SW.writeScidacFieldRecord(subspace[k], record);
+  SW.close();
+#endif
+}
+
+template <class Field>
+void loadSubspace(std::vector<Field> &subspace, std::string const fname){
+#ifdef HAVE_LIME
+  std::cout << Grid::GridLogMessage << "Loading subspace (" << subspace.size() << " vectors) from: " << fname << std::endl;
+  Grid::emptyUserRecord record;
+  Grid::ScidacReader SR;
+  SR.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++)
+    SR.readScidacFieldRecord(subspace[k], record);
+  SR.close();
+#endif
+}
+
+template<class Matrix,class Field>
+class PVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat;
+  Matrix &_PV;
+public:
+  PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV) {};
+  void OpDiag (const Field &in, Field &out) {    assert(0);  }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) {    assert(0);  }
+  void OpDirAll  (const Field &in, std::vector<Field> &out){    assert(0);  };
+  void Op     (const Field &in, Field &out){
+    Field tmp(in.Grid());
+    _Mat.M(in,tmp);
+    _PV.Mdag(tmp,out);
+  }
+  void AdjOp     (const Field &in, Field &out){
+    Field tmp(in.Grid());
+    _PV.M(in,tmp);
+    _Mat.Mdag(tmp,out);
+  }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+    HermOp(in,out);
+    ComplexD dot = innerProduct(in,out);
+    n1=real(dot);
+    n2=norm2(out);
+  }
+  void HermOp(const Field &in, Field &out){
+    Field tmp(in.Grid());
+    Op(in,tmp);
+    AdjOp(tmp,out);
+  }
+};
+
+template<class Matrix,class Field>
+class ShiftedPVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat;
+  Matrix &_PV;
+public:
+  RealD shift;
+  ShiftedPVdagMLinearOperator(RealD _shift,Matrix &Mat,Matrix &PV): shift(_shift),_Mat(Mat),_PV(PV){};
+  void OpDiag (const Field &in, Field &out) {    assert(0);  }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) {    assert(0);  }
+  void OpDirAll  (const Field &in, std::vector<Field> &out){    assert(0);  };
+  void Op     (const Field &in, Field &out){
+    Field tmp(in.Grid());
+    _Mat.M(in,tmp);
+    _PV.Mdag(tmp,out);
+    out = out + shift * in;
+  }
+  void AdjOp     (const Field &in, Field &out){
+    Field tmp(in.Grid());
+    _PV.M(tmp,out);
+    _Mat.Mdag(in,tmp);
+    out = out + shift * in;
+  }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){    assert(0);  }
+  void HermOp(const Field &in, Field &out){
+    Field tmp(in.Grid());
+    Op(in,tmp);
+    AdjOp(tmp,out);
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// Minimal multi-RHS function interface (preconditioner slot)
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class MrhsLinearFunction {
+public:
+  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
+};
+
+//////////////////////////////////////////////////////////////////////
+// Single-polynomial multi-RHS PGCR (non-Hermitian).
+//
+// Verbatim adaptation of PrecGeneralisedConjugateResidualNonHermitian
+// to std::vector<Field>: every innerProduct / norm2 is SUMMED over the
+// RHS index, so one alpha/beta per step is shared by all RHS -- the
+// single GCR on the enlarged block-diagonal system.
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class MrhsPGCRNonHermitian {
+public:
+  RealD   Tolerance;
+  Integer MaxIterations;
+  int mmax;
+  int nstep;
+  int steps;
+  int level;
+  int ZeroGuess = 0; int FirstCycle = 0;  // caller contract: zero guess => first-cycle r0 = src
+  std::string name = "Level 1";
+  LinearOperatorBase<Field> &Linop;
+  MrhsLinearFunction<Field> &Preconditioner;
+
+  void Level(int lv) { name = "Level " + std::to_string(lv); level=lv; };
+  void Name(std::string n) { name = n; };
+  void SetZeroGuess(int z) { ZeroGuess=z; };
+
+  MrhsPGCRNonHermitian(RealD tol,Integer maxit,
+                       LinearOperatorBase<Field> &_Linop,
+                       MrhsLinearFunction<Field> &Prec,
+                       int _mmax,int _nstep)
+    : Tolerance(tol), MaxIterations(maxit), Linop(_Linop), Preconditioner(Prec),
+      mmax(_mmax), nstep(_nstep) { level=1; }
+
+  ///////////////////////////////////////////////////////////////
+  // vector-of-fields linear algebra, reductions summed over rhs
+  ///////////////////////////////////////////////////////////////
+  static RealD vnorm2(std::vector<Field> &x){
+    RealD s=0.0; for(auto &f : x) s+=norm2(f); return s;
+  }
+  static ComplexD vinnerProduct(std::vector<Field> &x, std::vector<Field> &y){
+    ComplexD s(0.0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s;
+  }
+  static void vaxpy(std::vector<Field> &z, ComplexD a, std::vector<Field> &x, std::vector<Field> &y){
+    for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]);
+  }
+  void vOp(std::vector<Field> &in, std::vector<Field> &out){
+    for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]);
+  }
+
+  void operator() (std::vector<Field> &src, std::vector<Field> &psi){
+    RealD cp, ssq, rsq;
+    int nrhs = src.size();
+    GridBase *grid = src[0].Grid();
+
+    ssq=vnorm2(src);
+    rsq=Tolerance*Tolerance*ssq;
+
+    std::vector<Field> r(nrhs,grid);
+
+    GridStopWatch SolverTimer;
+    SolverTimer.Start();
+
+    steps=0;
+    FirstCycle=1;
+    for(int k=0;k<MaxIterations;k++){
+
+      cp=GCRnStep(src,psi,rsq);
+
+      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
+               <<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
+
+      if(cp<rsq){
+        SolverTimer.Stop();
+        vOp(psi,r);
+        for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
+        RealD tr=vnorm2(r);
+        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
+                 <<" MrhsPGCR: Converged on iteration "<<steps
+                 <<" computed residual "<<std::sqrt(cp/ssq)
+                 <<" true residual "<<std::sqrt(tr/ssq)
+                 <<" target "<<Tolerance<<std::endl;
+        std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
+                 <<" MrhsPGCR Time elapsed: Total "<<SolverTimer.Elapsed()<<std::endl;
+        // Per-RHS true residuals: the honest metric under the summed norm
+        for(int rr=0;rr<nrhs;rr++){
+          RealD rn = std::sqrt(norm2(r[rr])/norm2(src[rr]));
+          std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl;
+        }
+        return;
+      }
+    }
+    std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
+  }
+
+  RealD GCRnStep(std::vector<Field> &src, std::vector<Field> &psi, RealD rsq){
+
+    RealD cp;
+    ComplexD a, b, rq;
+    RealD zAAz;
+
+    int nrhs = src.size();
+    GridBase *grid = src[0].Grid();
+
+    std::vector<Field> r (nrhs,grid);
+    std::vector<Field> z (nrhs,grid);
+    std::vector<Field> Az(nrhs,grid);
+
+    ////////////////////////////////
+    // history for flexible orthog: [mmax][nrhs]
+    ////////////////////////////////
+    std::vector< std::vector<Field> > q(mmax, std::vector<Field>(nrhs,grid));
+    std::vector< std::vector<Field> > p(mmax, std::vector<Field>(nrhs,grid));
+    std::vector<RealD> qq(mmax);
+
+    std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
+
+    if (ZeroGuess && FirstCycle) {
+      for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; }
+    } else {
+      vOp(psi,Az);
+      for(int rr=0;rr<nrhs;rr++) r[rr] = src[rr]-Az[rr];
+      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
+               <<" MrhsPGCR true residual r = src - A psi   "<<vnorm2(r)<<std::endl;
+    }
+    FirstCycle=0;
+
+    Preconditioner(r,z);
+    vOp(z,Az);
+    zAAz=vnorm2(Az);
+
+    p[0]=z;
+    q[0]=Az;
+    qq[0]=zAAz;
+
+    cp=vnorm2(r);
+
+    for(int k=0;k<nstep;k++){
+
+      steps++;
+
+      int kp     = k+1;
+      int peri_k = k %mmax;
+      int peri_kp= kp%mmax;
+
+      rq = vinnerProduct(q[peri_k],r);
+      a  = rq/qq[peri_k];
+
+      vaxpy(psi,a,p[peri_k],psi);
+      vaxpy(r,-a,q[peri_k],r);
+      cp = vnorm2(r);
+
+      std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
+               <<" MrhsPGCR step["<<steps<<"]  resid "<<cp<<" target "<<rsq<<std::endl;
+
+      if((k==nstep-1)||(cp<rsq)){
+        return cp;
+      }
+
+      Preconditioner(r,z);
+      vOp(z,Az);
+      zAAz=vnorm2(Az);
+
+      q[peri_kp]=Az;
+      p[peri_kp]=z;
+
+      int northog = ((kp)>(mmax-1))?(mmax-1):(kp);
+      for(int back=0;back<northog;back++){
+        int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
+        b = -real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
+        vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]);
+        vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]);
+      }
+      qq[peri_kp]=vnorm2(q[peri_kp]);
+    }
+    GRID_ASSERT(0); // never reached
+    return cp;
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// Trivial multi-RHS preconditioner
+//////////////////////////////////////////////////////////////////////
+template<class Field>
+class TrivialMrhsPrecon : public MrhsLinearFunction<Field> {
+public:
+  virtual void operator()(std::vector<Field> &in, std::vector<Field> &out){
+    for(int r=0;r<(int)in.size();r++) out[r]=in[r];
+  }
+};
+
+//////////////////////////////////////////////////////////////////////
+// MultiRHS two-level V-cycle.
+//
+// Mirrors MGPreconditioner in Example_pvdagm:
+//   out = in (trivial pre)                      [per rhs]
+//   r1  = in - A out                            [per rhs]
+//   batched blockProject -> pack -> ONE mrhs coarse PGCR -> unpack
+//     -> batched blockPromote; out += correction
+//   r2  = in - A out                            [per rhs]
+//   per-RHS fine post-smoother; out += smooth(r2)
+//////////////////////////////////////////////////////////////////////
+template<class FineField, class MrhsCoarseVector, class FineSmoother>
+class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
+public:
+  typedef MrhsCoarseVector CoarseVector;   // same lattice type on Coarse5d and CoarseMrhs
+
+  LinearOperatorBase<FineField>            &_FineOperator;
+  FineSmoother                             &_PostSmoother;   // single-RHS smoother, looped
+  MultiRHSBlockProject<FineField>          &_Projector;
+  LinearFunction<CoarseVector>             &_CoarseSolve;    // PGCR on the 6D mrhs field
+  GridBase                                 *_CoarseGrid;     // Coarse5d (single rhs)
+  GridBase                                 *_CoarseGridMrhs; // 6D
+
+  MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp,
+                 FineSmoother                  &Post,
+                 MultiRHSBlockProject<FineField> &Projector,
+                 LinearFunction<CoarseVector>  &CoarseSolve,
+                 GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
+    : _FineOperator(FineOp), _PostSmoother(Post), _Projector(Projector),
+      _CoarseSolve(CoarseSolve), _CoarseGrid(CoarseGrid), _CoarseGridMrhs(CoarseGridMrhs) {}
+
+  virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
+    int nrhs = in.size();
+    GridBase *fgrid = in[0].Grid();
+    double t;
+
+    std::vector<FineField> vec1(nrhs,fgrid);
+    std::vector<FineField> vec2(nrhs,fgrid);
+
+    // Trivial pre-smoother: out = in (as in Example_pvdagm with simple_fine)
+    for(int r=0;r<nrhs;r++) out[r]=in[r];
+
+    // Residual
+    for(int r=0;r<nrhs;r++){
+      _FineOperator.Op(out[r],vec1[r]);
+      sub(vec1[r],in[r],vec1[r]);
+    }
+
+    // Batched fine->coarse, pack rhs into 6D field
+    std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid);
+    std::vector<CoarseVector> Csol_split(nrhs,_CoarseGrid);
+    CoarseVector CsrcMrhs(_CoarseGridMrhs);
+    CoarseVector CsolMrhs(_CoarseGridMrhs);
+
+    t=-usecond();
+    _Projector.blockProject(vec1,Csrc_split);
+    for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
+    t+=usecond();
+    std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
+
+    // ONE coarse solve for all rhs (GEMM coarse mults)
+    t=-usecond();
+    CsolMrhs=Zero();
+    _CoarseSolve(CsrcMrhs,CsolMrhs);
+    t+=usecond();
+    std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
+
+    // Unpack, batched coarse->fine, add correction
+    t=-usecond();
+    for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
+    _Projector.blockPromote(vec1,Csol_split);
+    for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
+    t+=usecond();
+    std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
+
+    // Residual
+    for(int r=0;r<nrhs;r++){
+      _FineOperator.Op(out[r],vec1[r]);
+      sub(vec1[r],in[r],vec1[r]);
+    }
+
+    // Per-RHS post-smoother (fine level has no batching win; memory-light)
+    t=-usecond();
+    for(int r=0;r<nrhs;r++){
+      vec2[r]=Zero();
+      _PostSmoother(vec1[r],vec2[r]);
+      add(out[r],out[r],vec2[r]);
+    }
+    t+=usecond();
+    std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms  ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
+  }
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+  ParseEnvironment();
+
+  const int Ls=24;
+  RealD M5=1.8;
+  RealD b=1.5;
+  RealD c=0.5;
+  const int nbasis = 60;
+  const int nrhs   = Nrhs;
+
+  GRID_ASSERT(nrhs % vComplex::Nsimd() == 0);
+
+  std::cout << GridLogMessage << "MultiRHS PVdagM MG: mass=" << mass << " Ls=" << Ls
+            << " nbasis=" << nbasis << " nrhs=" << nrhs << std::endl;
+
+  std::vector<int> lat_size {48, 48, 48, 96};
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(lat_size, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  // Blocking, env-overridable: BLOCK=2.2.2.2
+  Coordinate clatt = lat_size;
+  Coordinate Block({4,4,3,4});
+  if ( getenv("BLOCK") ) {
+    GridCmdOptionIntVector(std::string(getenv("BLOCK")),Block);
+    GRID_ASSERT(Block.size()==4);
+  }
+  for(int d=0;d<clatt.size();d++){
+    GRID_ASSERT(lat_size[d] % Block[d] == 0);
+    clatt[d] = lat_size[d]/Block[d];
+  }
+  std::cout << GridLogMessage << "Block " << Block << "  coarse lattice " << clatt << std::endl;
+
+  GridCartesian *Coarse4d =  SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian *Coarse5d =  SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
+
+  ////////////////////////////////////////////////////////////
+  // 6D multi-RHS coarse grid: rhs is dim 0, SIMD across rhs
+  // (pattern: tests/debug/Test_general_coarse_hdcg_phys48.cc)
+  ////////////////////////////////////////////////////////////
+  Coordinate mpi=GridDefaultMpi();
+  Coordinate rhMpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  Coordinate rhLatt({nrhs,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  Coordinate rhSimd({vComplex::Nsimd(),1, 1,1,1,1});
+  GridCartesian *CoarseMrhs = new GridCartesian(rhLatt,rhSimd,rhMpi);
+
+  GridParallelRNG RNG5(FGrid);  RNG5.SeedFixedIntegers({5,6,7,8});
+  GridParallelRNG RNG4(UGrid);  RNG4.SeedFixedIntegers({1,2,3,4});
+
+  LatticeGaugeField Umu(UGrid);
+  std::cout << GridLogMessage << "Reading gauge field" << std::endl;
+  FieldMetaData header;
+  std::string file("/ccs/home/poare/ckpoint_lat.1000");
+  NerscIO::readConfiguration(Umu,header,file);
+
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
+  MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
+
+  typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD>        PVdagM_t;
+  typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
+  typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis>  LittleDiracOperator;
+  typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
+  typedef LittleDiracOperator::CoarseVector                           CoarseVector;
+  typedef Aggregation<vSpinColourVector,vTComplex,nbasis>             Subspace;
+
+  PVdagM_t        PVdagM(Ddwf,Dpv);
+  ShiftedPVdagM_t ShiftedPVdagM(FineSmootherShift,Ddwf,Dpv);
+
+  NextToNearestStencilGeometry5D geom(Coarse5d);
+
+  ////////////////////////////////////////////////////////////
+  // Subspace: load from cache or generate
+  ////////////////////////////////////////////////////////////
+  std::string subspace_file = "/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb"
+                              + std::to_string(nbasis) + ".scidac";
+  if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
+
+  uint64_t file_exists = 0;
+  if ( UGrid->IsBoss() ) { std::ifstream f(subspace_file); file_exists = f.good() ? 1 : 0; }
+  UGrid->GlobalSum(file_exists);
+
+  const int cb = 0;
+  Subspace AggregatesGCR(Coarse5d,FGrid,cb);
+
+  if ( file_exists ) {
+    std::cout << GridLogMessage << "*** Loading subspace from disk ***" << std::endl;
+    loadSubspace(AggregatesGCR.subspace, subspace_file);
+  } else {
+    std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
+    AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
+    saveSubspace(AggregatesGCR.subspace, subspace_file);
+  }
+
+  ////////////////////////////////////////////////////////////
+  // Coarsen once (single-RHS machinery), import into mrhs op.
+  // NB CoarsenOperator block-orthogonalises subspace in place;
+  // ImportBasis AFTER so the projector matches the coarse op.
+  ////////////////////////////////////////////////////////////
+  MrhsLittleDiracOperator mrhsLittleDiracOpPV(geom,CoarseMrhs);
+  {
+    // Scope the single-RHS operator so its padded _A is freed after import.
+    // At small local volumes the depth-2 padded cell inflates ~8x
+    // (e.g. 2^4 blocking on 432 ranks: local 8x4x4x12 -> padded 12x8x8x16,
+    // ~23GB/GCD for _A alone -> OOM if kept alive).
+    LittleDiracOperator LittleDiracOpPV(geom,FGrid,Coarse5d);
+    LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesGCR);
+    mrhsLittleDiracOpPV.CopyMatrix(LittleDiracOpPV);
+  }
+
+  MultiRHSBlockProject<LatticeFermionD> MrhsProjector;
+  MrhsProjector.Allocate(nbasis,FGrid,Coarse5d);
+  MrhsProjector.ImportBasis(AggregatesGCR.subspace);
+
+  ////////////////////////////////////////////////////////////
+  // Solvers
+  ////////////////////////////////////////////////////////////
+  NonHermitianLinearOperator<MrhsLittleDiracOperator,CoarseVector> mrhsLinOpCoarse(mrhsLittleDiracOpPV);
+
+  TrivialPrecon<CoarseVector> simpleC;
+  PrecGeneralisedConjugateResidualNonHermitian<CoarseVector>
+    L2PGCRmrhs(CoarseSolverTol, CoarseSolverOrder/20, mrhsLinOpCoarse, simpleC, 20, 20);
+  L2PGCRmrhs.Level(2);
+  L2PGCRmrhs.Name("Couter");
+  L2PGCRmrhs.SetZeroGuess(1);          // caller zeroes CsolMrhs
+
+  TrivialPrecon<LatticeFermionD> simple_fine;
+  PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD>
+    SmootherGCR(0.0, 1, ShiftedPVdagM, simple_fine, FineSmootherOrder, FineSmootherOrder);
+  SmootherGCR.Level(1);
+  SmootherGCR.Name("Fsmoother");
+  SmootherGCR.SetZeroGuess(1);         // caller zeroes vec2[r]
+
+  typedef PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD> FineSmoother_t;
+  MrhsTwoLevelMG<LatticeFermionD,CoarseVector,FineSmoother_t>
+    TwoLevelPrecon(PVdagM, SmootherGCR, MrhsProjector, L2PGCRmrhs, Coarse5d, CoarseMrhs);
+
+  MrhsPGCRNonHermitian<LatticeFermionD>
+    L1PGCRmrhs(OuterTol, 1000, PVdagM, TwoLevelPrecon, OuterMmax, OuterNstep);
+  L1PGCRmrhs.Level(1);
+  L1PGCRmrhs.Name("Fouter");
+  L1PGCRmrhs.SetZeroGuess(1);          // sol[r]=Zero() at source setup
+
+  ////////////////////////////////////////////////////////////
+  // Sources and solve
+  ////////////////////////////////////////////////////////////
+  std::vector<LatticeFermionD> src(nrhs,FGrid);
+  std::vector<LatticeFermionD> sol(nrhs,FGrid);
+  for(int r=0;r<nrhs;r++){
+    gaussian(RNG5,src[r]);
+    sol[r]=Zero();
+  }
+
+  std::cout << GridLogMessage << "**********************************************" << std::endl;
+  std::cout << GridLogMessage << " MultiRHS two-level solve: " << nrhs << " RHS " << std::endl;
+  std::cout << GridLogMessage << "**********************************************" << std::endl;
+
+  GridStopWatch w; w.Start();
+  L1PGCRmrhs(src,sol);
+  w.Stop();
+
+  std::cout << GridLogMessage << "MultiRHS solve total " << w.Elapsed()
+            << "  (per RHS: " << w.useconds()/1.0e6/nrhs << " s)" << std::endl;
+
+  // Independent final verification, per RHS
+  {
+    LatticeFermionD Ax(FGrid);
+    RealD worst=0.0;
+    for(int r=0;r<nrhs;r++){
+      PVdagM.Op(sol[r],Ax);
+      Ax = Ax - src[r];
+      RealD rn = std::sqrt(norm2(Ax)/norm2(src[r]));
+      std::cout << GridLogMessage << "FINAL: rhs["<<r<<"] true residual = " << rn << std::endl;
+      worst = std::max(worst,rn);
+    }
+    std::cout << GridLogMessage << "FINAL: worst-case residual = " << worst << std::endl;
+  }
+
+  std::cout << GridLogMessage << "Done" << std::endl;
+  Grid_finalize();
+  return 0;
+}

examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc

@@ -0,0 +1,476 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// PVdagM three level multigrid on the V2 coarse operator.
+//
+// STAGE ONE: grids, types, subspace, and the L1 coarsening only. The L2
+// chain, the dense bottom and the solves are not here yet.
+//
+// Differences from Example_pvdagm_mrhs_3level_DenseCoarseMatrix.cc:
+//
+//  * The coarse space is UNVECTORISED (sComplexD). The fine space stays
+//    vectorised. MultiRHSBlockProject carries the mixed layout.
+//
+//  * One operator, not two. V1 needed GeneralCoarsenedMatrix to coarsen and
+//    MultiGeneralCoarsenedMatrix to apply, bridged by CopyMatrix. V2 does
+//    both, and single versus multiRHS is SetGrid on the same object with the
+//    matrix elements built once.
+//
+//  * Nrhs is unconstrained. V1 required nrhs % vComplex::Nsimd() == 0 because
+//    its multiRHS grid carried the SIMD in the rhs direction.
+//
+//  * CoarsenOperator takes the subspace vectors, not an Aggregation. It block
+//    orthonormalises them IN PLACE -- the vectors are far too large to copy
+//    defensively -- so rawNull is taken first and the RAW vectors are what
+//    define the L2 null space. Do not insert an Orthogonalise() anywhere:
+//    projecting a block-orthonormal vector onto its own block-orthonormalised
+//    aggregation gives e_k, and the near null content is silently gone. The
+//    ||<psi|psi> - I||_F guard below is what catches that.
+//
+// Env: LATT LS MASS NBASIS(compile time) NRHS BLOCK COARSEN_BATCH
+//      HOT_START CONFIG SUBSPACE_FILE V1_CHECK
+//
+
+#include <Grid/Grid.h>
+#include <Grid/lattice/PaddedCell.h>
+#include <Grid/stencil/GeneralLocalStencil.h>
+#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
+
+#include <memory>
+
+using namespace std;
+using namespace Grid;
+
+// Compile time so it can be cut down for laptop runs: -DNBASIS=8
+#ifndef NBASIS
+#define NBASIS 60
+#endif
+
+RealD mass      = 0.00078;
+int   Nrhs      = 12;
+int   Ls        = 24;
+int   CoarsenBatch = 9;
+std::vector<int> lat_size({48,48,48,96});
+
+void ParseEnvironment(void)
+{
+  if(getenv("MASS"))          mass        = atof(getenv("MASS"));
+  if(getenv("NRHS"))          Nrhs        = atoi(getenv("NRHS"));
+  if(getenv("LS"))            Ls          = atoi(getenv("LS"));
+  if(getenv("COARSEN_BATCH")) CoarsenBatch= atoi(getenv("COARSEN_BATCH"));
+  if(getenv("LATT")){
+    Coordinate l;
+    GridCmdOptionIntVector(std::string(getenv("LATT")),l);
+    GRID_ASSERT(l.size()==4);
+    for(int d=0;d<4;d++) lat_size[d]=l[d];
+  }
+
+  std::cout << GridLogMessage << "PARAM: LATT               "
+	    << lat_size[0]<<"."<<lat_size[1]<<"."<<lat_size[2]<<"."<<lat_size[3] << std::endl;
+  std::cout << GridLogMessage << "PARAM: LS                 " << Ls           << std::endl;
+  std::cout << GridLogMessage << "PARAM: MASS               " << mass         << std::endl;
+  std::cout << GridLogMessage << "PARAM: NBASIS             " << NBASIS       << std::endl;
+  std::cout << GridLogMessage << "PARAM: NRHS               " << Nrhs         << std::endl;
+  std::cout << GridLogMessage << "PARAM: COARSEN_BATCH      " << CoarsenBatch << std::endl;
+}
+
+template <class Field>
+void saveSubspace(std::vector<Field> &subspace, std::string const fname){
+#ifdef HAVE_LIME
+  Grid::emptyUserRecord record;
+  Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
+  SW.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++) SW.writeScidacFieldRecord(subspace[k], record);
+  SW.close();
+#endif
+}
+template <class Field>
+void loadSubspace(std::vector<Field> &subspace, std::string const fname){
+#ifdef HAVE_LIME
+  Grid::emptyUserRecord record;
+  Grid::ScidacReader SR;
+  SR.open(fname);
+  for (int k = 0; k < (int)subspace.size(); k++) SR.readScidacFieldRecord(subspace[k], record);
+  SR.close();
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////
+// A = PV^dag M (non-Hermitian)
+//////////////////////////////////////////////////////////////////////
+template<class Matrix,class Field>
+class PVdagMLinearOperator : public LinearOperatorBase<Field> {
+  Matrix &_Mat; Matrix &_PV;
+public:
+  PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV) {};
+  void OpDiag (const Field &in, Field &out) { assert(0); }
+  void OpDir  (const Field &in, Field &out,int dir,int disp) { assert(0); }
+  void OpDirAll  (const Field &in, std::vector<Field> &out){ assert(0); };
+  void Op     (const Field &in, Field &out){ Field tmp(in.Grid()); _Mat.M(in,tmp); _PV.Mdag(tmp,out); }
+  void AdjOp  (const Field &in, Field &out){ Field tmp(in.Grid()); _PV.M(in,tmp); _Mat.Mdag(tmp,out); }
+  void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ HermOp(in,out); ComplexD d=innerProduct(in,out); n1=real(d); n2=norm2(out); }
+  void HermOp(const Field &in, Field &out){ Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
+};
+
+//////////////////////////////////////////////////////////////////////
+// ||<v|v> - I||_F over a set of coarse vectors. ~0.23 means the raw near
+// null content survived the projection; ~sqrt(N_sites) means block
+// orthonormal vectors leaked in and every image collapsed to e_k.
+//////////////////////////////////////////////////////////////////////
+template<class CoarseField>
+RealD GramDefect(std::vector<CoarseField> &v)
+{
+  RealD s2=0.0;
+  for(int i=0;i<(int)v.size();i++){
+    for(int j=0;j<(int)v.size();j++){
+      ComplexD sij=TensorRemove(innerProduct(v[i],v[j]));
+      ComplexD d=sij-(i==j?ComplexD(1.0):ComplexD(0.0));
+      s2+=real(d)*real(d)+imag(d)*imag(d);
+    }
+  }
+  return std::sqrt(s2);
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+  ParseEnvironment();
+
+  RealD M5=1.8, b=1.5, c=0.5;
+  const int nbasis=NBASIS;
+  const int nrhs=Nrhs;
+  const int batch=CoarsenBatch;
+
+  Coordinate mpi = GridDefaultMpi();
+  Coordinate fsimd= GridDefaultSimd(Nd,vComplex::Nsimd());
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(lat_size,fsimd,mpi);
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  // Level 1 blocking (default 2^4)
+  Coordinate clatt = lat_size;
+  Coordinate Block({2,2,2,2});
+  if ( getenv("BLOCK") ){ GridCmdOptionIntVector(std::string(getenv("BLOCK")),Block); GRID_ASSERT(Block.size()==4); }
+  for(int d=0;d<4;d++){ GRID_ASSERT(lat_size[d]%Block[d]==0); clatt[d]=lat_size[d]/Block[d]; }
+  std::cout << GridLogMessage << "Block  " << Block  << "  coarse lattice " << clatt << std::endl;
+
+  //////////////////////////////////////////////////////////////////////
+  // The coarse space is unvectorised. The 5D coarse grid is built here
+  // rather than through SpaceTimeGrid so the SIMD layout is ours.
+  //////////////////////////////////////////////////////////////////////
+  Coordinate c5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  Coordinate c5simd({1,1,1,1,1});
+  Coordinate c5mpi ({1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  GridCartesian *Coarse5d = new GridCartesian(c5latt,c5simd,c5mpi);
+
+  // 6D coarse multiRHS grid: rhs is dim 0, undistributed and unvectorised.
+  // No divisibility constraint on nrhs, unlike V1.
+  Coordinate cmlatt({nrhs,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  Coordinate cmsimd({1,1,1,1,1,1});
+  Coordinate cmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  GridCartesian *CoarseMrhs = new GridCartesian(cmlatt,cmsimd,cmmpi);
+
+  // 6D coarse grid at the coarsening batch, used only while CoarsenOperator
+  // runs. The matrix elements survive the change back to nrhs.
+  Coordinate cblatt({batch,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+  GridCartesian *CoarseBatch = new GridCartesian(cblatt,cmsimd,cmmpi);
+
+  // 6D fine grid carrying the coarsening batch: fine SIMD layout preserved
+  Coordinate fmlatt({batch,Ls,lat_size[0],lat_size[1],lat_size[2],lat_size[3]});
+  Coordinate fmsimd({1,1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
+  Coordinate fmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
+  GridCartesian *FineMrhs = new GridCartesian(fmlatt,fmsimd,fmmpi);
+
+  std::cout << GridLogMessage << "Nsimd  fine " << FGrid->Nsimd()
+	    << "   coarse " << Coarse5d->Nsimd() << std::endl;
+
+  GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
+  GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
+
+  //////////////////////////////////////////////////////////////////////
+  // Gauge field
+  //////////////////////////////////////////////////////////////////////
+  LatticeGaugeField Umu(UGrid);
+  if ( getenv("HOT_START") ) {
+    std::cout << GridLogMessage << "Hot start gauge field" << std::endl;
+    SU<Nc>::HotConfiguration(RNG4,Umu);
+  } else {
+    std::string file("/ccs/home/poare/ckpoint_lat.1000");
+    if ( getenv("CONFIG") ) file = std::string(getenv("CONFIG"));
+    std::cout << GridLogMessage << "Reading gauge field " << file << std::endl;
+    FieldMetaData header;
+    NerscIO::readConfiguration(Umu,header,file);
+  }
+
+  MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
+  MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
+
+  typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD> PVdagM_t;
+  PVdagM_t PVdagM(Ddwf,Dpv);
+
+  //////////////////////////////////////////////////////////////////////
+  // Level 1 types: unvectorised coarse scalar
+  //////////////////////////////////////////////////////////////////////
+  typedef sTComplexD                                                          CComplexS;
+  typedef MultiGeneralCoarsenedOperatorV2<vSpinColourVector,CComplexS,nbasis> CoarseOperator;
+  typedef CoarseOperator::CoarseVector                                        CoarseVector;
+  typedef Aggregation<vSpinColourVector,CComplexS,nbasis>                     Subspace;
+
+  NextToNearestStencilGeometry5D geom(Coarse5d);
+
+  //////////////////////////////////////////////////////////////////////
+  // Subspace: load RAW (no Orthogonalise!), or generate.
+  //
+  // The Aggregation is scaffolding for CreateSubspaceGCR only. That runs
+  // entirely on the fine grid and ends in GlobalOrthonormalise, which is a
+  // whole-lattice Gram-Schmidt, so the coarse grid it holds is never
+  // dereferenced and may be the unvectorised one.
+  //////////////////////////////////////////////////////////////////////
+  std::string subspace_file = "subspace_nb" + std::to_string(nbasis) + ".scidac";
+  if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
+  uint64_t file_exists=0;
+  if ( UGrid->IsBoss() ){ std::ifstream f(subspace_file); file_exists=f.good()?1:0; }
+  UGrid->GlobalSum(file_exists);
+
+  const int cb=0;
+  Subspace AggregatesGCR(Coarse5d,FGrid,cb);
+  if ( file_exists ){
+    std::cout << GridLogMessage << "*** Loading subspace from disk (kept RAW) ***" << std::endl;
+    loadSubspace(AggregatesGCR.subspace, subspace_file);
+  } else {
+    std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
+    AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
+    saveSubspace(AggregatesGCR.subspace, subspace_file);
+  }
+
+  // RAW copy BEFORE CoarsenOperator block-orthonormalises in place.
+  std::vector<LatticeFermionD> rawNull(nbasis,FGrid);
+  for(int k=0;k<nbasis;k++) rawNull[k]=AggregatesGCR.subspace[k];
+
+  //////////////////////////////////////////////////////////////////////
+  // L1 coarsening. The fine operator is single RHS, so it is promoted to
+  // the 6D batch grid; a natively multiRHS fine operator would substitute
+  // here with no other change.
+  //////////////////////////////////////////////////////////////////////
+  CoarseOperator CoarseOpPV(geom,Coarse5d);
+  CoarseOpPV.SetGrid(CoarseBatch);
+
+  std::cout << GridLogMessage << "*** L1 CoarsenOperator, batch "<<batch<<" ***" << std::endl;
+  if ( getenv("MRHS_COARSEN") ) {
+    // Promote the single RHS operator and pack the batch: costs an
+    // ExtractSlice/InsertSlice pair per rhs. Here for the A/B only; this is
+    // the path a natively multiRHS fine operator would take.
+    MrhsPromotedOperator<LatticeFermionD> MrhsPVdagM(PVdagM,FGrid,batch);
+    CoarseOpPV.CoarsenOperator(MrhsPVdagM,FineMrhs,AggregatesGCR.subspace,Coarse5d);
+  } else {
+    // PVdagM is single RHS: apply it directly, batch on the coarse side.
+    CoarseOpPV.CoarsenOperator(PVdagM,AggregatesGCR.subspace,Coarse5d,batch);
+  }
+
+  // Stay on the batch grid: the L2 coarsening drives this operator at the
+  // batch. It is switched to the solve Nrhs once L2 is built.
+
+  //////////////////////////////////////////////////////////////////////
+  // psi_coarse = P^dag (RAW fine null) -> Galerkin images that carry the
+  // near null content, and are free: A_c (P psi) = P A psi.
+  //////////////////////////////////////////////////////////////////////
+  MultiRHSBlockProject<LatticeFermionD> MrhsProjector;
+  MrhsProjector.Allocate(nbasis,FGrid,Coarse5d);
+  MrhsProjector.ImportBasis(AggregatesGCR.subspace);   // block orthonormal basis
+
+  std::vector<CoarseVector> psi_coarse(nbasis,Coarse5d);
+  MrhsProjector.blockProject(rawNull,psi_coarse);      // RAW vectors in
+  rawNull.clear(); rawNull.shrink_to_fit();
+
+  {
+    RealD defect = GramDefect(psi_coarse);
+    RealD leak   = std::sqrt((double)Coarse5d->gSites());
+    std::cout << GridLogMessage << "GUARD: ||<psi_coarse|psi_coarse> - I||_F = " << defect
+	      << "   (~0.23 good; ~sqrt(N_coarse)=" << leak << " = e_k leak)" << std::endl;
+    GRID_ASSERT( defect < leak );
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // Optional cross check of the coarse matrix elements against the V1
+  // path, which needs a vectorised coarse space. Block Gram-Schmidt is
+  // idempotent, so V1 may re-orthonormalise the same vectors in place
+  // without a second copy of the subspace.
+  //////////////////////////////////////////////////////////////////////
+  if ( getenv("V1_CHECK") ) {
+
+    typedef GeneralCoarsenedMatrix     <vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
+    typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
+    typedef Aggregation                <vSpinColourVector,vTComplex,nbasis> SubspaceV;
+
+    Coordinate v5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    Coordinate v5simd({1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
+    GridCartesian *Coarse5dV = new GridCartesian(v5latt,v5simd,c5mpi);
+
+    int nrhs_v1 = vComplex::Nsimd();
+    Coordinate vmlatt({nrhs_v1,1,clatt[0],clatt[1],clatt[2],clatt[3]});
+    Coordinate vmsimd({vComplex::Nsimd(),1,1,1,1,1});
+    GridCartesian *CoarseMrhsV = new GridCartesian(vmlatt,vmsimd,cmmpi);
+
+    NextToNearestStencilGeometry5D geomV(Coarse5dV);
+
+    SubspaceV AggV(Coarse5dV,FGrid,cb);
+    for(int k=0;k<nbasis;k++) AggV.subspace[k]=AggregatesGCR.subspace[k];
+
+    LittleDiracOperator LittleDiracOpPV(geomV,FGrid,Coarse5dV);
+    std::cout << GridLogMessage << "*** V1 CoarsenOperator (cross check) ***" << std::endl;
+    LittleDiracOpPV.CoarsenOperator(PVdagM,AggV);
+
+    MrhsLittleDiracOperator mrhsV1(geomV,CoarseMrhsV);
+    mrhsV1.CopyMatrix(LittleDiracOpPV);
+
+    // BLAS_A is written by GridtoBLAS in lSite order and both sides carry
+    // the same scalar_object, so the two are directly comparable.
+    typedef MrhsLittleDiracOperator::calcMatrix calcMatrix;
+    int npoint = geom.npoint;
+    RealD num=0.0, den=0.0;
+    for(int p=0;p<npoint;p++){
+      int64_t sites = mrhsV1.BLAS_A[p].size();
+      GRID_ASSERT(sites == (int64_t)CoarseOpPV.BLAS_A[p].size());
+      std::vector<calcMatrix> h1(sites),h2(sites);
+      acceleratorCopyFromDevice(&mrhsV1.BLAS_A[p][0],    &h1[0],sites*sizeof(calcMatrix));
+      acceleratorCopyFromDevice(&CoarseOpPV.BLAS_A[p][0],&h2[0],sites*sizeof(calcMatrix));
+      ComplexD *w1=(ComplexD *)&h1[0];
+      ComplexD *w2=(ComplexD *)&h2[0];
+      int64_t words = sites*sizeof(calcMatrix)/sizeof(ComplexD);
+      for(int64_t i=0;i<words;i++){
+	ComplexD d=w1[i]-w2[i];
+	num += real(d)*real(d)+imag(d)*imag(d);
+	den += real(w1[i])*real(w1[i])+imag(w1[i])*imag(w1[i]);
+      }
+    }
+    std::cout << GridLogMessage << "V1_CHECK: |A_V1|^2 = " << den << std::endl;
+    std::cout << GridLogMessage << "V1_CHECK: |A_V1 - A_V2|^2 / |A_V1|^2 = " << num/den << std::endl;
+    GRID_ASSERT( den > 0.0 );
+    GRID_ASSERT( num/den < 1.0e-18 );
+  }
+
+  //////////////////////////////////////////////////////////////////////
+  // STAGE TWO: L2 -> L3.
+  //
+  // The fine operator here is V2 at L1, which is natively multiRHS, so the
+  // multiRHS driver applies with no promotion adapter: its D+1 grid IS the
+  // batch grid the L1 operator is currently set to.
+  //////////////////////////////////////////////////////////////////////
+  Coordinate cclatt = clatt;
+  Coordinate Block2({8,4,3,6});
+  if ( getenv("BLOCK2") ){ GridCmdOptionIntVector(std::string(getenv("BLOCK2")),Block2); GRID_ASSERT(Block2.size()==4); }
+  for(int d=0;d<4;d++){ GRID_ASSERT(clatt[d]%Block2[d]==0); cclatt[d]=clatt[d]/Block2[d]; }
+  std::cout << GridLogMessage << "Block2 " << Block2 << "  coarse-coarse lattice " << cclatt << std::endl;
+
+  Coordinate cc5latt({1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+  GridCartesian *CoarseCoarse5d = new GridCartesian(cc5latt,c5simd,c5mpi);
+
+  Coordinate ccmlatt({nrhs,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+  GridCartesian *CoarseCoarseMrhs = new GridCartesian(ccmlatt,cmsimd,cmmpi);
+
+  Coordinate ccblatt({batch,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
+  GridCartesian *CoarseCoarseBatch = new GridCartesian(ccblatt,cmsimd,cmmpi);
+
+  // Coarsening deepens the tensor nest by one iScalar
+  typedef CoarseVector::vector_object                                         CoarseSiteObj;
+  typedef iScalar<CComplexS>                                                  CComplexS2;
+  typedef MultiGeneralCoarsenedOperatorV2<CoarseSiteObj,CComplexS2,nbasis>    CoarseCoarseOperator;
+  typedef CoarseCoarseOperator::CoarseVector                                  CoarseCoarseVector;
+
+  NextToNearestStencilGeometry5D geom2(CoarseCoarse5d);
+
+  // RAW copy of the coarse null vectors, for the same reason as rawNull:
+  // the L2 CoarsenOperator block-orthonormalises its subspace in place, and
+  // the L3 basis must be defined by the vectors that still carry content.
+  std::vector<CoarseVector> rawPsi(nbasis,Coarse5d);
+  for(int k=0;k<nbasis;k++) rawPsi[k]=psi_coarse[k];
+
+  CoarseCoarseOperator CoarseOpL2(geom2,CoarseCoarse5d);
+  CoarseOpL2.SetGrid(CoarseCoarseBatch);
+
+  NonHermitianLinearOperator<CoarseOperator,CoarseVector> LinOpCoarse(CoarseOpPV);
+
+  std::cout << GridLogMessage << "*** L2 CoarsenOperator, batch "<<batch<<" ***" << std::endl;
+  CoarseOpL2.CoarsenOperator(LinOpCoarse,CoarseBatch,psi_coarse,CoarseCoarse5d);
+
+  //////////////////////////////////////////////////////////////////////
+  // Both operators to the solve Nrhs. The matrix elements are Nrhs
+  // independent and survive the change.
+  //////////////////////////////////////////////////////////////////////
+  CoarseOpPV.SetGrid(CoarseMrhs);
+  CoarseOpL2.SetGrid(CoarseCoarseMrhs);
+  std::cout << GridLogMessage << "L1 operator at Nrhs " << CoarseOpPV.Nrhs()
+	    << ",  L2 operator at Nrhs " << CoarseOpL2.Nrhs() << std::endl;
+
+  //////////////////////////////////////////////////////////////////////
+  // psi_cc from the RAW coarse null vectors, and the same guard
+  //////////////////////////////////////////////////////////////////////
+  MultiRHSBlockProject<CoarseVector> MrhsProjectorL2;
+  MrhsProjectorL2.Allocate(nbasis,Coarse5d,CoarseCoarse5d);
+  MrhsProjectorL2.ImportBasis(psi_coarse);            // block orthonormal basis
+
+  {
+    std::vector<CoarseCoarseVector> psi_cc(nbasis,CoarseCoarse5d);
+    MrhsProjectorL2.blockProject(rawPsi,psi_cc);      // RAW vectors in
+
+    RealD defect = GramDefect(psi_cc);
+    RealD leak   = std::sqrt((double)CoarseCoarse5d->gSites());
+    std::cout << GridLogMessage << "GUARD: ||<psi_cc|psi_cc> - I||_F = " << defect
+	      << "   (~0.23 good; ~sqrt(N_cc)=" << leak << " = e_k leak)" << std::endl;
+    GRID_ASSERT( defect < leak );
+  }
+  rawPsi.clear(); rawPsi.shrink_to_fit();
+
+  //////////////////////////////////////////////////////////////////////
+  // Both coarse operators apply on their solve grids
+  //////////////////////////////////////////////////////////////////////
+  {
+    GridParallelRNG cRNG(Coarse5d); cRNG.SeedFixedIntegers({3,4,5,6});
+    CoarseVector cin(CoarseMrhs), cout_(CoarseMrhs);
+    random(cRNG,cin);
+    CoarseOpPV.M(cin,cout_);
+    std::cout << GridLogMessage << "L1 apply |in|^2 = " << norm2(cin)
+	      << "   |M in|^2 = " << norm2(cout_) << std::endl;
+    GRID_ASSERT( norm2(cout_) > 0.0 );
+
+    GridParallelRNG ccRNG(CoarseCoarse5d); ccRNG.SeedFixedIntegers({7,8,9,10});
+    CoarseCoarseVector ccin(CoarseCoarseMrhs), ccout(CoarseCoarseMrhs);
+    random(ccRNG,ccin);
+    CoarseOpL2.M(ccin,ccout);
+    std::cout << GridLogMessage << "L2 apply |in|^2 = " << norm2(ccin)
+	      << "   |M in|^2 = " << norm2(ccout) << std::endl;
+    GRID_ASSERT( norm2(ccout) > 0.0 );
+  }
+
+  std::cout << GridLogMessage << "*** stage two complete: L1 and L2 coarse operators built ***" << std::endl;
+
+  Grid_finalize();
+}

scripts/filelist

@@ -10,8 +10,8 @@ CCFILES=`find . -name '*.cc' -not -path '*/instantiation/*/*' -not -path '*/gamm
 
 
 ZWILS_FERMION_FILES=` find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/ZWilsonImpl*' `
-WILS_FERMION_FILES=`  find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/WilsonImpl*' `
-STAG_FERMION_FILES=`  find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/StaggeredImpl*' `
+WILS_FERMION_FILES=`  find . -name '*.cc' -path '*/instantiation/*' \( -path '*/instantiation/WilsonImpl*' -o -path '*/instantiation/lexWilsonImpl*' \) `
+STAG_FERMION_FILES=`  find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/*StaggeredImpl*' `
 GP_FERMION_FILES=`    find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/Gparity*' `
 ADJ_FERMION_FILES=`   find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/WilsonAdj*' `
 TWOIND_FERMION_FILES=`find . -name '*.cc' -path '*/instantiation/*' -path '*/instantiation/WilsonTwoIndex*'`

tests/Makefile.am

@@ -1,4 +1,4 @@
-SUBDIRS = . core forces hmc solver debug smearing IO lanczos sp2n
+SUBDIRS = . core forces hmc solver debug smearing IO lanczos sp2n lexLattice
 
 if BUILD_CHROMA_REGRESSION
   SUBDIRS+= qdpxx

tests/Test_simd.cc

@@ -560,29 +560,198 @@ int main (int argc, char ** argv)
   auto latt_size   = GridDefaultLatt();
   auto simd_layout = GridDefaultSimd(4,vComplex::Nsimd());
   auto mpi_layout  = GridDefaultMpi();
-
+  Coordinate simd1_layout({1,1,1,1});
+  
   {
     std::cout << " Constructing Test({1,2,3,4,5,6}) " << std::endl;
     Coordinate Test({1,2,3,4,5,6});
     std::cout << " Test({1,2,3,4,5,6}) = " << Test <<std::endl;
   }
-  /*
-  {
-    Coordinate Test =  {1,2,3,4} ;
-    std::cout << " Test = {1,2,3,4} " << Test <<std::endl;
-  }
-  {
-    Coordinate Test {1,2,3,4};
-    std::cout << " Test {1,2,3,4} " << Test <<std::endl;
-  }
-  */
 
   GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
   std::vector<int> seeds({1,2,3,4});
 
   // Insist that operations on random scalars gives
   // identical results to on vectors.
+  std::cout << GridLogMessage <<"==================================="<<  std::endl;
+  std::cout << GridLogMessage <<"Testing sRealF "<<std::endl;
+  std::cout << GridLogMessage <<"==================================="<<  std::endl;
+  Tester<RealF,sRealF>(funcPlus());
+  Tester<RealF,sRealF>(funcMinus());
+  Tester<RealF,sRealF>(funcTimes());
+  Tester<RealF,sRealF>(funcDivide());
+  Tester<RealF,sRealF>(funcAdj());
+  Tester<RealF,sRealF>(funcConj());
+  Tester<RealF,sRealF>(funcInnerProduct());
+  ReductionTester<RealF,RealF,sRealF>(funcReduce());
 
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sRealF permutes (there are none) "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+
+  // Log2 iteration
+  for(int i=0;(1<<i)< sRealF::Nsimd();i++){
+    PermTester<RealF,sRealF>(funcPermute(i));
+  }
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sRealF exchanges "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+
+  // Log2 iteration
+  for(int i=0;(1<<i)< sRealF::Nsimd();i++){
+    ExchangeTester<RealF,sRealF>(funcExchange(i));
+  }
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sRealF rotate "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  for(int r=0;r<sRealF::Nsimd();r++){
+    PermTester<RealF,sRealF>(funcRotate(r));
+  }
+
+
+
+  std::cout << GridLogMessage <<"==================================="<<  std::endl;
+  std::cout << GridLogMessage <<"Testing sRealD "<<std::endl;
+  std::cout << GridLogMessage <<"==================================="<<  std::endl;
+
+  Tester<RealD,sRealD>(funcPlus());
+  Tester<RealD,sRealD>(funcMinus());
+  Tester<RealD,sRealD>(funcTimes());
+  Tester<RealD,sRealD>(funcDivide());
+  Tester<RealD,sRealD>(funcAdj());
+  Tester<RealD,sRealD>(funcConj());
+  Tester<RealD,sRealD>(funcInnerProduct());
+  ReductionTester<RealD,RealD,sRealD>(funcReduce());
+
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sRealD permutes "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+
+  // Log2 iteration
+  for(int i=0;(1<<i)< sRealD::Nsimd();i++){
+    PermTester<RealD,sRealD>(funcPermute(i));
+  }
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sRealD exchanges "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  // Log2 iteration
+  for(int i=0;(1<<i)< sRealD::Nsimd();i++){
+    ExchangeTester<RealD,sRealD>(funcExchange(i));
+  }
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sRealD rotate "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  for(int r=0;r<sRealD::Nsimd();r++){
+    PermTester<RealD,sRealD>(funcRotate(r));
+  }
+
+
+
+  std::cout << GridLogMessage <<"==================================="<<  std::endl;
+  std::cout << GridLogMessage <<"Testing sComplexF "<<std::endl;
+  std::cout << GridLogMessage <<"==================================="<<  std::endl;
+
+  Tester<ComplexF,sComplexF>(funcTimesI());
+  Tester<ComplexF,sComplexF>(funcTimesMinusI());
+  Tester<ComplexF,sComplexF>(funcPlus());
+  Tester<ComplexF,sComplexF>(funcMinus());
+  Tester<ComplexF,sComplexF>(funcTimes());
+  Tester<ComplexF,sComplexF>(funcConj());
+  Tester<ComplexF,sComplexF>(funcAdj());
+  Tester<ComplexF,sComplexF>(funcReal());
+  Tester<ComplexF,sComplexF>(funcImag());
+  Tester<ComplexF,sComplexF>(funcInnerProduct());
+  ReductionTester<ComplexF,ComplexF,sComplexF>(funcReduce());
+
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sComplexF permutes "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+
+  // Log2 iteration
+  for(int i=0;(1<<i)< sComplexF::Nsimd();i++){
+    PermTester<ComplexF,sComplexF>(funcPermute(i));
+  }
+
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sComplexF exchanges "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  // Log2 iteration
+  for(int i=0;(1<<i)< sComplexF::Nsimd();i++){
+    ExchangeTester<ComplexF,sComplexF>(funcExchange(i));
+  }
+
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sComplexF rotate "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  for(int r=0;r<sComplexF::Nsimd();r++){
+    PermTester<ComplexF,sComplexF>(funcRotate(r));
+  }
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sComplexD "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+
+
+  Tester<ComplexD,sComplexD>(funcTimesI());
+  Tester<ComplexD,sComplexD>(funcTimesMinusI());
+  Tester<ComplexD,sComplexD>(funcPlus());
+  Tester<ComplexD,sComplexD>(funcMinus());
+  Tester<ComplexD,sComplexD>(funcTimes());
+  Tester<ComplexD,sComplexD>(funcConj());
+  Tester<ComplexD,sComplexD>(funcAdj());
+  Tester<ComplexD, sComplexD>(funcReal());
+  Tester<ComplexD, sComplexD>(funcImag());
+
+  Tester<ComplexD, sComplexD>(funcInnerProduct());
+  ReductionTester<ComplexD, ComplexD, sComplexD>(funcReduce());
+
+  std::cout << GridLogMessage
+            << "===================================" << std::endl;
+  std::cout << GridLogMessage << "Testing sComplexD permutes " << std::endl;
+  std::cout << GridLogMessage
+            << "===================================" << std::endl;
+
+  // Log2 iteration
+  for (int i = 0; (1 << i) < sComplexD::Nsimd(); i++) {
+    PermTester<ComplexD, sComplexD>(funcPermute(i));
+  }
+
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sComplexD exchanges "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  // Log2 iteration
+  for(int i=0;(1<<i)< sComplexD::Nsimd();i++){
+    ExchangeTester<ComplexD,sComplexD>(funcExchange(i));
+  }
+
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sComplexD rotate "<<std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  for(int r=0;r<sComplexD::Nsimd();r++){
+    PermTester<ComplexD,sComplexD>(funcRotate(r));
+  }
+
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  std::cout<<GridLogMessage << "Testing sInteger                   "<<  std::endl;
+  std::cout<<GridLogMessage << "==================================="<<  std::endl;
+  IntTester(funcPlus());
+  IntTester(funcMinus());
+  IntTester(funcTimes());
+  IntReductionTester<Integer, Integer, sInteger>(funcReduce());
+
+
+
+  
+  
   std::cout << GridLogMessage <<"==================================="<<  std::endl;
   std::cout << GridLogMessage <<"Testing vRealF "<<std::endl;
   std::cout << GridLogMessage <<"==================================="<<  std::endl;
@@ -623,7 +792,6 @@ int main (int argc, char ** argv)
     PermTester<RealF,vRealF>(funcRotate(r));
   }
 
-
   std::cout << GridLogMessage <<"==================================="<<  std::endl;
   std::cout << GridLogMessage <<"Testing vRealD "<<std::endl;
   std::cout << GridLogMessage <<"==================================="<<  std::endl;

tests/debug/Test_coarse_v2.cc

@@ -0,0 +1,252 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/debug/Test_coarse_v2.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// MultiGeneralCoarsenedOperatorV2 against the existing mrhs coarse operator.
+//
+// V1 is constructed on the D+1 grid as now; V2 on the D dimensional grid,
+// with SetGrid() adopting the caller owned D+1 grid and building its padded
+// cell and neighbour table from the D dimensional stencil, with the Nrhs
+// factor multiplied in.
+//
+// Both are given identical matrix elements, so any difference in the apply is
+// the restructured neighbour table. The same geometry object is passed to
+// both: V1 adds one to skip for the rhs direction, V2 uses it as is on the D
+// dimensional grid, so both describe the same stencil over the D dimensions.
+//
+#include <Grid/Grid.h>
+
+using namespace Grid;
+
+const int nbasis = 8;
+
+typedef vSpinColourVector FineObj;
+typedef sTComplexD        CComplexT;   // unvectorised coarse space
+
+typedef MultiGeneralCoarsenedMatrix    <FineObj,CComplexT,nbasis> MrhsV1;
+typedef MultiGeneralCoarsenedOperatorV2<FineObj,CComplexT,nbasis> MrhsV2;
+
+////////////////////////////////////////////////////////////////////////
+// Identical random matrix elements into both operators
+////////////////////////////////////////////////////////////////////////
+template<class OpA,class OpB>
+void SeedMatrixElements(OpA &A,OpB &B,int npoint,GridSerialRNG &sRNG)
+{
+  typedef typename OpA::calcMatrix calcMatrix;
+
+  for(int p=0;p<npoint;p++){
+
+    GRID_ASSERT(A.BLAS_A[p].size() == B.BLAS_A[p].size());
+
+    int64_t sites = A.BLAS_A[p].size();
+    std::vector<calcMatrix> host(sites);
+
+    ComplexD *w = (ComplexD *)&host[0];
+    int64_t words = sites*sizeof(calcMatrix)/sizeof(ComplexD);
+    for(int64_t i=0;i<words;i++){
+      RealD re,im;
+      random(sRNG,re);
+      random(sRNG,im);
+      w[i] = ComplexD(re-0.5,im-0.5);
+    }
+
+    acceleratorCopyToDevice(&host[0],&A.BLAS_A[p][0],sites*sizeof(calcMatrix));
+    acceleratorCopyToDevice(&host[0],&B.BLAS_A[p][0],sites*sizeof(calcMatrix));
+  }
+}
+
+template<class Op>
+RealD MatrixChecksum(Op &O,int npoint)
+{
+  typedef typename Op::calcMatrix calcMatrix;
+  RealD sum=0.0;
+  for(int p=0;p<npoint;p++){
+    int64_t sites = O.BLAS_A[p].size();
+    std::vector<calcMatrix> host(sites);
+    acceleratorCopyFromDevice(&O.BLAS_A[p][0],&host[0],sites*sizeof(calcMatrix));
+    ComplexD *w = (ComplexD *)&host[0];
+    int64_t words = sites*sizeof(calcMatrix)/sizeof(ComplexD);
+    for(int64_t i=0;i<words;i++) sum += real(w[i])*real(w[i]) + imag(w[i])*imag(w[i]);
+  }
+  return sum;
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  const int nrhs = 4;
+
+  Coordinate clatt = GridDefaultLatt();
+  Coordinate csimd(Nd,1);                 // the coarse space is unvectorised
+  Coordinate cmpi  = GridDefaultMpi();
+
+  ////////////////////////////////////////////////
+  // D dimensional coarse grid, and D+1 for V1
+  ////////////////////////////////////////////////
+  GridCartesian *CoarseD = new GridCartesian(clatt,csimd,cmpi);
+
+  std::cout << GridLogMessage << "coarse D   grid "; for(int d=0;d<Nd;d++) std::cout<<clatt[d]<<" ";
+  std::cout << "  Nsimd " << CoarseD->Nsimd() << std::endl;
+
+  ////////////////////////////////////////////////
+  // One geometry object for both, and one D+1 grid
+  // owned here and shared by both operators: fields
+  // conform only across a shared grid object.
+  ////////////////////////////////////////////////
+  NextToNearestStencilGeometry4D geom(CoarseD);
+
+  Coordinate mlatt(1,nrhs), msimd(1,1), mmpi(1,1);
+  for(int d=0;d<Nd;d++){
+    mlatt.push_back(clatt[d]);
+    msimd.push_back(csimd[d]);
+    mmpi .push_back(cmpi[d]);
+  }
+  GridCartesian *CoarseMulti = new GridCartesian(mlatt,msimd,mmpi);
+
+  MrhsV2 OpV2(geom,CoarseD);
+  MrhsV1 OpV1(geom,CoarseMulti);
+  OpV2.SetGrid(CoarseMulti);
+
+  std::cout << GridLogMessage << "coarse D+1 grid nrhs " << nrhs
+	    << "  Nsimd " << CoarseMulti->Nsimd() << std::endl;
+
+  std::cout << GridLogMessage << "npoint V1 " << OpV1.geom.npoint
+	    << "   npoint V2 " << OpV2.geom.npoint << std::endl;
+  GRID_ASSERT(OpV1.geom.npoint == OpV2.geom.npoint);
+
+  int npoint = OpV1.geom.npoint;
+
+  ////////////////////////////////////////////////
+  // Identical matrix elements
+  ////////////////////////////////////////////////
+  GridSerialRNG sRNG; sRNG.SeedFixedIntegers(std::vector<int>({7,8,9,10}));
+  SeedMatrixElements(OpV1,OpV2,npoint,sRNG);
+
+  RealD ckV1 = MatrixChecksum(OpV1,npoint);
+  RealD ckV2 = MatrixChecksum(OpV2,npoint);
+  std::cout << GridLogMessage << "matrix element checksum V1 " << ckV1
+	    << "  V2 " << ckV2 << std::endl;
+  GRID_ASSERT( ckV1 == ckV2 );
+
+  ////////////////////////////////////////////////
+  // Same input, compare the applies
+  ////////////////////////////////////////////////
+  typedef MrhsV1::CoarseVector CoarseVector;
+  // RNG on the D dimensional grid fills any D+1 field: the rhs direction is
+  // undistributed and divides cleanly. One RNG serves every Nrhs.
+  GridParallelRNG pRNG(CoarseD); pRNG.SeedFixedIntegers(std::vector<int>({1,2,3,4}));
+
+  CoarseVector in  (CoarseMulti); random(pRNG,in);
+  CoarseVector out1(CoarseMulti);
+  CoarseVector out2(CoarseMulti);
+  CoarseVector err (CoarseMulti);
+
+  OpV1.M(in,out1);
+  OpV2.M(in,out2);
+
+  err = out1 - out2;
+  std::cout << GridLogMessage << "|V1 out|^2 = " << norm2(out1)
+	    << "   |V2 out|^2 = " << norm2(out2) << std::endl;
+  std::cout << GridLogMessage << "|V1 - V2|^2 = " << norm2(err) << std::endl;
+  GRID_ASSERT( norm2(out1) > 0.0 );
+  GRID_ASSERT( norm2(err) == 0.0 );
+
+  ////////////////////////////////////////////////
+  // SetGrid is idempotent on pointer identity
+  ////////////////////////////////////////////////
+  OpV2.SetGrid(CoarseMulti);
+  GRID_ASSERT( MatrixChecksum(OpV2,npoint) == ckV2 );
+  OpV2.M(in,out2);
+  err = out1 - out2;
+  GRID_ASSERT( norm2(err) == 0.0 );
+  std::cout << GridLogMessage << "SetGrid idempotent on identity" << std::endl;
+
+  ////////////////////////////////////////////////
+  // Move to a different Nrhs and back. The matrix
+  // elements are Nrhs independent and must survive
+  // both the release and the rebuild.
+  ////////////////////////////////////////////////
+  // Nrhs 1 is the single RHS case through the multiRHS path, and each slice
+  // of the Nrhs 4 apply must come back unchanged.
+  OpV2.M(in,out2);
+  for(int nr=2;nr>=1;nr--){
+    Coordinate latt2(1,nr), simd2(1,1), mpi2(1,1);
+    for(int d=0;d<Nd;d++){
+      latt2.push_back(clatt[d]);
+      simd2.push_back(csimd[d]);
+      mpi2 .push_back(cmpi[d]);
+    }
+    GridCartesian *CoarseMulti2 = new GridCartesian(latt2,simd2,mpi2);
+
+    OpV2.SetGrid(CoarseMulti2);
+    GRID_ASSERT( OpV2.Nrhs() == nr );
+    GRID_ASSERT( MatrixChecksum(OpV2,npoint) == ckV2 );
+
+    CoarseVector in2 (CoarseMulti2);
+    CoarseVector out(CoarseMulti2);
+    for(int r=0;r<nr;r++){
+      CoarseVector slice(CoarseD);
+      ExtractSliceFast(slice,in,r,0);
+      InsertSliceFast(slice,in2,r,0);
+    }
+    OpV2.M(in2,out);
+
+    RealD sdiff=0.0;
+    for(int r=0;r<nr;r++){
+      CoarseVector a(CoarseD),b(CoarseD),e(CoarseD);
+      ExtractSliceFast(a,out ,r,0);
+      ExtractSliceFast(b,out2,r,0);
+      e = a-b;
+      sdiff += norm2(e);
+    }
+    // Not bit exact: a different Nrhs is a different GEMM shape
+    std::cout << GridLogMessage << "Nrhs " << nr << " slices agree with Nrhs "
+	      << nrhs << " : |diff|^2/|out|^2 = " << sdiff/norm2(out) << std::endl;
+    GRID_ASSERT( norm2(out) > 0.0 );
+    GRID_ASSERT( sdiff/norm2(out) < 1.0e-20 );
+
+    OpV2.ReleaseGrid();
+    GRID_ASSERT( MatrixChecksum(OpV2,npoint) == ckV2 );  // survives release
+
+    delete CoarseMulti2;
+  }
+
+  OpV2.SetGrid(CoarseMulti);
+  GRID_ASSERT( OpV2.Nrhs() == nrhs );
+
+  OpV2.M(in,out2);
+  err = out1 - out2;
+  std::cout << GridLogMessage << "after Nrhs 4 -> 2 -> 1 -> release -> 4, |V1 - V2|^2 = "
+	    << norm2(err) << std::endl;
+  GRID_ASSERT( norm2(err) == 0.0 );
+
+  std::cout << GridLogMessage << "Test_coarse_v2: ALL PASS" << std::endl;
+
+  Grid_finalize();
+}

tests/debug/Test_coarse_v2_coarsen.cc

@@ -0,0 +1,242 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/debug/Test_coarse_v2_coarsen.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// Coarsen the same fine operator two ways and compare the matrix elements.
+//
+//   V1 : existing mrhs CoarsenOperator, vectorised coarse space matched to
+//        the fine SIMD layout, single RHS fine applications
+//   V2 : D+1 CoarsenOperator, unvectorised sComplexD coarse space, the batch
+//        of phased basis vectors carried in the rhs direction and applied
+//        through MrhsPromotedOperator
+//
+// BLAS_A is written by GridtoBLAS in lSite order, which does not depend on
+// the SIMD layout, so the two are directly comparable.
+//
+#include <Grid/Grid.h>
+
+using namespace Grid;
+
+const int nbasis = 8;
+const int batch  = 9;
+
+typedef vSpinColourVector FineObj;
+typedef vTComplex         CComplexV;   // vectorised coarse space, V1 reference
+typedef sTComplexD        CComplexS;   // unvectorised coarse space, V2
+
+typedef MultiGeneralCoarsenedMatrix    <FineObj,CComplexV,nbasis> MrhsV1;
+typedef MultiGeneralCoarsenedOperatorV2<FineObj,CComplexS,nbasis> MrhsV2;
+
+template<class Op>
+void ReadMatrix(Op &O,int npoint,std::vector<std::vector<typename Op::calcMatrix> > &host)
+{
+  typedef typename Op::calcMatrix calcMatrix;
+  host.resize(npoint);
+  for(int p=0;p<npoint;p++){
+    int64_t sites = O.BLAS_A[p].size();
+    host[p].resize(sites);
+    acceleratorCopyFromDevice(&O.BLAS_A[p][0],&host[p][0],sites*sizeof(calcMatrix));
+  }
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  Coordinate flatt = GridDefaultLatt();
+  Coordinate fsimd = GridDefaultSimd(Nd,vComplexD::Nsimd());
+  Coordinate fmpi  = GridDefaultMpi();
+
+  Coordinate block({2,2,2,2});
+  Coordinate clatt(Nd);
+  for(int d=0;d<Nd;d++){
+    GRID_ASSERT(flatt[d]%block[d]==0);
+    clatt[d] = flatt[d]/block[d];
+  }
+
+  GridCartesian         *FineGrid   = new GridCartesian(flatt,fsimd,fmpi);
+  GridRedBlackCartesian *FrbGrid    = SpaceTimeGrid::makeFourDimRedBlackGrid(FineGrid);
+
+  ////////////////////////////////////////////////
+  // V1 coarse space: SIMD layout matched to the fine
+  ////////////////////////////////////////////////
+  Coordinate cvsimd = GridDefaultSimd(Nd,CComplexV::Nsimd());
+  GridCartesian *CoarseV = new GridCartesian(clatt,cvsimd,fmpi);
+
+  // V1 puts all of the SIMD in the rhs direction. CoarsenOperator does not
+  // use the multiRHS grid; it only sizes BLAS_A, so one lane of rhs suffices.
+  int nrhs_v1 = CComplexV::Nsimd();
+  Coordinate v1latt(1,nrhs_v1),v1simd(1,CComplexV::Nsimd()),v1mpi(1,1);
+  for(int d=0;d<Nd;d++){
+    v1latt.push_back(clatt[d]);
+    v1simd.push_back(1);
+    v1mpi .push_back(fmpi[d]);
+  }
+  GridCartesian *CoarseVMulti = new GridCartesian(v1latt,v1simd,v1mpi);
+
+  ////////////////////////////////////////////////
+  // V2 coarse space: unvectorised
+  ////////////////////////////////////////////////
+  Coordinate cssimd(Nd,1);
+  GridCartesian *CoarseS = new GridCartesian(clatt,cssimd,fmpi);
+
+  Coordinate cmlatt(1,batch),cmsimd(1,1),cmmpi(1,1);
+  for(int d=0;d<Nd;d++){
+    cmlatt.push_back(clatt[d]);
+    cmsimd.push_back(1);
+    cmmpi .push_back(fmpi[d]);
+  }
+  GridCartesian *CoarseSMulti = new GridCartesian(cmlatt,cmsimd,cmmpi);
+
+  // D+1 fine grid carrying the batch
+  Coordinate fmlatt(1,batch), fmsimd(1,1), fmmpi(1,1);
+  for(int d=0;d<Nd;d++){
+    fmlatt.push_back(flatt[d]);
+    fmsimd.push_back(fsimd[d]);
+    fmmpi .push_back(fmpi[d]);
+  }
+  GridCartesian *FineGridMulti = new GridCartesian(fmlatt,fmsimd,fmmpi);
+
+  std::cout << GridLogMessage << "fine "<<flatt<<"   coarse "<<clatt<<"   batch "<<batch<<std::endl;
+  std::cout << GridLogMessage << "Nsimd   fine "<<FineGrid->Nsimd()
+	    << "   coarse V1 "<<CoarseV->Nsimd()
+	    << "   coarse V2 "<<CoarseS->Nsimd()<<std::endl;
+
+  ////////////////////////////////////////////////
+  // Fine operator
+  ////////////////////////////////////////////////
+  GridParallelRNG pRNG(FineGrid); pRNG.SeedFixedIntegers(std::vector<int>({1,2,3,4}));
+  LatticeGaugeFieldD Umu(FineGrid); SU<Nc>::HotConfiguration(pRNG,Umu);
+
+  RealD mass = 0.1;
+  WilsonFermionD Dw(Umu,*FineGrid,*FrbGrid,mass);
+  MdagMLinearOperator<WilsonFermionD,LatticeFermionD> HermOp(Dw);
+
+  ////////////////////////////////////////////////
+  // One random subspace, two copies: CoarsenOperator orthogonalises in place
+  ////////////////////////////////////////////////
+  Aggregation<FineObj,CComplexV,nbasis> Subspace(CoarseV,FineGrid,0);
+  std::vector<LatticeFermionD> subspace(nbasis,FineGrid);
+  for(int i=0;i<nbasis;i++){
+    random(pRNG,Subspace.subspace[i]);
+    subspace[i] = Subspace.subspace[i];
+  }
+
+  NextToNearestStencilGeometry4D geomV(CoarseV);
+  NextToNearestStencilGeometry4D geomS(CoarseS);
+
+  ////////////////////////////////////////////////
+  // V1 coarsening, matched layouts
+  ////////////////////////////////////////////////
+  MrhsV1 OpV1(geomV,CoarseVMulti);
+  std::cout << GridLogMessage << "V1 CoarsenOperator" << std::endl;
+  OpV1.CoarsenOperator(HermOp,Subspace,CoarseV);
+
+  ////////////////////////////////////////////////
+  // V2 coarsening, D+1 fine applications, unvectorised coarse
+  ////////////////////////////////////////////////
+  MrhsV2 OpV2(geomS,CoarseS);
+  OpV2.SetGrid(CoarseSMulti);
+
+  MrhsPromotedOperator<LatticeFermionD> MrhsHermOp(HermOp,FineGrid,batch);
+
+  std::cout << GridLogMessage << "V2 CoarsenOperator (D+1)" << std::endl;
+  OpV2.CoarsenOperator(MrhsHermOp,FineGridMulti,subspace,CoarseS);
+
+  ////////////////////////////////////////////////
+  // Compare matrix elements
+  ////////////////////////////////////////////////
+  int npoint = OpV1.geom.npoint;
+  GRID_ASSERT(npoint == OpV2.geom.npoint);
+  typedef MrhsV1::calcMatrix calcMatrix;
+  std::vector<std::vector<calcMatrix> > A1,A2;
+  ReadMatrix(OpV1,npoint,A1);
+  ReadMatrix(OpV2,npoint,A2);
+
+  RealD num=0.0, den=0.0;
+  for(int p=0;p<npoint;p++){
+    GRID_ASSERT(A1[p].size()==A2[p].size());
+    ComplexD *w1 = (ComplexD *)&A1[p][0];
+    ComplexD *w2 = (ComplexD *)&A2[p][0];
+    int64_t words = A1[p].size()*sizeof(calcMatrix)/sizeof(ComplexD);
+    for(int64_t i=0;i<words;i++){
+      ComplexD d = w1[i]-w2[i];
+      num += real(d)*real(d)+imag(d)*imag(d);
+      den += real(w1[i])*real(w1[i])+imag(w1[i])*imag(w1[i]);
+    }
+  }
+  std::cout << GridLogMessage << "|A_V1|^2 = " << den << std::endl;
+  std::cout << GridLogMessage << "|A_V1 - A_V2|^2 / |A_V1|^2 = " << num/den << std::endl;
+  GRID_ASSERT( den > 0.0 );
+  GRID_ASSERT( num/den < 1.0e-20 );
+
+  ////////////////////////////////////////////////
+  // Same coarsening through the single RHS variant: no mrhs packing, the
+  // batch assembled on the coarse side by the mixed blockProject. Block
+  // Gram-Schmidt is idempotent so the subspace may be reused in place.
+  ////////////////////////////////////////////////
+  MrhsV2 OpV2s(geomS,CoarseS);
+  OpV2s.SetGrid(CoarseSMulti);
+
+  std::cout << GridLogMessage << "V2 CoarsenOperator (single RHS fine op)" << std::endl;
+  OpV2s.CoarsenOperator(HermOp,subspace,CoarseS,batch);
+
+  std::vector<std::vector<calcMatrix> > A3;
+  ReadMatrix(OpV2s,npoint,A3);
+
+  RealD nums=0.0;
+  for(int p=0;p<npoint;p++){
+    GRID_ASSERT(A1[p].size()==A3[p].size());
+    ComplexD *w1 = (ComplexD *)&A1[p][0];
+    ComplexD *w3 = (ComplexD *)&A3[p][0];
+    int64_t words = A1[p].size()*sizeof(calcMatrix)/sizeof(ComplexD);
+    for(int64_t i=0;i<words;i++){
+      ComplexD d = w1[i]-w3[i];
+      nums += real(d)*real(d)+imag(d)*imag(d);
+    }
+  }
+  std::cout << GridLogMessage << "|A_V1 - A_V2srhs|^2 / |A_V1|^2 = " << nums/den << std::endl;
+  GRID_ASSERT( nums/den < 1.0e-20 );
+
+  ////////////////////////////////////////////////
+  // and V2 applies the matrix it just built
+  ////////////////////////////////////////////////
+  typedef MrhsV2::CoarseVector CoarseVectorS;
+  GridParallelRNG cRNG(CoarseS); cRNG.SeedFixedIntegers(std::vector<int>({5,6,7,8}));
+  CoarseVectorS in(CoarseSMulti);  random(cRNG,in);
+  CoarseVectorS out(CoarseSMulti);
+
+  OpV2.M(in,out);
+  std::cout << GridLogMessage << "|in|^2 = " << norm2(in)
+	    << "   |M_V2 in|^2 = " << norm2(out) << std::endl;
+  GRID_ASSERT( norm2(out) > 0.0 );
+
+  std::cout << GridLogMessage << "Test_coarse_v2_coarsen: ALL PASS" << std::endl;
+
+  Grid_finalize();
+}

tests/debug/Test_mrhs_blockproject_import.cc

@@ -0,0 +1,224 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/debug/Test_mrhs_blockproject_import.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// ImportFineGridMrhsVectors / ExportCoarseGridMrhsVectors against the vector
+// of single RHS routines they replace. The BLAS buffers must come out
+// identical, so the mrhs path is a pure removal of the unpack and repack.
+//
+#include <Grid/Grid.h>
+
+using namespace Grid;
+
+const int nbasis = 8;
+const int nrhs   = 4;
+
+typedef LatticeFermionD FineField;
+typedef iVector<sTComplexD,nbasis> CoarseSiteObj;   // unvectorised coarse space
+typedef Lattice<CoarseSiteObj>    CoarseField;
+
+template<class T>
+RealD BufDiff(deviceVector<T> &a,deviceVector<T> &b)
+{
+  GRID_ASSERT(a.size()==b.size());
+  std::vector<T> ha(a.size()),hb(b.size());
+  acceleratorCopyFromDevice(&a[0],&ha[0],a.size()*sizeof(T));
+  acceleratorCopyFromDevice(&b[0],&hb[0],b.size()*sizeof(T));
+  RealD num=0.0;
+  for(int64_t i=0;i<a.size();i++){
+    T d = ha[i]-hb[i];
+    num += real(d)*real(d)+imag(d)*imag(d);
+  }
+  return num;
+}
+template<class T>
+RealD BufNorm(deviceVector<T> &a)
+{
+  std::vector<T> ha(a.size());
+  acceleratorCopyFromDevice(&a[0],&ha[0],a.size()*sizeof(T));
+  RealD num=0.0;
+  for(int64_t i=0;i<a.size();i++) num += real(ha[i])*real(ha[i])+imag(ha[i])*imag(ha[i]);
+  return num;
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  Coordinate flatt = GridDefaultLatt();
+  Coordinate fsimd = GridDefaultSimd(Nd,vComplexD::Nsimd());
+  Coordinate fmpi  = GridDefaultMpi();
+
+  Coordinate block({2,2,2,2});
+  Coordinate clatt(Nd);
+  for(int d=0;d<Nd;d++) clatt[d]=flatt[d]/block[d];
+
+  GridCartesian *FineGrid   = new GridCartesian(flatt,fsimd,fmpi);
+  Coordinate csimd(Nd,1);                            // coarse space is unvectorised
+  GridCartesian *CoarseGrid = new GridCartesian(clatt,csimd,fmpi);
+
+  // D+1 grids, rhs innermost and unvectorised
+  Coordinate fml(1,nrhs),fms(1,1),fmm(1,1);
+  Coordinate cml(1,nrhs),cms(1,1),cmm(1,1);
+  for(int d=0;d<Nd;d++){
+    fml.push_back(flatt[d]); fms.push_back(fsimd[d]); fmm.push_back(fmpi[d]);
+    cml.push_back(clatt[d]); cms.push_back(csimd[d]); cmm.push_back(fmpi[d]);
+  }
+  GridCartesian *FineMulti   = new GridCartesian(fml,fms,fmm);
+  GridCartesian *CoarseMulti = new GridCartesian(cml,cms,cmm);
+
+  std::cout << GridLogMessage << "fine "<<flatt<<"  coarse "<<clatt<<"  nrhs "<<nrhs
+	    << "  fine Nsimd "<<FineGrid->Nsimd()<<"  coarse Nsimd "<<CoarseGrid->Nsimd()<<std::endl;
+
+  GridParallelRNG pRNG(FineGrid); pRNG.SeedFixedIntegers(std::vector<int>({1,2,3,4}));
+
+  MultiRHSBlockProject<FineField> Projector;
+  Projector.Allocate(nbasis,FineGrid,CoarseGrid);
+
+  ///////////////////////////////////////////////////////
+  // Same data as nrhs single RHS fields and as one D+1
+  ///////////////////////////////////////////////////////
+  std::vector<FineField> vecs(nrhs,FineGrid);
+  FineField mrhs(FineMulti);
+  for(int r=0;r<nrhs;r++){
+    random(pRNG,vecs[r]);
+    InsertSliceFast(vecs[r],mrhs,r,0);
+  }
+
+  int64_t fsz = FineGrid->lSites()*Projector.words*nrhs;
+  deviceVector<ComplexD> Fa(fsz), Fb(fsz);
+
+  Projector.ImportFineGridVectors    (vecs,Fa);
+  Projector.ImportFineGridMrhsVectors(mrhs,Fb);
+
+  RealD fn = BufNorm(Fa);
+  RealD fd = BufDiff(Fa,Fb);
+  std::cout << GridLogMessage << "fine import  |vector|^2 = " << fn
+	    << "   |vector - mrhs|^2 = " << fd << std::endl;
+  GRID_ASSERT( fn > 0.0 );
+  GRID_ASSERT( fd == 0.0 );
+
+  ///////////////////////////////////////////////////////
+  // Coarse export: fill the BLAS buffer, read it back
+  // both ways, and compare the resulting fields
+  ///////////////////////////////////////////////////////
+  int64_t csz = CoarseGrid->lSites()*nbasis*nrhs;
+  deviceVector<ComplexD> Cbuf(csz);
+  {
+    std::vector<ComplexD> host(csz);
+    GridSerialRNG sRNG; sRNG.SeedFixedIntegers(std::vector<int>({9,8,7,6}));
+    for(int64_t i=0;i<csz;i++){
+      RealD re,im; random(sRNG,re); random(sRNG,im);
+      host[i]=ComplexD(re-0.5,im-0.5);
+    }
+    acceleratorCopyToDevice(&host[0],&Cbuf[0],csz*sizeof(ComplexD));
+  }
+
+  std::vector<CoarseField> cvecs(nrhs,CoarseGrid);
+  CoarseField cmrhs(CoarseMulti);
+
+  Projector.ExportCoarseGridVectors    (cvecs,Cbuf);
+  Projector.ExportCoarseGridMrhsVectors(cmrhs,Cbuf);
+
+  RealD cn=0.0, cd=0.0;
+  for(int r=0;r<nrhs;r++){
+    CoarseField slice(CoarseGrid);
+    ExtractSliceFast(slice,cmrhs,r,0);
+    CoarseField e(CoarseGrid);
+    e = slice - cvecs[r];
+    cn += norm2(cvecs[r]);
+    cd += norm2(e);
+  }
+  std::cout << GridLogMessage << "coarse export |vector|^2 = " << cn
+	    << "   |vector - mrhs|^2 = " << cd << std::endl;
+  GRID_ASSERT( cn > 0.0 );
+  GRID_ASSERT( cd == 0.0 );
+
+  ///////////////////////////////////////////////////////
+  // All four blockProject orderings must agree
+  ///////////////////////////////////////////////////////
+  {
+    std::vector<FineField> basis(nbasis,FineGrid);
+    for(int b=0;b<nbasis;b++) random(pRNG,basis[b]);
+    Projector.ImportBasis(basis);
+
+    std::vector<CoarseField> cvv(nrhs,CoarseGrid);   // vector fine -> vector coarse
+    std::vector<CoarseField> cmv(nrhs,CoarseGrid);   // mrhs   fine -> vector coarse
+    CoarseField cvm(CoarseMulti);                    // vector fine -> mrhs   coarse
+    CoarseField cmm(CoarseMulti);                    // mrhs   fine -> mrhs   coarse
+
+    Projector.blockProject(vecs,cvv);
+    Projector.blockProject(mrhs,cmv);
+    Projector.blockProject(vecs,cvm);
+    Projector.blockProject(mrhs,cmm);
+
+    RealD ref=0.0, d_mv=0.0, d_vm=0.0, d_mm=0.0;
+    for(int r=0;r<nrhs;r++){
+      CoarseField svm(CoarseGrid),smm(CoarseGrid),e(CoarseGrid);
+      ExtractSliceFast(svm,cvm,r,0);
+      ExtractSliceFast(smm,cmm,r,0);
+      ref  += norm2(cvv[r]);
+      e = cmv[r]-cvv[r];  d_mv += norm2(e);
+      e = svm    -cvv[r]; d_vm += norm2(e);
+      e = smm    -cvv[r]; d_mm += norm2(e);
+    }
+    std::cout << GridLogMessage << "blockProject |vec->vec|^2 = " << ref << std::endl;
+    std::cout << GridLogMessage << "   mrhs->vec diff " << d_mv
+	      << "   vec->mrhs diff " << d_vm
+	      << "   mrhs->mrhs diff " << d_mm << std::endl;
+    GRID_ASSERT( ref > 0.0 );
+    GRID_ASSERT( d_mv == 0.0 );
+    GRID_ASSERT( d_vm == 0.0 );
+    GRID_ASSERT( d_mm == 0.0 );
+
+    ///////////////////////////////////////////////////////
+    // Promote must agree across orderings too
+    ///////////////////////////////////////////////////////
+    std::vector<FineField> fvv(nrhs,FineGrid);
+    FineField fmm(FineMulti);
+    Projector.blockPromote(fvv,cvv);
+    Projector.blockPromote(fmm,cmm);
+
+    RealD pref=0.0, pdiff=0.0;
+    for(int r=0;r<nrhs;r++){
+      FineField s(FineGrid),e(FineGrid);
+      ExtractSliceFast(s,fmm,r,0);
+      e = s - fvv[r];
+      pref  += norm2(fvv[r]);
+      pdiff += norm2(e);
+    }
+    std::cout << GridLogMessage << "blockPromote |vec|^2 = " << pref
+	      << "   mrhs diff " << pdiff << std::endl;
+    GRID_ASSERT( pref > 0.0 );
+    GRID_ASSERT( pdiff == 0.0 );
+  }
+
+  std::cout << GridLogMessage << "Test_mrhs_blockproject_import: ALL PASS" << std::endl;
+
+  Grid_finalize();
+}

tests/debug/Test_mrhs_promotion.cc

@@ -0,0 +1,157 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/debug/Test_mrhs_promotion.cc
+
+    Copyright (C) 2026
+
+Author: Peter Boyle <pboyle@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//
+// MrhsPromotedOperator: a D dimensional operator presented as D+1 with Nrhs.
+//
+// Each output slice must equal the D dimensional operator applied to the
+// corresponding input slice, bit for bit -- the promotion moves data, it does
+// not change arithmetic.  Slices are independent random fields so a mistake in
+// the slice indexing cannot pass.
+//
+#include <Grid/Grid.h>
+
+using namespace Grid;
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  const int Nrhs = 4;
+
+  Coordinate latt  = GridDefaultLatt();
+  Coordinate simd  = GridDefaultSimd(Nd,vComplexD::Nsimd());
+  Coordinate mpi   = GridDefaultMpi();
+
+  GridCartesian         *UGrid  = new GridCartesian(latt,simd,mpi);
+  GridRedBlackCartesian *UrbGrid= SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+
+  // D+1 grid with Nrhs in dimension 0
+  GridCartesian         *RGrid  = SpaceTimeGrid::makeFiveDimGrid(Nrhs,UGrid);
+
+  std::cout << GridLogMessage << "Nrhs = " << Nrhs
+	    << "  low grid " << UGrid->_ndimension << "d"
+	    << "  high grid " << RGrid->_ndimension << "d" << std::endl;
+
+  GridParallelRNG pRNG(UGrid); pRNG.SeedFixedIntegers(std::vector<int>({1,2,3,4}));
+
+  LatticeGaugeFieldD Umu(UGrid); SU<Nc>::HotConfiguration(pRNG,Umu);
+
+  RealD mass = 0.1;
+  WilsonFermionD Dw(Umu,*UGrid,*UrbGrid,mass);
+
+  MdagMLinearOperator<WilsonFermionD,LatticeFermionD> HermOp(Dw);
+
+  MrhsPromotedOperator<LatticeFermionD> MrhsOp(HermOp,UGrid,Nrhs);
+
+  //////////////////////////////////////////////////
+  // Independent random source on each slice
+  //////////////////////////////////////////////////
+  LatticeFermionD hi_in (RGrid);
+  LatticeFermionD hi_out(RGrid);
+  std::vector<LatticeFermionD> lo_in (Nrhs,UGrid);
+  std::vector<LatticeFermionD> lo_ref(Nrhs,UGrid);
+
+  for(int r=0;r<Nrhs;r++){
+    random(pRNG,lo_in[r]);
+    InsertSliceFast(lo_in[r],hi_in,r,0);
+  }
+
+  //////////////////////////////////////////////////
+  // Promoted application, and the reference
+  //////////////////////////////////////////////////
+  MrhsOp.Op(hi_in,hi_out);
+
+  for(int r=0;r<Nrhs;r++){
+    HermOp.Op(lo_in[r],lo_ref[r]);
+  }
+
+  RealD worst = 0.0;
+  for(int r=0;r<Nrhs;r++){
+    LatticeFermionD slice(UGrid);
+    LatticeFermionD err(UGrid);
+    ExtractSliceFast(slice,hi_out,r,0);
+    err = slice - lo_ref[r];
+    RealD n = norm2(err);
+    std::cout << GridLogMessage << "rhs["<<r<<"] |promoted - reference|^2 = " << n
+	      << "   |reference|^2 = " << norm2(lo_ref[r]) << std::endl;
+    worst = std::max(worst,n);
+    GRID_ASSERT( n == 0.0 );
+  }
+  std::cout << GridLogMessage << "worst slice difference " << worst << " (bit exact required)" << std::endl;
+
+  //////////////////////////////////////////////////
+  // Slices must be independent: perturbing one must
+  // change only that output slice
+  //////////////////////////////////////////////////
+  {
+    LatticeFermionD pert(UGrid);
+    random(pRNG,pert);
+    InsertSliceFast(pert,hi_in,1,0);
+    MrhsOp.Op(hi_in,hi_out);
+
+    for(int r=0;r<Nrhs;r++){
+      LatticeFermionD slice(UGrid);
+      LatticeFermionD err(UGrid);
+      ExtractSliceFast(slice,hi_out,r,0);
+      err = slice - lo_ref[r];
+      RealD n = norm2(err);
+      if ( r == 1 ) {
+	std::cout << GridLogMessage << "perturbed rhs[1] changed by " << n << std::endl;
+	GRID_ASSERT( n > 0.0 );
+      } else {
+	std::cout << GridLogMessage << "untouched rhs["<<r<<"] moved by " << n << std::endl;
+	GRID_ASSERT( n == 0.0 );
+      }
+    }
+  }
+
+  //////////////////////////////////////////////////
+  // AdjOp routes through the same slicing
+  //////////////////////////////////////////////////
+  {
+    for(int r=0;r<Nrhs;r++){
+      random(pRNG,lo_in[r]);
+      InsertSliceFast(lo_in[r],hi_in,r,0);
+      HermOp.AdjOp(lo_in[r],lo_ref[r]);
+    }
+    MrhsOp.AdjOp(hi_in,hi_out);
+    for(int r=0;r<Nrhs;r++){
+      LatticeFermionD slice(UGrid);
+      LatticeFermionD err(UGrid);
+      ExtractSliceFast(slice,hi_out,r,0);
+      err = slice - lo_ref[r];
+      GRID_ASSERT( norm2(err) == 0.0 );
+    }
+    std::cout << GridLogMessage << "AdjOp slices bit exact" << std::endl;
+  }
+
+  std::cout << GridLogMessage << "Test_mrhs_promotion: ALL PASS" << std::endl;
+
+  Grid_finalize();
+}