Merge branch 'feature/ddhmc' of https://github.com/paboyle/Grid into feature/ddhmc

Can simplify and make specific if performance matters

Grid/DisableWarnings.h

@@ -35,7 +35,7 @@ directory
 #if defined __GNUC__ && __GNUC__>=6
 #pragma GCC diagnostic ignored "-Wignored-attributes"
 #endif
-#if defined __GNUC__ 
+#if defined __GNUC__ && __GNUC__>=6
 #pragma GCC diagnostic ignored "-Wpsabi"
 #endif
 

Grid/algorithms/CoarsenedMatrix.h

@@ -358,7 +358,7 @@ public:
     autoView( in_v , in, AcceleratorRead);
     autoView( out_v , out, AcceleratorWrite);
     autoView( Stencil_v  , Stencil, AcceleratorRead);
-    int npoint = geom.npoint;
+    auto& geom_v = geom;
     typedef LatticeView<Cobj> Aview;
       
     Vector<Aview> AcceleratorViewContainer;
@@ -380,7 +380,7 @@ public:
       int ptype;
       StencilEntry *SE;
 
-      for(int point=0;point<npoint;point++){
+      for(int point=0;point<geom_v.npoint;point++){
 
 	SE=Stencil_v.GetEntry(ptype,point,ss);
 	  
@@ -424,7 +424,7 @@ public:
     autoView( in_v , in, AcceleratorRead);
     autoView( out_v , out, AcceleratorWrite);
     autoView( Stencil_v  , Stencil, AcceleratorRead);
-    int npoint = geom.npoint;
+    auto& geom_v = geom;
     typedef LatticeView<Cobj> Aview;
 
     Vector<Aview> AcceleratorViewContainer;
@@ -454,7 +454,7 @@ public:
       int ptype;
       StencilEntry *SE;
 
-      for(int p=0;p<npoint;p++){
+      for(int p=0;p<geom_v.npoint;p++){
         int point = points_p[p];
 
 	SE=Stencil_v.GetEntry(ptype,point,ss);

Grid/algorithms/LinearOperator.h

@@ -52,7 +52,6 @@ public:
   virtual void AdjOp  (const Field &in, Field &out) = 0; // Abstract base
   virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
   virtual void HermOp(const Field &in, Field &out)=0;
-  virtual ~LinearOperatorBase(){};
 };
 
 
@@ -224,9 +223,14 @@ class SchurOperatorBase :  public LinearOperatorBase<Field> {
     Mpc(in,tmp);
     MpcDag(tmp,out);
   }
+  virtual  void MpcMpcDag(const Field &in, Field &out) {
+    Field tmp(in.Grid());
+    tmp.Checkerboard() = in.Checkerboard();
+    MpcDag(in,tmp);
+    Mpc(tmp,out);
+  }
   virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
-    out.Checkerboard() = in.Checkerboard();
-    MpcDagMpc(in,out);
+    HermOp(in,out);
     ComplexD dot= innerProduct(in,out); 
     n1=real(dot);
     n2=norm2(out);
@@ -277,6 +281,16 @@ template<class Matrix,class Field>
       axpy(out,-1.0,tmp,out);
     }
 };
+// Mpc MpcDag system presented as the HermOp
+template<class Matrix,class Field>
+class SchurDiagMooeeDagOperator :  public SchurDiagMooeeOperator<Matrix,Field> {
+ public:
+  virtual void HermOp(const Field &in, Field &out){
+    out.Checkerboard() = in.Checkerboard();
+    this->MpcMpcDag(in,out);
+  }
+  SchurDiagMooeeDagOperator (Matrix &Mat): SchurDiagMooeeOperator<Matrix,Field>(Mat){};
+};
 template<class Matrix,class Field>
   class SchurDiagOneOperator :  public SchurOperatorBase<Field> {
  protected:
@@ -508,7 +522,7 @@ class SchurStaggeredOperator :  public SchurOperatorBase<Field> {
   virtual  void MpcDag   (const Field &in, Field &out){
     Mpc(in,out);
   }
-  virtual void MpcDagMpc(const Field &in, Field &out) {
+  virtual void MpcDagMpc(const Field &in, Field &out,RealD &ni,RealD &no) {
     assert(0);// Never need with staggered
   }
 };
@@ -586,7 +600,6 @@ class HermOpOperatorFunction : public OperatorFunction<Field> {
 template<typename Field>
 class PlainHermOp : public LinearFunction<Field> {
 public:
-  using LinearFunction<Field>::operator();
   LinearOperatorBase<Field> &_Linop;
       
   PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop) 
@@ -600,7 +613,6 @@ public:
 template<typename Field>
 class FunctionHermOp : public LinearFunction<Field> {
 public:
-  using LinearFunction<Field>::operator(); 
   OperatorFunction<Field>   & _poly;
   LinearOperatorBase<Field> &_Linop;
       

Grid/algorithms/Preconditioner.h

@@ -30,19 +30,13 @@ Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
-template<class Field> using Preconditioner =  LinearFunction<Field> ;
-
-/*
-template<class Field> class Preconditioner :  public LinearFunction<Field> {
-  using LinearFunction<Field>::operator();
+template<class Field> class Preconditioner :  public LinearFunction<Field> { 
   virtual void operator()(const Field &src, Field & psi)=0;
 };
-*/
 
 template<class Field> class TrivialPrecon :  public Preconditioner<Field> { 
 public:
-  using Preconditioner<Field>::operator();
-  virtual void operator()(const Field &src, Field & psi){
+  void operator()(const Field &src, Field & psi){
     psi = src;
   }
   TrivialPrecon(void){};

Grid/algorithms/SparseMatrix.h

@@ -48,7 +48,6 @@ public:
   virtual  void Mdiag    (const Field &in, Field &out)=0;
   virtual  void Mdir     (const Field &in, Field &out,int dir, int disp)=0;
   virtual  void MdirAll  (const Field &in, std::vector<Field> &out)=0;
-  virtual ~SparseMatrixBase() {};
 };
 
 /////////////////////////////////////////////////////////////////////////////////////////////
@@ -73,7 +72,7 @@ public:
   virtual  void MeooeDag    (const Field &in, Field &out)=0;
   virtual  void MooeeDag    (const Field &in, Field &out)=0;
   virtual  void MooeeInvDag (const Field &in, Field &out)=0;
-  virtual ~CheckerBoardedSparseMatrixBase() {};
+
 };
 
 NAMESPACE_END(Grid);

Grid/algorithms/iterative/BiCGSTABMixedPrec.h

@@ -36,8 +36,7 @@ NAMESPACE_BEGIN(Grid);
 template<class FieldD, class FieldF, typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0> 
 class MixedPrecisionBiCGSTAB : public LinearFunction<FieldD> 
 {
-  public:
-    using LinearFunction<FieldD>::operator();
+  public:                                                
     RealD   Tolerance;
     RealD   InnerTolerance; // Initial tolerance for inner CG. Defaults to Tolerance but can be changed
     Integer MaxInnerIterations;

Grid/algorithms/iterative/ConjugateGradient.h

@@ -102,7 +102,7 @@ public:
     // Check if guess is really REALLY good :)
     if (cp <= rsq) {
       TrueResidual = std::sqrt(a/ssq);
-      std::cout << GridLogMessage << "ConjugateGradient guess is converged already " << std::endl;
+      std::cout << GridLogMessage << "ConjugateGradient guess is converged already "<<TrueResidual<< " tol "<< Tolerance<< std::endl;
       IterationsToComplete = 0;	
       return;
     }

Grid/algorithms/iterative/ConjugateGradientMixedPrec.h

@@ -35,8 +35,7 @@ NAMESPACE_BEGIN(Grid);
     typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
     typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0> 
   class MixedPrecisionConjugateGradient : public LinearFunction<FieldD> {
-  public:
-    using LinearFunction<FieldD>::operator();
+  public:                                                
     RealD   Tolerance;
     RealD   InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
     Integer MaxInnerIterations;
@@ -53,23 +52,31 @@ NAMESPACE_BEGIN(Grid);
 
     //Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
     LinearFunction<FieldF> *guesser;
-    
-    MixedPrecisionConjugateGradient(RealD tol, 
+
+    MixedPrecisionConjugateGradient(RealD Tol,
+				    Integer maxinnerit, 
+				    Integer maxouterit, 
+				    GridBase* _sp_grid, 
+				    LinearOperatorBase<FieldF> &_Linop_f, 
+				    LinearOperatorBase<FieldD> &_Linop_d) :
+      MixedPrecisionConjugateGradient(Tol, Tol, maxinnerit, maxouterit, _sp_grid, _Linop_f, _Linop_d) {};
+
+    MixedPrecisionConjugateGradient(RealD Tol,
+				    RealD InnerTol,
 				    Integer maxinnerit, 
 				    Integer maxouterit, 
 				    GridBase* _sp_grid, 
 				    LinearOperatorBase<FieldF> &_Linop_f, 
 				    LinearOperatorBase<FieldD> &_Linop_d) :
       Linop_f(_Linop_f), Linop_d(_Linop_d),
-      Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), SinglePrecGrid(_sp_grid),
-      OuterLoopNormMult(100.), guesser(NULL){ };
+      Tolerance(Tol), InnerTolerance(InnerTol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), SinglePrecGrid(_sp_grid),
+      OuterLoopNormMult(100.), guesser(NULL){ assert(InnerTol < 1.0e-1);};
 
     void useGuesser(LinearFunction<FieldF> &g){
       guesser = &g;
     }
   
   void operator() (const FieldD &src_d_in, FieldD &sol_d){
-    std::cout << GridLogMessage << "MixedPrecisionConjugateGradient: Starting mixed precision CG with outer tolerance " << Tolerance << " and inner tolerance " << InnerTolerance << std::endl;
     TotalInnerIterations = 0;
 	
     GridStopWatch TotalTimer;
@@ -104,7 +111,6 @@ NAMESPACE_BEGIN(Grid);
     FieldF sol_f(SinglePrecGrid);
     sol_f.Checkerboard() = cb;
     
-    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting initial inner CG with tolerance " << inner_tol << std::endl;
     ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
     CG_f.ErrorOnNoConverge = false;
 
@@ -138,7 +144,6 @@ NAMESPACE_BEGIN(Grid);
 	(*guesser)(src_f, sol_f);
 
       //Inner CG
-      std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " << outer_iter << " starting inner CG with tolerance " << inner_tol << std::endl;
       CG_f.Tolerance = inner_tol;
       InnerCGtimer.Start();
       CG_f(Linop_f, src_f, sol_f);

Grid/algorithms/iterative/Deflation.h

@@ -33,19 +33,16 @@ namespace Grid {
 template<class Field>
 class ZeroGuesser: public LinearFunction<Field> {
 public:
-  using LinearFunction<Field>::operator();
     virtual void operator()(const Field &src, Field &guess) { guess = Zero(); };
 };
 template<class Field>
 class DoNothingGuesser: public LinearFunction<Field> {
 public:
-  using LinearFunction<Field>::operator();
   virtual void operator()(const Field &src, Field &guess) {  };
 };
 template<class Field>
 class SourceGuesser: public LinearFunction<Field> {
 public:
-  using LinearFunction<Field>::operator();
   virtual void operator()(const Field &src, Field &guess) { guess = src; };
 };
 
@@ -60,7 +57,6 @@ private:
   const unsigned int       N;
 
 public:
-  using LinearFunction<Field>::operator();
 
   DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval)
   : DeflatedGuesser(_evec, _eval, _evec.size())
@@ -91,7 +87,6 @@ private:
   const std::vector<RealD>       &eval_coarse;
 public:
   
-  using LinearFunction<FineField>::operator();
   LocalCoherenceDeflatedGuesser(const std::vector<FineField>   &_subspace,
 				const std::vector<CoarseField> &_evec_coarse,
 				const std::vector<RealD>       &_eval_coarse)

Grid/algorithms/iterative/LocalCoherenceLanczos.h

@@ -44,7 +44,6 @@ public:
 				  int, MinRes);    // Must restart
 };
 
-//This class is the input parameter class for some testing programs
 struct LocalCoherenceLanczosParams : Serializable {
 public:
   GRID_SERIALIZABLE_CLASS_MEMBERS(LocalCoherenceLanczosParams,
@@ -68,7 +67,6 @@ public:
 template<class Fobj,class CComplex,int nbasis>
 class ProjectedHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
 public:
-  using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
   typedef iVector<CComplex,nbasis >           CoarseSiteVector;
   typedef Lattice<CoarseSiteVector>           CoarseField;
   typedef Lattice<CComplex>   CoarseScalar; // used for inner products on fine field
@@ -99,7 +97,6 @@ public:
 template<class Fobj,class CComplex,int nbasis>
 class ProjectedFunctionHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
 public:
-  using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
   typedef iVector<CComplex,nbasis >           CoarseSiteVector;
   typedef Lattice<CoarseSiteVector>           CoarseField;
   typedef Lattice<CComplex>   CoarseScalar; // used for inner products on fine field
@@ -156,7 +153,6 @@ public:
       _coarse_relax_tol(coarse_relax_tol)  
   {    };
 
-  //evalMaxApprox: approximation of largest eval of the fine Chebyshev operator (suitably wrapped by block projection)
   int TestConvergence(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
   {
     CoarseField v(B);
@@ -183,16 +179,8 @@ public:
     if( (vv<eresid*eresid) ) conv = 1;
     return conv;
   }
-
-  //This function is called at the end of the coarse grid Lanczos. It promotes the coarse eigenvector 'B' to the fine grid,
-  //applies a smoother to the result then computes the computes the *fine grid* eigenvalue (output as 'eval').
-
-  //evalMaxApprox should be the approximation of the largest eval of the fine Hermop. However when this function is called by IRL it actually passes the largest eval of the *Chebyshev* operator (as this is the max approx used for the TestConvergence above)
-  //As the largest eval of the Chebyshev is typically several orders of magnitude larger this makes the convergence test pass even when it should not.
-  //We therefore ignore evalMaxApprox here and use a value of 1.0 (note this value is already used by TestCoarse)
-  int ReconstructEval(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)  
+  int ReconstructEval(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
   {
-    evalMaxApprox = 1.0; //cf above
     GridBase *FineGrid = _subspace[0].Grid();    
     int checkerboard   = _subspace[0].Checkerboard();
     FineField fB(FineGrid);fB.Checkerboard() =checkerboard;
@@ -211,13 +199,13 @@ public:
     eval   = vnum/vden;
     fv -= eval*fB;
     RealD vv = norm2(fv) / ::pow(evalMaxApprox,2.0);
-    if ( j > nbasis ) eresid = eresid*_coarse_relax_tol;
-    
+
     std::cout.precision(13);
     std::cout<<GridLogIRL  << "[" << std::setw(3)<<j<<"] "
 	     <<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
-	     <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv << " target " << eresid*eresid
+	     <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
 	     <<std::endl;
+    if ( j > nbasis ) eresid = eresid*_coarse_relax_tol;
     if( (vv<eresid*eresid) ) return 1;
     return 0;
   }
@@ -295,10 +283,6 @@ public:
     evals_coarse.resize(0);
   };
 
-  //The block inner product is the inner product on the fine grid locally summed over the blocks
-  //to give a Lattice<Scalar> on the coarse grid. This function orthnormalizes the fine-grid subspace
-  //vectors under the block inner product. This step must be performed after computing the fine grid
-  //eigenvectors and before computing the coarse grid eigenvectors.    
   void Orthogonalise(void ) {
     CoarseScalar InnerProd(_CoarseGrid);
     std::cout << GridLogMessage <<" Gramm-Schmidt pass 1"<<std::endl;
@@ -342,8 +326,6 @@ public:
     }
   }
 
-  //While this method serves to check the coarse eigenvectors, it also recomputes the eigenvalues from the smoothed reconstructed eigenvectors
-  //hence the smoother can be tuned after running the coarse Lanczos by using a different smoother here
   void testCoarse(RealD resid,ChebyParams cheby_smooth,RealD relax) 
   {
     assert(evals_fine.size() == nbasis);
@@ -392,31 +374,25 @@ public:
     evals_fine.resize(nbasis);
     subspace.resize(nbasis,_FineGrid);
   }
-
-
-  //cheby_op: Parameters of the fine grid Chebyshev polynomial used for the Lanczos acceleration
-  //cheby_smooth: Parameters of a separate Chebyshev polynomial used after the Lanczos has completed to smooth out high frequency noise in the reconstructed fine grid eigenvectors prior to computing the eigenvalue
-  //relax: Reconstructed eigenvectors (post smoothing) are naturally not as precise as true eigenvectors. This factor acts as a multiplier on the stopping condition when determining whether the results satisfy the user provided stopping condition
   void calcCoarse(ChebyParams cheby_op,ChebyParams cheby_smooth,RealD relax,
 		  int Nstop, int Nk, int Nm,RealD resid, 
 		  RealD MaxIt, RealD betastp, int MinRes)
   {
-    Chebyshev<FineField>                          Cheby(cheby_op); //Chebyshev of fine operator on fine grid
-    ProjectedHermOp<Fobj,CComplex,nbasis>         Op(_FineOp,subspace); //Fine operator on coarse grid with intermediate fine grid conversion
-    ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (Cheby,_FineOp,subspace); //Chebyshev of fine operator on coarse grid with intermediate fine grid conversion
+    Chebyshev<FineField>                          Cheby(cheby_op);
+    ProjectedHermOp<Fobj,CComplex,nbasis>         Op(_FineOp,subspace);
+    ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (Cheby,_FineOp,subspace);
     //////////////////////////////////////////////////////////////////////////////////////////////////
     // create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
     //////////////////////////////////////////////////////////////////////////////////////////////////
 
-    Chebyshev<FineField>                                           ChebySmooth(cheby_smooth); //lower order Chebyshev of fine operator on fine grid used to smooth regenerated eigenvectors
-    ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,subspace,relax); 
+    Chebyshev<FineField>                                           ChebySmooth(cheby_smooth);
+    ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,subspace,relax);
 
     evals_coarse.resize(Nm);
     evec_coarse.resize(Nm,_CoarseGrid);
 
     CoarseField src(_CoarseGrid);     src=1.0; 
 
-    //Note the "tester" here is also responsible for generating the fine grid eigenvalues which are output into the "evals_coarse" array
     ImplicitlyRestartedLanczos<CoarseField> IRL(ChebyOp,ChebyOp,ChebySmoothTester,Nstop,Nk,Nm,resid,MaxIt,betastp,MinRes);
     int Nconv=0;
     IRL.calc(evals_coarse,evec_coarse,src,Nconv,false);
@@ -427,14 +403,6 @@ public:
       std::cout << i << " Coarse eval = " << evals_coarse[i]  << std::endl;
     }
   }
-
-  //Get the fine eigenvector 'i' by reconstruction
-  void getFineEvecEval(FineField &evec, RealD &eval, const int i) const{
-    blockPromote(evec_coarse[i],evec,subspace);  
-    eval = evals_coarse[i];
-  }
-    
-    
 };
 
 NAMESPACE_END(Grid);

Grid/algorithms/iterative/PowerMethod.h

@@ -29,8 +29,6 @@ template<class Field> class PowerMethod
       RealD vnum = real(innerProduct(src_n,tmp)); // HermOp. 
       RealD vden = norm2(src_n); 
       RealD na = vnum/vden; 
-
-      std::cout << GridLogIterative << "PowerMethod: Current approximation of largest eigenvalue " << na << std::endl;
       
       if ( (fabs(evalMaxApprox/na - 1.0) < 0.001) || (i==_MAX_ITER_EST_-1) ) { 
  	evalMaxApprox = na; 

Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h

@@ -43,7 +43,7 @@ NAMESPACE_BEGIN(Grid);
 template<class Field>
 class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
 public:                                                
-  using LinearFunction<Field>::operator();
+
   RealD   Tolerance;
   Integer MaxIterations;
   int verbose;

Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h

@@ -43,7 +43,7 @@ NAMESPACE_BEGIN(Grid);
 template<class Field>
 class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
 public:                                                
-  using LinearFunction<Field>::operator();
+
   RealD   Tolerance;
   Integer MaxIterations;
   int verbose;
@@ -119,8 +119,7 @@ public:
   RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
 
     RealD cp;
-    ComplexD a, b;
-    //    ComplexD zAz;
+    ComplexD a, b, zAz;
     RealD zAAz;
     ComplexD rq;
 
@@ -147,7 +146,7 @@ public:
     //////////////////////////////////
     MatTimer.Start();
     Linop.Op(psi,Az);
-    //    zAz = innerProduct(Az,psi);
+    zAz = innerProduct(Az,psi);
     zAAz= norm2(Az);
     MatTimer.Stop();
     
@@ -171,7 +170,7 @@ public:
 
     LinalgTimer.Start();
 
-    //    zAz = innerProduct(Az,psi);
+    zAz = innerProduct(Az,psi);
     zAAz= norm2(Az);
 
     //p[0],q[0],qq[0] 
@@ -213,7 +212,7 @@ public:
       MatTimer.Start();
       Linop.Op(z,Az);
       MatTimer.Stop();
-      //      zAz = innerProduct(Az,psi);
+      zAz = innerProduct(Az,psi);
       zAAz= norm2(Az);
 
       LinalgTimer.Start();

Grid/algorithms/iterative/SchurRedBlack.h

@@ -40,7 +40,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
    *        (-MoeMee^{-1}   1 )   
    * L^{dag} = ( 1       Mee^{-dag} Moe^{dag} )
    *           ( 0       1                    )
-   * L^{-d}  = ( 1      -Mee^{-dag} Moe^{dag} )
+   * L^{-dag}= ( 1      -Mee^{-dag} Moe^{dag} )
    *           ( 0       1                    )
    *
    * U^-1 = (1   -Mee^{-1} Meo)
@@ -82,7 +82,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
    * c) M_oo^-dag Doo^{dag} Doo Moo^-1 phi_0 = M_oo^-dag (D_oo)^dag L^{-1}  eta_o
    *                              eta_o'     = M_oo^-dag (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
    *                              psi_o = M_oo^-1 phi_o
-   * TODO: Deflation 
+   *
+   *
    */
 namespace Grid {
 
@@ -97,6 +98,7 @@ namespace Grid {
   protected:
     typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
     OperatorFunction<Field> & _HermitianRBSolver;
+    
     int CBfactorise;
     bool subGuess;
     bool useSolnAsInitGuess; // if true user-supplied solution vector is used as initial guess for solver
@@ -219,13 +221,20 @@ namespace Grid {
 	/////////////////////////////////////////////////
 	// Check unprec residual if possible
 	/////////////////////////////////////////////////
-	if ( ! subGuess ) {
-	  _Matrix.M(out[b],resid); 
+	if ( ! subGuess ) {	  
+
+	  if ( this->adjoint() ) _Matrix.Mdag(out[b],resid); 
+	  else                   _Matrix.M(out[b],resid); 
+
 	  resid = resid-in[b];
 	  RealD ns = norm2(in[b]);
 	  RealD nr = norm2(resid);
 	
-	  std::cout<<GridLogMessage<< "SchurRedBlackBase solver true unprec resid["<<b<<"] "<<std::sqrt(nr/ns) << std::endl;
+	  std::cout<<GridLogMessage<< "SchurRedBlackBase adjoint "<< this->adjoint() << std::endl;
+	  if ( this->adjoint() ) 
+	    std::cout<<GridLogMessage<< "SchurRedBlackBase adjoint solver true unprec resid["<<b<<"] "<<std::sqrt(nr/ns) << std::endl;
+	  else                   
+	    std::cout<<GridLogMessage<< "SchurRedBlackBase solver true unprec resid["<<b<<"] "<<std::sqrt(nr/ns) << std::endl;
 	} else {
 	  std::cout<<GridLogMessage<< "SchurRedBlackBase Guess subtracted after solve["<<b<<"] " << std::endl;
 	}
@@ -279,12 +288,21 @@ namespace Grid {
 
       // Verify the unprec residual
       if ( ! subGuess ) {
-        _Matrix.M(out,resid); 
+
+	std::cout<<GridLogMessage<< "SchurRedBlackBase adjoint "<< this->adjoint() << std::endl;
+	
+	if ( this->adjoint() ) _Matrix.Mdag(out,resid); 
+	else                   _Matrix.M(out,resid); 
+
         resid = resid-in;
         RealD ns = norm2(in);
         RealD nr = norm2(resid);
 
-        std::cout<<GridLogMessage << "SchurRedBlackBase solver true unprec resid "<< std::sqrt(nr/ns) << std::endl;
+	  if ( this->adjoint() ) 
+	    std::cout<<GridLogMessage<< "SchurRedBlackBase adjoint solver true unprec resid "<<std::sqrt(nr/ns) << std::endl;
+	  else                   
+	    std::cout<<GridLogMessage<< "SchurRedBlackBase solver true unprec resid "<<std::sqrt(nr/ns) << std::endl;
+
       } else {
         std::cout << GridLogMessage << "SchurRedBlackBase Guess subtracted after solve." << std::endl;
       }
@@ -293,6 +311,7 @@ namespace Grid {
     /////////////////////////////////////////////////////////////
     // Override in derived. 
     /////////////////////////////////////////////////////////////
+    virtual bool adjoint(void) { return false; }
     virtual void RedBlackSource  (Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)                =0;
     virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)          =0;
     virtual void RedBlackSolve   (Matrix & _Matrix,const Field &src_o, Field &sol_o)                           =0;
@@ -646,6 +665,127 @@ namespace Grid {
         this->_HermitianRBSolver(_OpEO, src_o, sol_o); 
       }
   };
+
+  /*
+   * Red black Schur decomposition
+   *
+   *  M = (Mee Meo) =  (1             0 )   (Mee   0               )  (1 Mee^{-1} Meo)
+   *      (Moe Moo)    (Moe Mee^-1    1 )   (0   Moo-Moe Mee^-1 Meo)  (0   1         )
+   *                =         L                     D                     U
+   *
+   * L^-1 = (1              0 )
+   *        (-MoeMee^{-1}   1 )   
+   * L^{dag} = ( 1       Mee^{-dag} Moe^{dag} )
+   *           ( 0       1                    )
+   *
+   * U^-1 = (1   -Mee^{-1} Meo)
+   *        (0    1           )
+   * U^{dag} = ( 1                 0)
+   *           (Meo^dag Mee^{-dag} 1)
+   * U^{-dag} = (  1                 0)
+   *            (-Meo^dag Mee^{-dag} 1)
+   *
+   *
+   ***********************
+   *     M^dag psi = eta
+   ***********************
+   *
+   * Really for Mobius: (Wilson - easier to just use gamma 5 hermiticity)
+   *
+   *    Mdag psi     =         Udag  Ddag  Ldag psi = eta
+   *
+   * U^{-dag} = (  1                 0)
+   *            (-Meo^dag Mee^{-dag} 1)
+   *
+   *
+   * i)                D^dag phi =  (U^{-dag}  eta)
+   *                        eta'_e = eta_e
+   *                        eta'_o = (eta_o - Meo^dag Mee^{-dag} eta_e)
+   * 
+   *      phi_o = D_oo^-dag eta'_o = D_oo^-dag (eta_o - Meo^dag Mee^{-dag} eta_e)
+   *
+   *      phi_e = D_ee^-dag eta'_e = D_ee^-dag eta_e
+   * 
+   * Solve: 
+   *
+   *      D_oo D_oo^dag phi_o = D_oo (eta_o - Meo^dag Mee^{-dag} eta_e)
+   *
+   * ii) 
+   *      phi = L^dag psi => psi = L^-dag phi. 
+   *
+   * L^{-dag} = ( 1      -Mee^{-dag} Moe^{dag} )
+   *            ( 0       1                    )
+   *
+   *   => sol_e = M_ee^-dag * ( src_e - Moe^dag phi_o )...
+   *   => sol_o = phi_o
+   */
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Site diagonal has Mooee on it, but solve the Adjoint system
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  template<class Field> class SchurRedBlackDiagMooeeDagSolve : public SchurRedBlackBase<Field> {
+  public:
+    typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
+
+    virtual bool adjoint(void) { return true; }
+    SchurRedBlackDiagMooeeDagSolve(OperatorFunction<Field> &HermitianRBSolver,
+				   const bool initSubGuess = false,
+				   const bool _solnAsInitGuess = false)  
+      : SchurRedBlackBase<Field> (HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
+
+    //////////////////////////////////////////////////////
+    // Override RedBlack specialisation
+    //////////////////////////////////////////////////////
+    virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
+    {
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      Field   tmp(grid);
+      Field  Mtmp(grid);
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd ,src_o,src);
+      /////////////////////////////////////////////////////
+      // src_o = (source_o - Moe^dag MeeInvDag source_e)
+      /////////////////////////////////////////////////////
+      _Matrix.MooeeInvDag(src_e,tmp);  assert(  tmp.Checkerboard() ==Even);
+      _Matrix.MeooeDag   (tmp,Mtmp);   assert( Mtmp.Checkerboard() ==Odd);     
+      tmp=src_o-Mtmp;                  assert(  tmp.Checkerboard() ==Odd);     
+
+      // get the right Mpc
+      SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
+      _HermOpEO.Mpc(tmp,src_o);     assert(src_o.Checkerboard() ==Odd);
+    }
+    virtual void RedBlackSolve   (Matrix & _Matrix,const Field &src_o, Field &sol_o)
+    {
+      SchurDiagMooeeDagOperator<Matrix,Field> _HermOpEO(_Matrix);
+      this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); 
+    };
+    virtual void RedBlackSolve   (Matrix & _Matrix,const std::vector<Field> &src_o,  std::vector<Field> &sol_o)
+    {
+      SchurDiagMooeeDagOperator<Matrix,Field> _HermOpEO(_Matrix);
+      this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
+    }
+    virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
+    {
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      Field  sol_e(grid);
+      Field  tmp(grid);
+      ///////////////////////////////////////////////////
+      // sol_e = M_ee^-dag * ( src_e - Moe^dag phi_o )...
+      // sol_o = phi_o
+      ///////////////////////////////////////////////////
+      _Matrix.MeooeDag(sol_o,tmp);      assert(tmp.Checkerboard()==Even);
+      tmp = src_e-tmp;                  assert(tmp.Checkerboard()==Even);
+      _Matrix.MooeeInvDag(tmp,sol_e);   assert(sol_e.Checkerboard()==Even);
+      
+      setCheckerboard(sol,sol_e); assert(  sol_e.Checkerboard() ==Even);
+      setCheckerboard(sol,sol_o); assert(  sol_o.Checkerboard() ==Odd );
+    }
+  };
+
 }
 
 #endif

Grid/allocator/MemoryManager.cc

@@ -159,6 +159,7 @@ void MemoryManager::Init(void)
 
   char * str;
   int Nc;
+  int NcS;
   
   str= getenv("GRID_ALLOC_NCACHE_LARGE");
   if ( str ) {

Grid/allocator/MemoryManager.h

@@ -170,7 +170,6 @@ private:
 
  public:
   static void Print(void);
-  static void PrintState( void* CpuPtr);
   static int   isOpen   (void* CpuPtr);
   static void  ViewClose(void* CpuPtr,ViewMode mode);
   static void *ViewOpen (void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);

Grid/allocator/MemoryManagerCache.cc

@@ -474,32 +474,6 @@ int   MemoryManager::isOpen   (void* _CpuPtr)
   }
 }
 
-void MemoryManager::PrintState(void* _CpuPtr)
-{
-  uint64_t CpuPtr = (uint64_t)_CpuPtr;
-
-  if ( EntryPresent(CpuPtr) ){
-    auto AccCacheIterator = EntryLookup(CpuPtr);
-    auto & AccCache = AccCacheIterator->second;
-    std::string str;
-    if ( AccCache.state==Empty    ) str = std::string("Empty");
-    if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
-    if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
-    if ( AccCache.state==Consistent)str = std::string("Consistent");
-    if ( AccCache.state==EvictNext) str = std::string("EvictNext");
-
-    std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
-    std::cout << GridLogMessage << "0x"<<std::hex<<AccCache.CpuPtr<<std::dec
-    << "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
-    << "\t" << AccCache.cpuLock
-    << "\t" << AccCache.accLock
-    << "\t" << AccCache.LRU_valid<<std::endl;
-
-  } else {
-    std::cout << GridLogMessage << "No Entry in AccCache table." << std::endl; 
-  }
-}
-
 NAMESPACE_END(Grid);
 
 #endif

Grid/allocator/MemoryManagerShared.cc

@@ -16,10 +16,6 @@ uint64_t  MemoryManager::DeviceToHostXfer;
 void  MemoryManager::ViewClose(void* AccPtr,ViewMode mode){};
 void *MemoryManager::ViewOpen(void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint){ return CpuPtr; };
 int   MemoryManager::isOpen   (void* CpuPtr) { return 0;}
-void  MemoryManager::PrintState(void* CpuPtr)
-{
-std::cout << GridLogMessage << "Host<->Device memory movement not currently managed by Grid." << std::endl;
-};
 void  MemoryManager::Print(void){};
 void  MemoryManager::NotifyDeletion(void *ptr){};
 

Grid/communicator/Communicator_mpi3.cc

@@ -388,8 +388,8 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
     // TODO : make a OMP loop on CPU, call threaded bcopy
     void *shm = (void *) this->ShmBufferTranslate(dest,recv);
     assert(shm!=NULL);
-    //    std::cout <<"acceleratorCopyDeviceToDeviceAsynch"<< std::endl;
     acceleratorCopyDeviceToDeviceAsynch(xmit,shm,bytes);
+    acceleratorCopySynchronise(); // MPI prob slower
   }
 
   if ( CommunicatorPolicy == CommunicatorPolicySequential ) {
@@ -400,9 +400,6 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 }
 void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
 {
-  //   std::cout << "Copy Synchronised\n"<<std::endl;
-  acceleratorCopySynchronise();
-
   int nreq=list.size();
 
   if (nreq==0) return;

Grid/lattice/Lattice_base.h

@@ -88,13 +88,6 @@ public:
     LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
     accessor.ViewClose();
   }
-
-  // Helper function to print the state of this object in the AccCache
-  void PrintCacheState(void)
-  {
-    MemoryManager::PrintState(this->_odata);
-  }
-
   /////////////////////////////////////////////////////////////////////////////////
   // Return a view object that may be dereferenced in site loops.
   // The view is trivially copy constructible and may be copied to an accelerator device

Grid/lattice/Lattice_crc.h

@@ -29,6 +29,19 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 NAMESPACE_BEGIN(Grid);
 
+template<class vobj> void DumpSliceNorm(std::string s,Lattice<vobj> &f,int mu=-1)
+{
+  auto ff = localNorm2(f);
+  if ( mu==-1 ) mu = f.Grid()->Nd()-1;
+  typedef typename vobj::tensor_reduced normtype;
+  typedef typename normtype::scalar_object scalar;
+  std::vector<scalar> sff;
+  sliceSum(ff,sff,mu);
+  for(int t=0;t<sff.size();t++){
+    std::cout << s<<" "<<t<<" "<<sff[t]<<std::endl;
+  }
+}
+
 template<class vobj> uint32_t crc(Lattice<vobj> & buf)
 {
   autoView( buf_v , buf, CpuRead);

Grid/lattice/Lattice_reduction_gpu.h

@@ -42,6 +42,7 @@ void getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator
   std::cout << GridLogDebug << "\twarpSize            = " << warpSize << std::endl;
   std::cout << GridLogDebug << "\tsharedMemPerBlock   = " << sharedMemPerBlock << std::endl;
   std::cout << GridLogDebug << "\tmaxThreadsPerBlock  = " << maxThreadsPerBlock << std::endl;
+  std::cout << GridLogDebug << "\tmaxThreadsPerBlock  = " << warpSize << std::endl;
   std::cout << GridLogDebug << "\tmultiProcessorCount = " << multiProcessorCount << std::endl;
   
   if (warpSize != WARP_SIZE) {
@@ -51,10 +52,6 @@ void getNumBlocksAndThreads(const Iterator n, const size_t sizeofsobj, Iterator
   
   // let the number of threads in a block be a multiple of 2, starting from warpSize
   threads = warpSize;
-  if ( threads*sizeofsobj > sharedMemPerBlock ) {
-    std::cout << GridLogError << "The object is too large for the shared memory." << std::endl;
-    exit(EXIT_FAILURE);
-  }
   while( 2*threads*sizeofsobj < sharedMemPerBlock && 2*threads <= maxThreadsPerBlock ) threads *= 2;
   // keep all the streaming multiprocessors busy
   blocks = nextPow2(multiProcessorCount);

Grid/lattice/Lattice_rng.h

@@ -143,8 +143,8 @@ public:
 
   std::vector<RngEngine>                             _generators;
   std::vector<std::uniform_real_distribution<RealD> > _uniform;
-  std::vector<Grid::gaussian_distribution<RealD> >       _gaussian;
-  //  std::vector<std::discrete_distribution<int32_t> >   _bernoulli;
+  std::vector<Grid::gaussian_distribution<RealD> >    _gaussian;
+  std::vector<std::discrete_distribution<int32_t> >   _bernoulli;
   std::vector<std::uniform_int_distribution<uint32_t> > _uid;
 
   ///////////////////////
@@ -245,7 +245,7 @@ public:
     _generators.resize(1);
     _uniform.resize(1,std::uniform_real_distribution<RealD>{0,1});
     _gaussian.resize(1,gaussian_distribution<RealD>(0.0,1.0) );
-    //    _bernoulli.resize(1,std::discrete_distribution<int32_t>{1,1});
+    _bernoulli.resize(1,std::discrete_distribution<int32_t>{1,1});
     _uid.resize(1,std::uniform_int_distribution<uint32_t>() );
   }
 
@@ -359,7 +359,7 @@ public:
     _generators.resize(_vol);
     _uniform.resize(_vol,std::uniform_real_distribution<RealD>{0,1});
     _gaussian.resize(_vol,gaussian_distribution<RealD>(0.0,1.0) );
-    //    _bernoulli.resize(_vol,std::discrete_distribution<int32_t>{1,1});
+    _bernoulli.resize(_vol,std::discrete_distribution<int32_t>{1,1});
     _uid.resize(_vol,std::uniform_int_distribution<uint32_t>() );
   }
 
@@ -516,11 +516,11 @@ public:
 
 template <class vobj> inline void random(GridParallelRNG &rng,Lattice<vobj> &l)   { rng.fill(l,rng._uniform);  }
 template <class vobj> inline void gaussian(GridParallelRNG &rng,Lattice<vobj> &l) { rng.fill(l,rng._gaussian); }
-//template <class vobj> inline void bernoulli(GridParallelRNG &rng,Lattice<vobj> &l){ rng.fill(l,rng._bernoulli);}
+template <class vobj> inline void bernoulli(GridParallelRNG &rng,Lattice<vobj> &l){ rng.fill(l,rng._bernoulli);}
 
 template <class sobj> inline void random(GridSerialRNG &rng,sobj &l)   { rng.fill(l,rng._uniform  ); }
 template <class sobj> inline void gaussian(GridSerialRNG &rng,sobj &l) { rng.fill(l,rng._gaussian ); }
-//template <class sobj> inline void bernoulli(GridSerialRNG &rng,sobj &l){ rng.fill(l,rng._bernoulli); }
+template <class sobj> inline void bernoulli(GridSerialRNG &rng,sobj &l){ rng.fill(l,rng._bernoulli); }
 
 NAMESPACE_END(Grid);
 #endif

Grid/lattice/Lattice_transfer.h

@@ -85,76 +85,6 @@ template<class vobj> inline void setCheckerboard(Lattice<vobj> &full,const Latti
   });
 }
 
-template<class vobj> inline void acceleratorPickCheckerboard(int cb,Lattice<vobj> &half,const Lattice<vobj> &full, int checker_dim_half=0)
-{
-  half.Checkerboard() = cb;
-  autoView(half_v, half, AcceleratorWrite);
-  autoView(full_v, full, AcceleratorRead);
-  Coordinate rdim_full             = full.Grid()->_rdimensions;
-  Coordinate rdim_half             = half.Grid()->_rdimensions;
-  unsigned long ndim_half          = half.Grid()->_ndimension;
-  Coordinate checker_dim_mask_half = half.Grid()->_checker_dim_mask;
-  Coordinate ostride_half          = half.Grid()->_ostride;
-  accelerator_for(ss, full.Grid()->oSites(),full.Grid()->Nsimd(),{
-    
-    Coordinate coor;
-    int cbos;
-    int linear=0;
-
-    Lexicographic::CoorFromIndex(coor,ss,rdim_full);
-    assert(coor.size()==ndim_half);
-
-    for(int d=0;d<ndim_half;d++){ 
-      if(checker_dim_mask_half[d]) linear += coor[d];
-    }
-    cbos = (linear&0x1);
-
-    if (cbos==cb) {
-      int ssh=0;
-      for(int d=0;d<ndim_half;d++) {
-        if (d == checker_dim_half) ssh += ostride_half[d] * ((coor[d] / 2) % rdim_half[d]);
-        else ssh += ostride_half[d] * (coor[d] % rdim_half[d]);
-      }
-      coalescedWrite(half_v[ssh],full_v(ss));
-    }
-  });
-}
-template<class vobj> inline void acceleratorSetCheckerboard(Lattice<vobj> &full,const Lattice<vobj> &half, int checker_dim_half=0)
-{
-  int cb = half.Checkerboard();
-  autoView(half_v , half, AcceleratorRead);
-  autoView(full_v , full, AcceleratorWrite);
-  Coordinate rdim_full             = full.Grid()->_rdimensions;
-  Coordinate rdim_half             = half.Grid()->_rdimensions;
-  unsigned long ndim_half          = half.Grid()->_ndimension;
-  Coordinate checker_dim_mask_half = half.Grid()->_checker_dim_mask;
-  Coordinate ostride_half          = half.Grid()->_ostride;
-  accelerator_for(ss,full.Grid()->oSites(),full.Grid()->Nsimd(),{
-
-    Coordinate coor;
-    int cbos;
-    int linear=0;
-  
-    Lexicographic::CoorFromIndex(coor,ss,rdim_full);
-    assert(coor.size()==ndim_half);
-
-    for(int d=0;d<ndim_half;d++){ 
-      if(checker_dim_mask_half[d]) linear += coor[d];
-    }
-    cbos = (linear&0x1);
-
-    if (cbos==cb) {
-      int ssh=0;
-      for(int d=0;d<ndim_half;d++){
-        if (d == checker_dim_half) ssh += ostride_half[d] * ((coor[d] / 2) % rdim_half[d]);
-        else ssh += ostride_half[d] * (coor[d] % rdim_half[d]);
-      }
-      coalescedWrite(full_v[ss],half_v(ssh));
-    }
-
-  });
-}
-
 ////////////////////////////////////////////////////////////////////////////////////////////
 // Flexible Type Conversion for internal promotion to double as well as graceful
 // treatment of scalar-compatible types

Grid/parallelIO/IldgIO.h

@@ -576,8 +576,6 @@ class ScidacReader : public GridLimeReader {
     std::string rec_name(ILDG_BINARY_DATA);
     while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
       if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) )  ) {
-  // in principle should do the line below, but that breaks backard compatibility with old data
-  // skipPastObjectRecord(std::string(GRID_FIELD_NORM));
 	skipPastObjectRecord(std::string(SCIDAC_CHECKSUM));
 	return;
       }

Grid/qcd/QCD.h

@@ -104,6 +104,7 @@ template<typename vtype> using iSpinMatrix                = iScalar<iMatrix<iSca
 template<typename vtype> using iColourMatrix              = iScalar<iScalar<iMatrix<vtype, Nc> > > ;
 template<typename vtype> using iSpinColourMatrix          = iScalar<iMatrix<iMatrix<vtype, Nc>, Ns> >;
 template<typename vtype> using iLorentzColourMatrix       = iVector<iScalar<iMatrix<vtype, Nc> >, Nd > ;
+template<typename vtype> using iLorentzVector             = iVector<iScalar<iScalar<vtype> >, Nd > ;
 template<typename vtype> using iDoubleStoredColourMatrix  = iVector<iScalar<iMatrix<vtype, Nc> >, Nds > ;
 template<typename vtype> using iSpinVector                = iScalar<iVector<iScalar<vtype>, Ns> >;
 template<typename vtype> using iColourVector              = iScalar<iScalar<iVector<vtype, Nc> > >;
@@ -163,7 +164,16 @@ typedef iSpinColourSpinColourMatrix<vComplex >    vSpinColourSpinColourMatrix;
 typedef iSpinColourSpinColourMatrix<vComplexF>    vSpinColourSpinColourMatrixF;
 typedef iSpinColourSpinColourMatrix<vComplexD>    vSpinColourSpinColourMatrixD;
 
-// LorentzColour
+// LorentzVector
+typedef iLorentzVector<Complex  > LorentzVector;
+typedef iLorentzVector<ComplexF > LorentzVectorF;
+typedef iLorentzVector<ComplexD > LorentzVectorD;
+
+typedef iLorentzVector<vComplex > vLorentzVector;
+typedef iLorentzVector<vComplexF> vLorentzVectorF;
+typedef iLorentzVector<vComplexD> vLorentzVectorD;
+
+// LorentzColourMatrix
 typedef iLorentzColourMatrix<Complex  > LorentzColourMatrix;
 typedef iLorentzColourMatrix<ComplexF > LorentzColourMatrixF;
 typedef iLorentzColourMatrix<ComplexD > LorentzColourMatrixD;
@@ -288,6 +298,10 @@ typedef Lattice<vLorentzColourMatrix>  LatticeLorentzColourMatrix;
 typedef Lattice<vLorentzColourMatrixF> LatticeLorentzColourMatrixF;
 typedef Lattice<vLorentzColourMatrixD> LatticeLorentzColourMatrixD;
 
+typedef Lattice<vLorentzVector>  LatticeLorentzVector;
+typedef Lattice<vLorentzVectorF> LatticeLorentzVectorF;
+typedef Lattice<vLorentzVectorD> LatticeLorentzVectorD;
+
 // DoubleStored gauge field
 typedef Lattice<vDoubleStoredColourMatrix>  LatticeDoubleStoredColourMatrix;
 typedef Lattice<vDoubleStoredColourMatrixF> LatticeDoubleStoredColourMatrixF;

Grid/qcd/action/Action.h

@@ -30,8 +30,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#ifndef GRID_QCD_ACTION_H
-#define GRID_QCD_ACTION_H
+#pragma once
 
 ////////////////////////////////////////////
 // Abstract base interface
@@ -51,4 +50,4 @@ NAMESPACE_CHECK(Fermion);
 #include <Grid/qcd/action/pseudofermion/PseudoFermion.h>
 NAMESPACE_CHECK(PseudoFermion);
 
-#endif
+

Grid/qcd/action/ActionBase.h

@@ -40,6 +40,29 @@ class Action
 
 public:
   bool is_smeared = false;
+  RealD deriv_norm_sum;
+  RealD deriv_max_sum;
+  int   deriv_num;
+  RealD deriv_us;
+  RealD S_us;
+  RealD refresh_us;
+  void  reset_timer(void)        {
+    deriv_us = S_us = refresh_us = 0.0;
+    deriv_num=0;
+    deriv_norm_sum = deriv_max_sum=0.0;
+  }
+  void  deriv_log(RealD nrm, RealD max) { deriv_max_sum+=max; deriv_norm_sum+=nrm; deriv_num++;}
+  RealD deriv_max_average(void)         { return deriv_max_sum/deriv_num; };
+  RealD deriv_norm_average(void)        { return deriv_norm_sum/deriv_num; };
+  RealD deriv_timer(void)        { return deriv_us; };
+  RealD S_timer(void)            { return deriv_us; };
+  RealD refresh_timer(void)      { return deriv_us; };
+  void deriv_timer_start(void)   { deriv_us-=usecond(); }
+  void deriv_timer_stop(void)    { deriv_us+=usecond(); }
+  void refresh_timer_start(void) { refresh_us-=usecond(); }
+  void refresh_timer_stop(void)  { refresh_us+=usecond(); }
+  void S_timer_start(void)       { S_us-=usecond(); }
+  void S_timer_stop(void)        { S_us+=usecond(); }
   // Heatbath?
   virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
   virtual RealD S(const GaugeField& U) = 0;                             // evaluate the action

Grid/qcd/action/ActionCore.h

@@ -58,6 +58,8 @@ NAMESPACE_CHECK(Scalar);
 ////////////////////////////////////////////
 // Utility functions
 ////////////////////////////////////////////
+#include <Grid/qcd/action/domains/Domains.h>
+
 #include <Grid/qcd/utils/Metric.h>
 NAMESPACE_CHECK(Metric);
 #include <Grid/qcd/utils/CovariantLaplacian.h>

Grid/qcd/action/ActionParams.h

@@ -38,27 +38,32 @@ NAMESPACE_BEGIN(Grid);
 struct GparityWilsonImplParams {
   Coordinate twists; //Here the first Nd-1 directions are treated as "spatial", and a twist value of 1 indicates G-parity BCs in that direction. 
                      //mu=Nd-1 is assumed to be the time direction and a twist value of 1 indicates antiperiodic BCs
-  GparityWilsonImplParams() : twists(Nd, 0) {};
+  bool locally_periodic;
+  GparityWilsonImplParams() : twists(Nd, 0), locally_periodic(false) {};
 };
   
 struct WilsonImplParams {
   bool overlapCommsCompute;
+  bool locally_periodic;
   AcceleratorVector<Real,Nd> twist_n_2pi_L;
   AcceleratorVector<Complex,Nd> boundary_phases;
   WilsonImplParams()  {
     boundary_phases.resize(Nd, 1.0);
       twist_n_2pi_L.resize(Nd, 0.0);
+      locally_periodic = false;
   };
   WilsonImplParams(const AcceleratorVector<Complex,Nd> phi) : boundary_phases(phi), overlapCommsCompute(false) {
     twist_n_2pi_L.resize(Nd, 0.0);
+    locally_periodic = false;
   }
 };
 
 struct StaggeredImplParams {
-  StaggeredImplParams()  {};
+  bool locally_periodic;
+  StaggeredImplParams() : locally_periodic(false) {};
 };
   
-  struct OneFlavourRationalParams : Serializable {
+struct OneFlavourRationalParams : Serializable {
     GRID_SERIALIZABLE_CLASS_MEMBERS(OneFlavourRationalParams, 
 				    RealD, lo, 
 				    RealD, hi, 
@@ -66,8 +71,7 @@ struct StaggeredImplParams {
 				    RealD, tolerance, 
 				    int,   degree, 
 				    int,   precision,
-				    int,   BoundsCheckFreq,
-				    RealD, BoundsCheckTol);
+				    int,   BoundsCheckFreq);
     
   // MaxIter and tolerance, vectors??
     
@@ -78,16 +82,14 @@ struct StaggeredImplParams {
 				RealD tol      = 1.0e-8, 
                            	int _degree    = 10,
 				int _precision = 64,
-				int _BoundsCheckFreq=20,
-				double _BoundsCheckTol=1e-6)
+				int _BoundsCheckFreq=20)
       : lo(_lo),
 	hi(_hi),
 	MaxIter(_maxit),
 	tolerance(tol),
 	degree(_degree),
         precision(_precision),
-        BoundsCheckFreq(_BoundsCheckFreq),
-        BoundsCheckTol(_BoundsCheckTol){};
+        BoundsCheckFreq(_BoundsCheckFreq){};
   };
 
 

Grid/qcd/action/domains/DDHMCFilter.h

@@ -0,0 +1,52 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/DDHMC.h
+
+Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Christopher Kelly
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+
+NAMESPACE_BEGIN(Grid);
+////////////////////////////////////////////////////
+// DDHMC filter with sub-block size B[mu]
+////////////////////////////////////////////////////
+
+template<typename MomentaField>
+struct DDHMCFilter: public MomentumFilterBase<MomentaField>
+{
+  Coordinate Block;
+  int Width;
+  
+  DDHMCFilter(const Coordinate &_Block): Block(_Block) {}
+
+  void applyFilter(MomentaField &P) const override
+  {
+    DomainDecomposition Domains(Block);
+    Domains.ProjectDDHMC(P);
+  }
+};
+
+NAMESPACE_END(Grid);
+

Grid/qcd/action/domains/DirichletFilter.h

@@ -0,0 +1,98 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/momentum/DirichletFilter.h
+
+Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+
+////////////////////////////////////////////////////
+// Dirichlet filter with sub-block size B[mu]
+////////////////////////////////////////////////////
+#pragma once 
+
+#include <Grid/qcd/action/domains/DomainDecomposition.h>
+
+NAMESPACE_BEGIN(Grid);
+
+
+template<typename MomentaField>
+struct DirichletFilter: public MomentumFilterBase<MomentaField>
+{
+  Coordinate Block;
+  
+  DirichletFilter(const Coordinate &_Block): Block(_Block) {}
+
+  // Edge detect using domain projectors
+  void applyFilter (MomentaField &U) const override
+  {
+    DomainDecomposition Domains(Block);
+    GridBase *grid = U.Grid();
+    LatticeInteger  coor(grid);
+    LatticeInteger  face(grid);
+    LatticeInteger  one(grid);   one = 1;
+    LatticeInteger  zero(grid); zero = 0;
+    LatticeInteger  omega(grid);
+    LatticeInteger  omegabar(grid);
+    LatticeInteger  tmp(grid);
+
+    omega=one;    Domains.ProjectDomain(omega,0);
+    omegabar=one; Domains.ProjectDomain(omegabar,1);
+    
+    LatticeInteger nface(grid); nface=Zero();
+    
+    MomentaField projected(grid); projected=Zero();
+    typedef decltype(PeekIndex<LorentzIndex>(U,0)) MomentaLinkField;
+    MomentaLinkField  Umu(grid);
+    MomentaLinkField   zz(grid); zz=Zero();
+
+    int dims = grid->Nd();
+    Coordinate Global=grid->GlobalDimensions();
+    assert(dims==Nd);
+
+    for(int mu=0;mu<Nd;mu++){
+
+      if ( Block[mu]!=0 ) {
+
+	Umu = PeekIndex<LorentzIndex>(U,mu);
+
+	// Upper face 
+ 	tmp = Cshift(omegabar,mu,1);
+	tmp = tmp + omega;
+	face = where(tmp == Integer(2),one,zero );
+
+ 	tmp = Cshift(omega,mu,1);
+	tmp = tmp + omegabar;
+	face = where(tmp == Integer(2),one,face );
+
+	Umu = where(face,zz,Umu);
+
+	PokeIndex<LorentzIndex>(U, Umu, mu);
+      }
+    }
+  }
+  
+};
+
+NAMESPACE_END(Grid);

Grid/qcd/action/domains/DomainDecomposition.h

@@ -0,0 +1,187 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/domains/DomainDecomposition.h
+
+Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+
+////////////////////////////////////////////////////
+// Dirichlet filter with sub-block size B[mu]
+////////////////////////////////////////////////////
+#pragma once 
+
+
+NAMESPACE_BEGIN(Grid);
+
+
+struct DomainDecomposition
+{
+  Coordinate Block;
+  static constexpr RealD factor = 0.6;
+
+  DomainDecomposition(const Coordinate &_Block): Block(_Block){ assert(Block.size()==Nd);};
+  
+  template<class Field>
+  void ProjectDomain(Field &f,Integer domain)
+  {
+    GridBase *grid = f.Grid();
+    int dims = grid->Nd();
+    int isDWF= (dims==Nd+1);
+    assert((dims==Nd)||(dims==Nd+1));
+
+    Field   zz(grid);  zz = Zero();
+    LatticeInteger coor(grid);
+    LatticeInteger domaincoor(grid);
+    LatticeInteger mask(grid); mask = Integer(1);
+    LatticeInteger zi(grid);     zi = Integer(0);
+    for(int d=0;d<Nd;d++){
+      Integer B= Block[d];
+      if ( B ) {
+	LatticeCoordinate(coor,d+isDWF);
+	domaincoor = mod(coor,B);
+	mask = where(domaincoor==Integer(0),zi,mask);
+	mask = where(domaincoor==Integer(B-1),zi,mask);
+      }
+    }
+    if ( !domain )
+      f = where(mask==Integer(1),f,zz);
+    else 
+      f = where(mask==Integer(0),f,zz);
+  };
+  template<class GaugeField>
+  void ProjectDDHMC(GaugeField &U)
+  {
+    GridBase *grid = U.Grid();
+    Coordinate Global=grid->GlobalDimensions();
+    GaugeField zzz(grid); zzz = Zero();
+    LatticeInteger coor(grid); 
+
+    GaugeField Uorg(grid); Uorg = U;
+    
+    auto zzz_mu = PeekIndex<LorentzIndex>(zzz,0);
+    ////////////////////////////////////////////////////
+    // Zero BDY layers
+    ////////////////////////////////////////////////////
+    for(int mu=0;mu<Nd;mu++) {
+      Integer B1 = Block[mu];
+      if ( B1 && (B1 <= Global[mu]) ) {
+	LatticeCoordinate(coor,mu);
+
+
+	////////////////////////////////
+	// OmegaBar - zero all links contained in slice B-1,0 and
+	// mu links connecting to Omega
+	////////////////////////////////
+
+	U    = where(mod(coor,B1)==Integer(B1-1),zzz,U);
+	U    = where(mod(coor,B1)==Integer(0)   ,zzz,U); 
+
+	auto U_mu   = PeekIndex<LorentzIndex>(U,mu);
+	U_mu = where(mod(coor,B1)==Integer(B1-2),zzz_mu,U_mu); 
+	PokeIndex<LorentzIndex>(U, U_mu, mu);
+
+      }
+    }
+   
+    ////////////////////////////////////////////
+    // Omega interior slow the evolution
+    // Tricky as we need to take the smallest of values imposed by each cut
+    // Do them in order or largest to smallest and smallest writes last
+    ////////////////////////////////////////////
+    RealD f= factor;
+#if 0    
+    for(int mu=0;mu<Nd;mu++) {
+      Integer B1 = Block[mu];
+      if ( B1 && (B1 <= Global[mu]) ) {
+
+	auto U_mu   = PeekIndex<LorentzIndex>(U,mu);
+	auto Uorg_mu= PeekIndex<LorentzIndex>(Uorg,mu);
+	// In the plane
+	U = where(mod(coor,B1)==Integer(B1-5),Uorg*f,U); 
+	U = where(mod(coor,B1)==Integer(4)   ,Uorg*f,U); 
+
+	// Perp links
+       	U_mu = where(mod(coor,B1)==Integer(B1-6),Uorg_mu*f,U_mu);
+	U_mu = where(mod(coor,B1)==Integer(4)   ,Uorg_mu*f,U_mu);
+
+	PokeIndex<LorentzIndex>(U, U_mu, mu);
+      }
+    }
+#endif
+    for(int mu=0;mu<Nd;mu++) {
+      Integer B1 = Block[mu];
+      if ( B1 && (B1 <= Global[mu]) ) {
+
+	auto U_mu   = PeekIndex<LorentzIndex>(U,mu);
+	auto Uorg_mu= PeekIndex<LorentzIndex>(Uorg,mu);
+	// In the plane
+	U = where(mod(coor,B1)==Integer(B1-4),Uorg*f*f,U); 
+	U = where(mod(coor,B1)==Integer(3)   ,Uorg*f*f,U); 
+
+	// Perp links
+       	U_mu = where(mod(coor,B1)==Integer(B1-5),Uorg_mu*f*f,U_mu);
+	U_mu = where(mod(coor,B1)==Integer(3)   ,Uorg_mu*f*f,U_mu);
+
+	PokeIndex<LorentzIndex>(U, U_mu, mu);
+      }
+    }
+    for(int mu=0;mu<Nd;mu++) {
+      Integer B1 = Block[mu];
+      if ( B1 && (B1 <= Global[mu]) ) {
+
+	auto U_mu   = PeekIndex<LorentzIndex>(U,mu);
+	auto Uorg_mu= PeekIndex<LorentzIndex>(Uorg,mu);
+	// In the plane
+	U = where(mod(coor,B1)==Integer(B1-3),Uorg*f*f*f,U); 
+	U = where(mod(coor,B1)==Integer(2)   ,Uorg*f*f*f,U); 
+
+	// Perp links
+       	U_mu = where(mod(coor,B1)==Integer(B1-4),Uorg_mu*f*f*f,U_mu);
+	U_mu = where(mod(coor,B1)==Integer(2)   ,Uorg_mu*f*f*f,U_mu);
+
+	PokeIndex<LorentzIndex>(U, U_mu, mu);
+      }
+    }
+    for(int mu=0;mu<Nd;mu++) {
+      Integer B1 = Block[mu];
+      if ( B1 && (B1 <= Global[mu]) ) {
+
+	auto U_mu   = PeekIndex<LorentzIndex>(U,mu);
+	auto Uorg_mu= PeekIndex<LorentzIndex>(Uorg,mu);
+	// In the plane
+	U = where(mod(coor,B1)==Integer(B1-2),zzz,U); 
+	U = where(mod(coor,B1)==Integer(1)   ,zzz,U); 
+
+	// Perp links
+	U_mu = where(mod(coor,B1)==Integer(B1-3),Uorg_mu*f*f*f*f,U_mu);
+	U_mu = where(mod(coor,B1)==Integer(1)   ,Uorg_mu*f*f*f*f,U_mu);
+
+	PokeIndex<LorentzIndex>(U, U_mu, mu);
+      }
+    }
+  }
+};
+
+NAMESPACE_END(Grid);

Grid/qcd/action/domains/Domains.h

@@ -0,0 +1,39 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/momentum/Domains.h
+
+Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+			   /*  END LEGAL */
+
+////////////////////////////////////////////////////
+// Dirichlet filter with sub-block size B[mu]
+////////////////////////////////////////////////////
+#pragma once 
+
+#include <Grid/qcd/action/domains/DomainDecomposition.h>
+#include <Grid/qcd/action/domains/MomentumFilter.h>
+#include <Grid/qcd/action/domains/DirichletFilter.h>
+#include <Grid/qcd/action/domains/DDHMCFilter.h>
+

Grid/qcd/action/domains/MomentumFilter.h

@@ -28,8 +28,7 @@ directory
 *************************************************************************************/
 /*  END LEGAL */
 //--------------------------------------------------------------------
-#ifndef MOMENTUM_FILTER
-#define MOMENTUM_FILTER
+#pragma once 
 
 NAMESPACE_BEGIN(Grid);
 
@@ -37,7 +36,7 @@ NAMESPACE_BEGIN(Grid);
 
 template<typename MomentaField>
 struct MomentumFilterBase{
-  virtual void applyFilter(MomentaField &P) const;
+  virtual void applyFilter(MomentaField &P) const = 0;
 };
 
 //Do nothing
@@ -90,5 +89,3 @@ struct MomentumFilterApplyPhase: public MomentumFilterBase<MomentaField>{
 
 
 NAMESPACE_END(Grid);
-
-#endif

Grid/qcd/action/fermion/CayleyFermion5D.h

@@ -60,6 +60,8 @@ public:
   ///////////////////////////////////////////////////////////////
   virtual void Dminus(const FermionField &psi, FermionField &chi);
   virtual void DminusDag(const FermionField &psi, FermionField &chi);
+  virtual void ImportFourDimPseudoFermion(const FermionField &input,FermionField &imported);
+  virtual void ExportFourDimPseudoFermion(const FermionField &solution,FermionField &exported);
   virtual void ExportPhysicalFermionSolution(const FermionField &solution5d,FermionField &exported4d);
   virtual void ExportPhysicalFermionSource(const FermionField &solution5d, FermionField &exported4d);
   virtual void ImportPhysicalFermionSource(const FermionField &input4d,FermionField &imported5d);

Grid/qcd/action/fermion/CompactWilsonCloverFermion.h

@@ -1,240 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: ./lib/qcd/action/fermion/CompactWilsonCloverFermion.h
-
-    Copyright (C) 2020 - 2022
-
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
-    Author: Nils Meyer <nils.meyer@ur.de>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-/*  END LEGAL */
-
-#pragma once
-
-#include <Grid/qcd/action/fermion/WilsonCloverTypes.h>
-#include <Grid/qcd/action/fermion/WilsonCloverHelpers.h>
-
-NAMESPACE_BEGIN(Grid);
-
-// see Grid/qcd/action/fermion/WilsonCloverFermion.h for description
-//
-// Modifications done here:
-//
-// Original: clover term = 12x12 matrix per site
-//
-// But: Only two diagonal 6x6 hermitian blocks are non-zero (also true for original, verified by running)
-// Sufficient to store/transfer only the real parts of the diagonal and one triangular part
-// 2 * (6 + 15 * 2) = 72 real or 36 complex words to be stored/transfered
-//
-// Here: Above but diagonal as complex numbers, i.e., need to store/transfer
-// 2 * (6 * 2 + 15 * 2) = 84 real or 42 complex words
-//
-// Words per site and improvement compared to original (combined with the input and output spinors):
-//
-// - Original: 2*12 + 12*12 = 168 words -> 1.00 x less
-// - Minimal:  2*12 + 36    =  60 words -> 2.80 x less
-// - Here:     2*12 + 42    =  66 words -> 2.55 x less
-//
-// These improvements directly translate to wall-clock time
-//
-// Data layout:
-//
-// - diagonal and triangle part as separate lattice fields,
-//   this was faster than as 1 combined field on all tested machines
-// - diagonal: as expected
-// - triangle: store upper right triangle in row major order
-// - graphical:
-//        0  1  2  3  4
-//           5  6  7  8
-//              9 10 11 = upper right triangle indices
-//                12 13
-//                   14
-//     0
-//        1
-//           2
-//              3       = diagonal indices
-//                 4
-//                    5
-//     0
-//     1  5
-//     2  6  9          = lower left triangle indices
-//     3  7 10 12
-//     4  8 11 13 14
-//
-// Impact on total memory consumption:
-// - Original: (2 * 1 + 8 * 1/2) 12x12 matrices = 6 12x12 matrices = 864 complex words per site
-// - Here:     (2 * 1 + 4 * 1/2) diagonal parts = 4 diagonal parts =  24 complex words per site
-//           + (2 * 1 + 4 * 1/2) triangle parts = 4 triangle parts =  60 complex words per site
-//                                                                 =  84 complex words per site
-
-template<class Impl>
-class CompactWilsonCloverFermion : public WilsonFermion<Impl>,
-                                   public WilsonCloverHelpers<Impl>,
-                                   public CompactWilsonCloverHelpers<Impl> {
-  /////////////////////////////////////////////
-  // Sizes
-  /////////////////////////////////////////////
-
-public:
-
-  INHERIT_COMPACT_CLOVER_SIZES(Impl);
-
-  /////////////////////////////////////////////
-  // Type definitions
-  /////////////////////////////////////////////
-
-public:
-
-  INHERIT_IMPL_TYPES(Impl);
-  INHERIT_CLOVER_TYPES(Impl);
-  INHERIT_COMPACT_CLOVER_TYPES(Impl);
-
-  typedef WilsonFermion<Impl>              WilsonBase;
-  typedef WilsonCloverHelpers<Impl>        Helpers;
-  typedef CompactWilsonCloverHelpers<Impl> CompactHelpers;
-
-  /////////////////////////////////////////////
-  // Constructors
-  /////////////////////////////////////////////
-
-public:
-
-  CompactWilsonCloverFermion(GaugeField& _Umu,
-			    GridCartesian& Fgrid,
-			    GridRedBlackCartesian& Hgrid,
-			    const RealD _mass,
-			    const RealD _csw_r = 0.0,
-			    const RealD _csw_t = 0.0,
-			    const RealD _cF = 1.0,
-			    const WilsonAnisotropyCoefficients& clover_anisotropy = WilsonAnisotropyCoefficients(),
-			    const ImplParams& impl_p = ImplParams());
-
-  /////////////////////////////////////////////
-  // Member functions (implementing interface)
-  /////////////////////////////////////////////
-
-public:
-
-  virtual void Instantiatable() {};
-  int          ConstEE()     override { return 0; };
-  int          isTrivialEE() override { return 0; };
-
-  void Dhop(const FermionField& in, FermionField& out, int dag) override;
-
-  void DhopOE(const FermionField& in, FermionField& out, int dag) override;
-
-  void DhopEO(const FermionField& in, FermionField& out, int dag) override;
-
-  void DhopDir(const FermionField& in, FermionField& out, int dir, int disp) override;
-
-  void DhopDirAll(const FermionField& in, std::vector<FermionField>& out) /* override */;
-
-  void M(const FermionField& in, FermionField& out) override;
-
-  void Mdag(const FermionField& in, FermionField& out) override;
-
-  void Meooe(const FermionField& in, FermionField& out) override;
-
-  void MeooeDag(const FermionField& in, FermionField& out) override;
-
-  void Mooee(const FermionField& in, FermionField& out) override;
-
-  void MooeeDag(const FermionField& in, FermionField& out) override;
-
-  void MooeeInv(const FermionField& in, FermionField& out) override;
-
-  void MooeeInvDag(const FermionField& in, FermionField& out) override;
-
-  void Mdir(const FermionField& in, FermionField& out, int dir, int disp) override;
-
-  void MdirAll(const FermionField& in, std::vector<FermionField>& out) override;
-
-  void MDeriv(GaugeField& force, const FermionField& X, const FermionField& Y, int dag) override;
-
-  void MooDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag) override;
-
-  void MeeDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag) override;
-
-  /////////////////////////////////////////////
-  // Member functions (internals)
-  /////////////////////////////////////////////
-
-  void MooeeInternal(const FermionField&        in,
-                     FermionField&              out,
-                     const CloverDiagonalField& diagonal,
-                     const CloverTriangleField& triangle);
-
-  /////////////////////////////////////////////
-  // Helpers
-  /////////////////////////////////////////////
-
-  void ImportGauge(const GaugeField& _Umu) override;
-
-  /////////////////////////////////////////////
-  // Helpers
-  /////////////////////////////////////////////
-
-private:
-
-  template<class Field>
-  const MaskField* getCorrectMaskField(const Field &in) const {
-    if(in.Grid()->_isCheckerBoarded) {
-      if(in.Checkerboard() == Odd) {
-        return &this->BoundaryMaskOdd;
-      } else {
-        return &this->BoundaryMaskEven;
-      }
-    } else {
-      return &this->BoundaryMask;
-    }
-  }
-
-  template<class Field>
-  void ApplyBoundaryMask(Field& f) {
-    const MaskField* m = getCorrectMaskField(f); assert(m != nullptr);
-    assert(m != nullptr);
-    CompactHelpers::ApplyBoundaryMask(f, *m);
-  }
-
-  /////////////////////////////////////////////
-  // Member Data
-  /////////////////////////////////////////////
-
-public:
-
-  RealD csw_r;
-  RealD csw_t;
-  RealD cF;
-
-  bool open_boundaries;
-
-  CloverDiagonalField Diagonal,    DiagonalEven,    DiagonalOdd;
-  CloverDiagonalField DiagonalInv, DiagonalInvEven, DiagonalInvOdd;
-
-  CloverTriangleField Triangle,    TriangleEven,    TriangleOdd;
-  CloverTriangleField TriangleInv, TriangleInvEven, TriangleInvOdd;
-
-  FermionField Tmp;
-
-  MaskField BoundaryMask, BoundaryMaskEven, BoundaryMaskOdd;
-};
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/DirichletFermionOperator.h

@@ -0,0 +1,185 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/fermion/DirichletFermionOperator.h
+
+    Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+////////////////////////////////////////////////////////////////
+// Wrap a fermion operator in Dirichlet BC's at node boundary
+////////////////////////////////////////////////////////////////
+    
+template<class Impl>
+class DirichletFermionOperator : public FermionOperator<Impl>
+{
+public:
+
+  INHERIT_IMPL_TYPES(Impl);
+
+  // Data members
+  int CommsMode;
+  Coordinate Block;
+  DirichletFilter<GaugeField> Filter;
+  FermionOperator<Impl> & FermOp;
+  
+  // Constructor / bespoke
+  DirichletFermionOperator(FermionOperator<Impl> & _FermOp, Coordinate &_Block)
+    : FermOp(_FermOp), Block(_Block), Filter(Block)
+  {
+    // Save what the comms mode should be under normal BCs
+    CommsMode = WilsonKernelsStatic::Comms;
+    assert((CommsMode == WilsonKernelsStatic::CommsAndCompute)
+         ||(CommsMode == WilsonKernelsStatic::CommsThenCompute));
+
+    // Check the block size divides local lattice
+    GridBase *grid = FermOp.GaugeGrid();
+
+    int blocks_per_rank = 1;
+    Coordinate LocalDims = grid->LocalDimensions();
+    Coordinate GlobalDims= grid->GlobalDimensions();
+    assert(Block.size()==LocalDims.size());
+
+    for(int d=0;d<LocalDims.size();d++){
+      if (Block[d]&&(Block[d]<=GlobalDims[d])){
+	int r = LocalDims[d] % Block[d];
+	assert(r == 0);
+	blocks_per_rank *= (LocalDims[d] / Block[d]);
+      }
+    }
+    // Even blocks per node required // could be relaxed but inefficient use of hardware as idle nodes in boundary operator R
+    assert( blocks_per_rank != 0);
+
+    // Possible checks that SIMD lanes are used with full occupancy???
+  };
+  virtual ~DirichletFermionOperator(void) = default;
+
+  void DirichletOn(void)   {
+    assert(WilsonKernelsStatic::Comms!= WilsonKernelsStatic::CommsDirichlet);
+    //    WilsonKernelsStatic::Comms = WilsonKernelsStatic::CommsDirichlet;
+  }
+  void DirichletOff(void)  {
+    //    assert(WilsonKernelsStatic::Comms== WilsonKernelsStatic::CommsDirichlet);
+    //    WilsonKernelsStatic::Comms = CommsMode;
+  }
+
+  // Implement the full interface
+  virtual FermionField &tmp(void) { return FermOp.tmp(); };
+
+  virtual GridBase *FermionGrid(void)         { return FermOp.FermionGrid(); }
+  virtual GridBase *FermionRedBlackGrid(void) { return FermOp.FermionRedBlackGrid(); }
+  virtual GridBase *GaugeGrid(void)           { return FermOp.GaugeGrid(); }
+  virtual GridBase *GaugeRedBlackGrid(void)   { return FermOp.GaugeRedBlackGrid(); }
+  
+  // override multiply
+  virtual void  M    (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.M(in,out);    DirichletOff();  };
+  virtual void  Mdag (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.Mdag(in,out); DirichletOff();  };
+
+  // half checkerboard operaions
+  virtual void   Meooe       (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.Meooe(in,out);    DirichletOff(); };  
+  virtual void   MeooeDag    (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.MeooeDag(in,out); DirichletOff(); };
+  virtual void   Mooee       (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.Mooee(in,out);    DirichletOff(); };
+  virtual void   MooeeDag    (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.MooeeDag(in,out); DirichletOff(); };
+  virtual void   MooeeInv    (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.MooeeInv(in,out); DirichletOff(); };
+  virtual void   MooeeInvDag (const FermionField &in, FermionField &out) { DirichletOn(); FermOp.MooeeInvDag(in,out); DirichletOff(); };
+
+  // non-hermitian hopping term; half cb or both
+  virtual void Dhop  (const FermionField &in, FermionField &out,int dag) { DirichletOn(); FermOp.Dhop(in,out,dag);    DirichletOff(); };
+  virtual void DhopOE(const FermionField &in, FermionField &out,int dag) { DirichletOn(); FermOp.DhopOE(in,out,dag);  DirichletOff(); };
+  virtual void DhopEO(const FermionField &in, FermionField &out,int dag) { DirichletOn(); FermOp.DhopEO(in,out,dag);  DirichletOff(); };
+  virtual void DhopDir(const FermionField &in, FermionField &out,int dir,int disp) { DirichletOn(); FermOp.DhopDir(in,out,dir,disp);  DirichletOff(); };
+
+  // force terms; five routines; default to Dhop on diagonal
+  virtual void MDeriv  (GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.MDeriv(mat,U,V,dag);};
+  virtual void MoeDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.MoeDeriv(mat,U,V,dag);};
+  virtual void MeoDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.MeoDeriv(mat,U,V,dag);};
+  virtual void MooDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.MooDeriv(mat,U,V,dag);};
+  virtual void MeeDeriv(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.MeeDeriv(mat,U,V,dag);};
+
+  virtual void DhopDeriv  (GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.DhopDeriv(mat,U,V,dag);};
+  virtual void DhopDerivEO(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.DhopDerivEO(mat,U,V,dag);};
+  virtual void DhopDerivOE(GaugeField &mat,const FermionField &U,const FermionField &V,int dag){FermOp.DhopDerivOE(mat,U,V,dag);};
+
+  virtual void  Mdiag  (const FermionField &in, FermionField &out) { Mooee(in,out);};
+  virtual void  Mdir   (const FermionField &in, FermionField &out,int dir,int disp){FermOp.Mdir(in,out,dir,disp);};
+  virtual void  MdirAll(const FermionField &in, std::vector<FermionField> &out)    {FermOp.MdirAll(in,out);};
+
+  ///////////////////////////////////////////////
+  // Updates gauge field during HMC
+  ///////////////////////////////////////////////
+  DoubledGaugeField &GetDoubledGaugeField(void){ return FermOp.GetDoubledGaugeField(); };
+  DoubledGaugeField &GetDoubledGaugeFieldE(void){ return FermOp.GetDoubledGaugeFieldE(); };
+  DoubledGaugeField &GetDoubledGaugeFieldO(void){ return FermOp.GetDoubledGaugeFieldO(); };
+  virtual void ImportGauge(const GaugeField & _U)
+  {
+    GaugeField U = _U;
+    // Filter gauge field to apply Dirichlet
+    Filter.applyFilter(U);
+    FermOp.ImportGauge(U);
+  }
+  ///////////////////////////////////////////////
+  // Physical field import/export
+  ///////////////////////////////////////////////
+  virtual void Dminus(const FermionField &psi, FermionField &chi)    { FermOp.Dminus(psi,chi); }
+  virtual void DminusDag(const FermionField &psi, FermionField &chi) { FermOp.DminusDag(psi,chi); }
+  virtual void ImportFourDimPseudoFermion(const FermionField &input,FermionField &imported)   { FermOp.ImportFourDimPseudoFermion(input,imported);}
+  virtual void ExportFourDimPseudoFermion(const FermionField &solution,FermionField &exported){ FermOp.ExportFourDimPseudoFermion(solution,exported);}
+  virtual void ImportPhysicalFermionSource(const FermionField &input,FermionField &imported)  { FermOp.ImportPhysicalFermionSource(input,imported);}
+  virtual void ImportUnphysicalFermion(const FermionField &input,FermionField &imported)      { FermOp.ImportUnphysicalFermion(input,imported);}
+  virtual void ExportPhysicalFermionSolution(const FermionField &solution,FermionField &exported) {FermOp.ExportPhysicalFermionSolution(solution,exported);}
+  virtual void ExportPhysicalFermionSource(const FermionField &solution,FermionField &exported)   {FermOp.ExportPhysicalFermionSource(solution,exported);}
+  //////////////////////////////////////////////////////////////////////
+  // Should never be used
+  //////////////////////////////////////////////////////////////////////
+  virtual void MomentumSpacePropagator(FermionField &out,const FermionField &in,RealD _m,std::vector<double> twist) { assert(0);};
+  virtual void FreePropagator(const FermionField &in,FermionField &out,RealD mass,std::vector<Complex> boundary,std::vector<double> twist) {assert(0);}
+  virtual void FreePropagator(const FermionField &in,FermionField &out,RealD mass) { assert(0);}
+  virtual void ContractConservedCurrent(PropagatorField &q_in_1,
+					PropagatorField &q_in_2,
+					PropagatorField &q_out,
+					PropagatorField &phys_src,
+					Current curr_type,
+					unsigned int mu)
+  {assert(0);};
+  virtual void SeqConservedCurrent(PropagatorField &q_in, 
+				   PropagatorField &q_out,
+				   PropagatorField &phys_src,
+				   Current curr_type,
+				   unsigned int mu,
+				   unsigned int tmin, 
+				   unsigned int tmax,
+				   ComplexField &lattice_cmplx)
+  {assert(0);};
+      // Only reimplemented in Wilson5D 
+      // Default to just a zero correlation function
+  virtual void ContractJ5q(FermionField &q_in   ,ComplexField &J5q) { J5q=Zero(); };
+  virtual void ContractJ5q(PropagatorField &q_in,ComplexField &J5q) { J5q=Zero(); };
+  
+};
+
+
+NAMESPACE_END(Grid);
+

Grid/qcd/action/fermion/Fermion.h

@@ -53,7 +53,6 @@ NAMESPACE_CHECK(Wilson);
 #include <Grid/qcd/action/fermion/WilsonTMFermion.h>       // 4d wilson like
 NAMESPACE_CHECK(WilsonTM);
 #include <Grid/qcd/action/fermion/WilsonCloverFermion.h> // 4d wilson clover fermions
-#include <Grid/qcd/action/fermion/CompactWilsonCloverFermion.h> // 4d compact wilson clover fermions
 NAMESPACE_CHECK(WilsonClover);
 #include <Grid/qcd/action/fermion/WilsonFermion5D.h>     // 5d base used by all 5d overlap types
 NAMESPACE_CHECK(Wilson5D);
@@ -102,6 +101,12 @@ NAMESPACE_CHECK(WilsonTM5);
 #include <Grid/qcd/action/fermion/PauliVillarsInverters.h>
 #include <Grid/qcd/action/fermion/Reconstruct5Dprop.h>
 #include <Grid/qcd/action/fermion/MADWF.h>
+////////////////////////////////////////////////////////////////////
+// DDHMC related 
+////////////////////////////////////////////////////////////////////
+#include <Grid/qcd/action/fermion/DirichletFermionOperator.h>
+#include <Grid/qcd/action/fermion/SchurFactoredFermionOperator.h>
+
 NAMESPACE_CHECK(DWFutils);
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -154,23 +159,6 @@ typedef WilsonCloverFermion<WilsonTwoIndexAntiSymmetricImplR> WilsonCloverTwoInd
 typedef WilsonCloverFermion<WilsonTwoIndexAntiSymmetricImplF> WilsonCloverTwoIndexAntiSymmetricFermionF;
 typedef WilsonCloverFermion<WilsonTwoIndexAntiSymmetricImplD> WilsonCloverTwoIndexAntiSymmetricFermionD;
 
-// Compact Clover fermions
-typedef CompactWilsonCloverFermion<WilsonImplR> CompactWilsonCloverFermionR;
-typedef CompactWilsonCloverFermion<WilsonImplF> CompactWilsonCloverFermionF;
-typedef CompactWilsonCloverFermion<WilsonImplD> CompactWilsonCloverFermionD;
-
-typedef CompactWilsonCloverFermion<WilsonAdjImplR> CompactWilsonCloverAdjFermionR;
-typedef CompactWilsonCloverFermion<WilsonAdjImplF> CompactWilsonCloverAdjFermionF;
-typedef CompactWilsonCloverFermion<WilsonAdjImplD> CompactWilsonCloverAdjFermionD;
-
-typedef CompactWilsonCloverFermion<WilsonTwoIndexSymmetricImplR> CompactWilsonCloverTwoIndexSymmetricFermionR;
-typedef CompactWilsonCloverFermion<WilsonTwoIndexSymmetricImplF> CompactWilsonCloverTwoIndexSymmetricFermionF;
-typedef CompactWilsonCloverFermion<WilsonTwoIndexSymmetricImplD> CompactWilsonCloverTwoIndexSymmetricFermionD;
-
-typedef CompactWilsonCloverFermion<WilsonTwoIndexAntiSymmetricImplR> CompactWilsonCloverTwoIndexAntiSymmetricFermionR;
-typedef CompactWilsonCloverFermion<WilsonTwoIndexAntiSymmetricImplF> CompactWilsonCloverTwoIndexAntiSymmetricFermionF;
-typedef CompactWilsonCloverFermion<WilsonTwoIndexAntiSymmetricImplD> CompactWilsonCloverTwoIndexAntiSymmetricFermionD;
-
 // Domain Wall fermions
 typedef DomainWallFermion<WilsonImplR> DomainWallFermionR;
 typedef DomainWallFermion<WilsonImplF> DomainWallFermionF;

Grid/qcd/action/fermion/FermionCore.h

@@ -25,8 +25,7 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#ifndef  GRID_QCD_FERMION_CORE_H
-#define  GRID_QCD_FERMION_CORE_H
+#pragma once
 
 #include <Grid/GridCore.h>
 #include <Grid/GridQCDcore.h>
@@ -45,4 +44,3 @@ NAMESPACE_CHECK(FermionOperator);
 #include <Grid/qcd/action/fermion/StaggeredKernels.h>        //used by all wilson type fermions
 NAMESPACE_CHECK(Kernels);
 
-#endif

Grid/qcd/action/fermion/FermionOperator.h

@@ -140,6 +140,9 @@ public:
   // Updates gauge field during HMC
   ///////////////////////////////////////////////
   virtual void ImportGauge(const GaugeField & _U)=0;
+  virtual DoubledGaugeField &GetDoubledGaugeField(void)  =0;
+  virtual DoubledGaugeField &GetDoubledGaugeFieldE(void)  =0;
+  virtual DoubledGaugeField &GetDoubledGaugeFieldO(void)  =0;
 
   //////////////////////////////////////////////////////////////////////
   // Conserved currents, either contract at sink or insert sequentially.
@@ -171,6 +174,16 @@ public:
       ///////////////////////////////////////////////
       virtual void Dminus(const FermionField &psi, FermionField &chi)    { chi=psi; }
       virtual void DminusDag(const FermionField &psi, FermionField &chi) { chi=psi; }
+
+      virtual void ImportFourDimPseudoFermion(const FermionField &input,FermionField &imported)
+      {
+	imported = input;
+      };
+      virtual void ExportFourDimPseudoFermion(const FermionField &solution,FermionField &exported)
+      {
+	exported=solution;
+      };
+
       virtual void ImportPhysicalFermionSource(const FermionField &input,FermionField &imported)
       {
 	imported = input;

Grid/qcd/action/fermion/ImprovedStaggeredFermion.h

@@ -141,8 +141,11 @@ public:
   void ImportGauge(const GaugeField &_Uthin, const GaugeField &_Ufat);
   void ImportGaugeSimple(const GaugeField &_UUU    ,const GaugeField &_U);
   void ImportGaugeSimple(const DoubledGaugeField &_UUU,const DoubledGaugeField &_U);
-  DoubledGaugeField &GetU(void)   { return Umu ; } ;
-  DoubledGaugeField &GetUUU(void) { return UUUmu; };
+  virtual DoubledGaugeField &GetDoubledGaugeField(void)  override { return Umu; };
+  virtual DoubledGaugeField &GetDoubledGaugeFieldE(void) override { return UmuEven; };
+  virtual DoubledGaugeField &GetDoubledGaugeFieldO(void) override { return UmuOdd; };
+  virtual DoubledGaugeField &GetU(void)   { return Umu ; } ;
+  virtual DoubledGaugeField &GetUUU(void) { return UUUmu; };
   void CopyGaugeCheckerboards(void);
 
   ///////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/ImprovedStaggeredFermion5D.h

@@ -160,17 +160,20 @@ public:
 			       RealD _c1=1.0, RealD _c2=1.0,RealD _u0=1.0,
 			     const ImplParams &p= ImplParams());
     
-    // DoubleStore gauge field in operator
-    void ImportGauge      (const GaugeField &_Uthin ) { assert(0); }
+  // DoubleStore gauge field in operator
+  void ImportGauge      (const GaugeField &_Uthin ) { assert(0); }
   void ImportGauge(const GaugeField &_Uthin,const GaugeField &_Ufat);
-    void ImportGaugeSimple(const GaugeField &_UUU,const GaugeField &_U);
-    void ImportGaugeSimple(const DoubledGaugeField &_UUU,const DoubledGaugeField &_U);
-    // Give a reference; can be used to do an assignment or copy back out after import
-    // if Carleton wants to cache them and not use the ImportSimple
-    DoubledGaugeField &GetU(void)   { return Umu ; } ;
-    DoubledGaugeField &GetUUU(void) { return UUUmu; };
-    void CopyGaugeCheckerboards(void);
-    
+  void ImportGaugeSimple(const GaugeField &_UUU,const GaugeField &_U);
+  void ImportGaugeSimple(const DoubledGaugeField &_UUU,const DoubledGaugeField &_U);
+  // Give a reference; can be used to do an assignment or copy back out after import
+  // if Carleton wants to cache them and not use the ImportSimple
+  virtual DoubledGaugeField &GetDoubledGaugeField(void)  override { return Umu; };
+  virtual DoubledGaugeField &GetDoubledGaugeFieldE(void) override { return UmuEven; };
+  virtual DoubledGaugeField &GetDoubledGaugeFieldO(void) override { return UmuOdd; };
+  DoubledGaugeField &GetU(void)   { return Umu ; } ;
+  DoubledGaugeField &GetUUU(void) { return UUUmu; };
+  void CopyGaugeCheckerboards(void);
+  
   ///////////////////////////////////////////////////////////////
   // Data members require to support the functionality
   ///////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/NaiveStaggeredFermion.h

@@ -135,6 +135,9 @@ public:
 
   // DoubleStore impl dependent
   void ImportGauge      (const GaugeField &_U );
+  DoubledGaugeField &GetDoubledGaugeField(void){ return Umu; };
+  DoubledGaugeField &GetDoubledGaugeFieldE(void){ return UmuEven; };
+  DoubledGaugeField &GetDoubledGaugeFieldO(void){ return UmuOdd; };
   DoubledGaugeField &GetU(void)   { return Umu ; } ;
   void CopyGaugeCheckerboards(void);
 

Grid/qcd/action/fermion/SchurFactoredFermionOperator.h

@@ -0,0 +1,534 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/fermion/SchurFactoredFermionOperator.h
+
+    Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+#include <Grid/qcd/utils/MixedPrecisionOperatorFunction.h>
+#include <Grid/qcd/action/domains/Domains.h>
+
+NAMESPACE_BEGIN(Grid);
+
+  ////////////////////////////////////////////////////////
+  // Some explanation of class structure for domain decomposition:
+  //
+  // Need a dirichlet operator for two flavour determinant - acts on both Omega and OmegaBar.
+  //
+  // Possible gain if the global sums and CG are run independently?? Could measure this.
+  //
+  // Types of operations
+  //
+  // 1) assemble local det dOmega det dOmegaBar pseudofermion
+  //
+  // - DirichletFermionOperator - can either do a global solve, or independent/per cell coefficients.
+  //
+  // 2) assemble dOmegaInverse and dOmegaBarInverse in R
+  //
+  // - DirichletFermionOperator - can also be used to 
+  //                                       - need two or more cells per node. Options
+  //                                       - a) solve one cell at a time, no new code, CopyRegion and reduced /split Grids
+  //                                       - b) solve multiple cells in parallel. predicated dslash implementation
+  //
+  //                                       - b) has more parallelism, experience with block solver suggest might not be aalgorithmically inefficient
+  //                                         a) has more cache friendly and easier code.
+  //                                         b) is easy to implement in a "trial" or inefficient code with projection.
+  //
+  // 3)  Additional functionality for domain operations
+  //
+  // - SchurFactoredFermionOperator  - Need a DDHMC utility - whether used in two flavour or one flavour 
+  //
+  // - dBoundary - needs non-dirichlet operator
+  // - Contains one Dirichlet Op, and one non-Dirichlet op. Implements dBoundary etc...
+  // - The Dirichlet ops can be passed to dOmega(Bar) solvers etc...
+  //
+  ////////////////////////////////////////////////////////
+
+
+template<class ImplD,class ImplF>
+class SchurFactoredFermionOperator : public ImplD
+{
+  INHERIT_IMPL_TYPES(ImplD);
+  
+  typedef typename ImplF::FermionField FermionFieldF;
+  typedef typename ImplD::FermionField FermionFieldD;
+
+  typedef SchurDiagMooeeOperator<FermionOperator<ImplD>,FermionFieldD> LinearOperatorD;
+  typedef SchurDiagMooeeOperator<FermionOperator<ImplF>,FermionFieldF> LinearOperatorF;
+  typedef SchurDiagMooeeDagOperator<FermionOperator<ImplD>,FermionFieldD> LinearOperatorDagD;
+  typedef SchurDiagMooeeDagOperator<FermionOperator<ImplF>,FermionFieldF> LinearOperatorDagF;
+
+  typedef MixedPrecisionConjugateGradientOperatorFunction<FermionOperator<ImplD>,
+							  FermionOperator<ImplF>,
+							  LinearOperatorD,
+							  LinearOperatorF> MxPCG;
+
+  typedef MixedPrecisionConjugateGradientOperatorFunction<FermionOperator<ImplD>,
+							  FermionOperator<ImplF>,
+							  LinearOperatorDagD,
+							  LinearOperatorDagF> MxDagPCG;
+public:
+
+  GridBase *FermionGrid(void) { return PeriodicFermOpD.FermionGrid(); };
+  GridBase *GaugeGrid(void)   { return PeriodicFermOpD.GaugeGrid(); };
+  
+  FermionOperator<ImplD> & DirichletFermOpD;
+  FermionOperator<ImplF> & DirichletFermOpF;
+  FermionOperator<ImplD> & PeriodicFermOpD; 
+  FermionOperator<ImplF> & PeriodicFermOpF; 
+
+  LinearOperatorD DirichletLinOpD;
+  LinearOperatorF DirichletLinOpF;
+  LinearOperatorD PeriodicLinOpD;
+  LinearOperatorF PeriodicLinOpF;
+
+  LinearOperatorDagD DirichletLinOpDagD;
+  LinearOperatorDagF DirichletLinOpDagF;
+  LinearOperatorDagD PeriodicLinOpDagD;
+  LinearOperatorDagF PeriodicLinOpDagF;
+
+  // Can tinker with these in the pseudofermion for force vs. action solves
+  Integer maxinnerit;
+  Integer maxouterit;
+  RealD tol;
+  RealD tolinner;
+  
+  Coordinate Block;
+
+  DomainDecomposition Domains;
+
+  SchurFactoredFermionOperator(FermionOperator<ImplD>  & _PeriodicFermOpD,
+			       FermionOperator<ImplF>  & _PeriodicFermOpF,
+			       FermionOperator<ImplD>  & _DirichletFermOpD,
+			       FermionOperator<ImplF>  & _DirichletFermOpF,
+			       Coordinate &_Block)
+    : Block(_Block), Domains(Block),
+
+      PeriodicFermOpD(_PeriodicFermOpD),
+      PeriodicFermOpF(_PeriodicFermOpF),
+      DirichletFermOpD(_DirichletFermOpD),
+      DirichletFermOpF(_DirichletFermOpF),
+      DirichletLinOpD(DirichletFermOpD),
+      DirichletLinOpF(DirichletFermOpF),
+      PeriodicLinOpD(PeriodicFermOpD),
+      PeriodicLinOpF(PeriodicFermOpF),
+      DirichletLinOpDagD(DirichletFermOpD),
+      DirichletLinOpDagF(DirichletFermOpF),
+      PeriodicLinOpDagD(PeriodicFermOpD),
+      PeriodicLinOpDagF(PeriodicFermOpF)
+  {
+    tol=1.0e-10;
+    tolinner=1.0e-6;
+    maxinnerit=1000;
+    maxouterit=10;
+    assert(PeriodicFermOpD.FermionGrid() == DirichletFermOpD.FermionGrid());
+    assert(PeriodicFermOpF.FermionGrid() == DirichletFermOpF.FermionGrid());
+  };
+
+  enum Domain { Omega=0, OmegaBar=1 };
+
+  void ImportGauge(const GaugeField &Umu)
+  {
+    // Single precision will update in the mixed prec CG
+    PeriodicFermOpD.ImportGauge(Umu);
+    GaugeField dUmu(Umu.Grid());
+    dUmu=Umu;
+    //    DirchletBCs(dUmu);
+    DirichletFilter<GaugeField> Filter(Block);
+    Filter.applyFilter(dUmu);
+    DirichletFermOpD.ImportGauge(dUmu);
+  }
+
+/*
+  void ProjectBoundaryBothDomains (FermionField &f,int sgn)
+  {
+    assert((sgn==1)||(sgn==-1));
+    Real rsgn = sgn;
+
+    Gamma::Algebra Gmu [] = {
+      Gamma::Algebra::GammaX,
+      Gamma::Algebra::GammaY,
+      Gamma::Algebra::GammaZ,
+      Gamma::Algebra::GammaT
+    };
+
+    GridBase *grid = f.Grid();
+    LatticeInteger  coor(grid);
+    LatticeInteger  face(grid);
+    LatticeInteger  one(grid); one = 1;
+    LatticeInteger  zero(grid); zero = 0;
+    LatticeInteger nface(grid); nface=Zero();
+    
+    FermionField projected(grid); projected=Zero();
+    FermionField sp_proj  (grid);
+
+    int dims = grid->Nd();
+    int isDWF= (dims==Nd+1);
+    assert((dims==Nd)||(dims==Nd+1));
+    Coordinate Global=grid->GlobalDimensions();
+
+    for(int mu=0;mu<Nd;mu++){
+
+      if ( Block[mu] <= Global[mu+isDWF] ) {
+	// need to worry about DWF 5th dim first
+	LatticeCoordinate(coor,mu+isDWF); 
+      
+	face = where(mod(coor,Block[mu]) == Integer(0),one,zero );
+	nface = nface + face;
+
+	Gamma G(Gmu[mu]);
+	// Lower face receives (1-gamma)/2 in normal forward hopping term
+	sp_proj  = 0.5*(f-G*f*rsgn);
+	projected= where(face,sp_proj,projected);
+	//projected= where(face,f,projected);
+      
+	face = where(mod(coor,Block[mu]) == Integer(Block[mu]-1) ,one,zero );
+	nface = nface + face;
+
+	// Upper face receives (1+gamma)/2 in normal backward hopping term
+	sp_proj = 0.5*(f+G*f*rsgn);
+	projected= where(face,sp_proj,projected);
+	//projected= where(face,f,projected);
+      }
+      
+    }
+    // Initial Zero() where nface==0.
+    // Keep the spin projected faces where nface==1
+    // Full spinor where nface>=2
+    projected = where(nface>Integer(1),f,projected);
+    f=projected;
+  }
+*/
+  void ProjectBoundaryBothDomains (FermionField &f,int sgn)
+  {
+    assert((sgn==1)||(sgn==-1));
+    Real rsgn = sgn;
+
+    Gamma::Algebra Gmu [] = {
+      Gamma::Algebra::GammaX,
+      Gamma::Algebra::GammaY,
+      Gamma::Algebra::GammaZ,
+      Gamma::Algebra::GammaT
+    };
+
+    GridBase *grid = f.Grid();
+    LatticeInteger  coor(grid);
+    LatticeInteger  face(grid);
+    LatticeInteger  one(grid);   one = 1;
+    LatticeInteger  zero(grid); zero = 0;
+    LatticeInteger  omega(grid);
+    LatticeInteger  omegabar(grid);
+    LatticeInteger  tmp(grid);
+
+    omega=one;    Domains.ProjectDomain(omega,0);
+    omegabar=one; Domains.ProjectDomain(omegabar,1);
+    
+    LatticeInteger nface(grid); nface=Zero();
+    
+    FermionField projected(grid); projected=Zero();
+    FermionField sp_proj  (grid);
+
+    int dims = grid->Nd();
+    int isDWF= (dims==Nd+1);
+    assert((dims==Nd)||(dims==Nd+1));
+    Coordinate Global=grid->GlobalDimensions();
+
+    for(int mmu=0;mmu<Nd;mmu++){
+      Gamma G(Gmu[mmu]);
+
+      // need to worry about DWF 5th dim first
+      int mu = mmu+isDWF;
+      if ( Block[mmu] && (Block[mmu] <= Global[mu]) ) {
+
+	// Lower face receives (1-gamma)/2 in normal forward hopping term
+ 	tmp = Cshift(omegabar,mu,-1);
+	tmp = tmp + omega;
+	face = where(tmp == Integer(2),one,zero );
+
+ 	tmp = Cshift(omega,mu,-1);
+	tmp = tmp + omegabar;
+	face = where(tmp == Integer(2),one,face );
+
+	nface = nface + face;
+
+	sp_proj  = 0.5*(f-G*f*rsgn);
+	projected= where(face,sp_proj,projected);
+
+	// Upper face receives (1+gamma)/2 in normal backward hopping term
+ 	tmp = Cshift(omegabar,mu,1);
+	tmp = tmp + omega;
+	face = where(tmp == Integer(2),one,zero );
+
+ 	tmp = Cshift(omega,mu,1);
+	tmp = tmp + omegabar;
+	face = where(tmp == Integer(2),one,face );
+
+	nface = nface + face;
+
+	sp_proj = 0.5*(f+G*f*rsgn);
+	projected= where(face,sp_proj,projected);
+      }
+      
+    }
+    // Initial Zero() where nface==0.
+    // Keep the spin projected faces where nface==1
+    // Full spinor where nface>=2
+    projected = where(nface>Integer(1),f,projected);
+    f=projected;
+  }
+
+  void ProjectDomain(FermionField &f,int domain)
+  {
+/*
+    GridBase *grid = f.Grid();
+    int dims = grid->Nd();
+    int isDWF= (dims==Nd+1);
+    assert((dims==Nd)||(dims==Nd+1));
+
+    FermionField zz(grid); zz=Zero();
+    LatticeInteger coor(grid);
+    LatticeInteger domaincb(grid); domaincb=Zero();
+    for(int d=0;d<Nd;d++){
+      LatticeCoordinate(coor,d+isDWF);
+      domaincb = domaincb + div(coor,Block[d]);
+    }
+    f = where(mod(domaincb,2)==Integer(domain),f,zz);
+*/
+    Domains.ProjectDomain(f,domain);
+
+  };
+  void ProjectOmegaBar   (FermionField &f) {ProjectDomain(f,OmegaBar);}
+  void ProjectOmega      (FermionField &f) {ProjectDomain(f,Omega);}
+  // See my notes(!).
+  // Notation: Following Luscher, we introduce projectors $\hPdb$ with both spinor and space structure
+  // projecting all spinor elements in $\Omega$ connected by $\Ddb$ to $\bar{\Omega}$,
+  void ProjectBoundaryBar(FermionField &f)
+  {
+    ProjectBoundaryBothDomains(f,1);
+    ProjectOmega(f);
+  }
+  // and $\hPd$ projecting all spinor elements in $\bar{\Omega}$ connected by $\Dd$ to $\Omega$.
+  void ProjectBoundary   (FermionField &f)
+  {
+    ProjectBoundaryBothDomains(f,1);
+    ProjectOmegaBar(f);
+    //    DumpSliceNorm("ProjectBoundary",f,f.Grid()->Nd()-1);
+  };
+
+  void dBoundary    (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmegaBar(tmp);
+    PeriodicFermOpD.M(tmp,out);
+    ProjectOmega(out);
+  };
+  void dBoundaryDag (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmega(tmp);
+    PeriodicFermOpD.Mdag(tmp,out);
+    ProjectOmegaBar(out);
+  };
+  void dBoundaryBar (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmega(tmp);
+    PeriodicFermOpD.M(tmp,out);
+    ProjectOmegaBar(out);
+  };
+  void dBoundaryBarDag (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmegaBar(tmp);
+    PeriodicFermOpD.Mdag(tmp,out);
+    ProjectOmega(out);
+  };
+  void dOmega       (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmega(tmp);
+    DirichletFermOpD.M(tmp,out);
+    ProjectOmega(out);
+  };
+  void dOmegaBar    (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmegaBar(tmp);
+    DirichletFermOpD.M(tmp,out);
+    ProjectOmegaBar(out);
+  };
+  void dOmegaDag       (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmega(tmp);
+    DirichletFermOpD.Mdag(tmp,out);
+    ProjectOmega(out);
+  };
+  void dOmegaBarDag    (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmegaBar(tmp);
+    DirichletFermOpD.Mdag(tmp,out);
+    ProjectOmegaBar(out);
+  };
+  void dOmegaInv   (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmega(tmp);
+    dOmegaInvAndOmegaBarInv(tmp,out); // Inefficient warning
+    ProjectOmega(out);
+  };
+  void dOmegaBarInv(FermionField &in,FermionField &out)
+  {    
+    FermionField tmp(in);
+    ProjectOmegaBar(tmp);
+    dOmegaInvAndOmegaBarInv(tmp,out);
+    ProjectOmegaBar(out);
+  };
+  void dOmegaDagInv   (FermionField &in,FermionField &out)
+  {
+    FermionField tmp(in);
+    ProjectOmega(tmp);
+    dOmegaDagInvAndOmegaBarDagInv(tmp,out);
+    ProjectOmega(out);
+  };
+  void dOmegaBarDagInv(FermionField &in,FermionField &out)
+  {    
+    FermionField tmp(in);
+    ProjectOmegaBar(tmp);
+    dOmegaDagInvAndOmegaBarDagInv(tmp,out);
+    ProjectOmegaBar(out);
+  };
+  void dOmegaInvAndOmegaBarInv(FermionField &in,FermionField &out)
+  {
+    MxPCG OmegaSolver(tol,
+		      tolinner,
+		      maxinnerit,
+		      maxouterit,
+		      DirichletFermOpF.FermionRedBlackGrid(),
+		      DirichletFermOpF,
+		      DirichletFermOpD,
+		      DirichletLinOpF,
+		      DirichletLinOpD);
+    SchurRedBlackDiagMooeeSolve<FermionField> PrecSolve(OmegaSolver);
+    PrecSolve(DirichletFermOpD,in,out);
+  };
+  void dOmegaDagInvAndOmegaBarDagInv(FermionField &in,FermionField &out)
+  {
+    MxDagPCG OmegaDagSolver(tol,
+			    tolinner,
+			    maxinnerit,
+			    maxouterit,
+			    DirichletFermOpF.FermionRedBlackGrid(),
+			    DirichletFermOpF,
+			    DirichletFermOpD,
+			    DirichletLinOpDagF,
+			    DirichletLinOpDagD);
+    SchurRedBlackDiagMooeeDagSolve<FermionField> PrecSolve(OmegaDagSolver);
+    PrecSolve(DirichletFermOpD,in,out);
+  };
+
+  // Rdag = Pdbar - DdbarDag DomegabarDagInv  DdDag DomegaDagInv Pdbar 
+  void RDag(FermionField &in,FermionField &out)
+  {
+    FermionField tmp1(PeriodicFermOpD.FermionGrid());
+    FermionField tmp2(PeriodicFermOpD.FermionGrid());
+    out = in;
+    ProjectBoundaryBar(out);
+    dOmegaDagInv(out,tmp1);   
+    dBoundaryDag(tmp1,tmp2);   
+    dOmegaBarDagInv(tmp2,tmp1);
+    dBoundaryBarDag(tmp1,tmp2); 
+    out = out - tmp2;
+  };
+
+  // R = Pdbar - Pdbar DomegaInv Dd DomegabarInv Ddbar
+  void R(FermionField &in,FermionField &out)
+  {
+    FermionField tmp1(PeriodicFermOpD.FermionGrid());
+    FermionField tmp2(PeriodicFermOpD.FermionGrid());
+    out = in;
+    ProjectBoundaryBar(out);
+    dBoundaryBar(out,tmp1); 
+    dOmegaBarInv(tmp1,tmp2);
+    dBoundary(tmp2,tmp1);   
+    dOmegaInv(tmp1,tmp2);   
+    out = in - tmp2 ;       
+    ProjectBoundaryBar(out);
+    //    DumpSliceNorm("R",out,out.Grid()->Nd()-1);
+  };
+  
+  // R = Pdbar - Pdbar Dinv Ddbar 
+  void RInv(FermionField &in,FermionField &out)
+  {
+    FermionField tmp1(PeriodicFermOpD.FermionGrid());
+    dBoundaryBar(in,out);
+    Dinverse(out,tmp1);  
+    out =in -tmp1; 
+    ProjectBoundaryBar(out);
+  };
+  // R = Pdbar - DdbarDag DinvDag Pdbar 
+  void RDagInv(FermionField &in,FermionField &out)
+  {
+    FermionField tmp(PeriodicFermOpD.FermionGrid());
+    FermionField Pin(PeriodicFermOpD.FermionGrid());
+    Pin = in; ProjectBoundaryBar(Pin);
+    DinverseDag(Pin,out);  
+    dBoundaryBarDag(out,tmp);
+    out =Pin -tmp; 
+  };
+  // Non-dirichlet inverter using red-black preconditioning
+  void Dinverse(FermionField &in,FermionField &out)
+  {
+    MxPCG DSolver(tol,
+		  tolinner,
+		  maxinnerit,
+		  maxouterit,
+		  PeriodicFermOpF.FermionRedBlackGrid(),
+		  PeriodicFermOpF,
+		  PeriodicFermOpD,
+		  PeriodicLinOpF,
+		  PeriodicLinOpD);
+    SchurRedBlackDiagMooeeSolve<FermionField> Solve(DSolver);
+    Solve(PeriodicFermOpD,in,out);
+  }
+  void DinverseDag(FermionField &in,FermionField &out)
+  {
+    MxDagPCG DdagSolver(tol,
+			tolinner,
+			maxinnerit,
+			maxouterit,
+			PeriodicFermOpF.FermionRedBlackGrid(),
+			PeriodicFermOpF,
+			PeriodicFermOpD,
+			PeriodicLinOpDagF,
+			PeriodicLinOpDagD);
+    SchurRedBlackDiagMooeeDagSolve<FermionField> Solve(DdagSolver);
+    Solve(PeriodicFermOpD,in,out);
+  }
+};
+
+NAMESPACE_END(Grid);
+

Grid/qcd/action/fermion/WilsonCloverFermion.h

@@ -4,11 +4,10 @@
 
     Source file: ./lib/qcd/action/fermion/WilsonCloverFermion.h
 
-    Copyright (C) 2017 - 2022
+    Copyright (C) 2017
 
     Author: Guido Cossu <guido.cossu@ed.ac.uk>
     Author: David Preti <>
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -30,8 +29,7 @@
 
 #pragma once
 
-#include <Grid/qcd/action/fermion/WilsonCloverTypes.h>
-#include <Grid/qcd/action/fermion/WilsonCloverHelpers.h>
+#include <Grid/Grid.h>
 
 NAMESPACE_BEGIN(Grid);
 
@@ -52,15 +50,18 @@ NAMESPACE_BEGIN(Grid);
 //////////////////////////////////////////////////////////////////
 
 template <class Impl>
-class WilsonCloverFermion : public WilsonFermion<Impl>,
-                            public WilsonCloverHelpers<Impl>
+class WilsonCloverFermion : public WilsonFermion<Impl>
 {
 public:
+  // Types definitions
   INHERIT_IMPL_TYPES(Impl);
-  INHERIT_CLOVER_TYPES(Impl);
+  template <typename vtype>
+  using iImplClover = iScalar<iMatrix<iMatrix<vtype, Impl::Dimension>, Ns>>;
+  typedef iImplClover<Simd> SiteCloverType;
+  typedef Lattice<SiteCloverType> CloverFieldType;
 
-  typedef WilsonFermion<Impl>       WilsonBase;
-  typedef WilsonCloverHelpers<Impl> Helpers;
+public:
+  typedef WilsonFermion<Impl> WilsonBase;
 
   virtual int    ConstEE(void)     { return 0; };
   virtual void Instantiatable(void){};
@@ -71,7 +72,42 @@ public:
                       const RealD _csw_r = 0.0,
                       const RealD _csw_t = 0.0,
                       const WilsonAnisotropyCoefficients &clover_anisotropy = WilsonAnisotropyCoefficients(),
-                      const ImplParams &impl_p = ImplParams());
+                      const ImplParams &impl_p = ImplParams()) : WilsonFermion<Impl>(_Umu,
+                                                                                     Fgrid,
+                                                                                     Hgrid,
+                                                                                     _mass, impl_p, clover_anisotropy),
+                                                                 CloverTerm(&Fgrid),
+                                                                 CloverTermInv(&Fgrid),
+                                                                 CloverTermEven(&Hgrid),
+                                                                 CloverTermOdd(&Hgrid),
+                                                                 CloverTermInvEven(&Hgrid),
+                                                                 CloverTermInvOdd(&Hgrid),
+                                                                 CloverTermDagEven(&Hgrid),
+                                                                 CloverTermDagOdd(&Hgrid),
+                                                                 CloverTermInvDagEven(&Hgrid),
+                                                                 CloverTermInvDagOdd(&Hgrid)
+  {
+    assert(Nd == 4); // require 4 dimensions
+
+    if (clover_anisotropy.isAnisotropic)
+    {
+      csw_r = _csw_r * 0.5 / clover_anisotropy.xi_0;
+      diag_mass = _mass + 1.0 + (Nd - 1) * (clover_anisotropy.nu / clover_anisotropy.xi_0);
+    }
+    else
+    {
+      csw_r = _csw_r * 0.5;
+      diag_mass = 4.0 + _mass;
+    }
+    csw_t = _csw_t * 0.5;
+
+    if (csw_r == 0)
+      std::cout << GridLogWarning << "Initializing WilsonCloverFermion with csw_r = 0" << std::endl;
+    if (csw_t == 0)
+      std::cout << GridLogWarning << "Initializing WilsonCloverFermion with csw_t = 0" << std::endl;
+
+    ImportGauge(_Umu);
+  }
 
   virtual void M(const FermionField &in, FermionField &out);
   virtual void Mdag(const FermionField &in, FermionField &out);
@@ -88,21 +124,250 @@ public:
   void ImportGauge(const GaugeField &_Umu);
 
   // Derivative parts unpreconditioned pseudofermions
-  void MDeriv(GaugeField &force, const FermionField &X, const FermionField &Y, int dag);
+  void MDeriv(GaugeField &force, const FermionField &X, const FermionField &Y, int dag)
+  {
+    conformable(X.Grid(), Y.Grid());
+    conformable(X.Grid(), force.Grid());
+    GaugeLinkField force_mu(force.Grid()), lambda(force.Grid());
+    GaugeField clover_force(force.Grid());
+    PropagatorField Lambda(force.Grid());
 
-public:
+    // Guido: Here we are hitting some performance issues:
+    // need to extract the components of the DoubledGaugeField
+    // for each call
+    // Possible solution
+    // Create a vector object to store them? (cons: wasting space)
+    std::vector<GaugeLinkField> U(Nd, this->Umu.Grid());
+
+    Impl::extractLinkField(U, this->Umu);
+
+    force = Zero();
+    // Derivative of the Wilson hopping term
+    this->DhopDeriv(force, X, Y, dag);
+
+    ///////////////////////////////////////////////////////////
+    // Clover term derivative
+    ///////////////////////////////////////////////////////////
+    Impl::outerProductImpl(Lambda, X, Y);
+    //std::cout << "Lambda:" << Lambda << std::endl;
+
+    Gamma::Algebra sigma[] = {
+        Gamma::Algebra::SigmaXY,
+        Gamma::Algebra::SigmaXZ,
+        Gamma::Algebra::SigmaXT,
+        Gamma::Algebra::MinusSigmaXY,
+        Gamma::Algebra::SigmaYZ,
+        Gamma::Algebra::SigmaYT,
+        Gamma::Algebra::MinusSigmaXZ,
+        Gamma::Algebra::MinusSigmaYZ,
+        Gamma::Algebra::SigmaZT,
+        Gamma::Algebra::MinusSigmaXT,
+        Gamma::Algebra::MinusSigmaYT,
+        Gamma::Algebra::MinusSigmaZT};
+
+    /*
+      sigma_{\mu \nu}=
+      | 0         sigma[0]  sigma[1]  sigma[2] |
+      | sigma[3]    0       sigma[4]  sigma[5] |
+      | sigma[6]  sigma[7]     0      sigma[8] |
+      | sigma[9]  sigma[10] sigma[11]   0      |
+    */
+
+    int count = 0;
+    clover_force = Zero();
+    for (int mu = 0; mu < 4; mu++)
+    {
+      force_mu = Zero();
+      for (int nu = 0; nu < 4; nu++)
+      {
+        if (mu == nu)
+        continue;
+        
+        RealD factor;
+        if (nu == 4 || mu == 4)
+        {
+          factor = 2.0 * csw_t;
+        }
+        else
+        {
+          factor = 2.0 * csw_r;
+        }
+        PropagatorField Slambda = Gamma(sigma[count]) * Lambda; // sigma checked
+        Impl::TraceSpinImpl(lambda, Slambda);                   // traceSpin ok
+        force_mu -= factor*Cmunu(U, lambda, mu, nu);                   // checked
+        count++;
+      }
+
+      pokeLorentz(clover_force, U[mu] * force_mu, mu);
+    }
+    //clover_force *= csw;
+    force += clover_force;
+  }
+
+  // Computing C_{\mu \nu}(x) as in Eq.(B.39) in Zbigniew Sroczynski's PhD thesis
+  GaugeLinkField Cmunu(std::vector<GaugeLinkField> &U, GaugeLinkField &lambda, int mu, int nu)
+  {
+    conformable(lambda.Grid(), U[0].Grid());
+    GaugeLinkField out(lambda.Grid()), tmp(lambda.Grid());
+    // insertion in upper staple
+    // please check redundancy of shift operations
+
+    // C1+
+    tmp = lambda * U[nu];
+    out = Impl::ShiftStaple(Impl::CovShiftForward(tmp, nu, Impl::CovShiftBackward(U[mu], mu, Impl::CovShiftIdentityBackward(U[nu], nu))), mu);
+
+    // C2+
+    tmp = U[mu] * Impl::ShiftStaple(adj(lambda), mu);
+    out += Impl::ShiftStaple(Impl::CovShiftForward(U[nu], nu, Impl::CovShiftBackward(tmp, mu, Impl::CovShiftIdentityBackward(U[nu], nu))), mu);
+
+    // C3+
+    tmp = U[nu] * Impl::ShiftStaple(adj(lambda), nu);
+    out += Impl::ShiftStaple(Impl::CovShiftForward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, Impl::CovShiftIdentityBackward(tmp, nu))), mu);
+
+    // C4+
+    out += Impl::ShiftStaple(Impl::CovShiftForward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, Impl::CovShiftIdentityBackward(U[nu], nu))), mu) * lambda;
+
+    // insertion in lower staple
+    // C1-
+    out -= Impl::ShiftStaple(lambda, mu) * Impl::ShiftStaple(Impl::CovShiftBackward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, U[nu])), mu);
+
+    // C2-
+    tmp = adj(lambda) * U[nu];
+    out -= Impl::ShiftStaple(Impl::CovShiftBackward(tmp, nu, Impl::CovShiftBackward(U[mu], mu, U[nu])), mu);
+
+    // C3-
+    tmp = lambda * U[nu];
+    out -= Impl::ShiftStaple(Impl::CovShiftBackward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, tmp)), mu);
+
+    // C4-
+    out -= Impl::ShiftStaple(Impl::CovShiftBackward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, U[nu])), mu) * lambda;
+
+    return out;
+  }
+
+protected:
   // here fixing the 4 dimensions, make it more general?
 
   RealD csw_r;                                               // Clover coefficient - spatial
   RealD csw_t;                                               // Clover coefficient - temporal
   RealD diag_mass;                                           // Mass term
-  CloverField CloverTerm, CloverTermInv;                     // Clover term
-  CloverField CloverTermEven, CloverTermOdd;                 // Clover term EO
-  CloverField CloverTermInvEven, CloverTermInvOdd;           // Clover term Inv EO
-  CloverField CloverTermDagEven, CloverTermDagOdd;           // Clover term Dag EO
-  CloverField CloverTermInvDagEven, CloverTermInvDagOdd;     // Clover term Inv Dag EO
-};
+  CloverFieldType CloverTerm, CloverTermInv;                 // Clover term
+  CloverFieldType CloverTermEven, CloverTermOdd;             // Clover term EO
+  CloverFieldType CloverTermInvEven, CloverTermInvOdd;       // Clover term Inv EO
+  CloverFieldType CloverTermDagEven, CloverTermDagOdd;       // Clover term Dag EO
+  CloverFieldType CloverTermInvDagEven, CloverTermInvDagOdd; // Clover term Inv Dag EO
 
+ public:
+  // eventually these can be compressed into 6x6 blocks instead of the 12x12
+  // using the DeGrand-Rossi basis for the gamma matrices
+  CloverFieldType fillCloverYZ(const GaugeLinkField &F)
+  {
+    CloverFieldType T(F.Grid());
+    T = Zero();
+    autoView(T_v,T,AcceleratorWrite);
+    autoView(F_v,F,AcceleratorRead);
+    accelerator_for(i, CloverTerm.Grid()->oSites(),1,
+    {
+      T_v[i]()(0, 1) = timesMinusI(F_v[i]()());
+      T_v[i]()(1, 0) = timesMinusI(F_v[i]()());
+      T_v[i]()(2, 3) = timesMinusI(F_v[i]()());
+      T_v[i]()(3, 2) = timesMinusI(F_v[i]()());
+    });
+
+    return T;
+  }
+
+  CloverFieldType fillCloverXZ(const GaugeLinkField &F)
+  {
+    CloverFieldType T(F.Grid());
+    T = Zero();
+    
+    autoView(T_v, T,AcceleratorWrite);
+    autoView(F_v, F,AcceleratorRead);
+    accelerator_for(i, CloverTerm.Grid()->oSites(),1,
+    {
+      T_v[i]()(0, 1) = -F_v[i]()();
+      T_v[i]()(1, 0) = F_v[i]()();
+      T_v[i]()(2, 3) = -F_v[i]()();
+      T_v[i]()(3, 2) = F_v[i]()();
+    });
+
+    return T;
+  }
+
+  CloverFieldType fillCloverXY(const GaugeLinkField &F)
+  {
+    CloverFieldType T(F.Grid());
+    T = Zero();
+
+    autoView(T_v,T,AcceleratorWrite);
+    autoView(F_v,F,AcceleratorRead);
+    accelerator_for(i, CloverTerm.Grid()->oSites(),1,
+    {
+      T_v[i]()(0, 0) = timesMinusI(F_v[i]()());
+      T_v[i]()(1, 1) = timesI(F_v[i]()());
+      T_v[i]()(2, 2) = timesMinusI(F_v[i]()());
+      T_v[i]()(3, 3) = timesI(F_v[i]()());
+    });
+
+    return T;
+  }
+
+  CloverFieldType fillCloverXT(const GaugeLinkField &F)
+  {
+    CloverFieldType T(F.Grid());
+    T = Zero();
+
+    autoView( T_v , T, AcceleratorWrite);
+    autoView( F_v , F, AcceleratorRead);
+    accelerator_for(i, CloverTerm.Grid()->oSites(),1,
+    {
+      T_v[i]()(0, 1) = timesI(F_v[i]()());
+      T_v[i]()(1, 0) = timesI(F_v[i]()());
+      T_v[i]()(2, 3) = timesMinusI(F_v[i]()());
+      T_v[i]()(3, 2) = timesMinusI(F_v[i]()());
+    });
+
+    return T;
+  }
+
+  CloverFieldType fillCloverYT(const GaugeLinkField &F)
+  {
+    CloverFieldType T(F.Grid());
+    T = Zero();
+    
+    autoView( T_v ,T,AcceleratorWrite);
+    autoView( F_v ,F,AcceleratorRead);
+    accelerator_for(i, CloverTerm.Grid()->oSites(),1,
+    {
+      T_v[i]()(0, 1) = -(F_v[i]()());
+      T_v[i]()(1, 0) = (F_v[i]()());
+      T_v[i]()(2, 3) = (F_v[i]()());
+      T_v[i]()(3, 2) = -(F_v[i]()());
+    });
+
+    return T;
+  }
+
+  CloverFieldType fillCloverZT(const GaugeLinkField &F)
+  {
+    CloverFieldType T(F.Grid());
+
+    T = Zero();
+
+    autoView( T_v , T,AcceleratorWrite);
+    autoView( F_v , F,AcceleratorRead);
+    accelerator_for(i, CloverTerm.Grid()->oSites(),1,
+    {
+      T_v[i]()(0, 0) = timesI(F_v[i]()());
+      T_v[i]()(1, 1) = timesMinusI(F_v[i]()());
+      T_v[i]()(2, 2) = timesMinusI(F_v[i]()());
+      T_v[i]()(3, 3) = timesI(F_v[i]()());
+    });
+
+    return T;
+  }
+};
 NAMESPACE_END(Grid);
 
 

Grid/qcd/action/fermion/WilsonCloverHelpers.h

@@ -1,761 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: ./lib/qcd/action/fermion/WilsonCloverHelpers.h
-
-    Copyright (C) 2021 - 2022
-
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#pragma once
-
-// Helper routines that implement common clover functionality
-
-NAMESPACE_BEGIN(Grid);
-
-template<class Impl> class WilsonCloverHelpers {
-public:
-
-  INHERIT_IMPL_TYPES(Impl);
-  INHERIT_CLOVER_TYPES(Impl);
-
-  // Computing C_{\mu \nu}(x) as in Eq.(B.39) in Zbigniew Sroczynski's PhD thesis
-  static GaugeLinkField Cmunu(std::vector<GaugeLinkField> &U, GaugeLinkField &lambda, int mu, int nu)
-  {
-    conformable(lambda.Grid(), U[0].Grid());
-    GaugeLinkField out(lambda.Grid()), tmp(lambda.Grid());
-    // insertion in upper staple
-    // please check redundancy of shift operations
-
-    // C1+
-    tmp = lambda * U[nu];
-    out = Impl::ShiftStaple(Impl::CovShiftForward(tmp, nu, Impl::CovShiftBackward(U[mu], mu, Impl::CovShiftIdentityBackward(U[nu], nu))), mu);
-
-    // C2+
-    tmp = U[mu] * Impl::ShiftStaple(adj(lambda), mu);
-    out += Impl::ShiftStaple(Impl::CovShiftForward(U[nu], nu, Impl::CovShiftBackward(tmp, mu, Impl::CovShiftIdentityBackward(U[nu], nu))), mu);
-
-    // C3+
-    tmp = U[nu] * Impl::ShiftStaple(adj(lambda), nu);
-    out += Impl::ShiftStaple(Impl::CovShiftForward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, Impl::CovShiftIdentityBackward(tmp, nu))), mu);
-
-    // C4+
-    out += Impl::ShiftStaple(Impl::CovShiftForward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, Impl::CovShiftIdentityBackward(U[nu], nu))), mu) * lambda;
-
-    // insertion in lower staple
-    // C1-
-    out -= Impl::ShiftStaple(lambda, mu) * Impl::ShiftStaple(Impl::CovShiftBackward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, U[nu])), mu);
-
-    // C2-
-    tmp = adj(lambda) * U[nu];
-    out -= Impl::ShiftStaple(Impl::CovShiftBackward(tmp, nu, Impl::CovShiftBackward(U[mu], mu, U[nu])), mu);
-
-    // C3-
-    tmp = lambda * U[nu];
-    out -= Impl::ShiftStaple(Impl::CovShiftBackward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, tmp)), mu);
-
-    // C4-
-    out -= Impl::ShiftStaple(Impl::CovShiftBackward(U[nu], nu, Impl::CovShiftBackward(U[mu], mu, U[nu])), mu) * lambda;
-
-    return out;
-  }
-
-  static CloverField fillCloverYZ(const GaugeLinkField &F)
-  {
-    CloverField T(F.Grid());
-    T = Zero();
-    autoView(T_v,T,AcceleratorWrite);
-    autoView(F_v,F,AcceleratorRead);
-    accelerator_for(i, T.Grid()->oSites(),CloverField::vector_type::Nsimd(),
-    {
-      coalescedWrite(T_v[i]()(0, 1), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(1, 0), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(2, 3), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(3, 2), coalescedRead(timesMinusI(F_v[i]()())));
-    });
-
-    return T;
-  }
-
-  static CloverField fillCloverXZ(const GaugeLinkField &F)
-  {
-    CloverField T(F.Grid());
-    T = Zero();
-    
-    autoView(T_v, T,AcceleratorWrite);
-    autoView(F_v, F,AcceleratorRead);
-    accelerator_for(i, T.Grid()->oSites(),CloverField::vector_type::Nsimd(),
-    {
-      coalescedWrite(T_v[i]()(0, 1), coalescedRead(-F_v[i]()()));
-      coalescedWrite(T_v[i]()(1, 0), coalescedRead(F_v[i]()()));
-      coalescedWrite(T_v[i]()(2, 3), coalescedRead(-F_v[i]()()));
-      coalescedWrite(T_v[i]()(3, 2), coalescedRead(F_v[i]()()));
-    });
-
-    return T;
-  }
-
-  static CloverField fillCloverXY(const GaugeLinkField &F)
-  {
-    CloverField T(F.Grid());
-    T = Zero();
-
-    autoView(T_v,T,AcceleratorWrite);
-    autoView(F_v,F,AcceleratorRead);
-    accelerator_for(i, T.Grid()->oSites(),CloverField::vector_type::Nsimd(),
-    {
-      coalescedWrite(T_v[i]()(0, 0), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(1, 1), coalescedRead(timesI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(2, 2), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(3, 3), coalescedRead(timesI(F_v[i]()())));
-    });
-
-    return T;
-  }
-
-  static CloverField fillCloverXT(const GaugeLinkField &F)
-  {
-    CloverField T(F.Grid());
-    T = Zero();
-
-    autoView( T_v , T, AcceleratorWrite);
-    autoView( F_v , F, AcceleratorRead);
-    accelerator_for(i, T.Grid()->oSites(),CloverField::vector_type::Nsimd(),
-    {
-      coalescedWrite(T_v[i]()(0, 1), coalescedRead(timesI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(1, 0), coalescedRead(timesI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(2, 3), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(3, 2), coalescedRead(timesMinusI(F_v[i]()())));
-    });
-
-    return T;
-  }
-
-  static CloverField fillCloverYT(const GaugeLinkField &F)
-  {
-    CloverField T(F.Grid());
-    T = Zero();
-    
-    autoView( T_v ,T,AcceleratorWrite);
-    autoView( F_v ,F,AcceleratorRead);
-    accelerator_for(i, T.Grid()->oSites(),CloverField::vector_type::Nsimd(),
-    {
-      coalescedWrite(T_v[i]()(0, 1), coalescedRead(-(F_v[i]()())));
-      coalescedWrite(T_v[i]()(1, 0), coalescedRead((F_v[i]()())));
-      coalescedWrite(T_v[i]()(2, 3), coalescedRead((F_v[i]()())));
-      coalescedWrite(T_v[i]()(3, 2), coalescedRead(-(F_v[i]()())));
-    });
-
-    return T;
-  }
-
-  static CloverField fillCloverZT(const GaugeLinkField &F)
-  {
-    CloverField T(F.Grid());
-
-    T = Zero();
-
-    autoView( T_v , T,AcceleratorWrite);
-    autoView( F_v , F,AcceleratorRead);
-    accelerator_for(i, T.Grid()->oSites(),CloverField::vector_type::Nsimd(),
-    {
-      coalescedWrite(T_v[i]()(0, 0), coalescedRead(timesI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(1, 1), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(2, 2), coalescedRead(timesMinusI(F_v[i]()())));
-      coalescedWrite(T_v[i]()(3, 3), coalescedRead(timesI(F_v[i]()())));
-    });
-
-    return T;
-  }
-
-  template<class _Spinor>
-  static accelerator_inline void multClover(_Spinor& phi, const SiteClover& C, const _Spinor& chi) {
-    auto CC = coalescedRead(C);
-    mult(&phi, &CC, &chi);
-  }
-
-  template<class _SpinorField>
-  inline void multCloverField(_SpinorField& out, const CloverField& C, const _SpinorField& phi) {
-    const int Nsimd = SiteSpinor::Nsimd();
-    autoView(out_v, out, AcceleratorWrite);
-    autoView(phi_v, phi, AcceleratorRead);
-    autoView(C_v,   C,   AcceleratorRead);
-    typedef decltype(coalescedRead(out_v[0])) calcSpinor;
-    accelerator_for(sss,out.Grid()->oSites(),Nsimd,{
-      calcSpinor tmp;
-      multClover(tmp,C_v[sss],phi_v(sss));
-      coalescedWrite(out_v[sss],tmp);
-    });
-  }
-};
-
-
-template<class Impl> class CompactWilsonCloverHelpers {
-public:
-
-  INHERIT_COMPACT_CLOVER_SIZES(Impl);
-
-  INHERIT_IMPL_TYPES(Impl);
-  INHERIT_CLOVER_TYPES(Impl);
-  INHERIT_COMPACT_CLOVER_TYPES(Impl);
-
-  #if 0
-  static accelerator_inline typename SiteCloverTriangle::vector_type triangle_elem(const SiteCloverTriangle& triangle, int block, int i, int j) {
-    assert(i != j);
-    if(i < j) {
-      return triangle()(block)(triangle_index(i, j));
-    } else { // i > j
-      return conjugate(triangle()(block)(triangle_index(i, j)));
-    }
-  }
-  #else
-  template<typename vobj>
-  static accelerator_inline vobj triangle_elem(const iImplCloverTriangle<vobj>& triangle, int block, int i, int j) {
-    assert(i != j);
-    if(i < j) {
-      return triangle()(block)(triangle_index(i, j));
-    } else { // i > j
-      return conjugate(triangle()(block)(triangle_index(i, j)));
-    }
-  }
-  #endif
-
-  static accelerator_inline int triangle_index(int i, int j) {
-    if(i == j)
-      return 0;
-    else if(i < j)
-      return Nred * (Nred - 1) / 2 - (Nred - i) * (Nred - i - 1) / 2 + j - i - 1;
-    else // i > j
-      return Nred * (Nred - 1) / 2 - (Nred - j) * (Nred - j - 1) / 2 + i - j - 1;
-  }
-
-  static void MooeeKernel_gpu(int                        Nsite,
-                              int                        Ls,
-                              const FermionField&        in,
-                              FermionField&              out,
-                              const CloverDiagonalField& diagonal,
-                              const CloverTriangleField& triangle) {
-    autoView(diagonal_v, diagonal, AcceleratorRead);
-    autoView(triangle_v, triangle, AcceleratorRead);
-    autoView(in_v,       in,       AcceleratorRead);
-    autoView(out_v,      out,      AcceleratorWrite);
-
-    typedef decltype(coalescedRead(out_v[0])) CalcSpinor;
-
-    const uint64_t NN = Nsite * Ls;
-
-    accelerator_for(ss, NN, Simd::Nsimd(), {
-      int sF = ss;
-      int sU = ss/Ls;
-      CalcSpinor res;
-      CalcSpinor in_t = in_v(sF);
-      auto diagonal_t = diagonal_v(sU);
-      auto triangle_t = triangle_v(sU);
-      for(int block=0; block<Nhs; block++) {
-        int s_start = block*Nhs;
-        for(int i=0; i<Nred; i++) {
-          int si = s_start + i/Nc, ci = i%Nc;
-          res()(si)(ci) = diagonal_t()(block)(i) * in_t()(si)(ci);
-          for(int j=0; j<Nred; j++) {
-            if (j == i) continue;
-            int sj = s_start + j/Nc, cj = j%Nc;
-            res()(si)(ci) = res()(si)(ci) + triangle_elem(triangle_t, block, i, j) * in_t()(sj)(cj);
-          };
-        };
-      };
-      coalescedWrite(out_v[sF], res);
-    });
-  }
-
-  static void MooeeKernel_cpu(int                        Nsite,
-                              int                        Ls,
-                              const FermionField&        in,
-                              FermionField&              out,
-                              const CloverDiagonalField& diagonal,
-                              const CloverTriangleField& triangle) {
-    autoView(diagonal_v, diagonal, CpuRead);
-    autoView(triangle_v, triangle, CpuRead);
-    autoView(in_v,       in,       CpuRead);
-    autoView(out_v,      out,      CpuWrite);
-
-    typedef SiteSpinor CalcSpinor;
-
-#if defined(A64FX) || defined(A64FXFIXEDSIZE)
-#define PREFETCH_CLOVER(BASE) {                                     \
-    uint64_t base;                                                  \
-    int pf_dist_L1 = 1;                                             \
-    int pf_dist_L2 = -5; /* -> penalty -> disable */                \
-                                                                    \
-    if ((pf_dist_L1 >= 0) && (sU + pf_dist_L1 < Nsite)) {           \
-      base = (uint64_t)&diag_t()(pf_dist_L1+BASE)(0);               \
-      svprfd(svptrue_b64(), (int64_t*)(base +    0), SV_PLDL1STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base +  256), SV_PLDL1STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base +  512), SV_PLDL1STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base +  768), SV_PLDL1STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base + 1024), SV_PLDL1STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base + 1280), SV_PLDL1STRM); \
-    }                                                               \
-                                                                    \
-    if ((pf_dist_L2 >= 0) && (sU + pf_dist_L2 < Nsite)) {           \
-      base = (uint64_t)&diag_t()(pf_dist_L2+BASE)(0);               \
-      svprfd(svptrue_b64(), (int64_t*)(base +    0), SV_PLDL2STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base +  256), SV_PLDL2STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base +  512), SV_PLDL2STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base +  768), SV_PLDL2STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base + 1024), SV_PLDL2STRM); \
-      svprfd(svptrue_b64(), (int64_t*)(base + 1280), SV_PLDL2STRM); \
-    }                                                               \
-  }
-// TODO: Implement/generalize this for other architectures
-// I played around a bit on KNL (see below) but didn't bring anything
-// #elif defined(AVX512)
-// #define PREFETCH_CLOVER(BASE) {                              \
-//     uint64_t base;                                           \
-//     int pf_dist_L1 = 1;                                      \
-//     int pf_dist_L2 = +4;                                     \
-//                                                              \
-//     if ((pf_dist_L1 >= 0) && (sU + pf_dist_L1 < Nsite)) {    \
-//       base = (uint64_t)&diag_t()(pf_dist_L1+BASE)(0);        \
-//       _mm_prefetch((const char*)(base +    0), _MM_HINT_T0); \
-//       _mm_prefetch((const char*)(base +   64), _MM_HINT_T0); \
-//       _mm_prefetch((const char*)(base +  128), _MM_HINT_T0); \
-//       _mm_prefetch((const char*)(base +  192), _MM_HINT_T0); \
-//       _mm_prefetch((const char*)(base +  256), _MM_HINT_T0); \
-//       _mm_prefetch((const char*)(base +  320), _MM_HINT_T0); \
-//     }                                                        \
-//                                                              \
-//     if ((pf_dist_L2 >= 0) && (sU + pf_dist_L2 < Nsite)) {    \
-//       base = (uint64_t)&diag_t()(pf_dist_L2+BASE)(0);        \
-//       _mm_prefetch((const char*)(base +    0), _MM_HINT_T1); \
-//       _mm_prefetch((const char*)(base +   64), _MM_HINT_T1); \
-//       _mm_prefetch((const char*)(base +  128), _MM_HINT_T1); \
-//       _mm_prefetch((const char*)(base +  192), _MM_HINT_T1); \
-//       _mm_prefetch((const char*)(base +  256), _MM_HINT_T1); \
-//       _mm_prefetch((const char*)(base +  320), _MM_HINT_T1); \
-//     }                                                        \
-//   }
-#else
-#define PREFETCH_CLOVER(BASE)
-#endif
-
-    const uint64_t NN = Nsite * Ls;
-
-    thread_for(ss, NN, {
-      int sF = ss;
-      int sU = ss/Ls;
-      CalcSpinor res;
-      CalcSpinor in_t = in_v[sF];
-      auto diag_t     = diagonal_v[sU]; // "diag" instead of "diagonal" here to make code below easier to read
-      auto triangle_t = triangle_v[sU];
-
-      // upper half
-      PREFETCH_CLOVER(0);
-
-      auto in_cc_0_0 = conjugate(in_t()(0)(0)); // Nils: reduces number
-      auto in_cc_0_1 = conjugate(in_t()(0)(1)); // of conjugates from
-      auto in_cc_0_2 = conjugate(in_t()(0)(2)); // 30 to 20
-      auto in_cc_1_0 = conjugate(in_t()(1)(0));
-      auto in_cc_1_1 = conjugate(in_t()(1)(1));
-
-      res()(0)(0) =               diag_t()(0)( 0) * in_t()(0)(0)
-                  +           triangle_t()(0)( 0) * in_t()(0)(1)
-                  +           triangle_t()(0)( 1) * in_t()(0)(2)
-                  +           triangle_t()(0)( 2) * in_t()(1)(0)
-                  +           triangle_t()(0)( 3) * in_t()(1)(1)
-                  +           triangle_t()(0)( 4) * in_t()(1)(2);
-
-      res()(0)(1) =           triangle_t()(0)( 0) * in_cc_0_0;
-      res()(0)(1) =               diag_t()(0)( 1) * in_t()(0)(1)
-                  +           triangle_t()(0)( 5) * in_t()(0)(2)
-                  +           triangle_t()(0)( 6) * in_t()(1)(0)
-                  +           triangle_t()(0)( 7) * in_t()(1)(1)
-                  +           triangle_t()(0)( 8) * in_t()(1)(2)
-                  + conjugate(       res()(0)( 1));
-
-      res()(0)(2) =           triangle_t()(0)( 1) * in_cc_0_0
-                  +           triangle_t()(0)( 5) * in_cc_0_1;
-      res()(0)(2) =               diag_t()(0)( 2) * in_t()(0)(2)
-                  +           triangle_t()(0)( 9) * in_t()(1)(0)
-                  +           triangle_t()(0)(10) * in_t()(1)(1)
-                  +           triangle_t()(0)(11) * in_t()(1)(2)
-                  + conjugate(       res()(0)( 2));
-
-      res()(1)(0) =           triangle_t()(0)( 2) * in_cc_0_0
-                  +           triangle_t()(0)( 6) * in_cc_0_1
-                  +           triangle_t()(0)( 9) * in_cc_0_2;
-      res()(1)(0) =               diag_t()(0)( 3) * in_t()(1)(0)
-                  +           triangle_t()(0)(12) * in_t()(1)(1)
-                  +           triangle_t()(0)(13) * in_t()(1)(2)
-                  + conjugate(       res()(1)( 0));
-
-      res()(1)(1) =           triangle_t()(0)( 3) * in_cc_0_0
-                  +           triangle_t()(0)( 7) * in_cc_0_1
-                  +           triangle_t()(0)(10) * in_cc_0_2
-                  +           triangle_t()(0)(12) * in_cc_1_0;
-      res()(1)(1) =               diag_t()(0)( 4) * in_t()(1)(1)
-                  +           triangle_t()(0)(14) * in_t()(1)(2)
-                  + conjugate(       res()(1)( 1));
-
-      res()(1)(2) =           triangle_t()(0)( 4) * in_cc_0_0
-                  +           triangle_t()(0)( 8) * in_cc_0_1
-                  +           triangle_t()(0)(11) * in_cc_0_2
-                  +           triangle_t()(0)(13) * in_cc_1_0
-                  +           triangle_t()(0)(14) * in_cc_1_1;
-      res()(1)(2) =               diag_t()(0)( 5) * in_t()(1)(2)
-                  + conjugate(       res()(1)( 2));
-
-      vstream(out_v[sF]()(0)(0), res()(0)(0));
-      vstream(out_v[sF]()(0)(1), res()(0)(1));
-      vstream(out_v[sF]()(0)(2), res()(0)(2));
-      vstream(out_v[sF]()(1)(0), res()(1)(0));
-      vstream(out_v[sF]()(1)(1), res()(1)(1));
-      vstream(out_v[sF]()(1)(2), res()(1)(2));
-
-      // lower half
-      PREFETCH_CLOVER(1);
-
-      auto in_cc_2_0 = conjugate(in_t()(2)(0));
-      auto in_cc_2_1 = conjugate(in_t()(2)(1));
-      auto in_cc_2_2 = conjugate(in_t()(2)(2));
-      auto in_cc_3_0 = conjugate(in_t()(3)(0));
-      auto in_cc_3_1 = conjugate(in_t()(3)(1));
-
-      res()(2)(0) =               diag_t()(1)( 0) * in_t()(2)(0)
-                  +           triangle_t()(1)( 0) * in_t()(2)(1)
-                  +           triangle_t()(1)( 1) * in_t()(2)(2)
-                  +           triangle_t()(1)( 2) * in_t()(3)(0)
-                  +           triangle_t()(1)( 3) * in_t()(3)(1)
-                  +           triangle_t()(1)( 4) * in_t()(3)(2);
-
-      res()(2)(1) =           triangle_t()(1)( 0) * in_cc_2_0;
-      res()(2)(1) =               diag_t()(1)( 1) * in_t()(2)(1)
-                  +           triangle_t()(1)( 5) * in_t()(2)(2)
-                  +           triangle_t()(1)( 6) * in_t()(3)(0)
-                  +           triangle_t()(1)( 7) * in_t()(3)(1)
-                  +           triangle_t()(1)( 8) * in_t()(3)(2)
-                  + conjugate(       res()(2)( 1));
-
-      res()(2)(2) =           triangle_t()(1)( 1) * in_cc_2_0
-                  +           triangle_t()(1)( 5) * in_cc_2_1;
-      res()(2)(2) =               diag_t()(1)( 2) * in_t()(2)(2)
-                  +           triangle_t()(1)( 9) * in_t()(3)(0)
-                  +           triangle_t()(1)(10) * in_t()(3)(1)
-                  +           triangle_t()(1)(11) * in_t()(3)(2)
-                  + conjugate(       res()(2)( 2));
-
-      res()(3)(0) =           triangle_t()(1)( 2) * in_cc_2_0
-                  +           triangle_t()(1)( 6) * in_cc_2_1
-                  +           triangle_t()(1)( 9) * in_cc_2_2;
-      res()(3)(0) =               diag_t()(1)( 3) * in_t()(3)(0)
-                  +           triangle_t()(1)(12) * in_t()(3)(1)
-                  +           triangle_t()(1)(13) * in_t()(3)(2)
-                  + conjugate(       res()(3)( 0));
-
-      res()(3)(1) =           triangle_t()(1)( 3) * in_cc_2_0
-                  +           triangle_t()(1)( 7) * in_cc_2_1
-                  +           triangle_t()(1)(10) * in_cc_2_2
-                  +           triangle_t()(1)(12) * in_cc_3_0;
-      res()(3)(1) =               diag_t()(1)( 4) * in_t()(3)(1)
-                  +           triangle_t()(1)(14) * in_t()(3)(2)
-                  + conjugate(       res()(3)( 1));
-
-      res()(3)(2) =           triangle_t()(1)( 4) * in_cc_2_0
-                  +           triangle_t()(1)( 8) * in_cc_2_1
-                  +           triangle_t()(1)(11) * in_cc_2_2
-                  +           triangle_t()(1)(13) * in_cc_3_0
-                  +           triangle_t()(1)(14) * in_cc_3_1;
-      res()(3)(2) =               diag_t()(1)( 5) * in_t()(3)(2)
-                  + conjugate(       res()(3)( 2));
-
-      vstream(out_v[sF]()(2)(0), res()(2)(0));
-      vstream(out_v[sF]()(2)(1), res()(2)(1));
-      vstream(out_v[sF]()(2)(2), res()(2)(2));
-      vstream(out_v[sF]()(3)(0), res()(3)(0));
-      vstream(out_v[sF]()(3)(1), res()(3)(1));
-      vstream(out_v[sF]()(3)(2), res()(3)(2));
-    });
-  }
-
-  static void MooeeKernel(int                        Nsite,
-                          int                        Ls,
-                          const FermionField&        in,
-                          FermionField&              out,
-                          const CloverDiagonalField& diagonal,
-                          const CloverTriangleField& triangle) {
-#if defined(GRID_CUDA) || defined(GRID_HIP)
-    MooeeKernel_gpu(Nsite, Ls, in, out, diagonal, triangle);
-#else
-    MooeeKernel_cpu(Nsite, Ls, in, out, diagonal, triangle);
-#endif
-  }
-
-  static void Invert(const CloverDiagonalField& diagonal,
-                     const CloverTriangleField& triangle,
-                     CloverDiagonalField&       diagonalInv,
-                     CloverTriangleField&       triangleInv) {
-    conformable(diagonal, diagonalInv);
-    conformable(triangle, triangleInv);
-    conformable(diagonal, triangle);
-
-    diagonalInv.Checkerboard() = diagonal.Checkerboard();
-    triangleInv.Checkerboard() = triangle.Checkerboard();
-
-    GridBase* grid = diagonal.Grid();
-
-    long lsites = grid->lSites();
-
-    typedef typename SiteCloverDiagonal::scalar_object scalar_object_diagonal;
-    typedef typename SiteCloverTriangle::scalar_object scalar_object_triangle;
-
-    autoView(diagonal_v,  diagonal,  CpuRead);
-    autoView(triangle_v,  triangle,  CpuRead);
-    autoView(diagonalInv_v, diagonalInv, CpuWrite);
-    autoView(triangleInv_v, triangleInv, CpuWrite);
-
-    thread_for(site, lsites, { // NOTE: Not on GPU because of Eigen & (peek/poke)LocalSite
-      Eigen::MatrixXcd clover_inv_eigen = Eigen::MatrixXcd::Zero(Ns*Nc, Ns*Nc);
-      Eigen::MatrixXcd clover_eigen = Eigen::MatrixXcd::Zero(Ns*Nc, Ns*Nc);
-
-      scalar_object_diagonal diagonal_tmp     = Zero();
-      scalar_object_diagonal diagonal_inv_tmp = Zero();
-      scalar_object_triangle triangle_tmp     = Zero();
-      scalar_object_triangle triangle_inv_tmp = Zero();
-
-      Coordinate lcoor;
-      grid->LocalIndexToLocalCoor(site, lcoor);
-
-      peekLocalSite(diagonal_tmp, diagonal_v, lcoor);
-      peekLocalSite(triangle_tmp, triangle_v, lcoor);
-
-      // TODO: can we save time here by inverting the two 6x6 hermitian matrices separately?
-      for (long s_row=0;s_row<Ns;s_row++) {
-        for (long s_col=0;s_col<Ns;s_col++) {
-          if(abs(s_row - s_col) > 1 || s_row + s_col == 3) continue;
-          int block       = s_row / Nhs;
-          int s_row_block = s_row % Nhs;
-          int s_col_block = s_col % Nhs;
-          for (long c_row=0;c_row<Nc;c_row++) {
-            for (long c_col=0;c_col<Nc;c_col++) {
-              int i = s_row_block * Nc + c_row;
-              int j = s_col_block * Nc + c_col;
-              if(i == j)
-                clover_eigen(s_row*Nc+c_row, s_col*Nc+c_col) = static_cast<ComplexD>(TensorRemove(diagonal_tmp()(block)(i)));
-              else
-                clover_eigen(s_row*Nc+c_row, s_col*Nc+c_col) = static_cast<ComplexD>(TensorRemove(triangle_elem(triangle_tmp, block, i, j)));
-            }
-          }
-        }
-      }
-
-      clover_inv_eigen = clover_eigen.inverse();
-
-      for (long s_row=0;s_row<Ns;s_row++) {
-        for (long s_col=0;s_col<Ns;s_col++) {
-          if(abs(s_row - s_col) > 1 || s_row + s_col == 3) continue;
-          int block       = s_row / Nhs;
-          int s_row_block = s_row % Nhs;
-          int s_col_block = s_col % Nhs;
-          for (long c_row=0;c_row<Nc;c_row++) {
-            for (long c_col=0;c_col<Nc;c_col++) {
-              int i = s_row_block * Nc + c_row;
-              int j = s_col_block * Nc + c_col;
-              if(i == j)
-                diagonal_inv_tmp()(block)(i) = clover_inv_eigen(s_row*Nc+c_row, s_col*Nc+c_col);
-              else if(i < j)
-                triangle_inv_tmp()(block)(triangle_index(i, j)) = clover_inv_eigen(s_row*Nc+c_row, s_col*Nc+c_col);
-              else
-                continue;
-            }
-          }
-        }
-      }
-
-      pokeLocalSite(diagonal_inv_tmp, diagonalInv_v, lcoor);
-      pokeLocalSite(triangle_inv_tmp, triangleInv_v, lcoor);
-    });
-  }
-
-  static void ConvertLayout(const CloverField&   full,
-                            CloverDiagonalField& diagonal,
-                            CloverTriangleField& triangle) {
-    conformable(full, diagonal);
-    conformable(full, triangle);
-
-    diagonal.Checkerboard() = full.Checkerboard();
-    triangle.Checkerboard() = full.Checkerboard();
-
-    autoView(full_v,     full,     AcceleratorRead);
-    autoView(diagonal_v, diagonal, AcceleratorWrite);
-    autoView(triangle_v, triangle, AcceleratorWrite);
-
-    // NOTE: this function cannot be 'private' since nvcc forbids this for kernels
-    accelerator_for(ss, full.Grid()->oSites(), 1, {
-      for(int s_row = 0; s_row < Ns; s_row++) {
-        for(int s_col = 0; s_col < Ns; s_col++) {
-          if(abs(s_row - s_col) > 1 || s_row + s_col == 3) continue;
-          int block       = s_row / Nhs;
-          int s_row_block = s_row % Nhs;
-          int s_col_block = s_col % Nhs;
-          for(int c_row = 0; c_row < Nc; c_row++) {
-            for(int c_col = 0; c_col < Nc; c_col++) {
-              int i = s_row_block * Nc + c_row;
-              int j = s_col_block * Nc + c_col;
-              if(i == j)
-                diagonal_v[ss]()(block)(i) = full_v[ss]()(s_row, s_col)(c_row, c_col);
-              else if(i < j)
-                triangle_v[ss]()(block)(triangle_index(i, j)) = full_v[ss]()(s_row, s_col)(c_row, c_col);
-              else
-                continue;
-            }
-          }
-        }
-      }
-    });
-  }
-
-
-  static void ConvertLayout(const CloverDiagonalField& diagonal,
-                            const CloverTriangleField& triangle,
-                            CloverField&               full) {
-    conformable(full, diagonal);
-    conformable(full, triangle);
-
-    full.Checkerboard() = diagonal.Checkerboard();
-
-    full = Zero();
-
-    autoView(diagonal_v, diagonal, AcceleratorRead);
-    autoView(triangle_v, triangle, AcceleratorRead);
-    autoView(full_v,     full,     AcceleratorWrite);
-
-    // NOTE: this function cannot be 'private' since nvcc forbids this for kernels
-    accelerator_for(ss, full.Grid()->oSites(), 1, {
-      for(int s_row = 0; s_row < Ns; s_row++) {
-        for(int s_col = 0; s_col < Ns; s_col++) {
-          if(abs(s_row - s_col) > 1 || s_row + s_col == 3) continue;
-          int block       = s_row / Nhs;
-          int s_row_block = s_row % Nhs;
-          int s_col_block = s_col % Nhs;
-          for(int c_row = 0; c_row < Nc; c_row++) {
-            for(int c_col = 0; c_col < Nc; c_col++) {
-              int i = s_row_block * Nc + c_row;
-              int j = s_col_block * Nc + c_col;
-              if(i == j)
-                full_v[ss]()(s_row, s_col)(c_row, c_col) = diagonal_v[ss]()(block)(i);
-              else
-                full_v[ss]()(s_row, s_col)(c_row, c_col) = triangle_elem(triangle_v[ss], block, i, j);
-            }
-          }
-        }
-      }
-    });
-  }
-
-  static void ModifyBoundaries(CloverDiagonalField& diagonal, CloverTriangleField& triangle, RealD csw_t, RealD cF, RealD diag_mass) {
-    // Checks/grid
-    double t0 = usecond();
-    conformable(diagonal, triangle);
-    GridBase* grid = diagonal.Grid();
-
-    // Determine the boundary coordinates/sites
-    double t1 = usecond();
-    int t_dir = Nd - 1;
-    Lattice<iScalar<vInteger>> t_coor(grid);
-    LatticeCoordinate(t_coor, t_dir);
-    int T = grid->GlobalDimensions()[t_dir];
-
-    // Set off-diagonal parts at boundary to zero -- OK
-    double t2 = usecond();
-    CloverTriangleField zeroTriangle(grid);
-    zeroTriangle.Checkerboard() = triangle.Checkerboard();
-    zeroTriangle = Zero();
-    triangle = where(t_coor == 0,   zeroTriangle, triangle);
-    triangle = where(t_coor == T-1, zeroTriangle, triangle);
-
-    // Set diagonal to unity (scaled correctly) -- OK
-    double t3 = usecond();
-    CloverDiagonalField tmp(grid);
-    tmp.Checkerboard() = diagonal.Checkerboard();
-    tmp                = -1.0 * csw_t + diag_mass;
-    diagonal           = where(t_coor == 0,   tmp, diagonal);
-    diagonal           = where(t_coor == T-1, tmp, diagonal);
-
-    // Correct values next to boundary
-    double t4 = usecond();
-    if(cF != 1.0) {
-      tmp = cF - 1.0;
-      tmp += diagonal;
-      diagonal = where(t_coor == 1,   tmp, diagonal);
-      diagonal = where(t_coor == T-2, tmp, diagonal);
-    }
-
-    // Report timings
-    double t5 = usecond();
-#if 0
-    std::cout << GridLogMessage << "CompactWilsonCloverHelpers::ModifyBoundaries timings:"
-              << " checks = "          << (t1 - t0) / 1e6
-              << ", coordinate = "     << (t2 - t1) / 1e6
-              << ", off-diag zero = "  << (t3 - t2) / 1e6
-              << ", diagonal unity = " << (t4 - t3) / 1e6
-              << ", near-boundary = "  << (t5 - t4) / 1e6
-              << ", total = "          << (t5 - t0) / 1e6
-              << std::endl;
-#endif
-  }
-
-  template<class Field, class Mask>
-  static strong_inline void ApplyBoundaryMask(Field& f, const Mask& m) {
-    conformable(f, m);
-    auto grid  = f.Grid();
-    const uint32_t Nsite = grid->oSites();
-    const uint32_t Nsimd = grid->Nsimd();
-    autoView(f_v, f, AcceleratorWrite);
-    autoView(m_v, m, AcceleratorRead);
-    // NOTE: this function cannot be 'private' since nvcc forbids this for kernels
-    accelerator_for(ss, Nsite, Nsimd, {
-      coalescedWrite(f_v[ss], m_v(ss) * f_v(ss));
-    });
-  }
-
-  template<class MaskField>
-  static void SetupMasks(MaskField& full, MaskField& even, MaskField& odd) {
-    assert(even.Grid()->_isCheckerBoarded && even.Checkerboard() == Even);
-    assert(odd.Grid()->_isCheckerBoarded  && odd.Checkerboard()  == Odd);
-    assert(!full.Grid()->_isCheckerBoarded);
-
-    GridBase* grid = full.Grid();
-    int t_dir = Nd-1;
-    Lattice<iScalar<vInteger>> t_coor(grid);
-    LatticeCoordinate(t_coor, t_dir);
-    int T = grid->GlobalDimensions()[t_dir];
-
-    MaskField zeroMask(grid); zeroMask = Zero();
-    full = 1.0;
-    full = where(t_coor == 0,   zeroMask, full);
-    full = where(t_coor == T-1, zeroMask, full);
-
-    pickCheckerboard(Even, even, full);
-    pickCheckerboard(Odd,  odd,  full);
-  }
-};
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/WilsonCloverTypes.h

@@ -1,92 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: ./lib/qcd/action/fermion/WilsonCloverTypes.h
-
-    Copyright (C) 2021 - 2022
-
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#pragma once
-
-NAMESPACE_BEGIN(Grid);
-
-template<class Impl>
-class WilsonCloverTypes {
-public:
-  INHERIT_IMPL_TYPES(Impl);
-
-  template <typename vtype> using iImplClover = iScalar<iMatrix<iMatrix<vtype, Impl::Dimension>, Ns>>;
-
-  typedef iImplClover<Simd> SiteClover;
-
-  typedef Lattice<SiteClover> CloverField;
-};
-
-template<class Impl>
-class CompactWilsonCloverTypes {
-public:
-  INHERIT_IMPL_TYPES(Impl);
-
-  static_assert(Nd == 4 && Nc == 3 && Ns == 4 && Impl::Dimension == 3, "Wrong dimensions");
-
-  static constexpr int Nred      = Nc * Nhs;        // 6
-  static constexpr int Nblock    = Nhs;             // 2
-  static constexpr int Ndiagonal = Nred;            // 6
-  static constexpr int Ntriangle = (Nred - 1) * Nc; // 15
-
-  template<typename vtype> using iImplCloverDiagonal = iScalar<iVector<iVector<vtype, Ndiagonal>, Nblock>>;
-  template<typename vtype> using iImplCloverTriangle = iScalar<iVector<iVector<vtype, Ntriangle>, Nblock>>;
-
-  typedef iImplCloverDiagonal<Simd> SiteCloverDiagonal;
-  typedef iImplCloverTriangle<Simd> SiteCloverTriangle;
-  typedef iSinglet<Simd>            SiteMask;
-
-  typedef Lattice<SiteCloverDiagonal> CloverDiagonalField;
-  typedef Lattice<SiteCloverTriangle> CloverTriangleField;
-  typedef Lattice<SiteMask>           MaskField;
-};
-
-#define INHERIT_CLOVER_TYPES(Impl)                                 \
-  typedef typename WilsonCloverTypes<Impl>::SiteClover SiteClover; \
-  typedef typename WilsonCloverTypes<Impl>::CloverField CloverField;
-
-#define INHERIT_COMPACT_CLOVER_TYPES(Impl) \
-  typedef typename CompactWilsonCloverTypes<Impl>::SiteCloverDiagonal  SiteCloverDiagonal; \
-  typedef typename CompactWilsonCloverTypes<Impl>::SiteCloverTriangle  SiteCloverTriangle; \
-  typedef typename CompactWilsonCloverTypes<Impl>::SiteMask            SiteMask; \
-  typedef typename CompactWilsonCloverTypes<Impl>::CloverDiagonalField CloverDiagonalField; \
-  typedef typename CompactWilsonCloverTypes<Impl>::CloverTriangleField CloverTriangleField; \
-  typedef typename CompactWilsonCloverTypes<Impl>::MaskField           MaskField; \
-  /* ugly duplication but needed inside functionality classes */ \
-  template<typename vtype> using iImplCloverDiagonal = \
-    iScalar<iVector<iVector<vtype, CompactWilsonCloverTypes<Impl>::Ndiagonal>, CompactWilsonCloverTypes<Impl>::Nblock>>; \
-  template<typename vtype> using iImplCloverTriangle = \
-    iScalar<iVector<iVector<vtype, CompactWilsonCloverTypes<Impl>::Ntriangle>, CompactWilsonCloverTypes<Impl>::Nblock>>;
-
-#define INHERIT_COMPACT_CLOVER_SIZES(Impl)                                    \
-  static constexpr int Nred      = CompactWilsonCloverTypes<Impl>::Nred;      \
-  static constexpr int Nblock    = CompactWilsonCloverTypes<Impl>::Nblock;    \
-  static constexpr int Ndiagonal = CompactWilsonCloverTypes<Impl>::Ndiagonal; \
-  static constexpr int Ntriangle = CompactWilsonCloverTypes<Impl>::Ntriangle;
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/WilsonCompressor.h

@@ -303,9 +303,11 @@ public:
 		int npoints,
 		int checkerboard,
 		const std::vector<int> &directions,
-		const std::vector<int> &distances,Parameters p)  
-    : CartesianStencil<vobj,cobj,Parameters> (grid,npoints,checkerboard,directions,distances,p) 
-  { 
+		const std::vector<int> &distances,
+		bool locally_periodic,
+		Parameters p)  
+    : CartesianStencil<vobj,cobj,Parameters> (grid,npoints,checkerboard,directions,distances,locally_periodic,p)
+  {
     ZeroCountersi();
     surface_list.resize(0);
     this->same_node.resize(npoints);

Grid/qcd/action/fermion/WilsonFermion.h

@@ -146,8 +146,11 @@ public:
   void DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
                     const FermionField &in, FermionField &out, int dag);
 
+  void DhopInternalDirichletComms(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
+				  const FermionField &in, FermionField &out, int dag);
+  
   void DhopInternalOverlappedComms(StencilImpl &st, LebesgueOrder &lo, DoubledGaugeField &U,
-                    const FermionField &in, FermionField &out, int dag);
+				   const FermionField &in, FermionField &out, int dag);
 
   // Constructor
   WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
@@ -157,7 +160,10 @@ public:
 
   // DoubleStore impl dependent
   void ImportGauge(const GaugeField &_Umu);
-
+  DoubledGaugeField &GetDoubledGaugeField(void){ return Umu; };
+  DoubledGaugeField &GetDoubledGaugeFieldE(void){ return UmuEven; };
+  DoubledGaugeField &GetDoubledGaugeFieldO(void){ return UmuOdd; };
+  
   ///////////////////////////////////////////////////////////////
   // Data members require to support the functionality
   ///////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/WilsonFermion5D.h

@@ -165,7 +165,14 @@ public:
 			       const FermionField &in, 
 			       FermionField &out,
 			       int dag);
-    
+
+  void DhopInternalDirichletComms(StencilImpl & st,
+				  LebesgueOrder &lo,
+				  DoubledGaugeField &U,
+				  const FermionField &in, 
+				  FermionField &out,
+				  int dag);
+  
   // Constructors
   WilsonFermion5D(GaugeField &_Umu,
 		  GridCartesian         &FiveDimGrid,
@@ -174,19 +181,11 @@ public:
 		  GridRedBlackCartesian &FourDimRedBlackGrid,
 		  double _M5,const ImplParams &p= ImplParams());
     
-  // Constructors
-  /*
-    WilsonFermion5D(int simd, 
-    GaugeField &_Umu,
-    GridCartesian         &FiveDimGrid,
-    GridRedBlackCartesian &FiveDimRedBlackGrid,
-    GridCartesian         &FourDimGrid,
-    double _M5,const ImplParams &p= ImplParams());
-  */
-    
   // DoubleStore
   void ImportGauge(const GaugeField &_Umu);
-    
+  DoubledGaugeField &GetDoubledGaugeField(void){ return Umu; };
+  DoubledGaugeField &GetDoubledGaugeFieldE(void){ return UmuEven; };
+  DoubledGaugeField &GetDoubledGaugeFieldO(void){ return UmuOdd; };
   ///////////////////////////////////////////////////////////////
   // Data members require to support the functionality
   ///////////////////////////////////////////////////////////////

Grid/qcd/action/fermion/WilsonKernels.h

@@ -39,7 +39,7 @@ NAMESPACE_BEGIN(Grid);
 class WilsonKernelsStatic { 
 public:
   enum { OptGeneric, OptHandUnroll, OptInlineAsm };
-  enum { CommsAndCompute, CommsThenCompute };
+  enum { CommsAndCompute, CommsThenCompute, CommsDirichlet };
   static int Opt;  
   static int Comms;
 };

Grid/qcd/action/fermion/implementation/CayleyFermion5DImplementation.h

@@ -112,7 +112,6 @@ void CayleyFermion5D<Impl>::ImportUnphysicalFermion(const FermionField &input4d,
   axpby_ssp_pminus(tmp, 0., tmp, 1., tmp, Ls-1, Ls-1);
   imported5d=tmp;
 }
-
 template<class Impl>  
 void CayleyFermion5D<Impl>::ImportPhysicalFermionSource(const FermionField &input4d,FermionField &imported5d)
 {
@@ -127,6 +126,37 @@ void CayleyFermion5D<Impl>::ImportPhysicalFermionSource(const FermionField &inpu
   axpby_ssp_pminus(tmp, 0., tmp, 1., tmp, Ls-1, Ls-1);
   Dminus(tmp,imported5d);
 }
+////////////////////////////////////////////////////
+// Added for fourD pseudofermion det estimation
+////////////////////////////////////////////////////
+template<class Impl>  
+void CayleyFermion5D<Impl>::ImportFourDimPseudoFermion(const FermionField &input4d,FermionField &imported5d)
+{
+  int Ls = this->Ls;
+  FermionField tmp(this->FermionGrid());
+  conformable(imported5d.Grid(),this->FermionGrid());
+  conformable(input4d.Grid()   ,this->GaugeGrid());
+  tmp = Zero();
+  InsertSlice(input4d, tmp, 0   , 0);
+  InsertSlice(input4d, tmp, Ls-1, 0);
+  axpby_ssp_pminus(tmp, 0., tmp, 1., tmp, 0, 0);
+  axpby_ssp_pplus (tmp, 0., tmp, 1., tmp, Ls-1, Ls-1);
+  imported5d=tmp;
+}
+template<class Impl>  
+void CayleyFermion5D<Impl>::ExportFourDimPseudoFermion(const FermionField &solution5d,FermionField &exported4d)
+{
+  int Ls = this->Ls;
+  FermionField tmp(this->FermionGrid());
+  tmp = solution5d;
+  conformable(solution5d.Grid(),this->FermionGrid());
+  conformable(exported4d.Grid(),this->GaugeGrid());
+  axpby_ssp_pminus(tmp, 0., solution5d, 1., solution5d, 0, 0);
+  axpby_ssp_pplus (tmp, 1., tmp       , 1., solution5d, 0, Ls-1);
+  ExtractSlice(exported4d, tmp, 0, 0);
+}
+
+// Dminus
 template<class Impl>  
 void CayleyFermion5D<Impl>::Dminus(const FermionField &psi, FermionField &chi)
 {
@@ -828,7 +858,6 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
 
 #if (!defined(GRID_HIP))
   int tshift = (mu == Nd-1) ? 1 : 0;
-  unsigned int LLt    = GridDefaultLatt()[Tp];
   ////////////////////////////////////////////////
   // GENERAL CAYLEY CASE
   ////////////////////////////////////////////////
@@ -881,7 +910,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
   }
 
   std::vector<RealD> G_s(Ls,1.0);
-  RealD sign = 1.0; // sign flip for vector/tadpole
+  RealD sign = 1; // sign flip for vector/tadpole
   if ( curr_type == Current::Axial ) {
     for(int s=0;s<Ls/2;s++){
       G_s[s] = -1.0;
@@ -891,7 +920,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
     auto b=this->_b;
     auto c=this->_c;
     if ( b == 1 && c == 0 ) {
-      sign = -1.0;    
+      sign = -1;    
     }
     else {
       std::cerr << "Error: Tadpole implementation currently unavailable for non-Shamir actions." << std::endl;
@@ -935,13 +964,7 @@ void CayleyFermion5D<Impl>::SeqConservedCurrent(PropagatorField &q_in,
     tmp    = Cshift(tmp,mu,-1);
     Impl::multLinkField(Utmp,this->Umu,tmp,mu+Nd); // Adjoint link
     tmp = -G_s[s]*( Utmp + gmu*Utmp );
-    // Mask the time
-    if (tmax == LLt - 1 && tshift == 1){ // quick fix to include timeslice 0 if tmax + tshift is over the last timeslice
-      unsigned int t0 = 0;
-      tmp    = where(((lcoor==t0) || (lcoor>=tmin+tshift)),tmp,zz);
-    } else {
-      tmp    = where((lcoor>=tmin+tshift),tmp,zz);
-    }
+    tmp    = where((lcoor>=tmin+tshift),tmp,zz); // Mask the time 
     L_Q   += where((lcoor<=tmax+tshift),tmp,zz); // Position of current complicated
 
     InsertSlice(L_Q, q_out, s , 0);

Grid/qcd/action/fermion/implementation/CompactWilsonCloverFermionImplementation.h

@@ -1,363 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: ./lib/qcd/action/fermion/CompactWilsonCloverFermionImplementation.h
-
-    Copyright (C) 2017 - 2022
-
-    Author: paboyle <paboyle@ph.ed.ac.uk>
-    Author: Guido Cossu <guido.cossu@ed.ac.uk>
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Grid.h>
-#include <Grid/qcd/spin/Dirac.h>
-#include <Grid/qcd/action/fermion/CompactWilsonCloverFermion.h>
-
-NAMESPACE_BEGIN(Grid);
-template<class Impl>
-CompactWilsonCloverFermion<Impl>::CompactWilsonCloverFermion(GaugeField& _Umu,
-                                                             GridCartesian& Fgrid,
-                                                             GridRedBlackCartesian& Hgrid,
-                                                             const RealD _mass,
-                                                             const RealD _csw_r,
-                                                             const RealD _csw_t,
-                                                             const RealD _cF,
-                                                             const WilsonAnisotropyCoefficients& clover_anisotropy,
-                                                             const ImplParams& impl_p)
-  : WilsonBase(_Umu, Fgrid, Hgrid, _mass, impl_p, clover_anisotropy)
-  , csw_r(_csw_r)
-  , csw_t(_csw_t)
-  , cF(_cF)
-  , open_boundaries(impl_p.boundary_phases[Nd-1] == 0.0)
-  , Diagonal(&Fgrid),        Triangle(&Fgrid)
-  , DiagonalEven(&Hgrid),    TriangleEven(&Hgrid)
-  , DiagonalOdd(&Hgrid),     TriangleOdd(&Hgrid)
-  , DiagonalInv(&Fgrid),     TriangleInv(&Fgrid)
-  , DiagonalInvEven(&Hgrid), TriangleInvEven(&Hgrid)
-  , DiagonalInvOdd(&Hgrid),  TriangleInvOdd(&Hgrid)
-  , Tmp(&Fgrid)
-  , BoundaryMask(&Fgrid)
-  , BoundaryMaskEven(&Hgrid), BoundaryMaskOdd(&Hgrid)
-{
-  csw_r *= 0.5;
-  csw_t *= 0.5;
-  if (clover_anisotropy.isAnisotropic)
-    csw_r /= clover_anisotropy.xi_0;
-
-  ImportGauge(_Umu);
-  if (open_boundaries)
-    CompactHelpers::SetupMasks(this->BoundaryMask, this->BoundaryMaskEven, this->BoundaryMaskOdd);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::Dhop(const FermionField& in, FermionField& out, int dag) {
-  WilsonBase::Dhop(in, out, dag);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::DhopOE(const FermionField& in, FermionField& out, int dag) {
-  WilsonBase::DhopOE(in, out, dag);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::DhopEO(const FermionField& in, FermionField& out, int dag) {
-  WilsonBase::DhopEO(in, out, dag);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::DhopDir(const FermionField& in, FermionField& out, int dir, int disp) {
-  WilsonBase::DhopDir(in, out, dir, disp);
-  if(this->open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::DhopDirAll(const FermionField& in, std::vector<FermionField>& out) {
-  WilsonBase::DhopDirAll(in, out);
-  if(this->open_boundaries) {
-    for(auto& o : out) ApplyBoundaryMask(o);
-  }
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::M(const FermionField& in, FermionField& out) {
-  out.Checkerboard() = in.Checkerboard();
-  WilsonBase::Dhop(in, out, DaggerNo); // call base to save applying bc
-  Mooee(in, Tmp);
-  axpy(out, 1.0, out, Tmp);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::Mdag(const FermionField& in, FermionField& out) {
-  out.Checkerboard() = in.Checkerboard();
-  WilsonBase::Dhop(in, out, DaggerYes);  // call base to save applying bc
-  MooeeDag(in, Tmp);
-  axpy(out, 1.0, out, Tmp);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::Meooe(const FermionField& in, FermionField& out) {
-  WilsonBase::Meooe(in, out);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MeooeDag(const FermionField& in, FermionField& out) {
-  WilsonBase::MeooeDag(in, out);
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::Mooee(const FermionField& in, FermionField& out) {
-  if(in.Grid()->_isCheckerBoarded) {
-    if(in.Checkerboard() == Odd) {
-      MooeeInternal(in, out, DiagonalOdd, TriangleOdd);
-    } else {
-      MooeeInternal(in, out, DiagonalEven, TriangleEven);
-    }
-  } else {
-    MooeeInternal(in, out, Diagonal, Triangle);
-  }
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MooeeDag(const FermionField& in, FermionField& out) {
-  Mooee(in, out); // blocks are hermitian
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MooeeInv(const FermionField& in, FermionField& out) {
-  if(in.Grid()->_isCheckerBoarded) {
-    if(in.Checkerboard() == Odd) {
-      MooeeInternal(in, out, DiagonalInvOdd, TriangleInvOdd);
-    } else {
-      MooeeInternal(in, out, DiagonalInvEven, TriangleInvEven);
-    }
-  } else {
-    MooeeInternal(in, out, DiagonalInv, TriangleInv);
-  }
-  if(open_boundaries) ApplyBoundaryMask(out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MooeeInvDag(const FermionField& in, FermionField& out) {
-  MooeeInv(in, out); // blocks are hermitian
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::Mdir(const FermionField& in, FermionField& out, int dir, int disp) {
-  DhopDir(in, out, dir, disp);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MdirAll(const FermionField& in, std::vector<FermionField>& out) {
-  DhopDirAll(in, out);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MDeriv(GaugeField& force, const FermionField& X, const FermionField& Y, int dag) {
-  assert(!open_boundaries); // TODO check for changes required for open bc
-
-  // NOTE: code copied from original clover term
-  conformable(X.Grid(), Y.Grid());
-  conformable(X.Grid(), force.Grid());
-  GaugeLinkField force_mu(force.Grid()), lambda(force.Grid());
-  GaugeField clover_force(force.Grid());
-  PropagatorField Lambda(force.Grid());
-
-  // Guido: Here we are hitting some performance issues:
-  // need to extract the components of the DoubledGaugeField
-  // for each call
-  // Possible solution
-  // Create a vector object to store them? (cons: wasting space)
-  std::vector<GaugeLinkField> U(Nd, this->Umu.Grid());
-
-  Impl::extractLinkField(U, this->Umu);
-
-  force = Zero();
-  // Derivative of the Wilson hopping term
-  this->DhopDeriv(force, X, Y, dag);
-
-  ///////////////////////////////////////////////////////////
-  // Clover term derivative
-  ///////////////////////////////////////////////////////////
-  Impl::outerProductImpl(Lambda, X, Y);
-  //std::cout << "Lambda:" << Lambda << std::endl;
-
-  Gamma::Algebra sigma[] = {
-      Gamma::Algebra::SigmaXY,
-      Gamma::Algebra::SigmaXZ,
-      Gamma::Algebra::SigmaXT,
-      Gamma::Algebra::MinusSigmaXY,
-      Gamma::Algebra::SigmaYZ,
-      Gamma::Algebra::SigmaYT,
-      Gamma::Algebra::MinusSigmaXZ,
-      Gamma::Algebra::MinusSigmaYZ,
-      Gamma::Algebra::SigmaZT,
-      Gamma::Algebra::MinusSigmaXT,
-      Gamma::Algebra::MinusSigmaYT,
-      Gamma::Algebra::MinusSigmaZT};
-
-  /*
-    sigma_{\mu \nu}=
-    | 0         sigma[0]  sigma[1]  sigma[2] |
-    | sigma[3]    0       sigma[4]  sigma[5] |
-    | sigma[6]  sigma[7]     0      sigma[8] |
-    | sigma[9]  sigma[10] sigma[11]   0      |
-  */
-
-  int count = 0;
-  clover_force = Zero();
-  for (int mu = 0; mu < 4; mu++)
-  {
-    force_mu = Zero();
-    for (int nu = 0; nu < 4; nu++)
-    {
-      if (mu == nu)
-        continue;
-
-      RealD factor;
-      if (nu == 4 || mu == 4)
-      {
-        factor = 2.0 * csw_t;
-      }
-      else
-      {
-        factor = 2.0 * csw_r;
-      }
-      PropagatorField Slambda = Gamma(sigma[count]) * Lambda; // sigma checked
-      Impl::TraceSpinImpl(lambda, Slambda);                   // traceSpin ok
-      force_mu -= factor*Helpers::Cmunu(U, lambda, mu, nu);   // checked
-      count++;
-    }
-
-    pokeLorentz(clover_force, U[mu] * force_mu, mu);
-  }
-  //clover_force *= csw;
-  force += clover_force;
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MooDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag) {
-  assert(0);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MeeDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag) {
-  assert(0);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::MooeeInternal(const FermionField&        in,
-                    FermionField&              out,
-                    const CloverDiagonalField& diagonal,
-                    const CloverTriangleField& triangle) {
-  assert(in.Checkerboard() == Odd || in.Checkerboard() == Even);
-  out.Checkerboard() = in.Checkerboard();
-  conformable(in, out);
-  conformable(in, diagonal);
-  conformable(in, triangle);
-
-  CompactHelpers::MooeeKernel(diagonal.oSites(), 1, in, out, diagonal, triangle);
-}
-
-template<class Impl>
-void CompactWilsonCloverFermion<Impl>::ImportGauge(const GaugeField& _Umu) {
-  // NOTE: parts copied from original implementation
-
-  // Import gauge into base class
-  double t0 = usecond();
-  WilsonBase::ImportGauge(_Umu); // NOTE: called here and in wilson constructor -> performed twice, but can't avoid that
-
-  // Initialize temporary variables
-  double t1 = usecond();
-  conformable(_Umu.Grid(), this->GaugeGrid());
-  GridBase* grid = _Umu.Grid();
-  typename Impl::GaugeLinkField Bx(grid), By(grid), Bz(grid), Ex(grid), Ey(grid), Ez(grid);
-  CloverField TmpOriginal(grid);
-
-  // Compute the field strength terms mu>nu
-  double t2 = usecond();
-  WilsonLoops<Impl>::FieldStrength(Bx, _Umu, Zdir, Ydir);
-  WilsonLoops<Impl>::FieldStrength(By, _Umu, Zdir, Xdir);
-  WilsonLoops<Impl>::FieldStrength(Bz, _Umu, Ydir, Xdir);
-  WilsonLoops<Impl>::FieldStrength(Ex, _Umu, Tdir, Xdir);
-  WilsonLoops<Impl>::FieldStrength(Ey, _Umu, Tdir, Ydir);
-  WilsonLoops<Impl>::FieldStrength(Ez, _Umu, Tdir, Zdir);
-
-  // Compute the Clover Operator acting on Colour and Spin
-  // multiply here by the clover coefficients for the anisotropy
-  double t3 = usecond();
-  TmpOriginal  = Helpers::fillCloverYZ(Bx) * csw_r;
-  TmpOriginal += Helpers::fillCloverXZ(By) * csw_r;
-  TmpOriginal += Helpers::fillCloverXY(Bz) * csw_r;
-  TmpOriginal += Helpers::fillCloverXT(Ex) * csw_t;
-  TmpOriginal += Helpers::fillCloverYT(Ey) * csw_t;
-  TmpOriginal += Helpers::fillCloverZT(Ez) * csw_t;
-  TmpOriginal += this->diag_mass;
-
-  // Convert the data layout of the clover term
-  double t4 = usecond();
-  CompactHelpers::ConvertLayout(TmpOriginal, Diagonal, Triangle);
-
-  // Possible modify the boundary values
-  double t5 = usecond();
-  if(open_boundaries) CompactHelpers::ModifyBoundaries(Diagonal, Triangle, csw_t, cF, this->diag_mass);
-
-  // Invert the clover term in the improved layout
-  double t6 = usecond();
-  CompactHelpers::Invert(Diagonal, Triangle, DiagonalInv, TriangleInv);
-
-  // Fill the remaining clover fields
-  double t7 = usecond();
-  pickCheckerboard(Even, DiagonalEven,    Diagonal);
-  pickCheckerboard(Even, TriangleEven,    Triangle);
-  pickCheckerboard(Odd,  DiagonalOdd,     Diagonal);
-  pickCheckerboard(Odd,  TriangleOdd,     Triangle);
-  pickCheckerboard(Even, DiagonalInvEven, DiagonalInv);
-  pickCheckerboard(Even, TriangleInvEven, TriangleInv);
-  pickCheckerboard(Odd,  DiagonalInvOdd,  DiagonalInv);
-  pickCheckerboard(Odd,  TriangleInvOdd,  TriangleInv);
-
-  // Report timings
-  double t8 = usecond();
-#if 0
-  std::cout << GridLogMessage << "CompactWilsonCloverFermion::ImportGauge timings:"
-            << " WilsonFermion::Importgauge = " << (t1 - t0) / 1e6
-            << ", allocations = "               << (t2 - t1) / 1e6
-            << ", field strength = "            << (t3 - t2) / 1e6
-            << ", fill clover = "               << (t4 - t3) / 1e6
-            << ", convert = "                   << (t5 - t4) / 1e6
-            << ", boundaries = "                << (t6 - t5) / 1e6
-            << ", inversions = "                << (t7 - t6) / 1e6
-            << ", pick cbs = "                  << (t8 - t7) / 1e6
-            << ", total = "                     << (t8 - t0) / 1e6
-            << std::endl;
-#endif
-}
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/implementation/WilsonCloverFermionImplementation.h

@@ -2,13 +2,12 @@
 
     Grid physics library, www.github.com/paboyle/Grid
 
-    Source file: ./lib/qcd/action/fermion/WilsonCloverFermionImplementation.h
+    Source file: ./lib/qcd/action/fermion/WilsonCloverFermion.cc
 
-    Copyright (C) 2017 - 2022
+    Copyright (C) 2017
 
     Author: paboyle <paboyle@ph.ed.ac.uk>
     Author: Guido Cossu <guido.cossu@ed.ac.uk>
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -34,45 +33,6 @@
 
 NAMESPACE_BEGIN(Grid);
 
-template<class Impl>
-WilsonCloverFermion<Impl>::WilsonCloverFermion(GaugeField&                         _Umu,
-                                               GridCartesian&                      Fgrid,
-                                               GridRedBlackCartesian&              Hgrid,
-                                               const RealD                         _mass,
-                                               const RealD                         _csw_r,
-                                               const RealD                         _csw_t,
-                                               const WilsonAnisotropyCoefficients& clover_anisotropy,
-                                               const ImplParams&                   impl_p)
-  : WilsonFermion<Impl>(_Umu, Fgrid, Hgrid, _mass, impl_p, clover_anisotropy)
-  , CloverTerm(&Fgrid)
-  , CloverTermInv(&Fgrid)
-  , CloverTermEven(&Hgrid)
-  , CloverTermOdd(&Hgrid)
-  , CloverTermInvEven(&Hgrid)
-  , CloverTermInvOdd(&Hgrid)
-  , CloverTermDagEven(&Hgrid)
-  , CloverTermDagOdd(&Hgrid)
-  , CloverTermInvDagEven(&Hgrid)
-  , CloverTermInvDagOdd(&Hgrid) {
-  assert(Nd == 4); // require 4 dimensions
-
-  if(clover_anisotropy.isAnisotropic) {
-    csw_r     = _csw_r * 0.5 / clover_anisotropy.xi_0;
-    diag_mass = _mass + 1.0 + (Nd - 1) * (clover_anisotropy.nu / clover_anisotropy.xi_0);
-  } else {
-    csw_r     = _csw_r * 0.5;
-    diag_mass = 4.0 + _mass;
-  }
-  csw_t = _csw_t * 0.5;
-
-  if(csw_r == 0)
-    std::cout << GridLogWarning << "Initializing WilsonCloverFermion with csw_r = 0" << std::endl;
-  if(csw_t == 0)
-    std::cout << GridLogWarning << "Initializing WilsonCloverFermion with csw_t = 0" << std::endl;
-
-  ImportGauge(_Umu);
-}
-
 // *NOT* EO
 template <class Impl>
 void WilsonCloverFermion<Impl>::M(const FermionField &in, FermionField &out)
@@ -107,13 +67,10 @@ void WilsonCloverFermion<Impl>::Mdag(const FermionField &in, FermionField &out)
 template <class Impl>
 void WilsonCloverFermion<Impl>::ImportGauge(const GaugeField &_Umu)
 {
-  double t0 = usecond();
   WilsonFermion<Impl>::ImportGauge(_Umu);
-  double t1 = usecond();
   GridBase *grid = _Umu.Grid();
   typename Impl::GaugeLinkField Bx(grid), By(grid), Bz(grid), Ex(grid), Ey(grid), Ez(grid);
 
-  double t2 = usecond();
   // Compute the field strength terms mu>nu
   WilsonLoops<Impl>::FieldStrength(Bx, _Umu, Zdir, Ydir);
   WilsonLoops<Impl>::FieldStrength(By, _Umu, Zdir, Xdir);
@@ -122,22 +79,19 @@ void WilsonCloverFermion<Impl>::ImportGauge(const GaugeField &_Umu)
   WilsonLoops<Impl>::FieldStrength(Ey, _Umu, Tdir, Ydir);
   WilsonLoops<Impl>::FieldStrength(Ez, _Umu, Tdir, Zdir);
 
-  double t3 = usecond();
   // Compute the Clover Operator acting on Colour and Spin
   // multiply here by the clover coefficients for the anisotropy
-  CloverTerm  = Helpers::fillCloverYZ(Bx) * csw_r;
-  CloverTerm += Helpers::fillCloverXZ(By) * csw_r;
-  CloverTerm += Helpers::fillCloverXY(Bz) * csw_r;
-  CloverTerm += Helpers::fillCloverXT(Ex) * csw_t;
-  CloverTerm += Helpers::fillCloverYT(Ey) * csw_t;
-  CloverTerm += Helpers::fillCloverZT(Ez) * csw_t;
+  CloverTerm  = fillCloverYZ(Bx) * csw_r;
+  CloverTerm += fillCloverXZ(By) * csw_r;
+  CloverTerm += fillCloverXY(Bz) * csw_r;
+  CloverTerm += fillCloverXT(Ex) * csw_t;
+  CloverTerm += fillCloverYT(Ey) * csw_t;
+  CloverTerm += fillCloverZT(Ez) * csw_t;
   CloverTerm += diag_mass;
 
-  double t4 = usecond();
   int lvol = _Umu.Grid()->lSites();
   int DimRep = Impl::Dimension;
 
-  double t5 = usecond();
   {
     autoView(CTv,CloverTerm,CpuRead);
     autoView(CTIv,CloverTermInv,CpuWrite);
@@ -146,7 +100,7 @@ void WilsonCloverFermion<Impl>::ImportGauge(const GaugeField &_Umu)
       grid->LocalIndexToLocalCoor(site, lcoor);
       Eigen::MatrixXcd EigenCloverOp = Eigen::MatrixXcd::Zero(Ns * DimRep, Ns * DimRep);
       Eigen::MatrixXcd EigenInvCloverOp = Eigen::MatrixXcd::Zero(Ns * DimRep, Ns * DimRep);
-      typename SiteClover::scalar_object Qx = Zero(), Qxinv = Zero();
+      typename SiteCloverType::scalar_object Qx = Zero(), Qxinv = Zero();
       peekLocalSite(Qx, CTv, lcoor);
       //if (csw!=0){
       for (int j = 0; j < Ns; j++)
@@ -171,7 +125,6 @@ void WilsonCloverFermion<Impl>::ImportGauge(const GaugeField &_Umu)
     });
   }
 
-  double t6 = usecond();
   // Separate the even and odd parts
   pickCheckerboard(Even, CloverTermEven, CloverTerm);
   pickCheckerboard(Odd, CloverTermOdd, CloverTerm);
@@ -184,20 +137,6 @@ void WilsonCloverFermion<Impl>::ImportGauge(const GaugeField &_Umu)
 
   pickCheckerboard(Even, CloverTermInvDagEven, adj(CloverTermInv));
   pickCheckerboard(Odd, CloverTermInvDagOdd, adj(CloverTermInv));
-  double t7 = usecond();
-
-#if 0
-  std::cout << GridLogMessage << "WilsonCloverFermion::ImportGauge timings:"
-            << " WilsonFermion::Importgauge = " << (t1 - t0) / 1e6
-            << ", allocations = "               << (t2 - t1) / 1e6
-            << ", field strength = "            << (t3 - t2) / 1e6
-            << ", fill clover = "               << (t4 - t3) / 1e6
-            << ", misc = "                      << (t5 - t4) / 1e6
-            << ", inversions = "                << (t6 - t5) / 1e6
-            << ", pick cbs = "                  << (t7 - t6) / 1e6
-            << ", total = "                     << (t7 - t0) / 1e6
-            << std::endl;
-#endif
 }
 
 template <class Impl>
@@ -228,7 +167,7 @@ template <class Impl>
 void WilsonCloverFermion<Impl>::MooeeInternal(const FermionField &in, FermionField &out, int dag, int inv)
 {
   out.Checkerboard() = in.Checkerboard();
-  CloverField *Clover;
+  CloverFieldType *Clover;
   assert(in.Checkerboard() == Odd || in.Checkerboard() == Even);
 
   if (dag)
@@ -243,12 +182,12 @@ void WilsonCloverFermion<Impl>::MooeeInternal(const FermionField &in, FermionFie
       {
         Clover = (inv) ? &CloverTermInvDagEven : &CloverTermDagEven;
       }
-      Helpers::multCloverField(out, *Clover, in);
+      out = *Clover * in;
     }
     else
     {
       Clover = (inv) ? &CloverTermInv : &CloverTerm;
-      Helpers::multCloverField(out, *Clover, in); // don't bother with adj, hermitian anyway
+      out = adj(*Clover) * in;
     }
   }
   else
@@ -266,98 +205,18 @@ void WilsonCloverFermion<Impl>::MooeeInternal(const FermionField &in, FermionFie
         //  std::cout << "Calling clover term Even" << std::endl;
         Clover = (inv) ? &CloverTermInvEven : &CloverTermEven;
       }
-      Helpers::multCloverField(out, *Clover, in);
+      out = *Clover * in;
       //  std::cout << GridLogMessage << "*Clover.Checkerboard() "  << (*Clover).Checkerboard() << std::endl;
     }
     else
     {
       Clover = (inv) ? &CloverTermInv : &CloverTerm;
-      Helpers::multCloverField(out, *Clover, in);
+      out = *Clover * in;
     }
   }
+
 } // MooeeInternal
 
-// Derivative parts unpreconditioned pseudofermions
-template <class Impl>
-void WilsonCloverFermion<Impl>::MDeriv(GaugeField &force, const FermionField &X, const FermionField &Y, int dag)
-{
-  conformable(X.Grid(), Y.Grid());
-  conformable(X.Grid(), force.Grid());
-  GaugeLinkField force_mu(force.Grid()), lambda(force.Grid());
-  GaugeField clover_force(force.Grid());
-  PropagatorField Lambda(force.Grid());
-
-  // Guido: Here we are hitting some performance issues:
-  // need to extract the components of the DoubledGaugeField
-  // for each call
-  // Possible solution
-  // Create a vector object to store them? (cons: wasting space)
-  std::vector<GaugeLinkField> U(Nd, this->Umu.Grid());
-
-  Impl::extractLinkField(U, this->Umu);
-
-  force = Zero();
-  // Derivative of the Wilson hopping term
-  this->DhopDeriv(force, X, Y, dag);
-
-  ///////////////////////////////////////////////////////////
-  // Clover term derivative
-  ///////////////////////////////////////////////////////////
-  Impl::outerProductImpl(Lambda, X, Y);
-  //std::cout << "Lambda:" << Lambda << std::endl;
-
-  Gamma::Algebra sigma[] = {
-      Gamma::Algebra::SigmaXY,
-      Gamma::Algebra::SigmaXZ,
-      Gamma::Algebra::SigmaXT,
-      Gamma::Algebra::MinusSigmaXY,
-      Gamma::Algebra::SigmaYZ,
-      Gamma::Algebra::SigmaYT,
-      Gamma::Algebra::MinusSigmaXZ,
-      Gamma::Algebra::MinusSigmaYZ,
-      Gamma::Algebra::SigmaZT,
-      Gamma::Algebra::MinusSigmaXT,
-      Gamma::Algebra::MinusSigmaYT,
-      Gamma::Algebra::MinusSigmaZT};
-
-  /*
-    sigma_{\mu \nu}=
-    | 0         sigma[0]  sigma[1]  sigma[2] |
-    | sigma[3]    0       sigma[4]  sigma[5] |
-    | sigma[6]  sigma[7]     0      sigma[8] |
-    | sigma[9]  sigma[10] sigma[11]   0      |
-  */
-
-  int count = 0;
-  clover_force = Zero();
-  for (int mu = 0; mu < 4; mu++)
-  {
-    force_mu = Zero();
-    for (int nu = 0; nu < 4; nu++)
-    {
-      if (mu == nu)
-      continue;
-
-      RealD factor;
-      if (nu == 4 || mu == 4)
-      {
-        factor = 2.0 * csw_t;
-      }
-      else
-      {
-        factor = 2.0 * csw_r;
-      }
-      PropagatorField Slambda = Gamma(sigma[count]) * Lambda; // sigma checked
-      Impl::TraceSpinImpl(lambda, Slambda);                   // traceSpin ok
-      force_mu -= factor*Helpers::Cmunu(U, lambda, mu, nu);                   // checked
-      count++;
-    }
-
-    pokeLorentz(clover_force, U[mu] * force_mu, mu);
-  }
-  //clover_force *= csw;
-  force += clover_force;
-}
 
 // Derivative parts
 template <class Impl>

Grid/qcd/action/fermion/implementation/WilsonFermion5DImplementation.h

@@ -51,9 +51,9 @@ WilsonFermion5D<Impl>::WilsonFermion5D(GaugeField &_Umu,
   _FiveDimRedBlackGrid(&FiveDimRedBlackGrid),
   _FourDimGrid        (&FourDimGrid),
   _FourDimRedBlackGrid(&FourDimRedBlackGrid),
-  Stencil    (_FiveDimGrid,npoint,Even,directions,displacements,p),
-  StencilEven(_FiveDimRedBlackGrid,npoint,Even,directions,displacements,p), // source is Even
-  StencilOdd (_FiveDimRedBlackGrid,npoint,Odd ,directions,displacements,p), // source is Odd
+  Stencil    (_FiveDimGrid,npoint,Even,directions,displacements,p.locally_periodic,p),
+  StencilEven(_FiveDimRedBlackGrid,npoint,Even,directions,displacements,p.locally_periodic,p), // source is Even
+  StencilOdd (_FiveDimRedBlackGrid,npoint,Odd ,directions,displacements,p.locally_periodic,p), // source is Odd
   M5(_M5),
   Umu(_FourDimGrid),
   UmuEven(_FourDimRedBlackGrid),
@@ -361,10 +361,21 @@ void WilsonFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOrder &lo,
                                          const FermionField &in, FermionField &out,int dag)
 {
   DhopTotalTime-=usecond();
-  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
+
+  assert(  (WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute)
+	 ||(WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute)
+         ||(WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsDirichlet) );
+
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) {
     DhopInternalOverlappedComms(st,lo,U,in,out,dag);
-  else 
+  }
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute ) {
     DhopInternalSerialComms(st,lo,U,in,out,dag);
+  }
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsDirichlet ) {
+    DhopInternalDirichletComms(st,lo,U,in,out,dag);
+  }
+  
   DhopTotalTime+=usecond();
 }
 
@@ -431,6 +442,30 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
   DhopComputeTime2+=usecond();
 }
 
+template<class Impl>
+void WilsonFermion5D<Impl>::DhopInternalDirichletComms(StencilImpl & st, LebesgueOrder &lo,
+						       DoubledGaugeField & U,
+						       const FermionField &in, FermionField &out,int dag)
+{
+  Compressor compressor(dag);
+
+  int LLs = in.Grid()->_rdimensions[0];
+  int len =  U.Grid()->oSites();
+      
+  /////////////////////////////
+  // do the compute interior
+  /////////////////////////////
+  int Opt = WilsonKernelsStatic::Opt; // Why pass this. Kernels should know
+  DhopComputeTime-=usecond();
+  if (dag == DaggerYes) {
+    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),LLs,U.oSites(),in,out,1,0);
+  } else {
+    Kernels::DhopKernel   (Opt,st,U,st.CommBuf(),LLs,U.oSites(),in,out,1,0);
+  }
+  accelerator_barrier();
+  DhopComputeTime+=usecond();
+}
+
 
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopInternalSerialComms(StencilImpl & st, LebesgueOrder &lo,

Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h

@@ -47,9 +47,9 @@ WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
     Kernels(p),
     _grid(&Fgrid),
     _cbgrid(&Hgrid),
-    Stencil(&Fgrid, npoint, Even, directions, displacements,p),
-    StencilEven(&Hgrid, npoint, Even, directions,displacements,p),  // source is Even
-    StencilOdd(&Hgrid, npoint, Odd, directions,displacements,p),  // source is Odd
+    Stencil(&Fgrid, npoint, Even, directions, displacements,p.locally_periodic,p),
+    StencilEven(&Hgrid, npoint, Even, directions,displacements,p.locally_periodic,p),  // source is Even
+    StencilOdd(&Hgrid, npoint, Odd, directions,displacements,p.locally_periodic,p),  // source is Odd
     mass(_mass),
     Lebesgue(_grid),
     LebesgueEvenOdd(_cbgrid),
@@ -488,12 +488,21 @@ void WilsonFermion<Impl>::DhopInternal(StencilImpl &st, LebesgueOrder &lo,
                                        FermionField &out, int dag)
 {
   DhopTotalTime-=usecond();
-#ifdef GRID_OMP
-  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute )
+
+  assert(  (WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute)
+	 ||(WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute)
+         ||(WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsDirichlet) );
+
+
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) {
     DhopInternalOverlappedComms(st,lo,U,in,out,dag);
-  else
-#endif
+  }
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute ) {
     DhopInternalSerial(st,lo,U,in,out,dag);
+  }
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsDirichlet ) {
+    DhopInternalDirichletComms(st,lo,U,in,out,dag);
+  }
   DhopTotalTime+=usecond();
 }
 
@@ -562,6 +571,29 @@ void WilsonFermion<Impl>::DhopInternalOverlappedComms(StencilImpl &st, LebesgueO
   DhopComputeTime2+=usecond();
 };
 
+template <class Impl>
+void WilsonFermion<Impl>::DhopInternalDirichletComms(StencilImpl &st, LebesgueOrder &lo,
+						     DoubledGaugeField &U,
+						     const FermionField &in,
+						     FermionField &out, int dag)
+{
+  assert((dag == DaggerNo) || (dag == DaggerYes));
+
+  Compressor compressor(dag);
+  int len =  U.Grid()->oSites();
+
+  /////////////////////////////
+  // do the compute interior
+  /////////////////////////////
+  int Opt = WilsonKernelsStatic::Opt;
+  DhopComputeTime-=usecond();
+  if (dag == DaggerYes) {
+    Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+  } else {
+    Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0);
+  }
+  DhopComputeTime+=usecond();
+};
 
 template <class Impl>
 void WilsonFermion<Impl>::DhopInternalSerial(StencilImpl &st, LebesgueOrder &lo,

Grid/qcd/action/fermion/implementation/WilsonKernelsHandImplementation.h

@@ -77,23 +77,23 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define REGISTER
 
 #ifdef GRID_SIMT
-#define LOAD_CHIMU(Ptype)		\
+#define LOAD_CHIMU(ptype)		\
   {const SiteSpinor & ref (in[offset]);	\
-    Chimu_00=coalescedReadPermute<Ptype>(ref()(0)(0),perm,lane);	\
-    Chimu_01=coalescedReadPermute<Ptype>(ref()(0)(1),perm,lane);		\
-    Chimu_02=coalescedReadPermute<Ptype>(ref()(0)(2),perm,lane);		\
-    Chimu_10=coalescedReadPermute<Ptype>(ref()(1)(0),perm,lane);		\
-    Chimu_11=coalescedReadPermute<Ptype>(ref()(1)(1),perm,lane);		\
-    Chimu_12=coalescedReadPermute<Ptype>(ref()(1)(2),perm,lane);		\
-    Chimu_20=coalescedReadPermute<Ptype>(ref()(2)(0),perm,lane);		\
-    Chimu_21=coalescedReadPermute<Ptype>(ref()(2)(1),perm,lane);		\
-    Chimu_22=coalescedReadPermute<Ptype>(ref()(2)(2),perm,lane);		\
-    Chimu_30=coalescedReadPermute<Ptype>(ref()(3)(0),perm,lane);		\
-    Chimu_31=coalescedReadPermute<Ptype>(ref()(3)(1),perm,lane);		\
-    Chimu_32=coalescedReadPermute<Ptype>(ref()(3)(2),perm,lane);	}
+    Chimu_00=coalescedReadPermute<ptype>(ref()(0)(0),perm,lane);	\
+    Chimu_01=coalescedReadPermute<ptype>(ref()(0)(1),perm,lane);		\
+    Chimu_02=coalescedReadPermute<ptype>(ref()(0)(2),perm,lane);		\
+    Chimu_10=coalescedReadPermute<ptype>(ref()(1)(0),perm,lane);		\
+    Chimu_11=coalescedReadPermute<ptype>(ref()(1)(1),perm,lane);		\
+    Chimu_12=coalescedReadPermute<ptype>(ref()(1)(2),perm,lane);		\
+    Chimu_20=coalescedReadPermute<ptype>(ref()(2)(0),perm,lane);		\
+    Chimu_21=coalescedReadPermute<ptype>(ref()(2)(1),perm,lane);		\
+    Chimu_22=coalescedReadPermute<ptype>(ref()(2)(2),perm,lane);		\
+    Chimu_30=coalescedReadPermute<ptype>(ref()(3)(0),perm,lane);		\
+    Chimu_31=coalescedReadPermute<ptype>(ref()(3)(1),perm,lane);		\
+    Chimu_32=coalescedReadPermute<ptype>(ref()(3)(2),perm,lane);	}
 #define PERMUTE_DIR(dir) ;
 #else
-#define LOAD_CHIMU(Ptype)		\
+#define LOAD_CHIMU(ptype)		\
   {const SiteSpinor & ref (in[offset]);	\
     Chimu_00=ref()(0)(0);\
     Chimu_01=ref()(0)(1);\
@@ -109,12 +109,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     Chimu_32=ref()(3)(2);}
 
 #define PERMUTE_DIR(dir)			\
-  permute##dir(Chi_00,Chi_00);			\
-  permute##dir(Chi_01,Chi_01);			\
-  permute##dir(Chi_02,Chi_02);			\
-  permute##dir(Chi_10,Chi_10);			\
-  permute##dir(Chi_11,Chi_11);			\
-  permute##dir(Chi_12,Chi_12);
+  permute##dir(Chi_00,Chi_00);	\
+      permute##dir(Chi_01,Chi_01);\
+      permute##dir(Chi_02,Chi_02);\
+      permute##dir(Chi_10,Chi_10);	\
+      permute##dir(Chi_11,Chi_11);\
+      permute##dir(Chi_12,Chi_12);
 
 #endif
 
@@ -371,91 +371,88 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   result_32-= UChi_12;
 
 #define HAND_STENCIL_LEGB(PROJ,PERM,DIR,RECON)	\
-  {int ptype;					\
-   SE=st.GetEntry(ptype,DIR,ss);		\
-   auto offset = SE->_offset;			\
-   auto local  = SE->_is_local;			\
-   auto perm   = SE->_permute;			\
-   if ( local ) {				\
-     LOAD_CHIMU(PERM);				\
-     PROJ;					\
-     if ( perm) {				\
-       PERMUTE_DIR(PERM);			\
-     }						\
-   } else {					\
-     LOAD_CHI;					\
-   }						\
-   acceleratorSynchronise();			\
-   MULT_2SPIN(DIR);				\
-   RECON;					}
+  SE=st.GetEntry(ptype,DIR,ss);			\
+  offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
+  if ( local ) {				\
+    LOAD_CHIMU(PERM);				\
+    PROJ;					\
+    if ( perm) {				\
+      PERMUTE_DIR(PERM);			\
+    }						\
+  } else {					\
+    LOAD_CHI;					\
+  }						\
+  acceleratorSynchronise();			\
+  MULT_2SPIN(DIR);				\
+  RECON;					
 
-#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)		\
-  { SE=&st_p[DIR+8*ss];						\
-  auto ptype=st_perm[DIR];					\
-  auto offset = SE->_offset;					\
-  auto local  = SE->_is_local;					\
-  auto perm   = SE->_permute;					\
-  if ( local ) {						\
-    LOAD_CHIMU(PERM);						\
-    PROJ;							\
-    if ( perm) {						\
-      PERMUTE_DIR(PERM);					\
-    }								\
-  } else {							\
-    LOAD_CHI;							\
-  }								\
-  acceleratorSynchronise();					\
-  MULT_2SPIN(DIR);						\
-  RECON;					}
+#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)	\
+  SE=&st_p[DIR+8*ss];				\
+  ptype=st_perm[DIR];				\
+  offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
+  if ( local ) {				\
+    LOAD_CHIMU(PERM);				\
+    PROJ;					\
+    if ( perm) {				\
+      PERMUTE_DIR(PERM);			\
+    }						\
+  } else {					\
+    LOAD_CHI;					\
+  }						\
+  acceleratorSynchronise();			\
+  MULT_2SPIN(DIR);				\
+  RECON;					
 
 #define HAND_STENCIL_LEGA(PROJ,PERM,DIR,RECON)				\
-  { SE=&st_p[DIR+8*ss];							\
-    auto ptype=st_perm[DIR];						\
-    /*SE=st.GetEntry(ptype,DIR,ss);*/					\
-    auto offset = SE->_offset;						\
-    auto perm   = SE->_permute;						\
-    LOAD_CHIMU(PERM);							\
-    PROJ;								\
-    MULT_2SPIN(DIR);							\
-    RECON;					}
+  SE=&st_p[DIR+8*ss];							\
+  ptype=st_perm[DIR];							\
+ /*SE=st.GetEntry(ptype,DIR,ss);*/					\
+  offset = SE->_offset;				\
+  perm   = SE->_permute;			\
+  LOAD_CHIMU(PERM);				\
+  PROJ;						\
+  MULT_2SPIN(DIR);				\
+  RECON;					
 
 #define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON)	\
-  { int ptype;						\
-  SE=st.GetEntry(ptype,DIR,ss);				\
-  auto offset = SE->_offset;					\
-  auto local  = SE->_is_local;					\
-  auto perm   = SE->_permute;					\
-  if ( local ) {						\
-    LOAD_CHIMU(PERM);						\
-    PROJ;							\
-    if ( perm) {						\
-      PERMUTE_DIR(PERM);					\
-    }								\
-  } else if ( st.same_node[DIR] ) {				\
-    LOAD_CHI;							\
-  }								\
-  acceleratorSynchronise();					\
-  if (local || st.same_node[DIR] ) {				\
-    MULT_2SPIN(DIR);						\
-    RECON;							\
-  }								\
-  acceleratorSynchronise();			}
+  SE=st.GetEntry(ptype,DIR,ss);			\
+  offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
+  if ( local ) {				\
+    LOAD_CHIMU(PERM);				\
+    PROJ;					\
+    if ( perm) {				\
+      PERMUTE_DIR(PERM);			\
+    }						\
+  } else if ( st.same_node[DIR] ) {		\
+    LOAD_CHI;					\
+  }						\
+  acceleratorSynchronise();			\
+  if (local || st.same_node[DIR] ) {		\
+    MULT_2SPIN(DIR);				\
+    RECON;					\
+  }						\
+  acceleratorSynchronise();			
 
 #define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON)	\
-  { int ptype;						\
-  SE=st.GetEntry(ptype,DIR,ss);				\
-  auto offset = SE->_offset;				\
-  if((!SE->_is_local)&&(!st.same_node[DIR]) ) {		\
-    LOAD_CHI;						\
-    MULT_2SPIN(DIR);					\
-    RECON;						\
-    nmu++;						\
-  }							\
-  acceleratorSynchronise();			}
+  SE=st.GetEntry(ptype,DIR,ss);			\
+  offset = SE->_offset;				\
+  if((!SE->_is_local)&&(!st.same_node[DIR]) ) {	\
+    LOAD_CHI;					\
+    MULT_2SPIN(DIR);				\
+    RECON;					\
+    nmu++;					\
+  }						\
+  acceleratorSynchronise();			
 
-#define HAND_RESULT(ss)					\
-  {							\
-    SiteSpinor & ref (out[ss]);				\
+#define HAND_RESULT(ss)				\
+  {						\
+    SiteSpinor & ref (out[ss]);			\
     coalescedWrite(ref()(0)(0),result_00,lane);		\
     coalescedWrite(ref()(0)(1),result_01,lane);		\
     coalescedWrite(ref()(0)(2),result_02,lane);		\
@@ -566,6 +563,7 @@ WilsonKernels<Impl>::HandDhopSiteSycl(StencilVector st_perm,StencilEntry *st_p,
 
   HAND_DECLARATIONS(Simt);
 
+  int offset,local,perm, ptype;
   StencilEntry *SE;
   HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
   HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
@@ -595,7 +593,9 @@ WilsonKernels<Impl>::HandDhopSite(StencilView &st, DoubledGaugeFieldView &U,Site
 
   HAND_DECLARATIONS(Simt);
 
+  int offset,local,perm, ptype;
   StencilEntry *SE;
+
   HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
   HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
   HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
@@ -623,6 +623,8 @@ void WilsonKernels<Impl>::HandDhopSiteDag(StencilView &st,DoubledGaugeFieldView
   HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
+  int offset,local,perm, ptype;
+  
   HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
   HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
   HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
@@ -638,8 +640,8 @@ template<class Impl>  accelerator_inline void
 WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
-  //  auto st_p = st._entries_p;						
-  //  auto st_perm = st._permute_type;					
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
@@ -650,6 +652,7 @@ WilsonKernels<Impl>::HandDhopSiteInt(StencilView &st,DoubledGaugeFieldView &U,Si
 
   HAND_DECLARATIONS(Simt);
 
+  int offset,local,perm, ptype;
   StencilEntry *SE;
   ZERO_RESULT;
   HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
@@ -667,8 +670,8 @@ template<class Impl> accelerator_inline
 void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
-  //  auto st_p = st._entries_p;						
-  //  auto st_perm = st._permute_type;					
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
@@ -679,6 +682,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilView &st,DoubledGaugeFieldVi
   HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
+  int offset,local,perm, ptype;
   ZERO_RESULT;
   HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
   HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
@@ -695,8 +699,8 @@ template<class Impl>  accelerator_inline void
 WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor  *buf,
 					  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
-  //  auto st_p = st._entries_p;						
-  //  auto st_perm = st._permute_type;					
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
 // T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
@@ -707,7 +711,7 @@ WilsonKernels<Impl>::HandDhopSiteExt(StencilView &st,DoubledGaugeFieldView &U,Si
 
   HAND_DECLARATIONS(Simt);
 
-  //  int offset, ptype;
+  int offset, ptype;
   StencilEntry *SE;
   int nmu=0;
   ZERO_RESULT;
@@ -726,8 +730,8 @@ template<class Impl>  accelerator_inline
 void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldView &U,SiteHalfSpinor *buf,
 						  int ss,int sU,const FermionFieldView &in, FermionFieldView &out)
 {
-  //  auto st_p = st._entries_p;						
-  //  auto st_perm = st._permute_type;					
+  auto st_p = st._entries_p;						
+  auto st_perm = st._permute_type;					
   typedef typename Simd::scalar_type S;
   typedef typename Simd::vector_type V;
   typedef decltype( coalescedRead( in[0]()(0)(0) )) Simt;
@@ -738,7 +742,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilView &st,DoubledGaugeFieldVi
   HAND_DECLARATIONS(Simt);
 
   StencilEntry *SE;
-  //  int offset, ptype;
+  int offset, ptype;
   int nmu=0;
   ZERO_RESULT;
   HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);

Grid/qcd/action/fermion/instantiation/CompactWilsonCloverFermionInstantiation.cc.master

@@ -1,41 +0,0 @@
-/*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid
-
-    Source file: ./lib/ qcd/action/fermion/instantiation/CompactWilsonCloverFermionInstantiation.cc.master
-
-    Copyright (C) 2017 - 2022
-
-    Author: paboyle <paboyle@ph.ed.ac.uk>
-    Author: Guido Cossu <guido.cossu@ed.ac.uk>
-    Author: Daniel Richtmann <daniel.richtmann@gmail.com>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Grid.h>
-#include <Grid/qcd/spin/Dirac.h>
-#include <Grid/qcd/action/fermion/CompactWilsonCloverFermion.h>
-#include <Grid/qcd/action/fermion/implementation/CompactWilsonCloverFermionImplementation.h>
-
-NAMESPACE_BEGIN(Grid);
-
-#include "impl.h"
-template class CompactWilsonCloverFermion<IMPLEMENTATION>; 
-
-NAMESPACE_END(Grid);

Grid/qcd/action/fermion/instantiation/WilsonImplD/CompactWilsonCloverFermionInstantiationWilsonImplD.cc

@@ -1 +0,0 @@
-../CompactWilsonCloverFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/WilsonImplF/CompactWilsonCloverFermionInstantiationWilsonImplF.cc

@@ -1 +0,0 @@
-../CompactWilsonCloverFermionInstantiation.cc.master

Grid/qcd/action/fermion/instantiation/generate_instantiations.sh

@@ -40,7 +40,7 @@ EOF
 
 done
 
-CC_LIST="WilsonCloverFermionInstantiation CompactWilsonCloverFermionInstantiation WilsonFermionInstantiation WilsonKernelsInstantiation WilsonTMFermionInstantiation"
+CC_LIST="WilsonCloverFermionInstantiation WilsonFermionInstantiation WilsonKernelsInstantiation WilsonTMFermionInstantiation"
 
 for impl in $WILSON_IMPL_LIST
 do

Grid/qcd/action/gauge/GaugeImplTypes.h

@@ -61,7 +61,7 @@ NAMESPACE_BEGIN(Grid);
   typedef typename Impl::Field Field;
 
 // hardcodes the exponential approximation in the template
-template <class S, int Nrepresentation = Nc, int Nexp = 12 > class GaugeImplTypes {
+template <class S, int Nrepresentation = Nc, int Nexp = 20 > class GaugeImplTypes {
 public:
   typedef S Simd;
   typedef typename Simd::scalar_type scalar_type;
@@ -78,8 +78,6 @@ public:
   typedef Lattice<SiteLink>    LinkField; 
   typedef Lattice<SiteField>   Field;
 
-  typedef SU<Nrepresentation> Group;
-
   // Guido: we can probably separate the types from the HMC functions
   // this will create 2 kind of implementations
   // probably confusing the users
@@ -120,7 +118,7 @@ public:
     LinkField Pmu(P.Grid());
     Pmu = Zero();
     for (int mu = 0; mu < Nd; mu++) {
-      Group::GaussianFundamentalLieAlgebraMatrix(pRNG, Pmu);
+      SU<Nrepresentation>::GaussianFundamentalLieAlgebraMatrix(pRNG, Pmu);
       RealD scale = ::sqrt(HMC_MOMENTUM_DENOMINATOR) ;
       Pmu = Pmu*scale;
       PokeIndex<LorentzIndex>(P, Pmu, mu);
@@ -161,15 +159,15 @@ public:
   }
 
   static inline void HotConfiguration(GridParallelRNG &pRNG, Field &U) {
-    Group::HotConfiguration(pRNG, U);
+    SU<Nc>::HotConfiguration(pRNG, U);
   }
 
   static inline void TepidConfiguration(GridParallelRNG &pRNG, Field &U) {
-    Group::TepidConfiguration(pRNG, U);
+    SU<Nc>::TepidConfiguration(pRNG, U);
   }
 
   static inline void ColdConfiguration(GridParallelRNG &pRNG, Field &U) {
-    Group::ColdConfiguration(pRNG, U);
+    SU<Nc>::ColdConfiguration(pRNG, U);
   }
 };
 

Grid/qcd/action/gauge/GaugeImplementations.h

@@ -69,11 +69,6 @@ public:
     return PeriodicBC::ShiftStaple(Link,mu);
   }
 
-  //Same as Cshift for periodic BCs
-  static inline GaugeLinkField CshiftLink(const GaugeLinkField &Link, int mu, int shift){
-    return PeriodicBC::CshiftLink(Link,mu,shift);
-  }
-
   static inline bool isPeriodicGaugeField(void) { return true; }
 };
 
@@ -115,11 +110,6 @@ public:
       return PeriodicBC::CovShiftBackward(Link, mu, field);
   }
 
-  //If mu is a conjugate BC direction
-  //Out(x) = U^dag_\mu(x-mu)  | x_\mu != 0
-  //       = U^T_\mu(L-1)  | x_\mu == 0
-  //else
-  //Out(x) = U^dag_\mu(x-mu mod L)
   static inline GaugeLinkField
   CovShiftIdentityBackward(const GaugeLinkField &Link, int mu)
   {
@@ -139,13 +129,6 @@ public:
       return PeriodicBC::CovShiftIdentityForward(Link,mu);
   }
 
-
-  //If mu is a conjugate BC direction
-  //Out(x) = S_\mu(x+mu)  | x_\mu != L-1
-  //       = S*_\mu(x+mu)  | x_\mu == L-1
-  //else
-  //Out(x) = S_\mu(x+mu mod L)
-  //Note: While this is used for Staples it is also applicable for shifting gauge links or gauge transformation matrices
   static inline GaugeLinkField ShiftStaple(const GaugeLinkField &Link, int mu)
   {
     assert(_conjDirs.size() == Nd);
@@ -155,27 +138,6 @@ public:
       return PeriodicBC::ShiftStaple(Link,mu);
   }
 
-  //Boundary-aware C-shift of gauge links / gauge transformation matrices
-  //For conjugate BC direction
-  //shift = 1
-  //Out(x) = U_\mu(x+\hat\mu)  | x_\mu != L-1
-  //       = U*_\mu(0)  | x_\mu == L-1
-  //shift = -1
-  //Out(x) = U_\mu(x-mu)  | x_\mu != 0
-  //       = U*_\mu(L-1)  | x_\mu == 0
-  //else
-  //shift = 1
-  //Out(x) = U_\mu(x+\hat\mu mod L)
-  //shift = -1
-  //Out(x) = U_\mu(x-\hat\mu mod L)
-  static inline GaugeLinkField CshiftLink(const GaugeLinkField &Link, int mu, int shift){
-    assert(_conjDirs.size() == Nd);
-    if(_conjDirs[mu]) 
-      return ConjugateBC::CshiftLink(Link,mu,shift);
-    else     
-      return PeriodicBC::CshiftLink(Link,mu,shift);
-  }
-
   static inline void       setDirections(std::vector<int> &conjDirs) { _conjDirs=conjDirs; }
   static inline std::vector<int> getDirections(void) { return _conjDirs; }
   static inline bool isPeriodicGaugeField(void) { return false; }

Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourBosonPseudoFermion.h

@@ -0,0 +1,163 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/pseudofermion/DomainDecomposedTwoFlavourBoundaryBoson.h
+
+    Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////
+// Two flavour ratio
+///////////////////////////////////////
+template<class ImplD,class ImplF>
+class DomainDecomposedBoundaryTwoFlavourBosonPseudoFermion : public Action<typename ImplD::GaugeField> {
+public:
+  INHERIT_IMPL_TYPES(ImplD);
+
+private:
+  SchurFactoredFermionOperator<ImplD,ImplF> & NumOp;// the basic operator
+  RealD InnerStoppingCondition;
+  RealD ActionStoppingCondition;
+  RealD DerivativeStoppingCondition;
+  FermionField Phi; // the pseudo fermion field for this trajectory
+public:
+  DomainDecomposedBoundaryTwoFlavourBosonPseudoFermion(SchurFactoredFermionOperator<ImplD,ImplF>  &_NumOp,RealD _DerivativeTol, RealD _ActionTol, RealD _InnerTol=1.0e-6)
+    : NumOp(_NumOp), 
+      DerivativeStoppingCondition(_DerivativeTol),
+      ActionStoppingCondition(_ActionTol),
+      InnerStoppingCondition(_InnerTol),
+      Phi(_NumOp.FermionGrid()) {};
+
+  virtual std::string action_name(){return "DomainDecomposedBoundaryTwoFlavourBosonPseudoFermion";}
+
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    return sstream.str();
+  }  
+  
+  virtual void refresh(const GaugeField &U, GridSerialRNG& sRNG, GridParallelRNG& pRNG)
+  {
+    // P(phi) = e^{- phi^dag P^dag P phi}
+    //
+    // NumOp == P
+    //
+    // Take phi = P^{-1} eta  ; eta = P Phi
+    //
+    // P(eta) = e^{- eta^dag eta}
+    //
+    // e^{x^2/2 sig^2} => sig^2 = 0.5.
+    // 
+    // So eta should be of width sig = 1/sqrt(2) and must multiply by 0.707....
+    //
+    RealD scale = std::sqrt(0.5);
+
+    NumOp.tolinner=InnerStoppingCondition;
+    NumOp.tol=ActionStoppingCondition;
+    NumOp.ImportGauge(U);
+
+    FermionField eta(NumOp.FermionGrid());
+
+    gaussian(pRNG,eta);    eta=eta*scale;
+    
+    NumOp.ProjectBoundaryBar(eta);
+    //DumpSliceNorm("eta",eta);
+    NumOp.RInv(eta,Phi);
+
+    //DumpSliceNorm("Phi",Phi);
+
+  };
+
+  //////////////////////////////////////////////////////
+  // S = phi^dag Pdag P phi
+  //////////////////////////////////////////////////////
+  virtual RealD S(const GaugeField &U) {
+
+    NumOp.tolinner=InnerStoppingCondition;
+    NumOp.tol=ActionStoppingCondition;
+    NumOp.ImportGauge(U);
+
+    FermionField Y(NumOp.FermionGrid());
+
+    NumOp.R(Phi,Y);
+
+    RealD action = norm2(Y);
+
+    return action;
+  };
+
+  virtual void deriv(const GaugeField &U,GaugeField & dSdU)
+  {
+    NumOp.tolinner=InnerStoppingCondition;
+    NumOp.tol=DerivativeStoppingCondition;
+    NumOp.ImportGauge(U);
+
+    GridBase *fgrid = NumOp.FermionGrid();
+    GridBase *ugrid = NumOp.GaugeGrid();
+
+    FermionField  X(fgrid);
+    FermionField  Y(fgrid);
+    FermionField  tmp(fgrid);
+
+    GaugeField   force(ugrid);	
+
+    FermionField DobiDdbPhi(fgrid);      // Vector A in my notes
+    FermionField DoiDdDobiDdbPhi(fgrid); // Vector B in my notes
+    FermionField DoidP_Phi(fgrid);    // Vector E in my notes
+    FermionField DobidDddDoidP_Phi(fgrid);    // Vector F in my notes
+    
+    FermionField P_Phi(fgrid);
+    
+    // P term
+    NumOp.dBoundaryBar(Phi,tmp);
+    NumOp.dOmegaBarInv(tmp,DobiDdbPhi);        // Vector A
+    NumOp.dBoundary(DobiDdbPhi,tmp);
+    NumOp.dOmegaInv(tmp,DoiDdDobiDdbPhi);      // Vector B
+    P_Phi  = Phi - DoiDdDobiDdbPhi;
+    NumOp.ProjectBoundaryBar(P_Phi);
+    
+    // P^dag P term
+    NumOp.dOmegaDagInv(P_Phi,DoidP_Phi); // Vector E
+    NumOp.dBoundaryDag(DoidP_Phi,tmp);
+    NumOp.dOmegaBarDagInv(tmp,DobidDddDoidP_Phi);   // Vector F
+    NumOp.dBoundaryBarDag(DobidDddDoidP_Phi,tmp);
+
+    X = DobiDdbPhi;
+    Y = DobidDddDoidP_Phi;
+    NumOp.DirichletFermOpD.MDeriv(force,Y,X,DaggerNo);    dSdU=force;
+    NumOp.DirichletFermOpD.MDeriv(force,X,Y,DaggerYes);   dSdU=dSdU+force;
+
+    X = DoiDdDobiDdbPhi;
+    Y = DoidP_Phi;
+    NumOp.DirichletFermOpD.MDeriv(force,Y,X,DaggerNo);    dSdU=dSdU+force;
+    NumOp.DirichletFermOpD.MDeriv(force,X,Y,DaggerYes);   dSdU=dSdU+force;
+
+    dSdU *= -1.0;
+
+  };
+};
+
+NAMESPACE_END(Grid);
+

Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourPseudoFermion.h

@@ -0,0 +1,158 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/pseudofermion/DomainDecomposedTwoFlavourBoundary.h
+
+    Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////
+// Two flavour ratio
+///////////////////////////////////////
+template<class ImplD,class ImplF>
+class DomainDecomposedBoundaryTwoFlavourPseudoFermion : public Action<typename ImplD::GaugeField> {
+public:
+  INHERIT_IMPL_TYPES(ImplD);
+
+private:
+  SchurFactoredFermionOperator<ImplD,ImplF> & DenOp;// the basic operator
+  RealD ActionStoppingCondition;
+  RealD DerivativeStoppingCondition;
+  RealD InnerStoppingCondition;
+
+  FermionField Phi; // the pseudo fermion field for this trajectory
+
+  RealD refresh_action;
+public:
+  DomainDecomposedBoundaryTwoFlavourPseudoFermion(SchurFactoredFermionOperator<ImplD,ImplF>  &_DenOp,RealD _DerivativeTol, RealD _ActionTol, RealD _InnerTol = 1.0e-6 )
+    : DenOp(_DenOp),
+      DerivativeStoppingCondition(_DerivativeTol),
+      ActionStoppingCondition(_ActionTol),
+      InnerStoppingCondition(_InnerTol),
+      Phi(_DenOp.FermionGrid()) {};
+      
+  virtual std::string action_name(){return "DomainDecomposedBoundaryTwoFlavourPseudoFermion";}
+
+ 
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    return sstream.str();
+  }  
+  
+  virtual void refresh(const GaugeField &U, GridSerialRNG& sRNG, GridParallelRNG& pRNG)
+  {
+    // P(phi) = e^{- phi^dag Rdag^-1 R^-1 phi}
+    //
+    // DenOp == R
+    //
+    // Take phi = R eta  ; eta = R^-1 Phi
+    //
+    // P(eta) = e^{- eta^dag eta}
+    //
+    // e^{x^2/2 sig^2} => sig^2 = 0.5.
+    // 
+    // So eta should be of width sig = 1/sqrt(2) and must multiply by 0.707....
+    //
+    RealD scale = std::sqrt(0.5);
+
+    DenOp.tolinner=InnerStoppingCondition;
+    DenOp.tol     =ActionStoppingCondition;
+    DenOp.ImportGauge(U);
+
+    FermionField eta(DenOp.FermionGrid());
+
+    gaussian(pRNG,eta);    eta=eta*scale;
+    
+    DenOp.ProjectBoundaryBar(eta);
+    DenOp.R(eta,Phi);
+    //DumpSliceNorm("Phi",Phi);
+    refresh_action = norm2(eta);
+  };
+
+  //////////////////////////////////////////////////////
+  // S = phi^dag Rdag^-1 R^-1 phi
+  //////////////////////////////////////////////////////
+  virtual RealD S(const GaugeField &U) {
+
+    DenOp.tolinner=InnerStoppingCondition;
+    DenOp.tol=ActionStoppingCondition;
+    DenOp.ImportGauge(U);
+
+    FermionField X(DenOp.FermionGrid());
+
+    DenOp.RInv(Phi,X);
+
+    RealD action = norm2(X);
+
+    return action;
+  };
+
+  virtual void deriv(const GaugeField &U,GaugeField & dSdU)
+  {
+    DenOp.tolinner=InnerStoppingCondition;
+    DenOp.tol=DerivativeStoppingCondition;
+    DenOp.ImportGauge(U);
+
+    GridBase *fgrid = DenOp.FermionGrid();
+    GridBase *ugrid = DenOp.GaugeGrid();
+
+    FermionField  X(fgrid);
+    FermionField  Y(fgrid);
+    FermionField  tmp(fgrid);
+
+    GaugeField   force(ugrid);	
+
+    FermionField DiDdb_Phi(fgrid);      // Vector C in my notes
+    FermionField DidRinv_Phi(fgrid);    // Vector D in my notes
+    FermionField Rinv_Phi(fgrid);
+
+//   FermionField RinvDagRinv_Phi(fgrid);
+//   FermionField DdbdDidRinv_Phi(fgrid);
+
+    // R^-1 term
+    DenOp.dBoundaryBar(Phi,tmp);
+    DenOp.Dinverse(tmp,DiDdb_Phi);            // Vector C
+    Rinv_Phi = Phi - DiDdb_Phi;
+    DenOp.ProjectBoundaryBar(Rinv_Phi); 
+ 
+    // R^-dagger R^-1 term
+    DenOp.DinverseDag(Rinv_Phi,DidRinv_Phi); // Vector D
+/*
+    DenOp.dBoundaryBarDag(DidRinv_Phi,DdbdDidRinv_Phi);
+    RinvDagRinv_Phi = Rinv_Phi - DdbdDidRinv_Phi;
+    DenOp.ProjectBoundaryBar(RinvDagRinv_Phi);
+*/
+    X = DiDdb_Phi;
+    Y = DidRinv_Phi;
+    DenOp.PeriodicFermOpD.MDeriv(force,Y,X,DaggerNo);    dSdU=force;
+    DenOp.PeriodicFermOpD.MDeriv(force,X,Y,DaggerYes);   dSdU=dSdU+force;
+    DumpSliceNorm("force",dSdU);
+    dSdU *= -1.0;
+  };
+};
+
+NAMESPACE_END(Grid);
+

Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion.h

@@ -0,0 +1,237 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/pseudofermion/DomainDecomposedTwoFlavourBoundary.h
+
+    Copyright (C) 2021
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////
+// Two flavour ratio
+///////////////////////////////////////
+template<class ImplD,class ImplF>
+class DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion : public Action<typename ImplD::GaugeField> {
+public:
+  INHERIT_IMPL_TYPES(ImplD);
+
+private:
+  SchurFactoredFermionOperator<ImplD,ImplF> & NumOp;// the basic operator
+  SchurFactoredFermionOperator<ImplD,ImplF> & DenOp;// the basic operator
+
+  RealD InnerStoppingCondition;
+  RealD ActionStoppingCondition;
+  RealD DerivativeStoppingCondition;
+  
+  FermionField Phi; // the pseudo fermion field for this trajectory
+
+public:
+  DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion(SchurFactoredFermionOperator<ImplD,ImplF>  &_NumOp, 
+						       SchurFactoredFermionOperator<ImplD,ImplF>  &_DenOp,
+						       RealD _DerivativeTol, RealD _ActionTol, RealD _InnerTol=1.0e-6)
+    : NumOp(_NumOp), DenOp(_DenOp),
+      Phi(_NumOp.PeriodicFermOpD.FermionGrid()),
+      InnerStoppingCondition(_InnerTol),
+      DerivativeStoppingCondition(_DerivativeTol),
+      ActionStoppingCondition(_ActionTol)
+  {};
+      
+  virtual std::string action_name(){return "DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion";}
+ 
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    return sstream.str();
+  }  
+  
+  virtual void refresh(const GaugeField &U, GridSerialRNG& sRNG, GridParallelRNG& pRNG)
+  {
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    FermionField eta(NumOp.PeriodicFermOpD.FermionGrid());
+    FermionField tmp(NumOp.PeriodicFermOpD.FermionGrid());
+
+    // P(phi) = e^{- phi^dag P^dag Rdag^-1 R^-1 P phi}
+    //
+    // NumOp == P
+    // DenOp == R
+    //
+    // Take phi = P^{-1} R eta  ; eta = R^-1 P Phi
+    //
+    // P(eta) = e^{- eta^dag eta}
+    //
+    // e^{x^2/2 sig^2} => sig^2 = 0.5.
+    // 
+    // So eta should be of width sig = 1/sqrt(2) and must multiply by 0.707....
+    //
+    RealD scale = std::sqrt(0.5);
+
+    gaussian(pRNG,eta);    eta=eta*scale;
+    
+    NumOp.ProjectBoundaryBar(eta);
+    NumOp.tolinner=InnerStoppingCondition;
+    DenOp.tolinner=InnerStoppingCondition;
+    DenOp.tol = ActionStoppingCondition;
+    NumOp.tol = ActionStoppingCondition;
+    DenOp.R(eta,tmp);
+    NumOp.RInv(tmp,Phi);
+    DumpSliceNorm("Phi",Phi);
+
+  };
+
+  //////////////////////////////////////////////////////
+  // S = phi^dag Pdag Rdag^-1 R^-1 P phi
+  //////////////////////////////////////////////////////
+  virtual RealD S(const GaugeField &U) {
+
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    FermionField X(NumOp.PeriodicFermOpD.FermionGrid());
+    FermionField Y(NumOp.PeriodicFermOpD.FermionGrid());
+
+    NumOp.tolinner=InnerStoppingCondition;
+    DenOp.tolinner=InnerStoppingCondition;
+    DenOp.tol = ActionStoppingCondition;
+    NumOp.tol = ActionStoppingCondition;
+    NumOp.R(Phi,Y);
+    DenOp.RInv(Y,X);
+
+    RealD action = norm2(X);
+    //    std::cout << " DD boundary action is " <<action<<std::endl;
+
+    return action;
+  };
+
+  virtual void deriv(const GaugeField &U,GaugeField & dSdU)
+  {
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    GridBase *fgrid = NumOp.PeriodicFermOpD.FermionGrid();
+    GridBase *ugrid = NumOp.PeriodicFermOpD.GaugeGrid();
+
+    FermionField  X(fgrid);
+    FermionField  Y(fgrid);
+    FermionField  tmp(fgrid);
+
+    GaugeField   force(ugrid);	
+
+    FermionField DobiDdbPhi(fgrid);      // Vector A in my notes
+    FermionField DoiDdDobiDdbPhi(fgrid); // Vector B in my notes
+    FermionField DiDdbP_Phi(fgrid);      // Vector C in my notes
+    FermionField DidRinvP_Phi(fgrid);    // Vector D in my notes
+    FermionField DdbdDidRinvP_Phi(fgrid);
+    FermionField DoidRinvDagRinvP_Phi(fgrid);    // Vector E in my notes
+    FermionField DobidDddDoidRinvDagRinvP_Phi(fgrid);    // Vector F in my notes
+    
+    FermionField P_Phi(fgrid);
+    FermionField RinvP_Phi(fgrid);
+    FermionField RinvDagRinvP_Phi(fgrid);
+    FermionField PdagRinvDagRinvP_Phi(fgrid);
+
+    //    RealD action = S(U);
+    NumOp.tolinner=InnerStoppingCondition;
+    DenOp.tolinner=InnerStoppingCondition;
+    DenOp.tol = DerivativeStoppingCondition;
+    NumOp.tol = DerivativeStoppingCondition;
+    
+    // P term
+    NumOp.dBoundaryBar(Phi,tmp);
+    NumOp.dOmegaBarInv(tmp,DobiDdbPhi);        // Vector A
+    NumOp.dBoundary(DobiDdbPhi,tmp);
+    NumOp.dOmegaInv(tmp,DoiDdDobiDdbPhi);      // Vector B
+    P_Phi  = Phi - DoiDdDobiDdbPhi;
+    NumOp.ProjectBoundaryBar(P_Phi);
+
+    // R^-1 P term
+    DenOp.dBoundaryBar(P_Phi,tmp);
+    DenOp.Dinverse(tmp,DiDdbP_Phi);            // Vector C
+    RinvP_Phi = P_Phi - DiDdbP_Phi;
+    DenOp.ProjectBoundaryBar(RinvP_Phi); // Correct to here
+
+ 
+    // R^-dagger R^-1 P term
+    DenOp.DinverseDag(RinvP_Phi,DidRinvP_Phi); // Vector D
+    DenOp.dBoundaryBarDag(DidRinvP_Phi,DdbdDidRinvP_Phi);
+    RinvDagRinvP_Phi = RinvP_Phi - DdbdDidRinvP_Phi;
+    DenOp.ProjectBoundaryBar(RinvDagRinvP_Phi);
+
+    
+    // P^dag R^-dagger R^-1 P term
+    NumOp.dOmegaDagInv(RinvDagRinvP_Phi,DoidRinvDagRinvP_Phi); // Vector E
+    NumOp.dBoundaryDag(DoidRinvDagRinvP_Phi,tmp);
+    NumOp.dOmegaBarDagInv(tmp,DobidDddDoidRinvDagRinvP_Phi);   // Vector F
+    NumOp.dBoundaryBarDag(DobidDddDoidRinvDagRinvP_Phi,tmp);
+    PdagRinvDagRinvP_Phi = RinvDagRinvP_Phi- tmp;
+    NumOp.ProjectBoundaryBar(PdagRinvDagRinvP_Phi);
+
+    /*
+    std::cout << "S eval  "<< action << std::endl;
+    std::cout << "S - IP1 "<< innerProduct(Phi,PdagRinvDagRinvP_Phi) << std::endl;
+    std::cout << "S - IP2 "<< norm2(RinvP_Phi) << std::endl;
+
+    NumOp.R(Phi,tmp);
+    tmp = tmp - P_Phi;
+    std::cout << "diff1 "<<norm2(tmp) <<std::endl;
+    
+    
+    DenOp.RInv(P_Phi,tmp);
+    tmp = tmp - RinvP_Phi;
+    std::cout << "diff2 "<<norm2(tmp) <<std::endl;
+
+    DenOp.RDagInv(RinvP_Phi,tmp);
+    tmp  = tmp - RinvDagRinvP_Phi;
+    std::cout << "diff3 "<<norm2(tmp) <<std::endl;
+
+    DenOp.RDag(RinvDagRinvP_Phi,tmp);
+    tmp  = tmp - PdagRinvDagRinvP_Phi;
+    std::cout << "diff4 "<<norm2(tmp) <<std::endl;
+    */
+    
+    dSdU=Zero();
+
+    X = DobiDdbPhi;
+    Y = DobidDddDoidRinvDagRinvP_Phi;
+    NumOp.DirichletFermOpD.MDeriv(force,Y,X,DaggerNo);    dSdU=dSdU+force;
+    NumOp.DirichletFermOpD.MDeriv(force,X,Y,DaggerYes);   dSdU=dSdU+force;
+
+    X = DoiDdDobiDdbPhi;
+    Y = DoidRinvDagRinvP_Phi;
+    NumOp.DirichletFermOpD.MDeriv(force,Y,X,DaggerNo);    dSdU=dSdU+force;
+    NumOp.DirichletFermOpD.MDeriv(force,X,Y,DaggerYes);   dSdU=dSdU+force;
+
+    X = DiDdbP_Phi;
+    Y = DidRinvP_Phi;
+    DenOp.PeriodicFermOpD.MDeriv(force,Y,X,DaggerNo);    dSdU=dSdU+force;
+    DenOp.PeriodicFermOpD.MDeriv(force,X,Y,DaggerYes);   dSdU=dSdU+force;
+
+    dSdU *= -1.0;
+
+  };
+};
+
+NAMESPACE_END(Grid);
+

Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h

@@ -44,10 +44,6 @@ NAMESPACE_BEGIN(Grid);
   // Exact one flavour implementation of DWF determinant ratio //
   ///////////////////////////////////////////////////////////////
 
-  //Note: using mixed prec CG for the heatbath solver in this action class will not work
-  //      because the L, R operators must have their shift coefficients updated throughout the heatbath step
-  //      You will find that the heatbath solver simply won't converge.
-  //      To use mixed precision here use the ExactOneFlavourRatioMixedPrecHeatbathPseudoFermionAction variant below
   template<class Impl>
   class ExactOneFlavourRatioPseudoFermionAction : public Action<typename Impl::GaugeField>
   {
@@ -61,60 +57,37 @@ NAMESPACE_BEGIN(Grid);
       bool use_heatbath_forecasting;
       AbstractEOFAFermion<Impl>& Lop; // the basic LH operator
       AbstractEOFAFermion<Impl>& Rop; // the basic RH operator
-      SchurRedBlackDiagMooeeSolve<FermionField> SolverHBL;
-      SchurRedBlackDiagMooeeSolve<FermionField> SolverHBR;
+      SchurRedBlackDiagMooeeSolve<FermionField> SolverHB;
       SchurRedBlackDiagMooeeSolve<FermionField> SolverL;
       SchurRedBlackDiagMooeeSolve<FermionField> SolverR;
       SchurRedBlackDiagMooeeSolve<FermionField> DerivativeSolverL;
       SchurRedBlackDiagMooeeSolve<FermionField> DerivativeSolverR;
       FermionField Phi; // the pseudofermion field for this trajectory
 
-      RealD norm2_eta; //|eta|^2 where eta is the random gaussian field used to generate the pseudofermion field
-      bool initial_action; //true for the first call to S after refresh, for which the identity S = |eta|^2 holds provided the rational approx is good
     public:
 
-      //Used in the heatbath, refresh the shift coefficients of the L (LorR=0) or R (LorR=1) operator
-      virtual void heatbathRefreshShiftCoefficients(int LorR, RealD to){
-	AbstractEOFAFermion<Impl>&op = LorR == 0 ? Lop : Rop;
-	op.RefreshShiftCoefficients(to);
-      }
-
-
-      //Use the same solver for L,R in all cases
       ExactOneFlavourRatioPseudoFermionAction(AbstractEOFAFermion<Impl>& _Lop, 
 					      AbstractEOFAFermion<Impl>& _Rop,
 					      OperatorFunction<FermionField>& CG, 
 					      Params& p, 
 					      bool use_fc=false) 
-	: ExactOneFlavourRatioPseudoFermionAction(_Lop,_Rop,CG,CG,CG,CG,CG,CG,p,use_fc) {};
-
-      //Use the same solver for L,R in the heatbath but different solvers elsewhere
+	: ExactOneFlavourRatioPseudoFermionAction(_Lop,_Rop,CG,CG,CG,CG,CG,p,use_fc) {};
+	
       ExactOneFlavourRatioPseudoFermionAction(AbstractEOFAFermion<Impl>& _Lop, 
 					      AbstractEOFAFermion<Impl>& _Rop,
-					      OperatorFunction<FermionField>& HeatbathCG,
-					      OperatorFunction<FermionField>& ActionCGL, OperatorFunction<FermionField>& ActionCGR, 
-					      OperatorFunction<FermionField>& DerivCGL , OperatorFunction<FermionField>& DerivCGR, 
-					      Params& p, 
-					      bool use_fc=false)
-	: ExactOneFlavourRatioPseudoFermionAction(_Lop,_Rop,HeatbathCG,HeatbathCG, ActionCGL, ActionCGR, DerivCGL,DerivCGR,p,use_fc) {};
-
-      //Use different solvers for L,R in all cases
-      ExactOneFlavourRatioPseudoFermionAction(AbstractEOFAFermion<Impl>& _Lop, 
-					      AbstractEOFAFermion<Impl>& _Rop,
-					      OperatorFunction<FermionField>& HeatbathCGL, OperatorFunction<FermionField>& HeatbathCGR,
+					      OperatorFunction<FermionField>& HeatbathCG, 
 					      OperatorFunction<FermionField>& ActionCGL, OperatorFunction<FermionField>& ActionCGR, 
 					      OperatorFunction<FermionField>& DerivCGL , OperatorFunction<FermionField>& DerivCGR, 
 					      Params& p, 
 					      bool use_fc=false) : 
         Lop(_Lop), 
 	Rop(_Rop), 
-	SolverHBL(HeatbathCGL,false,true), SolverHBR(HeatbathCGR,false,true),
+	SolverHB(HeatbathCG,false,true),
 	SolverL(ActionCGL, false, true), SolverR(ActionCGR, false, true), 
 	DerivativeSolverL(DerivCGL, false, true), DerivativeSolverR(DerivCGR, false, true), 
 	Phi(_Lop.FermionGrid()), 
 	param(p), 
-	use_heatbath_forecasting(use_fc),
-	initial_action(false)
+        use_heatbath_forecasting(use_fc)
       {
         AlgRemez remez(param.lo, param.hi, param.precision);
 
@@ -124,8 +97,6 @@ NAMESPACE_BEGIN(Grid);
         PowerNegHalf.Init(remez, param.tolerance, true);
       };
 
-      const FermionField &getPhi() const{ return Phi; }
-
       virtual std::string action_name() { return "ExactOneFlavourRatioPseudoFermionAction"; }
 
       virtual std::string LogParameters() {
@@ -146,19 +117,6 @@ NAMESPACE_BEGIN(Grid);
         else{ for(int s=0; s<Ls; ++s){ axpby_ssp_pminus(out, 0.0, in, 1.0, in, s, s); } }
       }
 
-      virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
-        // P(eta_o) = e^{- eta_o^dag eta_o}
-        //
-        // e^{x^2/2 sig^2} => sig^2 = 0.5.
-        // 
-        RealD scale = std::sqrt(0.5);
-
-        FermionField eta    (Lop.FermionGrid());
-        gaussian(pRNG,eta); eta = eta * scale;
-
-	refresh(U,eta);
-      }
-
       // EOFA heatbath: see Eqn. (29) of arXiv:1706.05843
       // We generate a Gaussian noise vector \eta, and then compute
       //  \Phi = M_{\rm EOFA}^{-1/2} * \eta
@@ -166,10 +124,12 @@ NAMESPACE_BEGIN(Grid);
       //
       // As a check of rational require \Phi^dag M_{EOFA} \Phi == eta^dag M^-1/2^dag M M^-1/2 eta = eta^dag eta
       //
-     void refresh(const GaugeField &U, const FermionField &eta) {
+      virtual void refresh(const GaugeField& U, GridSerialRNG &sRNG, GridParallelRNG& pRNG)
+      {
         Lop.ImportGauge(U);
         Rop.ImportGauge(U);
 
+        FermionField eta         (Lop.FermionGrid());
         FermionField CG_src      (Lop.FermionGrid());
         FermionField CG_soln     (Lop.FermionGrid());
         FermionField Forecast_src(Lop.FermionGrid());
@@ -180,6 +140,11 @@ NAMESPACE_BEGIN(Grid);
         if(use_heatbath_forecasting){ prev_solns.reserve(param.degree); }
         ChronoForecast<AbstractEOFAFermion<Impl>, FermionField> Forecast;
 
+        // Seed with Gaussian noise vector (var = 0.5)
+        RealD scale = std::sqrt(0.5);
+        gaussian(pRNG,eta);
+        eta = eta * scale;
+
         // \Phi = ( \alpha_{0} + \sum_{k=1}^{N_{p}} \alpha_{l} * \gamma_{l} ) * \eta
         RealD N(PowerNegHalf.norm);
         for(int k=0; k<param.degree; ++k){ N += PowerNegHalf.residues[k] / ( 1.0 + PowerNegHalf.poles[k] ); }
@@ -195,16 +160,15 @@ NAMESPACE_BEGIN(Grid);
         tmp[1] = Zero();
         for(int k=0; k<param.degree; ++k){
           gamma_l = 1.0 / ( 1.0 + PowerNegHalf.poles[k] );
-          heatbathRefreshShiftCoefficients(0, -gamma_l);
-	  //Lop.RefreshShiftCoefficients(-gamma_l);
+          Lop.RefreshShiftCoefficients(-gamma_l);
           if(use_heatbath_forecasting){ // Forecast CG guess using solutions from previous poles
             Lop.Mdag(CG_src, Forecast_src);
             CG_soln = Forecast(Lop, Forecast_src, prev_solns);
-            SolverHBL(Lop, CG_src, CG_soln);
+            SolverHB(Lop, CG_src, CG_soln);
             prev_solns.push_back(CG_soln);
           } else {
             CG_soln = Zero(); // Just use zero as the initial guess
-	    SolverHBL(Lop, CG_src, CG_soln);
+            SolverHB(Lop, CG_src, CG_soln);
           }
           Lop.Dtilde(CG_soln, tmp[0]); // We actually solved Cayley preconditioned system: transform back
           tmp[1] = tmp[1] + ( PowerNegHalf.residues[k]*gamma_l*gamma_l*Lop.k ) * tmp[0];
@@ -223,16 +187,15 @@ NAMESPACE_BEGIN(Grid);
         if(use_heatbath_forecasting){ prev_solns.clear(); } // empirically, LH solns don't help for RH solves
         for(int k=0; k<param.degree; ++k){
           gamma_l = 1.0 / ( 1.0 + PowerNegHalf.poles[k] );
-	  heatbathRefreshShiftCoefficients(1, -gamma_l*PowerNegHalf.poles[k]);
-          //Rop.RefreshShiftCoefficients(-gamma_l*PowerNegHalf.poles[k]);
+          Rop.RefreshShiftCoefficients(-gamma_l*PowerNegHalf.poles[k]);
           if(use_heatbath_forecasting){
             Rop.Mdag(CG_src, Forecast_src);
             CG_soln = Forecast(Rop, Forecast_src, prev_solns);
-            SolverHBR(Rop, CG_src, CG_soln);
+            SolverHB(Rop, CG_src, CG_soln);
             prev_solns.push_back(CG_soln);
           } else {
             CG_soln = Zero();
-            SolverHBR(Rop, CG_src, CG_soln);
+            SolverHB(Rop, CG_src, CG_soln);
           }
           Rop.Dtilde(CG_soln, tmp[0]); // We actually solved Cayley preconditioned system: transform back
           tmp[1] = tmp[1] - ( PowerNegHalf.residues[k]*gamma_l*gamma_l*Rop.k ) * tmp[0];
@@ -242,119 +205,49 @@ NAMESPACE_BEGIN(Grid);
         Phi = Phi + tmp[1];
 
         // Reset shift coefficients for energy and force evals
-        //Lop.RefreshShiftCoefficients(0.0);
-        //Rop.RefreshShiftCoefficients(-1.0);
-	heatbathRefreshShiftCoefficients(0, 0.0);
-	heatbathRefreshShiftCoefficients(1, -1.0);
-
-	//Mark that the next call to S is the first after refresh
-	initial_action = true;
-
+        Lop.RefreshShiftCoefficients(0.0);
+        Rop.RefreshShiftCoefficients(-1.0);
 
 	// Bounds check
 	RealD EtaDagEta = norm2(eta);
-	norm2_eta = EtaDagEta;
-
 	//	RealD PhiDagMPhi= norm2(eta);
 
       };
 
-      void Meofa(const GaugeField& U,const FermionField &in, FermionField & out) 
+      void Meofa(const GaugeField& U,const FermionField &phi, FermionField & Mphi) 
       {
+#if 0
         Lop.ImportGauge(U);
         Rop.ImportGauge(U);
 
-        FermionField spProj_in(Lop.FermionGrid());
+        FermionField spProj_Phi(Lop.FermionGrid());
+	FermionField mPhi(Lop.FermionGrid());
         std::vector<FermionField> tmp(2, Lop.FermionGrid());
-	out = in;
+	mPhi = phi;
 	
         // LH term: S = S - k <\Phi| P_{-} \Omega_{-}^{\dagger} H(mf)^{-1} \Omega_{-} P_{-} |\Phi>
-        spProj(in, spProj_in, -1, Lop.Ls);
-        Lop.Omega(spProj_in, tmp[0], -1, 0);
+        spProj(Phi, spProj_Phi, -1, Lop.Ls);
+        Lop.Omega(spProj_Phi, tmp[0], -1, 0);
         G5R5(tmp[1], tmp[0]);
         tmp[0] = Zero();
         SolverL(Lop, tmp[1], tmp[0]);
         Lop.Dtilde(tmp[0], tmp[1]); // We actually solved Cayley preconditioned system: transform back
         Lop.Omega(tmp[1], tmp[0], -1, 1);
-	spProj(tmp[0], tmp[1], -1, Lop.Ls);
-
-	out = out -  Lop.k * tmp[1];
+	mPhi = mPhi -  Lop.k * innerProduct(spProj_Phi, tmp[0]).real();
 
         // RH term: S = S + k <\Phi| P_{+} \Omega_{+}^{\dagger} ( H(mb)
-        //               - \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{+} P_{+} |\Phi>
-        spProj(in, spProj_in, 1, Rop.Ls);
-        Rop.Omega(spProj_in, tmp[0], 1, 0);
+        //               - \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{-} P_{-} |\Phi>
+        spProj(Phi, spProj_Phi, 1, Rop.Ls);
+        Rop.Omega(spProj_Phi, tmp[0], 1, 0);
         G5R5(tmp[1], tmp[0]);
         tmp[0] = Zero();
         SolverR(Rop, tmp[1], tmp[0]);
         Rop.Dtilde(tmp[0], tmp[1]);
         Rop.Omega(tmp[1], tmp[0], 1, 1);
-	spProj(tmp[0], tmp[1], 1, Rop.Ls);
-
-        out = out + Rop.k * tmp[1];
+        action += Rop.k * innerProduct(spProj_Phi, tmp[0]).real();
+#endif
       }
 
-      //Due to the structure of EOFA, it is no more expensive to compute the inverse of Meofa
-      //To ensure correctness we can simply reuse the heatbath code but use the rational approx
-      //f(x) = 1/x   which corresponds to alpha_0=0,  alpha_1=1,  beta_1=0 => gamma_1=1
-      void MeofaInv(const GaugeField &U, const FermionField &in, FermionField &out) {
-        Lop.ImportGauge(U);
-        Rop.ImportGauge(U);
-
-        FermionField CG_src      (Lop.FermionGrid());
-        FermionField CG_soln     (Lop.FermionGrid());
-        std::vector<FermionField> tmp(2, Lop.FermionGrid());
-
-        // \Phi = ( \alpha_{0} + \sum_{k=1}^{N_{p}} \alpha_{l} * \gamma_{l} ) * \eta
-	// = 1 * \eta
-        out = in;
-
-        // LH terms:
-        // \Phi = \Phi + k \sum_{k=1}^{N_{p}} P_{-} \Omega_{-}^{\dagger} ( H(mf)
-        //          - \gamma_{l} \Delta_{-}(mf,mb) P_{-} )^{-1} \Omega_{-} P_{-} \eta
-        spProj(in, tmp[0], -1, Lop.Ls);
-        Lop.Omega(tmp[0], tmp[1], -1, 0);
-        G5R5(CG_src, tmp[1]);
-        {
-          heatbathRefreshShiftCoefficients(0, -1.); //-gamma_1 = -1.
-
-	  CG_soln = Zero(); // Just use zero as the initial guess
-	  SolverHBL(Lop, CG_src, CG_soln);
-
-          Lop.Dtilde(CG_soln, tmp[0]); // We actually solved Cayley preconditioned system: transform back
-          tmp[1] = Lop.k * tmp[0];
-        }
-        Lop.Omega(tmp[1], tmp[0], -1, 1);
-        spProj(tmp[0], tmp[1], -1, Lop.Ls);
-        out = out + tmp[1];
-
-        // RH terms:
-        // \Phi = \Phi - k \sum_{k=1}^{N_{p}} P_{+} \Omega_{+}^{\dagger} ( H(mb)
-        //          - \beta_l\gamma_{l} \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{+} P_{+} \eta
-        spProj(in, tmp[0], 1, Rop.Ls);
-        Rop.Omega(tmp[0], tmp[1], 1, 0);
-        G5R5(CG_src, tmp[1]);
-        {
-	  heatbathRefreshShiftCoefficients(1, 0.); //-gamma_1 * beta_1 = 0
-
-	  CG_soln = Zero();
-	  SolverHBR(Rop, CG_src, CG_soln);
-
-          Rop.Dtilde(CG_soln, tmp[0]); // We actually solved Cayley preconditioned system: transform back
-          tmp[1] = - Rop.k * tmp[0];
-        }
-        Rop.Omega(tmp[1], tmp[0], 1, 1);
-        spProj(tmp[0], tmp[1], 1, Rop.Ls);
-        out = out + tmp[1];
-
-        // Reset shift coefficients for energy and force evals
-	heatbathRefreshShiftCoefficients(0, 0.0);
-	heatbathRefreshShiftCoefficients(1, -1.0);
-      };
-
-
-
-
       // EOFA action: see Eqn. (10) of arXiv:1706.05843
       virtual RealD S(const GaugeField& U)
       {
@@ -378,7 +271,7 @@ NAMESPACE_BEGIN(Grid);
         action -= Lop.k * innerProduct(spProj_Phi, tmp[0]).real();
 
         // RH term: S = S + k <\Phi| P_{+} \Omega_{+}^{\dagger} ( H(mb)
-        //               - \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{+} P_{+} |\Phi>
+        //               - \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{-} P_{-} |\Phi>
         spProj(Phi, spProj_Phi, 1, Rop.Ls);
         Rop.Omega(spProj_Phi, tmp[0], 1, 0);
         G5R5(tmp[1], tmp[0]);
@@ -388,26 +281,6 @@ NAMESPACE_BEGIN(Grid);
         Rop.Omega(tmp[1], tmp[0], 1, 1);
         action += Rop.k * innerProduct(spProj_Phi, tmp[0]).real();
 
-	if(initial_action){
-	  //For the first call to S after refresh,  S = |eta|^2. We can use this to ensure the rational approx is good
-	  RealD diff = action - norm2_eta;
-
-	  //S_init = eta^dag M^{-1/2} M M^{-1/2} eta
-	  //S_init - eta^dag eta =  eta^dag ( M^{-1/2} M M^{-1/2} - 1 ) eta
-
-	  //If approximate solution
-	  //S_init - eta^dag eta =  eta^dag ( [M^{-1/2}+\delta M^{-1/2}] M [M^{-1/2}+\delta M^{-1/2}] - 1 ) eta
-	  //               \approx  eta^dag ( \delta M^{-1/2} M^{1/2} + M^{1/2}\delta M^{-1/2} ) eta
-	  // We divide out |eta|^2 to remove source scaling but the tolerance on this check should still be somewhat higher than the actual approx tolerance
-	  RealD test = fabs(diff)/norm2_eta; //test the quality of the rational approx
-
-	  std::cout << GridLogMessage << action_name() << " initial action " << action << " expect " << norm2_eta << "; diff " << diff << std::endl;
-	  std::cout << GridLogMessage << action_name() << "[ eta^dag ( M^{-1/2} M M^{-1/2} - 1 ) eta ]/|eta^2| = " << test << "  expect 0 (tol " << param.BoundsCheckTol << ")" << std::endl;
-
-	  assert( ( test < param.BoundsCheckTol ) && " Initial action check failed" );
-	  initial_action = false;
-	}
-
         return action;
       };
 
@@ -456,40 +329,6 @@ NAMESPACE_BEGIN(Grid);
       };
   };
 
-  template<class ImplD, class ImplF>
-  class ExactOneFlavourRatioMixedPrecHeatbathPseudoFermionAction : public ExactOneFlavourRatioPseudoFermionAction<ImplD>{
-  public:
-    INHERIT_IMPL_TYPES(ImplD);
-    typedef OneFlavourRationalParams Params;
-
-  private:
-    AbstractEOFAFermion<ImplF>& LopF; // the basic LH operator
-    AbstractEOFAFermion<ImplF>& RopF; // the basic RH operator
-
-  public:
-    
-    virtual std::string action_name() { return "ExactOneFlavourRatioMixedPrecHeatbathPseudoFermionAction"; }
-    
-    //Used in the heatbath, refresh the shift coefficients of the L (LorR=0) or R (LorR=1) operator
-    virtual void heatbathRefreshShiftCoefficients(int LorR, RealD to){
-      AbstractEOFAFermion<ImplF> &op = LorR == 0 ? LopF : RopF;
-      op.RefreshShiftCoefficients(to);
-      this->ExactOneFlavourRatioPseudoFermionAction<ImplD>::heatbathRefreshShiftCoefficients(LorR,to);
-    }
-    
-    ExactOneFlavourRatioMixedPrecHeatbathPseudoFermionAction(AbstractEOFAFermion<ImplF>& _LopF, 
-							     AbstractEOFAFermion<ImplF>& _RopF,
-							     AbstractEOFAFermion<ImplD>& _LopD, 
-							     AbstractEOFAFermion<ImplD>& _RopD,
-							     OperatorFunction<FermionField>& HeatbathCGL, OperatorFunction<FermionField>& HeatbathCGR,
-							     OperatorFunction<FermionField>& ActionCGL, OperatorFunction<FermionField>& ActionCGR, 
-							     OperatorFunction<FermionField>& DerivCGL , OperatorFunction<FermionField>& DerivCGR, 
-							     Params& p, 
-							     bool use_fc=false) : 
-    LopF(_LopF), RopF(_RopF), ExactOneFlavourRatioPseudoFermionAction<ImplD>(_LopD, _RopD, HeatbathCGL, HeatbathCGR, ActionCGL, ActionCGR, DerivCGL, DerivCGR, p, use_fc){}
-  };
-
-
 NAMESPACE_END(Grid);
 
 #endif

Grid/qcd/action/pseudofermion/PseudoFermion.h

@@ -26,8 +26,7 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#ifndef QCD_PSEUDOFERMION_AGGREGATE_H
-#define QCD_PSEUDOFERMION_AGGREGATE_H
+#pragma once
 
 // Rational functions
 #include <Grid/qcd/action/pseudofermion/Bounds.h>
@@ -44,5 +43,10 @@ directory
 #include <Grid/qcd/action/pseudofermion/GeneralEvenOddRationalRatioMixedPrec.h>
 #include <Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h>
 #include <Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h>
+#include <Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourPseudoFermion.h>
+#include <Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourBosonPseudoFermion.h>
+#include <Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion.h>
+
+
+
 
-#endif

Grid/qcd/action/pseudofermion/TwoFlavour.h

@@ -98,6 +98,7 @@ public:
 
     FermOp.ImportGauge(U);
     FermOp.Mdag(eta, Phi);
+    std::cout << GridLogMessage << "Pseudofermion action refresh " << norm2(eta) << std::endl;
   };
 
   //////////////////////////////////////////////////////

Grid/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h

@@ -50,6 +50,8 @@ NAMESPACE_BEGIN(Grid);
       FermionField PhiOdd;   // the pseudo fermion field for this trajectory
       FermionField PhiEven;  // the pseudo fermion field for this trajectory
 
+      virtual void refreshRestrict(FermionField &eta) {};
+      
     public:
       TwoFlavourEvenOddRatioPseudoFermionAction(FermionOperator<Impl>  &_NumOp, 
                                                 FermionOperator<Impl>  &_DenOp, 
@@ -60,7 +62,8 @@ NAMESPACE_BEGIN(Grid);
       TwoFlavourEvenOddRatioPseudoFermionAction(FermionOperator<Impl>  &_NumOp, 
                                                 FermionOperator<Impl>  &_DenOp, 
                                                 OperatorFunction<FermionField> & DS,
-                                                OperatorFunction<FermionField> & AS, OperatorFunction<FermionField> & HS) :
+                                                OperatorFunction<FermionField> & AS,
+						OperatorFunction<FermionField> & HS) :
       NumOp(_NumOp), 
       DenOp(_DenOp), 
       DerivativeSolver(DS), 
@@ -83,9 +86,6 @@ NAMESPACE_BEGIN(Grid);
 	return sstream.str();
       } 
 
-      //Access the fermion field
-      const FermionField &getPhiOdd() const{ return PhiOdd; }
-
       virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
         // P(eta_o) = e^{- eta_o^dag eta_o}
         //
@@ -96,6 +96,7 @@ NAMESPACE_BEGIN(Grid);
         FermionField eta    (NumOp.FermionGrid());
         gaussian(pRNG,eta); eta = eta * scale;
 
+	refreshRestrict(eta); // Used by DDHMC
 	refresh(U,eta);
       }
 	
@@ -131,6 +132,7 @@ NAMESPACE_BEGIN(Grid);
 
         //PhiOdd =PhiOdd*scale;
         //PhiEven=PhiEven*scale;
+	std::cout << GridLogMessage<<" TwoFlavourEvenOddRatio Expect action to be "<<norm2(etaOdd) + norm2(etaEven)<<std::endl;
         
       };
 
@@ -165,6 +167,8 @@ NAMESPACE_BEGIN(Grid);
         DenOp.MooeeInvDag(X,Y);
         action = action + norm2(Y);
 
+	std::cout << GridLogMessage<<" TwoFlavourEvenOddRatio action is "<<action<<std::endl;
+
         return action;
       };
 
@@ -177,7 +181,7 @@ NAMESPACE_BEGIN(Grid);
 
         NumOp.ImportGauge(U);
         DenOp.ImportGauge(U);
-
+	
         SchurDifferentiableOperator<Impl> Mpc(DenOp);
         SchurDifferentiableOperator<Impl> Vpc(NumOp);
 
@@ -212,7 +216,7 @@ NAMESPACE_BEGIN(Grid);
         assert(DenOp.ConstEE() == 1);
 
         dSdU = -dSdU;
-        
+
       };
     };
 NAMESPACE_END(Grid);

Grid/qcd/action/pseudofermion/TwoFlavourRatio.h

@@ -99,7 +99,7 @@ public:
     NumOp.M(tmp,Phi);               // Vdag^-1 Mdag eta
 
     Phi=Phi*scale;
-	
+    std::cout << GridLogMessage<<" TwoFlavourRatio Expect action to be "<<norm2(eta)*scale*scale<<std::endl;
   };
 
   //////////////////////////////////////////////////////
@@ -121,6 +121,7 @@ public:
     DenOp.M(X,Y);                  // Y=  Mdag^-1 Vdag phi
 
     RealD action = norm2(Y);
+    std::cout << GridLogMessage<<" TwoFlavourRatio action is "<<action<<std::endl;
 
     return action;
   };

Grid/qcd/action/pseudofermion/TwoFlavourRatio4DPseudoFermion.h

@@ -0,0 +1,197 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/pseudofermion/TwoFlavourRatio.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////
+// Two flavour ratio
+///////////////////////////////////////
+template<class Impl>
+class TwoFlavourRatio4DPseudoFermionAction : public Action<typename Impl::GaugeField> {
+public:
+  INHERIT_IMPL_TYPES(Impl);
+
+private:
+  FermionOperator<Impl> & NumOp;// the basic operator
+  FermionOperator<Impl> & DenOp;// the basic operator
+
+  OperatorFunction<FermionField> &DerivativeSolver;
+  OperatorFunction<FermionField> &ActionSolver;
+
+  FermionField phi4; // the pseudo fermion field for this trajectory
+
+public:
+  TwoFlavourRatio4DPseudoFermionAction(FermionOperator<Impl>  &_NumOp, 
+				       FermionOperator<Impl>  &_DenOp, 
+				       OperatorFunction<FermionField> & DS,
+				       OperatorFunction<FermionField> & AS
+				       ) : NumOp(_NumOp),
+					   DenOp(_DenOp),
+					   DerivativeSolver(DS),
+					   ActionSolver(AS),
+					   phi4(_NumOp.GaugeGrid())
+  {};
+      
+  virtual std::string action_name(){return "TwoFlavourRatio4DPseudoFermionAction";}
+
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+    return sstream.str();
+  }  
+      
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
+
+    // P(phi) = e^{- phi^dag (V^dag M^-dag)_11  (M^-1 V)_11 phi}
+    //
+    // NumOp == V
+    // DenOp == M
+    //
+    // Take phi = (V^{-1} M)_11 eta  ; eta = (M^{-1} V)_11 Phi
+    //
+    // P(eta) = e^{- eta^dag eta}
+    //
+    // e^{x^2/2 sig^2} => sig^2 = 0.5.
+    // 
+    // So eta should be of width sig = 1/sqrt(2) and must multiply by 0.707....
+    //
+    RealD scale = std::sqrt(0.5);
+
+    FermionField eta4(NumOp.GaugeGrid());
+    FermionField eta5(NumOp.FermionGrid());
+    FermionField tmp(NumOp.FermionGrid());
+    FermionField phi5(NumOp.FermionGrid());
+
+    gaussian(pRNG,eta4);
+    NumOp.ImportFourDimPseudoFermion(eta4,eta5);
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    MdagMLinearOperator<FermionOperator<Impl> ,FermionField> MdagMOp(NumOp);
+
+    DenOp.M(eta5,phi5);               // M eta
+    NumOp.Mdag(phi5,tmp);            // Vdag M eta
+    phi5 = Zero();
+    ActionSolver(MdagMOp,tmp,phi5);  // (VdagV)^-1 M eta = V^-1 Vdag^-1 Vdag M eta = V^-1 M eta
+    phi5=phi5*scale;
+
+    // Project to 4d
+    NumOp.ExportFourDimPseudoFermion(phi5,phi4);
+      
+  };
+
+  //////////////////////////////////////////////////////
+  // S = phi^dag (V^dag M^-dag)_11  (M^-1 V)_11 phi
+  //////////////////////////////////////////////////////
+  virtual RealD S(const GaugeField &U) {
+
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    FermionField Y4(NumOp.GaugeGrid());
+    FermionField X(NumOp.FermionGrid());
+    FermionField Y(NumOp.FermionGrid());
+    FermionField phi5(NumOp.FermionGrid());
+	
+    MdagMLinearOperator<FermionOperator<Impl> ,FermionField> MdagMOp(DenOp);
+
+    NumOp.ImportFourDimPseudoFermion(phi4,phi5);
+    NumOp.M(phi5,Y);              // Y= V phi
+    DenOp.Mdag(Y,X);              // X= Mdag V phi
+    Y=Zero();
+    ActionSolver(MdagMOp,X,Y);    // Y= (MdagM)^-1 Mdag Vdag phi = M^-1 V phi
+
+    NumOp.ExportFourDimPseudoFermion(Y,Y4);
+
+    RealD action = norm2(Y4);
+
+    return action;
+  };
+
+  //////////////////////////////////////////////////////
+  // dS/du = 2 Re phi^dag (V^dag M^-dag)_11  (M^-1 d V)_11  phi
+  //       - 2 Re phi^dag (dV^dag M^-dag)_11  (M^-1 dM M^-1 V)_11  phi
+  //////////////////////////////////////////////////////
+  virtual void deriv(const GaugeField &U,GaugeField & dSdU) {
+
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    MdagMLinearOperator<FermionOperator<Impl> ,FermionField> MdagMOp(DenOp);
+
+
+    FermionField  X(NumOp.FermionGrid());
+    FermionField  Y(NumOp.FermionGrid());
+    FermionField       phi(NumOp.FermionGrid());
+    FermionField      Vphi(NumOp.FermionGrid());
+    FermionField  MinvVphi(NumOp.FermionGrid());
+    FermionField      tmp4(NumOp.GaugeGrid());
+    FermionField  MdagInvMinvVphi(NumOp.FermionGrid());
+
+    GaugeField   force(NumOp.GaugeGrid());	
+
+    //Y=V phi
+    //X = (Mdag V phi
+    //Y = (Mdag M)^-1 Mdag V phi = M^-1 V Phi
+    NumOp.ImportFourDimPseudoFermion(phi4,phi);
+    NumOp.M(phi,Vphi);               //  V phi
+    DenOp.Mdag(Vphi,X);              // X=  Mdag V phi
+    Y=Zero();
+    DerivativeSolver(MdagMOp,X,MinvVphi);// M^-1 V phi
+
+    // Projects onto the physical space and back
+    NumOp.ExportFourDimPseudoFermion(MinvVphi,tmp4);
+    NumOp.ImportFourDimPseudoFermion(tmp4,Y);
+    
+    X=Zero();
+    DerivativeSolver(MdagMOp,Y,X);// X = (MdagM)^-1 proj M^-1 V phi
+    DenOp.M(X,MdagInvMinvVphi);
+    
+    // phi^dag (Vdag Mdag^-1) (M^-1 dV)  phi
+    NumOp.MDeriv(force ,MdagInvMinvVphi , phi, DaggerNo );  dSdU=force;
+  
+    // phi^dag (dVdag Mdag^-1) (M^-1 V)  phi
+    NumOp.MDeriv(force , phi, MdagInvMinvVphi ,DaggerYes  );  dSdU=dSdU+force;
+
+    //    - 2 Re phi^dag (dV^dag M^-dag)_11  (M^-1 dM M^-1 V)_11  phi
+    DenOp.MDeriv(force,MdagInvMinvVphi,MinvVphi,DaggerNo);   dSdU=dSdU-force;
+    DenOp.MDeriv(force,MinvVphi,MdagInvMinvVphi,DaggerYes);  dSdU=dSdU-force;
+
+    dSdU *= -1.0; 
+    //dSdU = - Ta(dSdU);
+    
+  };
+};
+
+NAMESPACE_END(Grid);
+
+

Grid/qcd/action/pseudofermion/TwoFlavourRatioEO4DPseudoFermion.h

@@ -0,0 +1,203 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/qcd/action/pseudofermion/TwoFlavourRatio.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#pragma once
+
+NAMESPACE_BEGIN(Grid);
+
+///////////////////////////////////////
+// Two flavour ratio
+///////////////////////////////////////
+template<class Impl>
+class TwoFlavourRatioEO4DPseudoFermionAction : public Action<typename Impl::GaugeField> {
+public:
+  INHERIT_IMPL_TYPES(Impl);
+
+private:
+  typedef FermionOperator<Impl> FermOp;
+  FermionOperator<Impl> & NumOp;// the basic operator
+  FermionOperator<Impl> & DenOp;// the basic operator
+
+  OperatorFunction<FermionField> &DerivativeSolver;
+  OperatorFunction<FermionField> &DerivativeDagSolver;
+  OperatorFunction<FermionField> &ActionSolver;
+  OperatorFunction<FermionField> &HeatbathSolver;
+
+  FermionField phi4; // the pseudo fermion field for this trajectory
+
+public:
+  TwoFlavourRatioEO4DPseudoFermionAction(FermionOperator<Impl>  &_NumOp, 
+					 FermionOperator<Impl>  &_DenOp, 
+					 OperatorFunction<FermionField> & DS,
+					 OperatorFunction<FermionField> & AS ) : 
+    TwoFlavourRatioEO4DPseudoFermionAction(_NumOp,_DenOp, DS,DS,AS,AS) {};
+  TwoFlavourRatioEO4DPseudoFermionAction(FermionOperator<Impl>  &_NumOp, 
+					 FermionOperator<Impl>  &_DenOp, 
+					 OperatorFunction<FermionField> & DS,
+					 OperatorFunction<FermionField> & DDS,
+					 OperatorFunction<FermionField> & AS,
+					 OperatorFunction<FermionField> & HS
+				       ) : NumOp(_NumOp),
+					   DenOp(_DenOp),
+					   DerivativeSolver(DS),
+					   DerivativeDagSolver(DDS),
+					   ActionSolver(AS),
+					   HeatbathSolver(HS),
+					   phi4(_NumOp.GaugeGrid())
+  {};
+      
+  virtual std::string action_name(){return "TwoFlavourRatioEO4DPseudoFermionAction";}
+
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+    return sstream.str();
+  }  
+      
+  virtual void refresh(const GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG& pRNG) {
+
+    // P(phi) = e^{- phi^dag (V^dag M^-dag)_11  (M^-1 V)_11 phi}
+    //
+    // NumOp == V
+    // DenOp == M
+    //
+    // Take phi = (V^{-1} M)_11 eta  ; eta = (M^{-1} V)_11 Phi
+    //
+    // P(eta) = e^{- eta^dag eta}
+    //
+    // e^{x^2/2 sig^2} => sig^2 = 0.5.
+    // 
+    // So eta should be of width sig = 1/sqrt(2) and must multiply by 0.707....
+    //
+    RealD scale = std::sqrt(0.5);
+
+    FermionField eta4(NumOp.GaugeGrid());
+    FermionField eta5(NumOp.FermionGrid());
+    FermionField tmp(NumOp.FermionGrid());
+    FermionField phi5(NumOp.FermionGrid());
+
+    gaussian(pRNG,eta4);
+    NumOp.ImportFourDimPseudoFermion(eta4,eta5);
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    SchurRedBlackDiagMooeeSolve<FermionField> PrecSolve(HeatbathSolver);
+
+    DenOp.M(eta5,tmp);               // M eta
+    PrecSolve(NumOp,tmp,phi5);  // phi = V^-1 M eta
+    phi5=phi5*scale;
+    std::cout << GridLogMessage << "4d pf refresh "<< norm2(phi5)<<"\n";
+    // Project to 4d
+    NumOp.ExportFourDimPseudoFermion(phi5,phi4);
+      
+  };
+
+  //////////////////////////////////////////////////////
+  // S = phi^dag (V^dag M^-dag)_11  (M^-1 V)_11 phi
+  //////////////////////////////////////////////////////
+  virtual RealD S(const GaugeField &U) {
+
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    FermionField Y4(NumOp.GaugeGrid());
+    FermionField X(NumOp.FermionGrid());
+    FermionField Y(NumOp.FermionGrid());
+    FermionField phi5(NumOp.FermionGrid());
+	
+    MdagMLinearOperator<FermionOperator<Impl> ,FermionField> MdagMOp(DenOp);
+    SchurRedBlackDiagMooeeSolve<FermionField> PrecSolve(ActionSolver);
+
+    NumOp.ImportFourDimPseudoFermion(phi4,phi5);
+    NumOp.M(phi5,X);              // X= V phi
+    PrecSolve(DenOp,X,Y);    // Y= (MdagM)^-1 Mdag Vdag phi = M^-1 V phi
+    NumOp.ExportFourDimPseudoFermion(Y,Y4);
+
+    RealD action = norm2(Y4);
+
+    return action;
+  };
+
+  //////////////////////////////////////////////////////
+  // dS/du = 2 Re phi^dag (V^dag M^-dag)_11  (M^-1 d V)_11  phi
+  //       - 2 Re phi^dag (dV^dag M^-dag)_11  (M^-1 dM M^-1 V)_11  phi
+  //////////////////////////////////////////////////////
+  virtual void deriv(const GaugeField &U,GaugeField & dSdU) {
+
+    NumOp.ImportGauge(U);
+    DenOp.ImportGauge(U);
+
+    FermionField  X(NumOp.FermionGrid());
+    FermionField  Y(NumOp.FermionGrid());
+    FermionField       phi(NumOp.FermionGrid());
+    FermionField      Vphi(NumOp.FermionGrid());
+    FermionField  MinvVphi(NumOp.FermionGrid());
+    FermionField      tmp4(NumOp.GaugeGrid());
+    FermionField  MdagInvMinvVphi(NumOp.FermionGrid());
+
+    GaugeField   force(NumOp.GaugeGrid());	
+
+    //Y=V phi
+    //X = (Mdag V phi
+    //Y = (Mdag M)^-1 Mdag V phi = M^-1 V Phi
+    NumOp.ImportFourDimPseudoFermion(phi4,phi);
+    NumOp.M(phi,Vphi);               //  V phi
+    SchurRedBlackDiagMooeeSolve<FermionField> PrecSolve(DerivativeSolver);
+    PrecSolve(DenOp,Vphi,MinvVphi);// M^-1 V phi
+    std::cout << GridLogMessage << "4d deriv solve "<< norm2(MinvVphi)<<"\n";
+
+    // Projects onto the physical space and back
+    NumOp.ExportFourDimPseudoFermion(MinvVphi,tmp4);
+    NumOp.ImportFourDimPseudoFermion(tmp4,Y);
+
+    SchurRedBlackDiagMooeeDagSolve<FermionField> PrecDagSolve(DerivativeDagSolver);
+    // X = proj M^-dag V phi
+    // Need an adjoint solve
+    PrecDagSolve(DenOp,Y,MdagInvMinvVphi);
+    std::cout << GridLogMessage << "4d deriv solve dag "<< norm2(MdagInvMinvVphi)<<"\n";
+    
+    // phi^dag (Vdag Mdag^-1) (M^-1 dV)  phi
+    NumOp.MDeriv(force ,MdagInvMinvVphi , phi, DaggerNo );  dSdU=force;
+  
+    // phi^dag (dVdag Mdag^-1) (M^-1 V)  phi
+    NumOp.MDeriv(force , phi, MdagInvMinvVphi ,DaggerYes  );  dSdU=dSdU+force;
+
+    //    - 2 Re phi^dag (dV^dag M^-dag)_11  (M^-1 dM M^-1 V)_11  phi
+    DenOp.MDeriv(force,MdagInvMinvVphi,MinvVphi,DaggerNo);   dSdU=dSdU-force;
+    DenOp.MDeriv(force,MinvVphi,MdagInvMinvVphi,DaggerYes);  dSdU=dSdU-force;
+
+    dSdU *= -1.0; 
+    //dSdU = - Ta(dSdU);
+    
+  };
+};
+
+NAMESPACE_END(Grid);
+
+

Grid/qcd/hmc/GenericHMCrunner.h

@@ -151,22 +151,12 @@ public:
       Resources.GetCheckPointer()->CheckpointRestore(Parameters.StartTrajectory, U,
 						     Resources.GetSerialRNG(),
 						     Resources.GetParallelRNG());
-    } else if (Parameters.StartingType == "CheckpointStartReseed") {
-      // Same as CheckpointRestart but reseed the RNGs using the fixed integer seeding used for ColdStart and HotStart
-      // Useful for creating new evolution streams from an existing stream
-      
-      // WARNING: Unfortunately because the checkpointer doesn't presently allow us to separately restore the RNG and gauge fields we have to load
-      // an existing RNG checkpoint first; make sure one is available and named correctly
-      Resources.GetCheckPointer()->CheckpointRestore(Parameters.StartTrajectory, U,
-						     Resources.GetSerialRNG(),
-						     Resources.GetParallelRNG());
-      Resources.SeedFixedIntegers();      
     } else {
       // others
       std::cout << GridLogError << "Unrecognized StartingType\n";
       std::cout
 	<< GridLogError
-	<< "Valid [HotStart, ColdStart, TepidStart, CheckpointStart, CheckpointStartReseed]\n";
+	<< "Valid [HotStart, ColdStart, TepidStart, CheckpointStart]\n";
       exit(1);
     }
   }
@@ -186,6 +176,9 @@ private:
     typedef IntegratorType<SmearingPolicy> TheIntegrator;
     TheIntegrator MDynamics(UGrid, Parameters.MD, TheAction, Smearing);
 
+    // Sets the momentum filter
+    MDynamics.setMomentumFilter(*(Resources.GetMomentumFilter()));
+
     Smearing.set_Field(U);
 
     HybridMonteCarlo<TheIntegrator> HMC(Parameters, MDynamics,

Grid/qcd/hmc/HMC.h

@@ -34,6 +34,7 @@ directory
 			    * @brief Classes for Hybrid Monte Carlo update
 			    *
 			    * @author Guido Cossu
+			    * @author Peter Boyle
 			    */
 			   //--------------------------------------------------------------------
 #pragma once
@@ -115,22 +116,17 @@ private:
 
     random(sRNG, rn_test);
 
-    std::cout << GridLogHMC
-              << "--------------------------------------------------\n";
-    std::cout << GridLogHMC << "exp(-dH) = " << prob
-              << "  Random = " << rn_test << "\n";
-    std::cout << GridLogHMC
-              << "Acc. Probability = " << ((prob < 1.0) ? prob : 1.0) << "\n";
+    std::cout << GridLogHMC << "--------------------------------------------------\n";
+    std::cout << GridLogHMC << "exp(-dH) = " << prob << "  Random = " << rn_test << "\n";
+    std::cout << GridLogHMC << "Acc. Probability = " << ((prob < 1.0) ? prob : 1.0) << "\n";
 
     if ((prob > 1.0) || (rn_test <= prob)) {  // accepted
       std::cout << GridLogHMC << "Metropolis_test -- ACCEPTED\n";
-      std::cout << GridLogHMC
-                << "--------------------------------------------------\n";
+      std::cout << GridLogHMC << "--------------------------------------------------\n";
       return true;
     } else {  // rejected
       std::cout << GridLogHMC << "Metropolis_test -- REJECTED\n";
-      std::cout << GridLogHMC
-                << "--------------------------------------------------\n";
+      std::cout << GridLogHMC << "--------------------------------------------------\n";
       return false;
     }
   }
@@ -139,19 +135,68 @@ private:
   // Evolution
   /////////////////////////////////////////////////////////
   RealD evolve_hmc_step(Field &U) {
-    TheIntegrator.refresh(U, sRNG, pRNG);  // set U and initialize P and phi's
 
-    RealD H0 = TheIntegrator.S(U);  // initial state action
+    GridBase *Grid = U.Grid();
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // Mainly for DDHMC perform a random translation of U modulo volume
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+    std::cout << GridLogMessage << "Random shifting gauge field by [";
+    for(int d=0;d<Grid->Nd();d++) {
+
+      int L = Grid->GlobalDimensions()[d];
+
+      RealD rn_uniform;  random(sRNG, rn_uniform);
+
+      int shift = (int) (rn_uniform*L);
+
+      std::cout << shift;
+      if(d<Grid->Nd()-1) std::cout <<",";
+      else               std::cout <<"]\n";
+      
+      U = Cshift(U,d,shift);
+    }
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+
+    TheIntegrator.reset_timer();
+    
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // set U and initialize P and phi's
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+    std::cout << GridLogMessage << "Refresh momenta and pseudofermions";
+    TheIntegrator.refresh(U, sRNG, pRNG);  
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // initial state action
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+    std::cout << GridLogMessage << "Compute initial action";
+    RealD H0 = TheIntegrator.S(U);  
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
 
     std::streamsize current_precision = std::cout.precision();
     std::cout.precision(15);
     std::cout << GridLogHMC << "Total H before trajectory = " << H0 << "\n";
     std::cout.precision(current_precision);
 
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+    std::cout << GridLogMessage << " Molecular Dynamics evolution ";
     TheIntegrator.integrate(U);
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
 
-    RealD H1 = TheIntegrator.S(U);  // updated state action
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // updated state action
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
+    std::cout << GridLogMessage << "Compute final action";
+    RealD H1 = TheIntegrator.S(U);  
+    std::cout << GridLogMessage << "--------------------------------------------------\n";
 
+
+    
     ///////////////////////////////////////////////////////////
     if(0){
       std::cout << "------------------------- Reversibility test" << std::endl;
@@ -163,17 +208,16 @@ private:
     }
     ///////////////////////////////////////////////////////////
 
-
     std::cout.precision(15);
-    std::cout << GridLogHMC << "Total H after trajectory  = " << H1
-	      << "  dH = " << H1 - H0 << "\n";
+
+    std::cout << GridLogHMC << "--------------------------------------------------\n";
+    std::cout << GridLogHMC << "Total H after trajectory  = " << H1 << "  dH = " << H1 - H0 << "\n";
+    std::cout << GridLogHMC << "--------------------------------------------------\n";
+
     std::cout.precision(current_precision);
     
     return (H1 - H0);
   }
-  
-
-  
 
 public:
   /////////////////////////////////////////
@@ -195,8 +239,11 @@ public:
 
     // Actual updates (evolve a copy Ucopy then copy back eventually)
     unsigned int FinalTrajectory = Params.Trajectories + Params.NoMetropolisUntil + Params.StartTrajectory;
+
     for (int traj = Params.StartTrajectory; traj < FinalTrajectory; ++traj) {
+
       std::cout << GridLogHMC << "-- # Trajectory = " << traj << "\n";
+
       if (traj < Params.StartTrajectory + Params.NoMetropolisUntil) {
       	std::cout << GridLogHMC << "-- Thermalization" << std::endl;
       }
@@ -216,11 +263,10 @@ public:
       if (accept)
         Ucur = Ucopy; 
       
-     
-      
       double t1=usecond();
       std::cout << GridLogHMC << "Total time for trajectory (s): " << (t1-t0)/1e6 << std::endl;
 
+      TheIntegrator.print_timer();
 
       for (int obs = 0; obs < Observables.size(); obs++) {
       	std::cout << GridLogDebug << "Observables # " << obs << std::endl;

Grid/qcd/hmc/HMCModules.h

@@ -80,9 +80,7 @@ public:
       std::cout << GridLogError << "Seeds not initialized" << std::endl;
       exit(1);
     }
-    std::cout << GridLogMessage << "Reseeding serial RNG with seed vector " << SerialSeeds << std::endl;
     sRNG_.SeedFixedIntegers(SerialSeeds);
-    std::cout << GridLogMessage << "Reseeding parallel RNG with seed vector " << ParallelSeeds << std::endl;
     pRNG_->SeedFixedIntegers(ParallelSeeds);
   }
 };

Grid/qcd/hmc/HMCResourceManager.h

@@ -72,6 +72,8 @@ class HMCResourceManager {
   typedef HMCModuleBase< BaseHmcCheckpointer<ImplementationPolicy> > CheckpointerBaseModule;
   typedef HMCModuleBase< HmcObservable<typename ImplementationPolicy::Field> > ObservableBaseModule;
   typedef ActionModuleBase< Action<typename ImplementationPolicy::Field>, GridModule > ActionBaseModule;
+  typedef typename ImplementationPolicy::Field MomentaField;
+  typedef typename ImplementationPolicy::Field Field;  
 
   // Named storage for grid pairs (std + red-black)
   std::unordered_map<std::string, GridModule> Grids;
@@ -80,6 +82,9 @@ class HMCResourceManager {
   // SmearingModule<ImplementationPolicy> Smearing;
   std::unique_ptr<CheckpointerBaseModule> CP;
 
+  // Momentum filter
+  std::unique_ptr<MomentumFilterBase<typename ImplementationPolicy::Field> > Filter;
+  
   // A vector of HmcObservable modules
   std::vector<std::unique_ptr<ObservableBaseModule> > ObservablesList;
 
@@ -90,6 +95,7 @@ class HMCResourceManager {
 
   bool have_RNG;
   bool have_CheckPointer;
+  bool have_Filter;
 
   // NOTE: operator << is not overloaded for std::vector<string> 
   // so this function is necessary
@@ -101,7 +107,7 @@ class HMCResourceManager {
 
 
 public:
-  HMCResourceManager() : have_RNG(false), have_CheckPointer(false) {}
+  HMCResourceManager() : have_RNG(false), have_CheckPointer(false), have_Filter(false) {}
 
   template <class ReaderClass, class vector_type = vComplex >
   void initialize(ReaderClass &Read){
@@ -129,6 +135,7 @@ public:
     RNGModuleParameters RNGpar(Read);
     SetRNGSeeds(RNGpar);
 
+  
     // Observables
     auto &ObsFactory = HMC_ObservablesModuleFactory<observable_string, typename ImplementationPolicy::Field, ReaderClass>::getInstance(); 
     Read.push(observable_string);// here must check if existing...
@@ -208,6 +215,16 @@ public:
     AddGrid(s, Mod);
   }
 
+  void SetMomentumFilter( MomentumFilterBase<typename ImplementationPolicy::Field> * MomFilter) {
+    assert(have_Filter==false);
+    Filter = std::unique_ptr<MomentumFilterBase<typename ImplementationPolicy::Field> >(MomFilter);
+    have_Filter = true;
+  }
+  MomentumFilterBase<typename ImplementationPolicy::Field> *GetMomentumFilter(void) {
+    if ( !have_Filter)
+      SetMomentumFilter(new MomentumFilterNone<typename ImplementationPolicy::Field>());
+    return Filter.get();
+  }
 
   GridCartesian* GetCartesian(std::string s = "") {
     if (s.empty()) s = Grids.begin()->first;

Grid/qcd/hmc/UsingHMC.md

@@ -1,63 +1,61 @@
-# Using HMC in Grid
+Using HMC in Grid version 0.5.1
 
-These are the instructions to use the Generalised HMC on Grid as of commit `749b802`.
-Disclaimer: Grid is still under active development so any information here can be changed in future releases.
+These are the instructions to use the Generalised HMC on Grid version 0.5.1.
+Disclaimer: GRID is still under active development so any information here can be changed in future releases.
 
 
-## Command line options
-
-(relevant file `GenericHMCrunner.h`)
+Command line options
+===================
+(relevant file GenericHMCrunner.h)
 The initial configuration can be changed at the command line using 
-`--StartingType STARTING_TYPE`, where `STARTING_TYPE` is one of
-`HotStart`, `ColdStart`, `TepidStart`, and `CheckpointStart`.
-Default: `--StartingType HotStart`
+--StartType <your choice>
+valid choices, one among these
+HotStart, ColdStart, TepidStart, CheckpointStart
+default: HotStart
 
-Example:
-```
-./My_hmc_exec  --StartingType HotStart
-```
+example
+./My_hmc_exec  --StartType HotStart
 
-The `CheckpointStart` option uses the prefix for the configurations and rng seed files defined in your executable and the initial configuration is specified by
-`--StartingTrajectory STARTING_TRAJECTORY`, where `STARTING_TRAJECTORY` is an integer.
-Default: `--StartingTrajectory 0`
+The CheckpointStart option uses the prefix for the configurations and rng seed files defined in your executable and the initial configuration is specified by
+--StartTrajectory <integer>
+default: 0
 
 The number of trajectories for a specific run are specified at command line by
-`--Trajectories TRAJECTORIES`, where `TRAJECTORIES` is an integer.
-Default: `--Trajectories 1`
+--Trajectories <integer>
+default: 1
 
 The number of thermalization steps (i.e. steps when the Metropolis acceptance check is turned off) is specified by
-`--Thermalizations THERMALIZATIONS`, where `THERMALIZATIONS` is an integer.
-Default: `--Thermalizations 10`
+--Thermalizations <integer>
+default: 10
+
 
 Any other parameter is defined in the source for the executable.
 
-## HMC controls
+HMC controls
+===========
 
 The lines 
 
-```
   std::vector<int> SerSeed({1, 2, 3, 4, 5});
   std::vector<int> ParSeed({6, 7, 8, 9, 10});
-```
 
 define the seeds for the serial and the parallel RNG.
 
 The line 
 
-```
   TheHMC.MDparameters.set(20, 1.0);// MDsteps, traj length
-```
 
 declares the number of molecular dynamics steps and the total trajectory length.
 
 
-## Actions
+Actions
+======
 
-Action names are defined in the directory `Grid/qcd/action`.
+Action names are defined in the file
+lib/qcd/Actions.h
 
-Gauge actions list (from `Grid/qcd/action/gauge/Gauge.h`):
+Gauge actions list:
 
-```
 WilsonGaugeActionR;
 WilsonGaugeActionF;
 WilsonGaugeActionD;
@@ -70,9 +68,8 @@ IwasakiGaugeActionD;
 SymanzikGaugeActionR;
 SymanzikGaugeActionF;
 SymanzikGaugeActionD;
-```
 
-```
+
 ConjugateWilsonGaugeActionR;
 ConjugateWilsonGaugeActionF;
 ConjugateWilsonGaugeActionD;
@@ -85,23 +82,26 @@ ConjugateIwasakiGaugeActionD;
 ConjugateSymanzikGaugeActionR;
 ConjugateSymanzikGaugeActionF;
 ConjugateSymanzikGaugeActionD;
-```
 
-Each of these action accepts one single parameter at creation time (beta).
-Example for creating a Symanzik action with beta=4.0
 
-```
-  SymanzikGaugeActionR(4.0)
-```
-
-Scalar actions list (from `Grid/qcd/action/scalar/Scalar.h`):
-
-```
 ScalarActionR;
 ScalarActionF;
 ScalarActionD;
-```
 
-The suffixes `R`, `F`, `D` in the action names refer to the `Real`
-(the precision is defined at compile time by the `--enable-precision` flag in the configure),
-`Float` and `Double`, that force the precision of the action to be 32, 64 bit respectively.
+
+each of these action accept one single parameter at creation time (beta).
+Example for creating a Symanzik action with beta=4.0
+
+	SymanzikGaugeActionR(4.0)
+
+The suffixes R,F,D in the action names refer to the Real
+(the precision is defined at compile time by the --enable-precision flag in the configure),
+Float and Double, that force the precision of the action to be 32, 64 bit respectively.
+
+
+
+
+
+
+
+

Grid/qcd/hmc/integrators/Integrator.h

@@ -33,7 +33,6 @@ directory
 #define INTEGRATOR_INCLUDED
 
 #include <memory>
-#include "MomentumFilter.h"
 
 NAMESPACE_BEGIN(Grid);
 
@@ -67,6 +66,7 @@ public:
 template <class FieldImplementation, class SmearingPolicy, class RepresentationPolicy>
 class Integrator {
 protected:
+
   typedef typename FieldImplementation::Field MomentaField;  //for readability
   typedef typename FieldImplementation::Field Field;
 
@@ -119,6 +119,7 @@ protected:
     }
   } update_P_hireps{};
 
+ 
   void update_P(MomentaField& Mom, Field& U, int level, double ep) {
     // input U actually not used in the fundamental case
     // Fundamental updates, include smearing
@@ -130,31 +131,45 @@ protected:
 
       Field& Us = Smearer.get_U(as[level].actions.at(a)->is_smeared);
       double start_force = usecond();
+      as[level].actions.at(a)->deriv_timer_start();
       as[level].actions.at(a)->deriv(Us, force);  // deriv should NOT include Ta
+      as[level].actions.at(a)->deriv_timer_stop();
 
       std::cout << GridLogIntegrator << "Smearing (on/off): " << as[level].actions.at(a)->is_smeared << std::endl;
+      auto name = as[level].actions.at(a)->action_name();
       if (as[level].actions.at(a)->is_smeared) Smearer.smeared_force(force);
+      DumpSliceNorm("force before Ta",force,Nd-1);
       force = FieldImplementation::projectForce(force); // Ta for gauge fields
       double end_force = usecond();
 
-      Real force_abs = std::sqrt(norm2(force)/U.Grid()->gSites()); //average per-site norm.  nb. norm2(latt) = \sum_x norm2(latt[x]) 
+      DumpSliceNorm("force before filter",force,Nd-1);
+      MomFilter->applyFilter(force);
+      
+      Real force_abs   = std::sqrt(norm2(force)/U.Grid()->gSites()); //average per-site norm.  nb. norm2(latt) = \sum_x norm2(latt[x]) 
       Real impulse_abs = force_abs * ep * HMC_MOMENTUM_DENOMINATOR;    
 
-      Real max_force_abs = std::sqrt(maxLocalNorm2(force));
-      Real max_impulse_abs = max_force_abs * ep * HMC_MOMENTUM_DENOMINATOR;    
+      Real force_max   = std::sqrt(maxLocalNorm2(force));
+      Real impulse_max = force_max * ep * HMC_MOMENTUM_DENOMINATOR;    
+
+      as[level].actions.at(a)->deriv_log(force_abs,force_max);
+      
+      std::cout << GridLogIntegrator<< "["<<level<<"]["<<a<<"] Force average: " << force_abs <<" "<<name<<std::endl;
+      std::cout << GridLogIntegrator<< "["<<level<<"]["<<a<<"] Force max    : " << force_max <<" "<<name<<std::endl;
+      std::cout << GridLogIntegrator<< "["<<level<<"]["<<a<<"] Fdt average  : " << impulse_abs <<" "<<name<<std::endl;
+      std::cout << GridLogIntegrator<< "["<<level<<"]["<<a<<"] Fdt max      : " << impulse_max <<" "<<name<<std::endl;
 
-      std::cout << GridLogIntegrator << "["<<level<<"]["<<a<<"] Force average: " << force_abs << " Max force: " << max_force_abs << " Time step: " << ep << " Impulse average: " << impulse_abs << " Max impulse: " << max_impulse_abs << std::endl;
       Mom -= force * ep* HMC_MOMENTUM_DENOMINATOR;; 
       double end_full = usecond();
       double time_full  = (end_full - start_full) / 1e3;
       double time_force = (end_force - start_force) / 1e3;
       std::cout << GridLogMessage << "["<<level<<"]["<<a<<"] P update elapsed time: " << time_full << " ms (force: " << time_force << " ms)"  << std::endl;
+
+      DumpSliceNorm("force after filter",force,Nd-1);
     }
 
     // Force from the other representations
     as[level].apply(update_P_hireps, Representations, Mom, U, ep);
 
-    MomFilter->applyFilter(Mom);
   }
 
   void update_U(Field& U, double ep) 
@@ -168,8 +183,12 @@ protected:
   
   void update_U(MomentaField& Mom, Field& U, double ep) 
   {
+    MomentaField MomFiltered(Mom.Grid());
+    MomFiltered = Mom;
+    MomFilter->applyFilter(MomFiltered);
+
     // exponential of Mom*U in the gauge fields case
-    FieldImplementation::update_field(Mom, U, ep);
+    FieldImplementation::update_field(MomFiltered, U, ep);
 
     // Update the smeared fields, can be implemented as observer
     Smearer.set_Field(U);
@@ -212,6 +231,66 @@ public:
   const MomentaField & getMomentum() const{ return P; }
   
 
+  void reset_timer(void)
+  {
+    for (int level = 0; level < as.size(); ++level) {
+      for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
+        as[level].actions.at(actionID)->reset_timer();
+      }
+    }
+  }
+  void print_timer(void)
+  {
+    std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::" << std::endl;
+    std::cout << GridLogMessage << " Refresh cumulative timings "<<std::endl;
+    std::cout << GridLogMessage << "--------------------------- "<<std::endl;
+    for (int level = 0; level < as.size(); ++level) {
+      for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
+	std::cout << GridLogMessage 
+		  << as[level].actions.at(actionID)->action_name()
+		  <<"["<<level<<"]["<< actionID<<"] "
+		  << as[level].actions.at(actionID)->refresh_us*1.0e-6<<" s"<< std::endl;
+      }
+    }
+    std::cout << GridLogMessage << "--------------------------- "<<std::endl;
+    std::cout << GridLogMessage << " Action cumulative timings "<<std::endl;
+    std::cout << GridLogMessage << "--------------------------- "<<std::endl;
+    for (int level = 0; level < as.size(); ++level) {
+      for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
+	std::cout << GridLogMessage 
+		  << as[level].actions.at(actionID)->action_name()
+		  <<"["<<level<<"]["<< actionID<<"] "
+		  << as[level].actions.at(actionID)->S_us*1.0e-6<<" s"<< std::endl;
+      }
+    }
+    std::cout << GridLogMessage << "--------------------------- "<<std::endl;
+    std::cout << GridLogMessage << " Force cumulative timings "<<std::endl;
+    std::cout << GridLogMessage << "------------------------- "<<std::endl;
+    for (int level = 0; level < as.size(); ++level) {
+      for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
+	std::cout << GridLogMessage 
+		  << as[level].actions.at(actionID)->action_name()
+		  <<"["<<level<<"]["<< actionID<<"] "
+		  << as[level].actions.at(actionID)->deriv_us*1.0e-6<<" s"<< std::endl;
+      }
+    }
+    std::cout << GridLogMessage << "--------------------------- "<<std::endl;
+    std::cout << GridLogMessage << " Force average size "<<std::endl;
+    std::cout << GridLogMessage << "------------------------- "<<std::endl;
+    for (int level = 0; level < as.size(); ++level) {
+      for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
+	std::cout << GridLogMessage 
+		  << as[level].actions.at(actionID)->action_name()
+		  <<"["<<level<<"]["<< actionID<<"] : "
+		  <<" force max " << as[level].actions.at(actionID)->deriv_max_average()
+		  <<" norm "      << as[level].actions.at(actionID)->deriv_norm_average()
+		  <<" calls "     << as[level].actions.at(actionID)->deriv_num
+		  << std::endl;
+      }
+    }
+    std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::"<< std::endl;
+  }
+  
   void print_parameters()
   {
     std::cout << GridLogMessage << "[Integrator] Name : "<< integrator_name() << std::endl;
@@ -230,7 +309,6 @@ public:
       }
     }
     std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::"<< std::endl;
-
   }
 
   void reverse_momenta()
@@ -255,19 +333,15 @@ public:
   void refresh(Field& U,  GridSerialRNG & sRNG, GridParallelRNG& pRNG) 
   {
     assert(P.Grid() == U.Grid());
-    std::cout << GridLogIntegrator << "Integrator refresh" << std::endl;
+    std::cout << GridLogIntegrator << "Integrator refresh\n";
 
-    std::cout << GridLogIntegrator << "Generating momentum" << std::endl;
     FieldImplementation::generate_momenta(P, sRNG, pRNG);
 
     // Update the smeared fields, can be implemented as observer
     // necessary to keep the fields updated even after a reject
     // of the Metropolis
-    std::cout << GridLogIntegrator << "Updating smeared fields" << std::endl;
     Smearer.set_Field(U);
     // Set the (eventual) representations gauge fields
-
-    std::cout << GridLogIntegrator << "Updating representations" << std::endl;
     Representations.update(U);
 
     // The Smearer is attached to a pointer of the gauge field
@@ -277,16 +351,19 @@ public:
       for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
         // get gauge field from the SmearingPolicy and
         // based on the boolean is_smeared in actionID
-	std::cout << GridLogIntegrator << "Refreshing integrator level " << level << " index " << actionID << std::endl;
+	auto name = as[level].actions.at(actionID)->action_name();
+        std::cout << GridLogMessage << "refresh [" << level << "][" << actionID << "] "<<name << std::endl;
+
         Field& Us = Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
+	as[level].actions.at(actionID)->refresh_timer_start();
         as[level].actions.at(actionID)->refresh(Us, sRNG, pRNG);
+	as[level].actions.at(actionID)->refresh_timer_stop();
       }
 
       // Refresh the higher representation actions
       as[level].apply(refresh_hireps, Representations, sRNG, pRNG);
     }
 
-    MomFilter->applyFilter(P);
   }
 
   // to be used by the actionlevel class to iterate
@@ -321,7 +398,9 @@ public:
         // based on the boolean is_smeared in actionID
         Field& Us = Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
         std::cout << GridLogMessage << "S [" << level << "][" << actionID << "] action eval " << std::endl;
+	        as[level].actions.at(actionID)->S_timer_start();
         Hterm = as[level].actions.at(actionID)->S(Us);
+   	        as[level].actions.at(actionID)->S_timer_stop();
         std::cout << GridLogMessage << "S [" << level << "][" << actionID << "] H = " << Hterm << std::endl;
         H += Hterm;
       }

Grid/qcd/observables/topological_charge.h

@@ -99,7 +99,7 @@ public:
 	// using wilson flow by default here
 	WilsonFlow<PeriodicGimplR> WF(Pars.Smearing.steps, Pars.Smearing.step_size, Pars.Smearing.meas_interval);
 	WF.smear_adaptive(Usmear, U, Pars.Smearing.maxTau);
-	Real T0   = WF.energyDensityPlaquette(Pars.Smearing.maxTau, Usmear);
+	Real T0   = WF.energyDensityPlaquette(Usmear);
 	std::cout << GridLogMessage << std::setprecision(std::numeric_limits<Real>::digits10 + 1)
 		  << "T0                : [ " << traj << " ] "<< T0 << std::endl;
       }

Grid/qcd/smearing/WilsonFlow.h

@@ -7,7 +7,6 @@ Source file: ./lib/qcd/modules/plaquette.h
 Copyright (C) 2017
 
 Author: Guido Cossu <guido.cossu@ed.ac.uk>
-Author: Christopher Kelly <ckelly@bnl.gov>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -34,44 +33,28 @@ NAMESPACE_BEGIN(Grid);
 
 template <class Gimpl>
 class WilsonFlow: public Smear<Gimpl>{
-public:
-  //Store generic measurements to take during smearing process using std::function
-  typedef std::function<void(int, RealD, const typename Gimpl::GaugeField &)> FunctionType;  //int: step,  RealD: flow time,  GaugeField : the gauge field
-  
-private:
   unsigned int Nstep;
-  RealD epsilon; //for regular smearing this is the time step, for adaptive it is the initial time step
- 
-  std::vector< std::pair<int, FunctionType> > functions; //The int maps to the measurement frequency
+  unsigned int measure_interval;
+  mutable RealD epsilon, taus;
+
 
   mutable WilsonGaugeAction<Gimpl> SG;
 
-  //Evolve the gauge field by 1 step and update tau
-  void evolve_step(typename Gimpl::GaugeField &U, RealD &tau) const;
-  //Evolve the gauge field by 1 step and update tau and the current time step eps
-  void evolve_step_adaptive(typename Gimpl::GaugeField&U, RealD &tau, RealD &eps, RealD maxTau) const;
+  void evolve_step(typename Gimpl::GaugeField&) const;
+  void evolve_step_adaptive(typename Gimpl::GaugeField&, RealD);
+  RealD tau(unsigned int t)const {return epsilon*(t+1.0); }
 
 public:
   INHERIT_GIMPL_TYPES(Gimpl)
 
-  void resetActions(){ functions.clear(); }
-
-  void addMeasurement(int meas_interval, FunctionType meas){ functions.push_back({meas_interval, meas}); }
-
-  //Set the class to perform the default measurements: 
-  //the plaquette energy density every step
-  //the plaquette topological charge every 'topq_meas_interval' steps
-  //and output to stdout
-  void setDefaultMeasurements(int topq_meas_interval = 1);
-
   explicit WilsonFlow(unsigned int Nstep, RealD epsilon, unsigned int interval = 1):
   Nstep(Nstep),
     epsilon(epsilon),
+    measure_interval(interval),
     SG(WilsonGaugeAction<Gimpl>(3.0)) {
     // WilsonGaugeAction with beta 3.0
     assert(epsilon > 0.0);
     LogMessage();
-    setDefaultMeasurements(interval);
   }
 
   void LogMessage() {
@@ -90,29 +73,9 @@ public:
     // undefined for WilsonFlow
   }
 
-  void smear_adaptive(GaugeField&, const GaugeField&, RealD maxTau) const;
-
-  //Compute t^2 <E(t)> for time t from the plaquette
-  static RealD energyDensityPlaquette(const RealD t, const GaugeField& U);
-
-  //Compute t^2 <E(t)> for time t from the 1x1 cloverleaf form
-  //t is the Wilson flow time
-  static RealD energyDensityCloverleaf(const RealD t, const GaugeField& U);
-  
-  //Evolve the gauge field by Nstep steps of epsilon and return the energy density computed every interval steps
-  //The smeared field is output as V
-  std::vector<RealD> flowMeasureEnergyDensityPlaquette(GaugeField &V, const GaugeField& U, int measure_interval = 1);
-
-  //Version that does not return the smeared field
-  std::vector<RealD> flowMeasureEnergyDensityPlaquette(const GaugeField& U, int measure_interval = 1);
-
-
-  //Evolve the gauge field by Nstep steps of epsilon and return the Cloverleaf energy density computed every interval steps
-  //The smeared field is output as V
-  std::vector<RealD> flowMeasureEnergyDensityCloverleaf(GaugeField &V, const GaugeField& U, int measure_interval = 1);
-
-  //Version that does not return the smeared field
-  std::vector<RealD> flowMeasureEnergyDensityCloverleaf(const GaugeField& U, int measure_interval = 1);
+  void smear_adaptive(GaugeField&, const GaugeField&, RealD maxTau);
+  RealD energyDensityPlaquette(unsigned int step, const GaugeField& U) const;
+  RealD energyDensityPlaquette(const GaugeField& U) const;
 };
 
 
@@ -120,7 +83,7 @@ public:
 // Implementations
 ////////////////////////////////////////////////////////////////////////////////
 template <class Gimpl>
-void WilsonFlow<Gimpl>::evolve_step(typename Gimpl::GaugeField &U, RealD &tau) const{
+void WilsonFlow<Gimpl>::evolve_step(typename Gimpl::GaugeField &U) const{
   GaugeField Z(U.Grid());
   GaugeField tmp(U.Grid());
   SG.deriv(U, Z);
@@ -136,13 +99,12 @@ void WilsonFlow<Gimpl>::evolve_step(typename Gimpl::GaugeField &U, RealD &tau) c
   SG.deriv(U, tmp); Z += tmp;                 // 4/3*(17/36*Z0 -8/9*Z1) +Z2
   Z *= 3.0/4.0;                               // Z = 17/36*Z0 -8/9*Z1 +3/4*Z2
   Gimpl::update_field(Z, U, -2.0*epsilon);    // V(t+e) = exp(ep*Z)*W2
-  tau += epsilon;
 }
 
 template <class Gimpl>
-void WilsonFlow<Gimpl>::evolve_step_adaptive(typename Gimpl::GaugeField &U, RealD &tau, RealD &eps, RealD maxTau) const{
-  if (maxTau - tau < eps){
-    eps = maxTau-tau;
+void WilsonFlow<Gimpl>::evolve_step_adaptive(typename Gimpl::GaugeField &U, RealD maxTau) {
+  if (maxTau - taus < epsilon){
+    epsilon = maxTau-taus;
   }
   //std::cout << GridLogMessage << "Integration epsilon : " << epsilon << std::endl;
   GaugeField Z(U.Grid());
@@ -152,151 +114,95 @@ void WilsonFlow<Gimpl>::evolve_step_adaptive(typename Gimpl::GaugeField &U, Real
   SG.deriv(U, Z);
   Zprime = -Z;
   Z *= 0.25;                                  // Z0 = 1/4 * F(U)
-  Gimpl::update_field(Z, U, -2.0*eps);    // U = W1 = exp(ep*Z0)*W0
+  Gimpl::update_field(Z, U, -2.0*epsilon);    // U = W1 = exp(ep*Z0)*W0
 
   Z *= -17.0/8.0;
   SG.deriv(U, tmp); Z += tmp;                 // -17/32*Z0 +Z1
   Zprime += 2.0*tmp;
   Z *= 8.0/9.0;                               // Z = -17/36*Z0 +8/9*Z1
-  Gimpl::update_field(Z, U, -2.0*eps);    // U_= W2 = exp(ep*Z)*W1
+  Gimpl::update_field(Z, U, -2.0*epsilon);    // U_= W2 = exp(ep*Z)*W1
     
 
   Z *= -4.0/3.0;
   SG.deriv(U, tmp); Z += tmp;                 // 4/3*(17/36*Z0 -8/9*Z1) +Z2
   Z *= 3.0/4.0;                               // Z = 17/36*Z0 -8/9*Z1 +3/4*Z2
-  Gimpl::update_field(Z, U, -2.0*eps);    // V(t+e) = exp(ep*Z)*W2
+  Gimpl::update_field(Z, U, -2.0*epsilon);    // V(t+e) = exp(ep*Z)*W2
 
   // Ramos 
-  Gimpl::update_field(Zprime, Uprime, -2.0*eps); // V'(t+e) = exp(ep*Z')*W0
+  Gimpl::update_field(Zprime, Uprime, -2.0*epsilon); // V'(t+e) = exp(ep*Z')*W0
   // Compute distance as norm^2 of the difference
   GaugeField diffU = U - Uprime;
   RealD diff = norm2(diffU);
   // adjust integration step
     
-  tau += eps;
+  taus += epsilon;
   //std::cout << GridLogMessage << "Adjusting integration step with distance: " << diff << std::endl;
     
-  eps = eps*0.95*std::pow(1e-4/diff,1./3.);
+  epsilon = epsilon*0.95*std::pow(1e-4/diff,1./3.);
   //std::cout << GridLogMessage << "New epsilon : " << epsilon << std::endl;
 
 }
 
-
 template <class Gimpl>
-RealD WilsonFlow<Gimpl>::energyDensityPlaquette(const RealD t, const GaugeField& U){
-  static WilsonGaugeAction<Gimpl> SG(3.0);
-  return 2.0 * t * t * SG.S(U)/U.Grid()->gSites();
-}
-
-//Compute t^2 <E(t)> for time from the 1x1 cloverleaf form
-template <class Gimpl>
-RealD WilsonFlow<Gimpl>::energyDensityCloverleaf(const RealD t, const GaugeField& U){
-  typedef typename Gimpl::GaugeLinkField GaugeMat;
-  typedef typename Gimpl::GaugeField GaugeLorentz;
-
-  assert(Nd == 4);
-  //E = 1/2 tr( F_munu F_munu )
-  //However as  F_numu = -F_munu, only need to sum the trace of the squares of the following 6 field strengths:
-  //F_01 F_02 F_03   F_12 F_13  F_23
-  GaugeMat F(U.Grid());
-  LatticeComplexD R(U.Grid());
-  R = Zero();
-  
-  for(int mu=0;mu<3;mu++){
-    for(int nu=mu+1;nu<4;nu++){
-      WilsonLoops<Gimpl>::FieldStrength(F, U, mu, nu);
-      R = R + trace(F*F);
-    }
-  }
-  ComplexD out = sum(R);
-  out = t*t*out / RealD(U.Grid()->gSites());
-  return -real(out); //minus sign necessary for +ve energy
-}
-
-
-template <class Gimpl>
-std::vector<RealD> WilsonFlow<Gimpl>::flowMeasureEnergyDensityPlaquette(GaugeField &V, const GaugeField& U, int measure_interval){
-  std::vector<RealD> out;
-  resetActions();
-  addMeasurement(measure_interval, [&out](int step, RealD t, const typename Gimpl::GaugeField &U){ 
-      std::cout << GridLogMessage << "[WilsonFlow] Computing plaquette energy density for step " << step << std::endl;
-      out.push_back( energyDensityPlaquette(t,U) );
-    });      
-  smear(V,U);
-  return out;
+RealD WilsonFlow<Gimpl>::energyDensityPlaquette(unsigned int step, const GaugeField& U) const {
+  RealD td = tau(step);
+  return 2.0 * td * td * SG.S(U)/U.Grid()->gSites();
 }
 
 template <class Gimpl>
-std::vector<RealD> WilsonFlow<Gimpl>::flowMeasureEnergyDensityPlaquette(const GaugeField& U, int measure_interval){
-  GaugeField V(U);
-  return flowMeasureEnergyDensityPlaquette(V,U, measure_interval);
+RealD WilsonFlow<Gimpl>::energyDensityPlaquette(const GaugeField& U) const {
+  return 2.0 * taus * taus * SG.S(U)/U.Grid()->gSites();
 }
 
-template <class Gimpl>
-std::vector<RealD> WilsonFlow<Gimpl>::flowMeasureEnergyDensityCloverleaf(GaugeField &V, const GaugeField& U, int measure_interval){
-  std::vector<RealD> out;
-  resetActions();
-  addMeasurement(measure_interval, [&out](int step, RealD t, const typename Gimpl::GaugeField &U){ 
-      std::cout << GridLogMessage << "[WilsonFlow] Computing Cloverleaf energy density for step " << step << std::endl;
-      out.push_back( energyDensityCloverleaf(t,U) );
-    });      
-  smear(V,U);
-  return out;
-}
-
-template <class Gimpl>
-std::vector<RealD> WilsonFlow<Gimpl>::flowMeasureEnergyDensityCloverleaf(const GaugeField& U, int measure_interval){
-  GaugeField V(U);
-  return flowMeasureEnergyDensityCloverleaf(V,U, measure_interval);
-}
-
-
 
 //#define WF_TIMING 
+
+
+
 template <class Gimpl>
-void WilsonFlow<Gimpl>::smear(GaugeField& out, const GaugeField& in) const{
+void WilsonFlow<Gimpl>::smear(GaugeField& out, const GaugeField& in) const {
   out = in;
-  RealD taus = 0.;
-  for (unsigned int step = 1; step <= Nstep; step++) { //step indicates the number of smearing steps applied at the time of measurement
+  for (unsigned int step = 1; step <= Nstep; step++) {
     auto start = std::chrono::high_resolution_clock::now();
-    evolve_step(out, taus);
+    evolve_step(out);
     auto end = std::chrono::high_resolution_clock::now();
     std::chrono::duration<double> diff = end - start;
 #ifdef WF_TIMING
     std::cout << "Time to evolve " << diff.count() << " s\n";
 #endif
-    //Perform measurements
-    for(auto const &meas : functions)
-      if( step % meas.first == 0 ) meas.second(step,taus,out);
+    std::cout << GridLogMessage << "[WilsonFlow] Energy density (plaq) : "
+		  << step << "  " << tau(step) << "  " 
+	      << energyDensityPlaquette(step,out) << std::endl;
+    if( step % measure_interval == 0){
+      std::cout << GridLogMessage << "[WilsonFlow] Top. charge           : "
+		<< step << "  " 
+		<< WilsonLoops<PeriodicGimplR>::TopologicalCharge(out) << std::endl;
+    }
   }
 }
 
 template <class Gimpl>
-void WilsonFlow<Gimpl>::smear_adaptive(GaugeField& out, const GaugeField& in, RealD maxTau) const{
+void WilsonFlow<Gimpl>::smear_adaptive(GaugeField& out, const GaugeField& in, RealD maxTau){
   out = in;
-  RealD taus = 0.;
-  RealD eps = epsilon;
+  taus = epsilon;
   unsigned int step = 0;
   do{
     step++;
     //std::cout << GridLogMessage << "Evolution time :"<< taus << std::endl;
-    evolve_step_adaptive(out, taus, eps, maxTau);
-    //Perform measurements
-    for(auto const &meas : functions)
-      if( step % meas.first == 0 ) meas.second(step,taus,out);
+    evolve_step_adaptive(out, maxTau);
+    std::cout << GridLogMessage << "[WilsonFlow] Energy density (plaq) : "
+		  << step << "  " << taus << "  "
+	      << energyDensityPlaquette(out) << std::endl;
+    if( step % measure_interval == 0){
+      std::cout << GridLogMessage << "[WilsonFlow] Top. charge           : "
+		<< step << "  " 
+		<< WilsonLoops<PeriodicGimplR>::TopologicalCharge(out) << std::endl;
+    }
   } while (taus < maxTau);
-}
 
-template <class Gimpl>
-void WilsonFlow<Gimpl>::setDefaultMeasurements(int topq_meas_interval){
-  addMeasurement(1, [](int step, RealD t, const typename Gimpl::GaugeField &U){
-      std::cout << GridLogMessage << "[WilsonFlow] Energy density (plaq) : "  << step << "  " << t << "  " << energyDensityPlaquette(t,U) << std::endl;
-    });
-  addMeasurement(topq_meas_interval, [](int step, RealD t, const typename Gimpl::GaugeField &U){
-      std::cout << GridLogMessage << "[WilsonFlow] Top. charge           : "  << step << "  " << WilsonLoops<Gimpl>::TopologicalCharge(U) << std::endl;
-    });
-}
 
 
+}
+
 NAMESPACE_END(Grid);
 

Grid/qcd/utils/CovariantCshift.h

@@ -88,12 +88,6 @@ namespace PeriodicBC {
     return CovShiftBackward(Link,mu,arg);
   }
 
-  //Boundary-aware C-shift of gauge links / gauge transformation matrices
-  template<class gauge> Lattice<gauge>
-  CshiftLink(const Lattice<gauge> &Link, int mu, int shift)
-  {
-    return Cshift(Link, mu, shift);
-  }
 
 }
 
@@ -164,9 +158,6 @@ namespace ConjugateBC {
     //    std::cout<<"Gparity::CovCshiftBackward mu="<<mu<<std::endl;
     return Cshift(tmp,mu,-1);// moves towards positive mu
   }
-
-  //Out(x) = U^dag_\mu(x-mu)  | x_\mu != 0
-  //       = U^T_\mu(L-1)  | x_\mu == 0
   template<class gauge> Lattice<gauge>
   CovShiftIdentityBackward(const Lattice<gauge> &Link, int mu) {
     GridBase *grid = Link.Grid();
@@ -185,16 +176,13 @@ namespace ConjugateBC {
     return Link;
   }
 
-  //Out(x) = S_\mu(x+\hat\mu)  | x_\mu != L-1
-  //       = S*_\mu(0)  | x_\mu == L-1
-  //Note: While this is used for Staples it is also applicable for shifting gauge links or gauge transformation matrices
   template<class gauge> Lattice<gauge>
   ShiftStaple(const Lattice<gauge> &Link, int mu)
   {
     GridBase *grid = Link.Grid();
     int Lmu = grid->GlobalDimensions()[mu] - 1;
 
-    Lattice<iScalar<vInteger>> coor(grid);
+    Lattice<iScalar<vInteger> > coor(grid);
     LatticeCoordinate(coor, mu);
 
     Lattice<gauge> tmp(grid);
@@ -220,35 +208,6 @@ namespace ConjugateBC {
     return CovShiftBackward(Link,mu,arg);
   }
 
-  //Boundary-aware C-shift of gauge links / gauge transformation matrices
-  //shift = 1
-  //Out(x) = U_\mu(x+\hat\mu)  | x_\mu != L-1
-  //       = U*_\mu(0)  | x_\mu == L-1
-  //shift = -1
-  //Out(x) = U_\mu(x-mu)  | x_\mu != 0
-  //       = U*_\mu(L-1)  | x_\mu == 0
-  template<class gauge> Lattice<gauge>
-  CshiftLink(const Lattice<gauge> &Link, int mu, int shift)
-  {
-    GridBase *grid = Link.Grid();
-    int Lmu = grid->GlobalDimensions()[mu] - 1;
-
-    Lattice<iScalar<vInteger>> coor(grid);
-    LatticeCoordinate(coor, mu);
-
-    Lattice<gauge> tmp(grid);
-    if(shift == 1){
-      tmp = Cshift(Link, mu, 1);
-      tmp = where(coor == Lmu, conjugate(tmp), tmp);
-      return tmp;
-    }else if(shift == -1){
-      tmp = Link;
-      tmp = where(coor == Lmu, conjugate(tmp), tmp);
-      return Cshift(tmp, mu, -1);
-    }else assert(0 && "Invalid shift value");
-    return tmp; //shuts up the compiler fussing about the return type
-  }
-
 }
 
 

Grid/qcd/utils/GaugeFix.h

@@ -40,46 +40,27 @@ public:
   typedef typename Gimpl::GaugeLinkField GaugeMat;
   typedef typename Gimpl::GaugeField GaugeLorentz;
 
-  //A_\mu(x) = -i Ta(U_\mu(x) )   where Ta(U) = 1/2( U - U^dag ) - 1/2N tr(U - U^dag)  is the traceless antihermitian part. This is an O(A^3) approximation to the logarithm of U
-  static void GaugeLinkToLieAlgebraField(const GaugeMat &U, GaugeMat &A) {
-    Complex cmi(0.0,-1.0);
-    A = Ta(U) * cmi;
+  static void GaugeLinkToLieAlgebraField(const std::vector<GaugeMat> &U,std::vector<GaugeMat> &A) {
+    for(int mu=0;mu<Nd;mu++){
+      Complex cmi(0.0,-1.0);
+      A[mu] = Ta(U[mu]) * cmi;
+    }
   }
-  
-  //The derivative of the Lie algebra field
-  static void DmuAmu(const std::vector<GaugeMat> &U, GaugeMat &dmuAmu,int orthog) {
-    GridBase* grid = U[0].Grid();
-    GaugeMat Ax(grid);
-    GaugeMat Axm1(grid);
-    GaugeMat Utmp(grid);
-
+  static void DmuAmu(const std::vector<GaugeMat> &A,GaugeMat &dmuAmu,int orthog) {
     dmuAmu=Zero();
     for(int mu=0;mu<Nd;mu++){
       if ( mu != orthog ) {
-	//Rather than define functionality to work out how the BCs apply to A_\mu we simply use the BC-aware Cshift to the gauge links and compute A_\mu(x) and A_\mu(x-1) separately
-	//Ax = A_\mu(x)
-	GaugeLinkToLieAlgebraField(U[mu], Ax);
-	
-	//Axm1 = A_\mu(x_\mu-1)
-	Utmp = Gimpl::CshiftLink(U[mu], mu, -1);
-	GaugeLinkToLieAlgebraField(Utmp, Axm1);
-	
-	//Derivative
-	dmuAmu = dmuAmu + Ax - Axm1;
+	dmuAmu = dmuAmu + A[mu] - Cshift(A[mu],mu,-1);
       }
     }
   }  
 
-  //Fix the gauge field Umu
-  //0 < alpha < 1 is related to the step size, cf https://arxiv.org/pdf/1405.5812.pdf
-  static void SteepestDescentGaugeFix(GaugeLorentz &Umu, Real alpha,int maxiter,Real Omega_tol, Real Phi_tol,bool Fourier=false,int orthog=-1) {
+  static void SteepestDescentGaugeFix(GaugeLorentz &Umu,Real & alpha,int maxiter,Real Omega_tol, Real Phi_tol,bool Fourier=false,int orthog=-1) {
     GridBase *grid = Umu.Grid();
     GaugeMat xform(grid);
     SteepestDescentGaugeFix(Umu,xform,alpha,maxiter,Omega_tol,Phi_tol,Fourier,orthog);
   }
-
-  //Fix the gauge field Umu and also return the gauge transformation from the original gauge field, xform
-  static void SteepestDescentGaugeFix(GaugeLorentz &Umu,GaugeMat &xform, Real alpha,int maxiter,Real Omega_tol, Real Phi_tol,bool Fourier=false,int orthog=-1) {
+  static void SteepestDescentGaugeFix(GaugeLorentz &Umu,GaugeMat &xform,Real & alpha,int maxiter,Real Omega_tol, Real Phi_tol,bool Fourier=false,int orthog=-1) {
 
     GridBase *grid = Umu.Grid();
 
@@ -141,24 +122,27 @@ public:
 
       }
     }
-    assert(0 && "Gauge fixing did not converge within the specified number of iterations");
   };
-  static Real SteepestDescentStep(std::vector<GaugeMat> &U,GaugeMat &xform, Real alpha, GaugeMat & dmuAmu,int orthog) {
+  static Real SteepestDescentStep(std::vector<GaugeMat> &U,GaugeMat &xform,Real & alpha, GaugeMat & dmuAmu,int orthog) {
     GridBase *grid = U[0].Grid();
 
+    std::vector<GaugeMat> A(Nd,grid);
     GaugeMat g(grid);
-    ExpiAlphaDmuAmu(U,g,alpha,dmuAmu,orthog);
+
+    GaugeLinkToLieAlgebraField(U,A);
+    ExpiAlphaDmuAmu(A,g,alpha,dmuAmu,orthog);
+
 
     Real vol = grid->gSites();
     Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
 
     xform = g*xform ;
-    SU<Nc>::GaugeTransform<Gimpl>(U,g);
+    SU<Nc>::GaugeTransform(U,g);
 
     return trG;
   }
 
-  static Real FourierAccelSteepestDescentStep(std::vector<GaugeMat> &U,GaugeMat &xform, Real alpha, GaugeMat & dmuAmu,int orthog) {
+  static Real FourierAccelSteepestDescentStep(std::vector<GaugeMat> &U,GaugeMat &xform,Real & alpha, GaugeMat & dmuAmu,int orthog) {
 
     GridBase *grid = U[0].Grid();
 
@@ -173,7 +157,11 @@ public:
 
     GaugeMat g(grid);
     GaugeMat dmuAmu_p(grid);
-    DmuAmu(U,dmuAmu,orthog);
+    std::vector<GaugeMat> A(Nd,grid);
+
+    GaugeLinkToLieAlgebraField(U,A);
+
+    DmuAmu(A,dmuAmu,orthog);
 
     std::vector<int> mask(Nd,1);
     for(int mu=0;mu<Nd;mu++) if (mu==orthog) mask[mu]=0;
@@ -217,16 +205,16 @@ public:
     Real trG = TensorRemove(sum(trace(g))).real()/vol/Nc;
 
     xform = g*xform ;
-    SU<Nc>::GaugeTransform<Gimpl>(U,g);
+    SU<Nc>::GaugeTransform(U,g);
 
     return trG;
   }
 
-  static void ExpiAlphaDmuAmu(const std::vector<GaugeMat> &U,GaugeMat &g, Real alpha, GaugeMat &dmuAmu,int orthog) {
+  static void ExpiAlphaDmuAmu(const std::vector<GaugeMat> &A,GaugeMat &g,Real & alpha, GaugeMat &dmuAmu,int orthog) {
     GridBase *grid = g.Grid();
     Complex cialpha(0.0,-alpha);
     GaugeMat ciadmam(grid);
-    DmuAmu(U,dmuAmu,orthog);
+    DmuAmu(A,dmuAmu,orthog);
     ciadmam = dmuAmu*cialpha;
     SU<Nc>::taExp(ciadmam,g);
   }  

Grid/qcd/utils/MixedPrecisionOperatorFunction.h

@@ -0,0 +1,111 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: 
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#pragma once
+
+NAMESPACE_BEGIN(Grid); 
+
+template<class FermionOperatorD, class FermionOperatorF, class SchurOperatorD, class  SchurOperatorF> 
+class MixedPrecisionConjugateGradientOperatorFunction : public OperatorFunction<typename FermionOperatorD::FermionField> {
+
+ public:
+    typedef typename FermionOperatorD::FermionField FieldD;
+    typedef typename FermionOperatorF::FermionField FieldF;
+
+    using OperatorFunction<FieldD>::operator();
+
+    RealD   Tolerance;
+    RealD   InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
+    Integer MaxInnerIterations;
+    Integer MaxOuterIterations;
+    GridBase* SinglePrecGrid;
+    RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
+
+    FermionOperatorF &FermOpF;
+    FermionOperatorD &FermOpD;;
+    SchurOperatorF &LinOpF;
+    SchurOperatorD &LinOpD;
+
+    Integer TotalInnerIterations; //Number of inner CG iterations
+    Integer TotalOuterIterations; //Number of restarts
+    Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
+
+     MixedPrecisionConjugateGradientOperatorFunction(RealD tol, RealD tolInner,
+						    Integer maxinnerit, 
+						    Integer maxouterit,
+						    GridBase *_SinglePrecGrid,
+                                                    FermionOperatorF &_FermOpF,
+                                                    FermionOperatorD &_FermOpD,
+						    SchurOperatorF   &_LinOpF,
+						    SchurOperatorD   &_LinOpD) : 
+      LinOpF(_LinOpF),
+      LinOpD(_LinOpD),
+      FermOpF(_FermOpF),
+      FermOpD(_FermOpD),
+      Tolerance(tol), 
+      InnerTolerance(tolInner), 
+      MaxInnerIterations(maxinnerit), 
+      MaxOuterIterations(maxouterit), 
+      SinglePrecGrid(_SinglePrecGrid),
+      OuterLoopNormMult(100.) 
+  { assert(tolInner<0.01);    };
+
+    void operator()(LinearOperatorBase<FieldD> &LinOpU, const FieldD &src, FieldD &psi)
+    {
+
+      SchurOperatorD * SchurOpU = static_cast<SchurOperatorD *>(&LinOpU);
+      
+      // Assumption made in code to extract gauge field
+      // We could avoid storing LinopD reference alltogether ?
+      assert(&(SchurOpU->_Mat)==&(LinOpD._Mat));
+
+      ////////////////////////////////////////////////////////////////////////////////////
+      // Moving this to a Clone method of fermion operator would allow to duplicate the 
+      // physics parameters and decrease gauge field copies
+      ////////////////////////////////////////////////////////////////////////////////////
+      auto &Umu_d = FermOpD.GetDoubledGaugeField();
+      auto &Umu_f = FermOpF.GetDoubledGaugeField();
+      auto &Umu_fe= FermOpF.GetDoubledGaugeFieldE();
+      auto &Umu_fo= FermOpF.GetDoubledGaugeFieldO();
+      precisionChange(Umu_f,Umu_d);
+      pickCheckerboard(Even,Umu_fe,Umu_f);
+      pickCheckerboard(Odd ,Umu_fo,Umu_f);
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // Make a mixed precision conjugate gradient
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // Could assume red black solver here and remove the SinglePrecGrid parameter???
+      MixedPrecisionConjugateGradient<FieldD,FieldF> MPCG(Tolerance, InnerTolerance,MaxInnerIterations,MaxOuterIterations,SinglePrecGrid,LinOpF,LinOpD);
+
+      std::cout << GridLogMessage << "Calling mixed precision Conjugate Gradient src "<<norm2(src) <<std::endl;
+      psi=Zero();
+      MPCG(src,psi);
+    }
+  };
+NAMESPACE_END(Grid); 

Grid/qcd/utils/SUn.h

@@ -694,32 +694,32 @@ public:
  * Adjoint rep gauge xform
  */
 
-  template<typename Gimpl>
-  static void GaugeTransform(typename Gimpl::GaugeField &Umu, typename Gimpl::GaugeLinkField &g){
+  template<typename GaugeField,typename GaugeMat>
+  static void GaugeTransform( GaugeField &Umu, GaugeMat &g){
     GridBase *grid = Umu.Grid();
     conformable(grid,g.Grid());
 
-    typename Gimpl::GaugeLinkField U(grid);
-    typename Gimpl::GaugeLinkField ag(grid); ag = adj(g);
+    GaugeMat U(grid);
+    GaugeMat ag(grid); ag = adj(g);
 
     for(int mu=0;mu<Nd;mu++){
       U= PeekIndex<LorentzIndex>(Umu,mu);
-      U = g*U*Gimpl::CshiftLink(ag, mu, 1); //BC-aware
+      U = g*U*Cshift(ag, mu, 1);
       PokeIndex<LorentzIndex>(Umu,U,mu);
     }
   }
-  template<typename Gimpl>
-  static void GaugeTransform( std::vector<typename Gimpl::GaugeLinkField> &U, typename Gimpl::GaugeLinkField &g){
+  template<typename GaugeMat>
+  static void GaugeTransform( std::vector<GaugeMat> &U, GaugeMat &g){
     GridBase *grid = g.Grid();
-    typename Gimpl::GaugeLinkField ag(grid); ag = adj(g);
+    GaugeMat ag(grid); ag = adj(g);
     for(int mu=0;mu<Nd;mu++){
-      U[mu] = g*U[mu]*Gimpl::CshiftLink(ag, mu, 1); //BC-aware
+      U[mu] = g*U[mu]*Cshift(ag, mu, 1);
     }
   }
-  template<typename Gimpl>
-  static void RandomGaugeTransform(GridParallelRNG &pRNG, typename Gimpl::GaugeField &Umu, typename Gimpl::GaugeLinkField &g){
+  template<typename GaugeField,typename GaugeMat>
+  static void RandomGaugeTransform(GridParallelRNG &pRNG, GaugeField &Umu, GaugeMat &g){
     LieRandomize(pRNG,g,1.0);
-    GaugeTransform<Gimpl>(Umu,g);
+    GaugeTransform(Umu,g);
   }
 
   // Projects the algebra components a lattice matrix (of dimension ncol*ncol -1 )

Grid/qcd/utils/WilsonLoops.h

@@ -125,56 +125,6 @@ public:
     return sumplaq / vol / faces / Nc; // Nd , Nc dependent... FIXME
   }
 
-  //////////////////////////////////////////////////
-  // sum over all spatial planes of plaquette
-  //////////////////////////////////////////////////
-  static void siteSpatialPlaquette(ComplexField &Plaq,
-                            const std::vector<GaugeMat> &U) {
-    ComplexField sitePlaq(U[0].Grid());
-    Plaq = Zero();
-    for (int mu = 1; mu < Nd-1; mu++) {
-      for (int nu = 0; nu < mu; nu++) {
-        traceDirPlaquette(sitePlaq, U, mu, nu);
-        Plaq = Plaq + sitePlaq;
-      }
-    }
-  }
-
-  ////////////////////////////////////
-  // sum over all x,y,z and over all spatial planes of plaquette
-  //////////////////////////////////////////////////
-  static std::vector<RealD> timesliceSumSpatialPlaquette(const GaugeLorentz &Umu) {
-    std::vector<GaugeMat> U(Nd, Umu.Grid());
-    // inefficient here
-    for (int mu = 0; mu < Nd; mu++) {
-      U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
-    }
-
-    ComplexField Plaq(Umu.Grid());
-
-    siteSpatialPlaquette(Plaq, U);
-    typedef typename ComplexField::scalar_object sobj;
-    std::vector<sobj> Tq;
-    sliceSum(Plaq, Tq, Nd-1);
-
-    std::vector<Real> out(Tq.size());
-    for(int t=0;t<Tq.size();t++) out[t] = TensorRemove(Tq[t]).real();
-    return out;
-  }
-  
-  //////////////////////////////////////////////////
-  // average over all x,y,z and over all spatial planes of plaquette
-  //////////////////////////////////////////////////
-  static std::vector<RealD> timesliceAvgSpatialPlaquette(const GaugeLorentz &Umu) {
-    std::vector<RealD> sumplaq = timesliceSumSpatialPlaquette(Umu);
-    int Lt = Umu.Grid()->FullDimensions()[Nd-1];
-    assert(sumplaq.size() == Lt);
-    double vol = Umu.Grid()->gSites() / Lt;
-    double faces = (1.0 * (Nd - 1)* (Nd - 2)) / 2.0;
-    for(int t=0;t<Lt;t++)
-      sumplaq[t] = sumplaq[t] / vol / faces / Nc; // Nd , Nc dependent... FIXME
-    return sumplaq;
-  }
 
   //////////////////////////////////////////////////
   // average over all x,y,z the temporal loop
@@ -413,11 +363,11 @@ public:
     GaugeMat u = PeekIndex<LorentzIndex>(Umu, mu);  // some redundant copies
     GaugeMat vu = v*u;
       //FS = 0.25*Ta(u*v + Cshift(vu, mu, -1));
-      FS = (u*v + Gimpl::CshiftLink(vu, mu, -1));
+      FS = (u*v + Cshift(vu, mu, -1));
       FS = 0.125*(FS - adj(FS));
   }
 
-  static Real TopologicalCharge(const GaugeLorentz &U){
+  static Real TopologicalCharge(GaugeLorentz &U){
     // 4d topological charge
     assert(Nd==4);
     // Bx = -iF(y,z), By = -iF(z,y), Bz = -iF(x,y)
@@ -440,203 +390,6 @@ public:
   }
 
 
-  //Clover-leaf Wilson loop combination for arbitrary mu-extent M and nu extent N,  mu >= nu
-  //cf  https://arxiv.org/pdf/hep-lat/9701012.pdf Eq 7  for 1x2 Wilson loop    
-  //Clockwise ordering
-  static void CloverleafMxN(GaugeMat &FS, const GaugeMat &Umu, const GaugeMat &Unu, int mu, int nu, int M, int N){  
-#define Fmu(A) Gimpl::CovShiftForward(Umu, mu, A)
-#define Bmu(A) Gimpl::CovShiftBackward(Umu, mu, A)
-#define Fnu(A) Gimpl::CovShiftForward(Unu, nu, A)
-#define Bnu(A) Gimpl::CovShiftBackward(Unu, nu, A)
-#define FmuI Gimpl::CovShiftIdentityForward(Umu, mu)
-#define BmuI Gimpl::CovShiftIdentityBackward(Umu, mu)
-#define FnuI Gimpl::CovShiftIdentityForward(Unu, nu)
-#define BnuI Gimpl::CovShiftIdentityBackward(Unu, nu)
-
-    //Upper right loop
-    GaugeMat tmp = BmuI;
-    for(int i=1;i<M;i++)
-      tmp = Bmu(tmp);
-    for(int j=0;j<N;j++)
-      tmp = Bnu(tmp);
-    for(int i=0;i<M;i++)
-      tmp = Fmu(tmp);
-    for(int j=0;j<N;j++)
-      tmp = Fnu(tmp);
-      
-    FS = tmp;
-
-    //Upper left loop
-    tmp = BnuI;
-    for(int j=1;j<N;j++)
-      tmp = Bnu(tmp);
-    for(int i=0;i<M;i++)
-      tmp = Fmu(tmp);
-    for(int j=0;j<N;j++)
-      tmp = Fnu(tmp);
-    for(int i=0;i<M;i++)
-      tmp = Bmu(tmp);
-      
-    FS = FS + tmp;
-
-    //Lower right loop
-    tmp = FnuI;
-    for(int j=1;j<N;j++)
-      tmp = Fnu(tmp);
-    for(int i=0;i<M;i++)
-      tmp = Bmu(tmp);
-    for(int j=0;j<N;j++)
-      tmp = Bnu(tmp);
-    for(int i=0;i<M;i++)
-      tmp = Fmu(tmp);
-      
-    FS = FS + tmp;
-
-    //Lower left loop
-    tmp = FmuI;
-    for(int i=1;i<M;i++)
-      tmp = Fmu(tmp);
-    for(int j=0;j<N;j++)
-      tmp = Fnu(tmp);
-    for(int i=0;i<M;i++)
-      tmp = Bmu(tmp);
-    for(int j=0;j<N;j++)
-      tmp = Bnu(tmp);
-
-    FS = FS + tmp;
-
-#undef Fmu
-#undef Bmu
-#undef Fnu
-#undef Bnu
-#undef FmuI
-#undef BmuI
-#undef FnuI
-#undef BnuI
-  }
-
-  //Field strength from MxN Wilson loop
-  //Note F_numu = - F_munu
-  static void FieldStrengthMxN(GaugeMat &FS, const GaugeLorentz &U, int mu, int nu, int M, int N){  
-    GaugeMat Umu = PeekIndex<LorentzIndex>(U, mu);
-    GaugeMat Unu = PeekIndex<LorentzIndex>(U, nu);
-    if(M == N){
-      GaugeMat F(Umu.Grid());
-      CloverleafMxN(F, Umu, Unu, mu, nu, M, N);
-      FS = 0.125 * ( F - adj(F) );
-    }else{
-      //Average over both orientations
-      GaugeMat horizontal(Umu.Grid()), vertical(Umu.Grid());
-      CloverleafMxN(horizontal, Umu, Unu, mu, nu, M, N);
-      CloverleafMxN(vertical, Umu, Unu, mu, nu, N, M);
-      FS = 0.0625 * ( horizontal - adj(horizontal) + vertical - adj(vertical) );
-    }
-  }
-
-  //Topological charge contribution from MxN Wilson loops
-  //cf  https://arxiv.org/pdf/hep-lat/9701012.pdf  Eq 6
-  //output is the charge by timeslice: sum over timeslices to obtain the total
-  static std::vector<Real> TimesliceTopologicalChargeMxN(const GaugeLorentz &U, int M, int N){
-    assert(Nd == 4);
-    std::vector<std::vector<GaugeMat*> > F(Nd,std::vector<GaugeMat*>(Nd,nullptr));
-    //Note F_numu = - F_munu
-    //hence we only need to loop over mu,nu,rho,sigma that aren't related by permuting mu,nu  or rho,sigma
-    //Use nu > mu
-    for(int mu=0;mu<Nd-1;mu++){
-      for(int nu=mu+1; nu<Nd; nu++){
-	F[mu][nu] = new GaugeMat(U.Grid());
-	FieldStrengthMxN(*F[mu][nu], U, mu, nu, M, N);
-      }
-    }
-    Real coeff = -1./(32 * M_PI*M_PI * M*M * N*N); //overall sign to match CPS and Grid conventions, possibly related to time direction = 3 vs 0
-
-    static const int combs[3][4] = { {0,1,2,3}, {0,2,1,3}, {0,3,1,2} };
-    static const int signs[3] = { 1, -1, 1 }; //epsilon_{mu nu rho sigma}
-
-    ComplexField fsum(U.Grid());
-    fsum = Zero();
-    for(int c=0;c<3;c++){
-      int mu = combs[c][0], nu = combs[c][1], rho = combs[c][2], sigma = combs[c][3];
-      int eps = signs[c];
-      fsum = fsum + (8. * coeff * eps) * trace( (*F[mu][nu]) * (*F[rho][sigma]) ); 
-    }
-
-    for(int mu=0;mu<Nd-1;mu++)
-      for(int nu=mu+1; nu<Nd; nu++)
-	delete F[mu][nu];
-    
-    typedef typename ComplexField::scalar_object sobj;
-    std::vector<sobj> Tq;
-    sliceSum(fsum, Tq, Nd-1);
-
-    std::vector<Real> out(Tq.size());
-    for(int t=0;t<Tq.size();t++) out[t] = TensorRemove(Tq[t]).real();
-    return out;
-  }
-  static Real TopologicalChargeMxN(const GaugeLorentz &U, int M, int N){
-    std::vector<Real> Tq = TimesliceTopologicalChargeMxN(U,M,N);
-    Real out(0);
-    for(int t=0;t<Tq.size();t++) out += Tq[t];
-    return out;
-  }
-
-  //Generate the contributions to the 5Li topological charge from Wilson loops of the following sizes
-  //Use coefficients from hep-lat/9701012
-  //1x1 : c1=(19.-55.*c5)/9.
-  //2x2 : c2=(1-64.*c5)/9.
-  //1x2 : c3=(-64.+640.*c5)/45.
-  //1x3 : c4=1./5.-2.*c5
-  //3x3 : c5=1./20.
-  //Output array outer index contains the loops in the above order
-  //Inner index is the time coordinate
-  static std::vector<std::vector<Real> > TimesliceTopologicalCharge5LiContributions(const GaugeLorentz &U){
-    static const int exts[5][2] = { {1,1}, {2,2}, {1,2}, {1,3}, {3,3} };       
-    std::vector<std::vector<Real> > out(5);
-    for(int i=0;i<5;i++){	
-      out[i] = TimesliceTopologicalChargeMxN(U,exts[i][0],exts[i][1]);
-    }
-    return out;
-  }   
-
-  static std::vector<Real> TopologicalCharge5LiContributions(const GaugeLorentz &U){   
-    static const int exts[5][2] = { {1,1}, {2,2}, {1,2}, {1,3}, {3,3} };
-    std::vector<Real> out(5);
-    std::cout << GridLogMessage << "Computing topological charge" << std::endl;
-    for(int i=0;i<5;i++){
-      out[i] = TopologicalChargeMxN(U,exts[i][0],exts[i][1]);
-      std::cout << GridLogMessage << exts[i][0] << "x" << exts[i][1] << " Wilson loop contribution " << out[i] << std::endl;
-    }
-    return out;
-  }
-
-  //Compute the 5Li topological charge
-  static std::vector<Real> TimesliceTopologicalCharge5Li(const GaugeLorentz &U){
-    std::vector<std::vector<Real> > loops = TimesliceTopologicalCharge5LiContributions(U);
-
-    double c5=1./20.;
-    double c4=1./5.-2.*c5;
-    double c3=(-64.+640.*c5)/45.;
-    double c2=(1-64.*c5)/9.;
-    double c1=(19.-55.*c5)/9.;
-
-    int Lt = loops[0].size();
-    std::vector<Real> out(Lt,0.);
-    for(int t=0;t<Lt;t++)
-      out[t] += c1*loops[0][t] + c2*loops[1][t] + c3*loops[2][t] + c4*loops[3][t] + c5*loops[4][t];
-    return out;
-  }
-
-  static Real TopologicalCharge5Li(const GaugeLorentz &U){
-    std::vector<Real> Qt = TimesliceTopologicalCharge5Li(U);
-    Real Q = 0.;
-    for(int t=0;t<Qt.size();t++) Q += Qt[t];
-    std::cout << GridLogMessage << "5Li Topological charge: " << Q << std::endl;
-    return Q;
-  }
-
-
-
-
   //////////////////////////////////////////////////////
   // Similar to above for rectangle is required
   //////////////////////////////////////////////////////

Grid/stencil/Stencil.h

@@ -263,7 +263,8 @@ public:
   int face_table_computed;
   std::vector<commVector<std::pair<int,int> > > face_table ;
   Vector<int> surface_list;
-
+  bool locally_periodic;
+  
   stencilVector<StencilEntry>  _entries; // Resident in managed memory
   commVector<StencilEntry>     _entries_device; // Resident in managed memory
   std::vector<Packet> Packets;
@@ -320,14 +321,15 @@ public:
     int ld              = _grid->_ldimensions[dimension];
     int rd              = _grid->_rdimensions[dimension];
     int simd_layout     = _grid->_simd_layout[dimension];
-    int comm_dim        = _grid->_processors[dimension] >1 ;
+    int comm_dim        = _grid->_processors[dimension] >1 && (!locally_periodic);
 
-    //    int recv_from_rank;
-    //    int xmit_to_rank;
+    int recv_from_rank;
+    int xmit_to_rank;
 
     if ( ! comm_dim ) return 1;
     if ( displacement == 0 ) return 1;
     return 0;
+
   }
 
   //////////////////////////////////////////
@@ -473,7 +475,7 @@ public:
 
     // the permute type
     int simd_layout     = _grid->_simd_layout[dimension];
-    int comm_dim        = _grid->_processors[dimension] >1 ;
+    int comm_dim        = _grid->_processors[dimension] >1 && (!locally_periodic);
     int splice_dim      = _grid->_simd_layout[dimension]>1 && (comm_dim);
 
     int is_same_node = 1;
@@ -657,6 +659,20 @@ public:
 		   const std::vector<int> &directions,
 		   const std::vector<int> &distances,
 		   Parameters p)
+    : CartesianStencil(grid,
+		       npoints,
+		       checkerboard,
+		       directions,
+		       distances,
+		       false,
+		       p){};
+  CartesianStencil(GridBase *grid,
+		   int npoints,
+		   int checkerboard,
+		   const std::vector<int> &directions,
+		   const std::vector<int> &distances,
+		   bool _locally_periodic,
+		   Parameters p)
     : shm_bytes_thr(npoints),
       comm_bytes_thr(npoints),
       comm_enter_thr(npoints),
@@ -665,6 +681,7 @@ public:
   {
     face_table_computed=0;
     _grid    = grid;
+    this->locally_periodic=_locally_periodic;
     this->parameters=p;
     /////////////////////////////////////
     // Initialise the base
@@ -690,6 +707,8 @@ public:
       int point = i;
 
       int dimension    = directions[i];
+      assert(dimension>=0 && dimension<_grid->Nd());
+
       int displacement = distances[i];
       int shift = displacement;
 
@@ -703,7 +722,7 @@ public:
       // the permute type
       //////////////////////////
       int simd_layout     = _grid->_simd_layout[dimension];
-      int comm_dim        = _grid->_processors[dimension] >1 ;
+      int comm_dim        = _grid->_processors[dimension] >1 && (!locally_periodic);
       int splice_dim      = _grid->_simd_layout[dimension]>1 && (comm_dim);
       int rotate_dim      = _grid->_simd_layout[dimension]>2;
 
@@ -817,7 +836,7 @@ public:
     int pd              = _grid->_processors[dimension];
     int simd_layout     = _grid->_simd_layout[dimension];
     int comm_dim        = _grid->_processors[dimension] >1 ;
-
+    assert(locally_periodic==false);
     assert(comm_dim==1);
     int shift = (shiftpm + fd) %fd;
     assert(shift>=0);
@@ -997,6 +1016,7 @@ public:
     int pd              = _grid->_processors[dimension];
     int simd_layout     = _grid->_simd_layout[dimension];
     int comm_dim        = _grid->_processors[dimension] >1 ;
+    assert(locally_periodic==false);
     assert(simd_layout==1);
     assert(comm_dim==1);
     assert(shift>=0);
@@ -1089,6 +1109,7 @@ public:
     int pd              = _grid->_processors[dimension];
     int simd_layout     = _grid->_simd_layout[dimension];
     int comm_dim        = _grid->_processors[dimension] >1 ;
+    assert(locally_periodic==false);
     assert(comm_dim==1);
     // This will not work with a rotate dim
     assert(simd_layout==maxl);

Grid/tensors/Tensor_exp.h

@@ -52,12 +52,17 @@ template<class vtype, int N> accelerator_inline iVector<vtype, N> Exponentiate(c
   return ret;
 }
 
-
-
 // Specialisation: Cayley-Hamilton exponential for SU(3)
-#ifndef GRID_CUDA
+#if 0
 template<class vtype, typename std::enable_if< GridTypeMapper<vtype>::TensorLevel == 0>::type * =nullptr> 
-accelerator_inline iMatrix<vtype,3> Exponentiate(const iMatrix<vtype,3> &arg, RealD alpha  , Integer Nexp = DEFAULT_MAT_EXP )
+accelerator_inline iMatrix<vtype,3> Exponentiated(const iMatrix<vtype,3> &arg, RealD alpha  , Integer Nexp = DEFAULT_MAT_EXP )
+{
+  return ExponentiateCayleyHamilton(arg,alpha);
+}
+#endif
+
+template<class vtype, typename std::enable_if< GridTypeMapper<vtype>::TensorLevel == 0>::type * =nullptr> 
+accelerator_inline iMatrix<vtype,3> ExponentiateCayleyHamilton(const iMatrix<vtype,3> &arg, RealD alpha )
 {
   // for SU(3) 2x faster than the std implementation using Nexp=12
   // notice that it actually computes
@@ -115,8 +120,6 @@ accelerator_inline iMatrix<vtype,3> Exponentiate(const iMatrix<vtype,3> &arg, Re
 
   return (f0 * unit + timesMinusI(f1) * arg*alpha - f2 * iQ2);
 }
-#endif
-
 
 // General exponential
 template<class vtype,int N, typename std::enable_if< GridTypeMapper<vtype>::TensorLevel == 0 >::type * =nullptr> 
@@ -129,8 +132,8 @@ accelerator_inline iMatrix<vtype,N> Exponentiate(const iMatrix<vtype,N> &arg, Re
   typedef iMatrix<vtype,N> mat;
   mat unit(1.0);
   mat temp(unit);
-  for(int i=Nexp; i>=1;--i){
-    temp *= alpha/RealD(i);
+  for(int n=Nexp; n>=1;--n){
+    temp *= alpha/RealD(n);
     temp = unit + temp*arg;
   }
   return temp;

Grid/tensors/Tensor_traits.h

@@ -47,20 +47,20 @@ NAMESPACE_BEGIN(Grid);
   class TypePair {
   public:
     T _internal[2];
-    accelerator TypePair<T>& operator=(const Grid::Zero& o) {
+    TypePair<T>& operator=(const Grid::Zero& o) {
       _internal[0] = Zero();
       _internal[1] = Zero();
       return *this;
     }
 
-    accelerator TypePair<T> operator+(const TypePair<T>& o) const {
+    TypePair<T> operator+(const TypePair<T>& o) const {
       TypePair<T> r;
       r._internal[0] = _internal[0] + o._internal[0];
       r._internal[1] = _internal[1] + o._internal[1];
       return r;
     }
 
-    accelerator TypePair<T>& operator+=(const TypePair<T>& o) {
+    TypePair<T>& operator+=(const TypePair<T>& o) {
       _internal[0] += o._internal[0];
       _internal[1] += o._internal[1];
       return *this;

Grid/threads/Accelerator.cc

@@ -84,8 +84,7 @@ void acceleratorInit(void)
   // IBM Jsrun makes cuda Device numbering screwy and not match rank
   if ( world_rank == 0 ) {
     printf("AcceleratorCudaInit: using default device \n");
-    printf("AcceleratorCudaInit: assume user either uses\n");
-    printf("AcceleratorCudaInit: a) IBM jsrun, or \n");
+    printf("AcceleratorCudaInit: assume user either uses a) IBM jsrun, or \n");
     printf("AcceleratorCudaInit: b) invokes through a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding \n");
     printf("AcceleratorCudaInit: Configure options --enable-setdevice=no \n");
   }
@@ -96,7 +95,7 @@ void acceleratorInit(void)
 #endif
 
   cudaSetDevice(device);
-  cudaStreamCreate(&copyStream);
+
   const int len=64;
   char busid[len];
   if( rank == world_rank ) { 
@@ -110,7 +109,6 @@ void acceleratorInit(void)
 
 #ifdef GRID_HIP
 hipDeviceProp_t *gpu_props;
-hipStream_t copyStream;
 void acceleratorInit(void)
 {
   int nDevices = 1;
@@ -168,25 +166,16 @@ void acceleratorInit(void)
 #ifdef GRID_DEFAULT_GPU
   if ( world_rank == 0 ) {
     printf("AcceleratorHipInit: using default device \n");
-    printf("AcceleratorHipInit: assume user or srun sets ROCR_VISIBLE_DEVICES and numa binding \n");
-    printf("AcceleratorHipInit: Configure options --enable-setdevice=no \n");
+    printf("AcceleratorHipInit: assume user either uses a wrapping script to set CUDA_VISIBLE_DEVICES, UCX_NET_DEVICES, and numa binding \n");
+    printf("AcceleratorHipInit: Configure options --enable-summit, --enable-select-gpu=no \n");
   }
-  int device = 0;
 #else
   if ( world_rank == 0 ) {
     printf("AcceleratorHipInit: rank %d setting device to node rank %d\n",world_rank,rank);
-    printf("AcceleratorHipInit: Configure options --enable-setdevice=yes \n");
+    printf("AcceleratorHipInit: Configure options --enable-select-gpu=yes \n");
   }
-  int device = rank;
+  hipSetDevice(rank);
 #endif
-  hipSetDevice(device);
-  hipStreamCreate(&copyStream);
-  const int len=64;
-  char busid[len];
-  if( rank == world_rank ) { 
-    hipDeviceGetPCIBusId(busid, len, device);
-    printf("local rank %d device %d bus id: %s\n", rank, device, busid);
-  }
   if ( world_rank == 0 )  printf("AcceleratorHipInit: ================================================\n");
 }
 #endif

Grid/threads/Accelerator.h

@@ -95,7 +95,6 @@ void     acceleratorInit(void);
 //////////////////////////////////////////////
 
 #ifdef GRID_CUDA
-
 #include <cuda.h>
 
 #ifdef __CUDA_ARCH__
@@ -206,8 +205,7 @@ inline void *acceleratorAllocShared(size_t bytes)
   auto err = cudaMallocManaged((void **)&ptr,bytes);
   if( err != cudaSuccess ) {
     ptr = (void *) NULL;
-    printf(" cudaMallocManaged failed for %lu %s \n",bytes,cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
+    printf(" cudaMallocManaged failed for %lu %s \n",bytes,cudaGetErrorString(err));
   }
   return ptr;
 };
@@ -217,53 +215,20 @@ inline void *acceleratorAllocDevice(size_t bytes)
   auto err = cudaMalloc((void **)&ptr,bytes);
   if( err != cudaSuccess ) {
     ptr = (void *) NULL;
-    printf(" cudaMalloc failed for %lu %s \n",bytes,cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
+    printf(" cudaMalloc failed for %lu %s \n",bytes,cudaGetErrorString(err));
   }
   return ptr;
 };
-inline void acceleratorFreeShared(void *ptr){
-  auto err = cudaFree(ptr);
-  if( err != cudaSuccess ) {
-    printf(" cudaFree(Shared) failed %s \n",cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
-  }
-};
-inline void acceleratorFreeDevice(void *ptr){
-  auto err = cudaFree(ptr);
-  if( err != cudaSuccess ) {
-    printf(" cudaFree(Device) failed %s \n",cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
-  }
-};
-inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)  {
-  auto err = cudaMemcpy(to,from,bytes, cudaMemcpyHostToDevice);
-  if( err != cudaSuccess ) {
-    printf(" cudaMemcpy(host->device) failed for %lu %s \n",bytes,cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
-  }
-}
-inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes){
-  auto err = cudaMemcpy(to,from,bytes, cudaMemcpyDeviceToHost);
-  if( err != cudaSuccess ) {
-    printf(" cudaMemcpy(device->host) failed for %lu %s \n",bytes,cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
-  }
-}
-inline void acceleratorMemSet(void *base,int value,size_t bytes) {
-  auto err = cudaMemset(base,value,bytes);
-  if( err != cudaSuccess ) {
-    printf(" cudaMemSet failed for %lu %s \n",bytes,cudaGetErrorString(err)); fflush(stdout);
-    if (acceleratorAbortOnGpuError) assert(err==cudaSuccess);
-  }
-}
-
+inline void acceleratorFreeShared(void *ptr){ cudaFree(ptr);};
+inline void acceleratorFreeDevice(void *ptr){ cudaFree(ptr);};
+inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)  { cudaMemcpy(to,from,bytes, cudaMemcpyHostToDevice);}
+inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes){ cudaMemcpy(to,from,bytes, cudaMemcpyDeviceToHost);}
+inline void acceleratorMemSet(void *base,int value,size_t bytes) { cudaMemset(base,value,bytes);}
 inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes) // Asynch
 {
   cudaMemcpyAsync(to,from,bytes, cudaMemcpyDeviceToDevice,copyStream);
 }
 inline void acceleratorCopySynchronise(void) { cudaStreamSynchronize(copyStream); };
-
 inline int  acceleratorIsCommunicable(void *ptr)
 {
   //  int uvm=0;
@@ -340,7 +305,7 @@ inline void acceleratorFreeDevice(void *ptr){free(ptr,*theGridAccelerator);};
 inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes)  {
   theGridAccelerator->memcpy(to,from,bytes);
 }
-inline void acceleratorCopySynchronise(void) {  theGridAccelerator->wait(); std::cout<<"acceleratorCopySynchronise() wait "<<std::endl; }
+inline void acceleratorCopySynchronise(void) {  theGridAccelerator->wait(); }
 inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)  { theGridAccelerator->memcpy(to,from,bytes); theGridAccelerator->wait();}
 inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes){ theGridAccelerator->memcpy(to,from,bytes); theGridAccelerator->wait();}
 inline void acceleratorMemSet(void *base,int value,size_t bytes) { theGridAccelerator->memset(base,value,bytes); theGridAccelerator->wait();}
@@ -371,11 +336,10 @@ NAMESPACE_BEGIN(Grid);
 #define accelerator        __host__ __device__
 #define accelerator_inline __host__ __device__ inline
 
-extern hipStream_t copyStream;
 /*These routines define mapping from thread grid to loop & vector lane indexing */
 accelerator_inline int acceleratorSIMTlane(int Nsimd) {
 #ifdef GRID_SIMT
-  return hipThreadIdx_x; 
+  return hipThreadIdx_z; 
 #else
   return 0;
 #endif
@@ -389,41 +353,19 @@ accelerator_inline int acceleratorSIMTlane(int Nsimd) {
       { __VA_ARGS__;}							\
     };									\
     int nt=acceleratorThreads();					\
-    dim3 hip_threads(nsimd, nt, 1);					 \
-    dim3 hip_blocks ((num1+nt-1)/nt,num2,1); \
-    if(hip_threads.x * hip_threads.y * hip_threads.z <= 64){ \
-      hipLaunchKernelGGL(LambdaApply64,hip_blocks,hip_threads,		\
-            0,0,						\
-            num1,num2,nsimd, lambda);				\
-    } else { \
-      hipLaunchKernelGGL(LambdaApply,hip_blocks,hip_threads,		\
-            0,0,						\
-            num1,num2,nsimd, lambda);				\
-    } \
+    dim3 hip_threads(nt,1,nsimd);					\
+    dim3 hip_blocks ((num1+nt-1)/nt,num2,1);				\
+    hipLaunchKernelGGL(LambdaApply,hip_blocks,hip_threads,		\
+		       0,0,						\
+		       num1,num2,nsimd,lambda);				\
   }
 
-
 template<typename lambda>  __global__
-__launch_bounds__(64,1)
-void LambdaApply64(uint64_t numx, uint64_t numy, uint64_t numz, lambda Lambda)
-{
-  // Following the same scheme as CUDA for now
-  uint64_t x = threadIdx.y + blockDim.y*blockIdx.x;
-  uint64_t y = threadIdx.z + blockDim.z*blockIdx.y;
-  uint64_t z = threadIdx.x;
-  if ( (x < numx) && (y<numy) && (z<numz) ) {
-    Lambda(x,y,z);
-  }
-}
-
-template<typename lambda>  __global__
-__launch_bounds__(1024,1)
 void LambdaApply(uint64_t numx, uint64_t numy, uint64_t numz, lambda Lambda)
 {
-  // Following the same scheme as CUDA for now
-  uint64_t x = threadIdx.y + blockDim.y*blockIdx.x;
-  uint64_t y = threadIdx.z + blockDim.z*blockIdx.y;
-  uint64_t z = threadIdx.x;
+  uint64_t x = hipThreadIdx_x + hipBlockDim_x*hipBlockIdx_x;
+  uint64_t y = hipThreadIdx_y + hipBlockDim_y*hipBlockIdx_y;
+  uint64_t z = hipThreadIdx_z ;//+ hipBlockDim_z*hipBlockIdx_z;
   if ( (x < numx) && (y<numy) && (z<numz) ) {
     Lambda(x,y,z);
   }
@@ -468,16 +410,10 @@ inline void acceleratorFreeShared(void *ptr){ hipFree(ptr);};
 inline void acceleratorFreeDevice(void *ptr){ hipFree(ptr);};
 inline void acceleratorCopyToDevice(void *from,void *to,size_t bytes)  { hipMemcpy(to,from,bytes, hipMemcpyHostToDevice);}
 inline void acceleratorCopyFromDevice(void *from,void *to,size_t bytes){ hipMemcpy(to,from,bytes, hipMemcpyDeviceToHost);}
-//inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes)  { hipMemcpy(to,from,bytes, hipMemcpyDeviceToDevice);}
-//inline void acceleratorCopySynchronise(void) {  }
+inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes)  { hipMemcpy(to,from,bytes, hipMemcpyDeviceToDevice);}
+inline void acceleratorCopySynchronise(void) {  }
 inline void acceleratorMemSet(void *base,int value,size_t bytes) { hipMemset(base,value,bytes);}
 
-inline void acceleratorCopyDeviceToDeviceAsynch(void *from,void *to,size_t bytes) // Asynch
-{
-  hipMemcpyAsync(to,from,bytes, hipMemcpyDeviceToDevice,copyStream);
-}
-inline void acceleratorCopySynchronise(void) { hipStreamSynchronize(copyStream); };
-
 #endif
 
 //////////////////////////////////////////////
@@ -548,12 +484,18 @@ inline void acceleratorFreeCpu  (void *ptr){free(ptr);};
 ///////////////////////////////////////////////////
 // Synchronise across local threads for divergence resynch
 ///////////////////////////////////////////////////
-accelerator_inline void acceleratorSynchronise(void)  // Only Nvidia needs 
+accelerator_inline void acceleratorSynchronise(void) 
 {
 #ifdef GRID_SIMT
 #ifdef GRID_CUDA
   __syncwarp();
 #endif
+#ifdef GRID_SYCL
+  //cl::sycl::detail::workGroupBarrier();
+#endif
+#ifdef GRID_HIP
+  __syncthreads();
+#endif
 #endif
   return;
 }

Grid/util/Coordinate.h

@@ -88,7 +88,7 @@ public:
 // Coordinate class, maxdims = 8 for now.
 ////////////////////////////////////////////////////////////////
 #define GRID_MAX_LATTICE_DIMENSION (8)
-#define GRID_MAX_SIMD              (32)
+#define GRID_MAX_SIMD              (16)
 
 static constexpr int MaxDims = GRID_MAX_LATTICE_DIMENSION;
 

Grid/util/Init.cc

@@ -167,13 +167,6 @@ void GridCmdOptionInt(std::string &str,int & val)
   return;
 }
 
-void GridCmdOptionFloat(std::string &str,float & val)
-{
-  std::stringstream ss(str);
-  ss>>val;
-  return;
-}
-
 
 void GridParseLayout(char **argv,int argc,
 		     Coordinate &latt_c,
@@ -534,7 +527,6 @@ void Grid_init(int *argc,char ***argv)
 void Grid_finalize(void)
 {
 #if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT)
-  MPI_Barrier(MPI_COMM_WORLD);
   MPI_Finalize();
   Grid_unquiesce_nodes();
 #endif

Grid/util/Init.h

@@ -57,7 +57,6 @@ void GridCmdOptionCSL(std::string str,std::vector<std::string> & vec);
 template<class VectorInt>
 void GridCmdOptionIntVector(const std::string &str,VectorInt & vec);
 void GridCmdOptionInt(std::string &str,int & val);
-void GridCmdOptionFloat(std::string &str,float & val);
 
 
 void GridParseLayout(char **argv,int argc,

HMC/Mobius2f.cc

@@ -0,0 +1,170 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_EODWFRatio.cc
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionR FermionAction;
+  typedef typename FermionAction::FermionField FermionField;
+
+  typedef Grid::XmlReader       Serialiser;
+
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper;
+  //  MD.name    = std::string("Leap Frog");
+  //  typedef GenericHMCRunner<ForceGradient> HMCWrapper;
+  //  MD.name    = std::string("Force Gradient");
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;
+  MD.name    = std::string("MinimumNorm2");
+  MD.MDsteps = 12;
+  MD.trajL   = 1.0;
+
+  HMCparameters HMCparams;
+  HMCparams.StartTrajectory  = 17;
+  HMCparams.Trajectories     = 200;
+  HMCparams.NoMetropolisUntil=  0;
+  // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  //  HMCparams.StartingType     =std::string("ColdStart");
+  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_2fDWF_lat";
+  CPparams.rng_prefix    = "ckpoint_2fDWF_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  const int Ls      = 16;
+  Real beta         = 2.13;
+  Real light_mass   = 0.01;
+  Real pv_mass      = 1.0;
+  RealD M5  = 1.8;
+  RealD b   = 1.0;
+  RealD c   = 0.0;
+
+  std::vector<Real> hasenbusch({ 0.1, 0.4, 0.7 });
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+
+  double StoppingCondition = 1e-10;
+  double MaxCGIterations = 30000;
+  ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
+
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(8);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> *> Quotients;
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    std::cout << GridLogMessage << " 2f quotient Action  "<< light_num[h] << " / " << light_den[h]<< std::endl;
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, Params));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, Params));
+    Quotients.push_back   (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],CG,CG));
+  }
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    Level1.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  Level2.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+  std::cout << GridLogMessage << " Action complete "<< std::endl;
+
+  /////////////////////////////////////////////////////////////
+  // HMC parameters are serialisable
+
+  std::cout << GridLogMessage << " Running the HMC "<< std::endl;
+  TheHMC.Run();  // no smearing
+
+  Grid_finalize();
+} // main
+
+
+

HMC/Mobius2f_DDHMC_mixed.cc

@@ -0,0 +1,386 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+nnSource file: 
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+
+NAMESPACE_BEGIN(Grid);
+
+template<class Impl>
+class DomainLocalTwoFlavourEvenOddRatioPseudoFermionAction
+  : public TwoFlavourEvenOddRatioPseudoFermionAction<Impl>
+{
+public:
+  INHERIT_IMPL_TYPES(Impl);
+  Coordinate Block;
+  DomainDecomposition Domains;
+  DomainLocalTwoFlavourEvenOddRatioPseudoFermionAction(FermionOperator<Impl>  &_NumOp, 
+						       FermionOperator<Impl>  &_DenOp, 
+						       OperatorFunction<FermionField> & DS,
+						       OperatorFunction<FermionField> & AS,
+						       OperatorFunction<FermionField> & HS,
+						       Coordinate &_Block ) :
+    Block(_Block),
+    Domains(_Block),
+    TwoFlavourEvenOddRatioPseudoFermionAction<Impl>(_NumOp,_DenOp,DS,AS,HS)
+    {};
+  virtual void refreshRestrict(FermionField &eta)
+  {
+    Domains.ProjectDomain(eta,0);
+    DumpSliceNorm("refresh Restrict eta",eta);
+  };
+};
+
+#define MIXED_PRECISION
+
+NAMESPACE_END(Grid);
+
+int main(int argc, char **argv)
+{
+  using namespace Grid;
+
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef WilsonImplR FimplD;
+  typedef WilsonImplF FimplF;
+  typedef FermionOperator<FimplF> FermionOperatorF;
+  typedef FermionOperator<FimplD> FermionOperatorD;
+  typedef MobiusFermionR FermionActionD;
+  typedef MobiusFermionF FermionActionF;
+  typedef DirichletFermionOperator<WilsonImplR> DirichletFermionD;
+  typedef DirichletFermionOperator<WilsonImplF> DirichletFermionF;
+
+  typedef MobiusEOFAFermionR FermionEOFAAction;
+  typedef typename FermionActionD::FermionField FermionFieldD;
+  typedef typename FermionActionF::FermionField FermionFieldF;
+  
+  typedef SchurDiagMooeeOperator<FermionOperator<FimplF>,FermionFieldF> LinearOperatorF;
+  typedef SchurDiagMooeeOperator<FermionOperator<FimplD>,FermionFieldD> LinearOperatorD;
+  typedef SchurDiagMooeeDagOperator<FermionOperator<FimplF>,FermionFieldF> LinearOperatorDagF;
+  typedef SchurDiagMooeeDagOperator<FermionOperator<FimplD>,FermionFieldD> LinearOperatorDagD;
+  
+  typedef Grid::XmlReader       Serialiser;
+  
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper; 
+  //  MD.name    = std::string("Leap Frog");
+  //  typedef GenericHMCRunner<ForceGradient> HMCWrapper; 
+  //  MD.name    = std::string("Force Gradient");
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper; 
+  /*
+  MD.name    = std::string("MinimumNorm2");
+  MD.MDsteps = 4; // dH = 0.08
+  //  MD.MDsteps = 3; // dH = 0.8
+  MD.trajL   = 1.0;
+  */
+  
+  HMCparameters HMCparams;
+  {
+    XmlReader  HMCrd("HMCparameters.xml");
+    read(HMCrd,"HMCparameters",HMCparams);
+    std::cout << GridLogMessage<< HMCparams <<std::endl;
+  }
+  HMCWrapper TheHMC(HMCparams);
+  /*
+  HMCparams.StartTrajectory  = 66;
+  HMCparams.Trajectories     = 200;
+  HMCparams.NoMetropolisUntil=  0;
+  //  "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  // HMCparams.StartingType     =std::string("ColdStart");
+  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  */
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+  
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_DDHMC_lat";
+  CPparams.rng_prefix    = "ckpoint_DDHMC_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Momentum Dirichlet
+  Coordinate Block({0,0,0,24});
+  
+  TheHMC.Resources.SetMomentumFilter(new DDHMCFilter<WilsonImplR::Field>(Block));
+  // Construct observables
+  // here there is too much indirection 
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  const int Ls      = 16;
+  Real beta         = 2.13;
+  //  Real light_mass   = 0.04;
+  Real light_mass   = 0.01;
+  Real pv_mass      = 1.0;
+  RealD M5  = 1.8;
+  RealD b   = 1.0; 
+  RealD c   = 0.0;
+
+  std::vector<Real> hasenbusch({ 0.1, 0.4, 0.7 });
+  
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  Coordinate latt  = GridDefaultLatt();
+  Coordinate mpi   = GridDefaultMpi();
+  Coordinate simdF = GridDefaultSimd(Nd,vComplexF::Nsimd());
+  Coordinate simdD = GridDefaultSimd(Nd,vComplexD::Nsimd());
+
+  auto GridPtrF   = SpaceTimeGrid::makeFourDimGrid(latt,simdF,mpi);
+  auto GridRBPtrF = SpaceTimeGrid::makeFourDimRedBlackGrid(GridPtrF);
+  auto FGridF     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtrF);
+  auto FrbGridF   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtrF);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+  LatticeGaugeFieldF UF(GridPtrF);
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionActionD::ImplParams Params(boundary);
+  FermionActionD::ImplParams DirichletParams(boundary);
+  DirichletParams.locally_periodic=true;
+  
+  double ActionStoppingCondition     = 1e-10;
+  double DerivativeStoppingCondition = 1e-10;
+  //  double BoundaryDerivativeStoppingCondition = 1e-10; decent acceptance
+  double BoundaryDerivativeStoppingCondition = 1e-7;   // decent acceptance
+  //  double BoundaryDerivativeStoppingCondition = 1e-6;  // bit bigger not huge
+  //  double BoundaryDerivativeStoppingCondition = 1e-5; // Large dH poor acceptance
+  double MaxCGIterations = 30000;
+
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(3);
+  ActionLevel<HMCWrapper::Field> Level3(8);
+
+  ConjugateGradient<FermionFieldD>      ActionCG(ActionStoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionFieldD>  DerivativeCG(DerivativeStoppingCondition,MaxCGIterations);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  //////////////////////////////////////////////////////////////
+  // Forced to replicate the MxPCG and DenominatorsF etc.. because
+  // there is no convenient way to "Clone" physics params from double op
+  // into single op for any operator pair.
+  // Same issue prevents using MxPCG in the Heatbath step
+  //////////////////////////////////////////////////////////////
+
+  /////////////////////////////////////////////////
+  // These are consumed/owned by the Dirichlet wrappers
+  /////////////////////////////////////////////////
+  std::vector<FermionActionD *> DNumeratorsD;
+  std::vector<FermionActionF *> DNumeratorsF;
+  std::vector<FermionActionD *> DDenominatorsD;
+  std::vector<FermionActionF *> DDenominatorsF;
+
+  /////////////////////////////////////////////////
+  // Dirichlet wrappers
+  /////////////////////////////////////////////////
+  std::vector<DirichletFermionD *> DirichletNumeratorsD;
+  std::vector<DirichletFermionF *> DirichletNumeratorsF;
+  std::vector<DirichletFermionD *> DirichletDenominatorsD;
+  std::vector<DirichletFermionF *> DirichletDenominatorsF;
+  
+  std::vector<DomainLocalTwoFlavourEvenOddRatioPseudoFermionAction<FimplD> *> Quotients;
+
+  typedef MixedPrecisionConjugateGradientOperatorFunction<FermionOperatorD,
+							  FermionOperatorF,
+							  LinearOperatorD,
+							  LinearOperatorF> MxPCG;
+  std::vector<MxPCG *> ActionMPCG;
+  std::vector<MxPCG *> MPCG;
+  std::vector<LinearOperatorD *> LinOpD;
+  std::vector<LinearOperatorF *> LinOpF; 
+
+  int MX_inner = 1000;
+  RealD MX_tol = 1.0e-5;
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+
+    std::cout << GridLogMessage << " 2f quotient Action  "<< light_num[h] << " / " << light_den[h]<< std::endl;
+
+    DNumeratorsD.push_back (new FermionActionD(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, DirichletParams));
+    DNumeratorsF.push_back (new FermionActionF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_num[h],M5,b,c, DirichletParams));
+    DDenominatorsD.push_back(new FermionActionD(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, DirichletParams));
+    DDenominatorsF.push_back(new FermionActionF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[h],M5,b,c, DirichletParams));
+
+    DirichletNumeratorsD.push_back  (new  DirichletFermionD(*DNumeratorsD[h],Block));
+    DirichletNumeratorsF.push_back  (new  DirichletFermionF(*DNumeratorsF[h],Block));
+    DirichletDenominatorsD.push_back(new  DirichletFermionD(*DDenominatorsD[h],Block));
+    DirichletDenominatorsF.push_back(new  DirichletFermionF(*DDenominatorsF[h],Block));
+
+    // Dirichlet Schur even odd MpsDagMpc operators on local domains
+    LinOpD.push_back(new LinearOperatorD(*DirichletDenominatorsD[h]));
+    LinOpF.push_back(new LinearOperatorF(*DirichletDenominatorsF[h]));
+
+    // Derivative
+    MPCG.push_back(new MxPCG(DerivativeStoppingCondition,MX_tol,
+			     MX_inner,
+			     MaxCGIterations,
+			     FrbGridF,
+			     *DirichletDenominatorsF[h],*DirichletDenominatorsD[h],
+			     *LinOpF[h], *LinOpD[h]) );
+
+    // Action
+    ActionMPCG.push_back(new MxPCG(ActionStoppingCondition,MX_tol,
+				   MX_inner,
+				   MaxCGIterations,
+				   FrbGridF,
+				   *DirichletDenominatorsF[h],*DirichletDenominatorsD[h],
+				   *LinOpF[h], *LinOpD[h]) );
+    
+    ////////////////////////////////////////////////////////////////////////////
+    // Standard CG for 2f force
+    ////////////////////////////////////////////////////////////////////////////
+    Quotients.push_back   (new
+			   DomainLocalTwoFlavourEvenOddRatioPseudoFermionAction<FimplD>
+			   (*DirichletNumeratorsD[h],
+			    *DirichletDenominatorsD[h],
+			    *MPCG[h],
+			    *ActionMPCG[h],
+			    ActionCG,Block));
+
+    Level2.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Boundary action
+  /////////////////////////////////////////////////////////////
+
+  int l_idx = 0;
+  int pv_idx = n_hasenbusch;
+  RealD h_mass = 0.012;
+  std::cout << GridLogMessage<<" Boundary action masses " <<light_num[l_idx]<<" / "<<light_den[pv_idx]<<std::endl;
+
+
+  // OmegaBar cross domain boundary and is used in Boundary operator, so no locally_periodic hack in the boundary det
+  // Dirichlet is applied in gauge link only. OmegaBar solve is too expensive. Monitor cost.
+  FermionActionD    PeriNumeratorD  (U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[pv_idx],M5,b,c, Params);
+  FermionActionF    PeriNumeratorF  (UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_num[pv_idx],M5,b,c, Params);
+  FermionActionD    DirichletNumeratorDD(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[pv_idx],M5,b,c, Params);
+  FermionActionF    DirichletNumeratorFF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_num[pv_idx],M5,b,c, Params);
+  DirichletFermionD DirichletNumeratorD  (DirichletNumeratorDD,Block);
+  DirichletFermionF DirichletNumeratorF  (DirichletNumeratorFF,Block);
+
+  FermionActionD    PeriDenominatorD(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[l_idx] ,M5,b,c, Params);
+  FermionActionF    PeriDenominatorF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[l_idx] ,M5,b,c, Params);
+  FermionActionD    DirichletDenominatorDD(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[l_idx] ,M5,b,c, Params);
+  FermionActionF    DirichletDenominatorFF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[l_idx] ,M5,b,c, Params);
+  DirichletFermionD DirichletDenominatorD(DirichletDenominatorDD,Block);
+  DirichletFermionF DirichletDenominatorF(DirichletDenominatorFF,Block);
+
+  FermionActionD    PeriHasenD  (U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,h_mass ,M5,b,c, Params);
+  FermionActionF    PeriHasenF  (UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,h_mass,M5,b,c, Params);
+  FermionActionD    DHasenD(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,h_mass,M5,b,c, Params);
+  FermionActionF    DHasenF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,h_mass,M5,b,c, Params);
+  DirichletFermionD DirichletHasenD(DHasenD,Block);
+  DirichletFermionF DirichletHasenF(DHasenF,Block);
+  
+  SchurFactoredFermionOperator<FimplD,FimplF> BoundaryNumerator(PeriNumeratorD,PeriNumeratorF,
+								DirichletNumeratorD,DirichletNumeratorF,
+								Block);
+
+  SchurFactoredFermionOperator<FimplD,FimplF> BoundaryDenominator(PeriDenominatorD,PeriDenominatorF,
+								  DirichletDenominatorD,DirichletDenominatorF,
+								  Block);
+
+  SchurFactoredFermionOperator<FimplD,FimplF> BoundaryHasen(PeriHasenD,PeriHasenF,
+							    DirichletHasenD,DirichletHasenF,
+							    Block);
+
+#if 1
+  std::cout << GridLogMessage << " Boundary NO ratio "<< std::endl;
+  MX_tol = 1.0e-5;
+  Level1.push_back(new
+		   DomainDecomposedBoundaryTwoFlavourPseudoFermion<FimplD,FimplF>
+		   (BoundaryDenominator,
+		    BoundaryDerivativeStoppingCondition,ActionStoppingCondition,MX_tol));
+  Level1.push_back(new
+		   DomainDecomposedBoundaryTwoFlavourBosonPseudoFermion<FimplD,FimplF>
+		   (BoundaryNumerator,
+		    BoundaryDerivativeStoppingCondition,ActionStoppingCondition,MX_tol));
+#else
+  Level1.push_back(new
+		   DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion<FimplD,FimplF>
+		   (BoundaryNumerator,
+		    BoundaryDenominator,
+		    BoundaryDerivativeStoppingCondition,ActionStoppingCondition));
+#endif
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  Level3.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+  TheHMC.TheAction.push_back(Level3);
+  std::cout << GridLogMessage << " Action complete "<< std::endl;
+
+  /////////////////////////////////////////////////////////////
+  // HMC parameters are serialisable
+  std::cout << GridLogMessage << " Running the HMC "<< std::endl;
+  TheHMC.Run();  // no smearing
+
+  Grid_finalize();
+} // main
+
+
+

HMC/Mobius2p1fEOFA.cc

@@ -33,137 +33,8 @@ directory
 #ifdef GRID_DEFAULT_PRECISION_DOUBLE
 #define MIXED_PRECISION
 #endif
+#include <Grid/qcd/utils/MixedPrecisionOperatorFunction.h>
 
-NAMESPACE_BEGIN(Grid);
-
-  /*
-   * Need a plan for gauge field update for mixed precision in HMC                      (2x speed up)
-   *    -- Store the single prec action operator.
-   *    -- Clone the gauge field from the operator function argument.
-   *    -- Build the mixed precision operator dynamically from the passed operator and single prec clone.
-   */
-
-  template<class FermionOperatorD, class FermionOperatorF, class SchurOperatorD, class  SchurOperatorF> 
-  class MixedPrecisionConjugateGradientOperatorFunction : public OperatorFunction<typename FermionOperatorD::FermionField> {
-  public:
-    typedef typename FermionOperatorD::FermionField FieldD;
-    typedef typename FermionOperatorF::FermionField FieldF;
-
-    using OperatorFunction<FieldD>::operator();
-
-    RealD   Tolerance;
-    RealD   InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
-    Integer MaxInnerIterations;
-    Integer MaxOuterIterations;
-    GridBase* SinglePrecGrid4; //Grid for single-precision fields
-    GridBase* SinglePrecGrid5; //Grid for single-precision fields
-    RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
-
-    FermionOperatorF &FermOpF;
-    FermionOperatorD &FermOpD;;
-    SchurOperatorF &LinOpF;
-    SchurOperatorD &LinOpD;
-
-    Integer TotalInnerIterations; //Number of inner CG iterations
-    Integer TotalOuterIterations; //Number of restarts
-    Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
-
-    MixedPrecisionConjugateGradientOperatorFunction(RealD tol, 
-						    Integer maxinnerit, 
-						    Integer maxouterit, 
-						    GridBase* _sp_grid4, 
-						    GridBase* _sp_grid5, 
-						    FermionOperatorF &_FermOpF,
-						    FermionOperatorD &_FermOpD,
-						    SchurOperatorF   &_LinOpF,
-						    SchurOperatorD   &_LinOpD): 
-      LinOpF(_LinOpF),
-      LinOpD(_LinOpD),
-      FermOpF(_FermOpF),
-      FermOpD(_FermOpD),
-      Tolerance(tol), 
-      InnerTolerance(tol), 
-      MaxInnerIterations(maxinnerit), 
-      MaxOuterIterations(maxouterit), 
-      SinglePrecGrid4(_sp_grid4),
-      SinglePrecGrid5(_sp_grid5),
-      OuterLoopNormMult(100.) 
-    { 
-      /* Debugging instances of objects; references are stored
-      std::cout << GridLogMessage << " Mixed precision CG wrapper LinOpF " <<std::hex<< &LinOpF<<std::dec <<std::endl;
-      std::cout << GridLogMessage << " Mixed precision CG wrapper LinOpD " <<std::hex<< &LinOpD<<std::dec <<std::endl;
-      std::cout << GridLogMessage << " Mixed precision CG wrapper FermOpF " <<std::hex<< &FermOpF<<std::dec <<std::endl;
-      std::cout << GridLogMessage << " Mixed precision CG wrapper FermOpD " <<std::hex<< &FermOpD<<std::dec <<std::endl;
-      */
-    };
-
-    void operator()(LinearOperatorBase<FieldD> &LinOpU, const FieldD &src, FieldD &psi) {
-
-      std::cout << GridLogMessage << " Mixed precision CG wrapper operator() "<<std::endl;
-
-      SchurOperatorD * SchurOpU = static_cast<SchurOperatorD *>(&LinOpU);
-      
-      //      std::cout << GridLogMessage << " Mixed precision CG wrapper operator() FermOpU " <<std::hex<< &(SchurOpU->_Mat)<<std::dec <<std::endl;
-      //      std::cout << GridLogMessage << " Mixed precision CG wrapper operator() FermOpD " <<std::hex<< &(LinOpD._Mat) <<std::dec <<std::endl;
-      // Assumption made in code to extract gauge field
-      // We could avoid storing LinopD reference alltogether ?
-      assert(&(SchurOpU->_Mat)==&(LinOpD._Mat));
-
-      ////////////////////////////////////////////////////////////////////////////////////
-      // Must snarf a single precision copy of the gauge field in Linop_d argument
-      ////////////////////////////////////////////////////////////////////////////////////
-      typedef typename FermionOperatorF::GaugeField GaugeFieldF;
-      typedef typename FermionOperatorF::GaugeLinkField GaugeLinkFieldF;
-      typedef typename FermionOperatorD::GaugeField GaugeFieldD;
-      typedef typename FermionOperatorD::GaugeLinkField GaugeLinkFieldD;
-
-      GridBase * GridPtrF = SinglePrecGrid4;
-      GridBase * GridPtrD = FermOpD.Umu.Grid();
-      GaugeFieldF     U_f  (GridPtrF);
-      GaugeLinkFieldF Umu_f(GridPtrF);
-      //      std::cout << " Dim gauge field "<<GridPtrF->Nd()<<std::endl; // 4d
-      //      std::cout << " Dim gauge field "<<GridPtrD->Nd()<<std::endl; // 4d
-
-      ////////////////////////////////////////////////////////////////////////////////////
-      // Moving this to a Clone method of fermion operator would allow to duplicate the 
-      // physics parameters and decrease gauge field copies
-      ////////////////////////////////////////////////////////////////////////////////////
-      GaugeLinkFieldD Umu_d(GridPtrD);
-      for(int mu=0;mu<Nd*2;mu++){ 
-	Umu_d = PeekIndex<LorentzIndex>(FermOpD.Umu, mu);
-	precisionChange(Umu_f,Umu_d);
-	PokeIndex<LorentzIndex>(FermOpF.Umu, Umu_f, mu);
-      }
-      pickCheckerboard(Even,FermOpF.UmuEven,FermOpF.Umu);
-      pickCheckerboard(Odd ,FermOpF.UmuOdd ,FermOpF.Umu);
-
-      ////////////////////////////////////////////////////////////////////////////////////
-      // Could test to make sure that LinOpF and LinOpD agree to single prec?
-      ////////////////////////////////////////////////////////////////////////////////////
-      /*
-      GridBase *Fgrid = psi._grid;
-      FieldD tmp2(Fgrid);
-      FieldD tmp1(Fgrid);
-      LinOpU.Op(src,tmp1);
-      LinOpD.Op(src,tmp2);
-      std::cout << " Double gauge field "<< norm2(FermOpD.Umu)<<std::endl;
-      std::cout << " Single gauge field "<< norm2(FermOpF.Umu)<<std::endl;
-      std::cout << " Test of operators "<<norm2(tmp1)<<std::endl;
-      std::cout << " Test of operators "<<norm2(tmp2)<<std::endl;
-      tmp1=tmp1-tmp2;
-      std::cout << " Test of operators diff "<<norm2(tmp1)<<std::endl;
-      */
-
-      ////////////////////////////////////////////////////////////////////////////////////
-      // Make a mixed precision conjugate gradient
-      ////////////////////////////////////////////////////////////////////////////////////
-      MixedPrecisionConjugateGradient<FieldD,FieldF> MPCG(Tolerance,MaxInnerIterations,MaxOuterIterations,SinglePrecGrid5,LinOpF,LinOpD);
-      std::cout << GridLogMessage << "Calling mixed precision Conjugate Gradient" <<std::endl;
-      MPCG(src,psi);
-    }
-  };
-
-NAMESPACE_END(Grid);
 
 int main(int argc, char **argv) {
   using namespace Grid;
@@ -190,18 +61,18 @@ int main(int argc, char **argv) {
   //  MD.name    = std::string("Leap Frog");
   typedef GenericHMCRunner<ForceGradient> HMCWrapper; 
   MD.name    = std::string("Force Gradient");
-  //  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper; 
-  //  MD.name    = std::string("MinimumNorm2");
-  MD.MDsteps = 6;
+  //typedef GenericHMCRunner<MinimumNorm2> HMCWrapper; 
+  //MD.name    = std::string("MinimumNorm2");
+  MD.MDsteps = 15;
   MD.trajL   = 1.0;
   
   HMCparameters HMCparams;
-  HMCparams.StartTrajectory  = 590;
+  HMCparams.StartTrajectory  = 0;
   HMCparams.Trajectories     = 1000;
-  HMCparams.NoMetropolisUntil=  0;
+  HMCparams.NoMetropolisUntil=  10;
   //  "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
-  //  HMCparams.StartingType     =std::string("ColdStart");
-  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.StartingType     =std::string("ColdStart");
+  //HMCparams.StartingType     =std::string("CheckpointStart");
   HMCparams.MD = MD;
   HMCWrapper TheHMC(HMCparams);
 
@@ -209,9 +80,9 @@ int main(int argc, char **argv) {
   TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
   
   CheckpointerParameters CPparams;
-  CPparams.config_prefix = "ckpoint_EODWF_lat";
-  CPparams.rng_prefix    = "ckpoint_EODWF_rng";
-  CPparams.saveInterval  = 10;
+  CPparams.config_prefix = "ckpoint_EOFA_lat";
+  CPparams.rng_prefix    = "ckpoint_EOFA_rng";
+  CPparams.saveInterval  = 1;
   CPparams.format        = "IEEE64BIG";
   TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
@@ -226,16 +97,16 @@ int main(int argc, char **argv) {
   TheHMC.Resources.AddObservable<PlaqObs>();
   //////////////////////////////////////////////
 
-  const int Ls      = 16;
+  const int Ls      = 24;
   Real beta         = 2.13;
-  Real light_mass   = 0.01;
-  Real strange_mass = 0.04;
+  Real light_mass   = 0.005;
+  Real strange_mass = 0.0362;
   Real pv_mass      = 1.0;
   RealD M5  = 1.8;
-  RealD b   = 1.0; 
-  RealD c   = 0.0;
+  RealD b   = 1.5; 
+  RealD c   = 0.5;
 
-  std::vector<Real> hasenbusch({ 0.1, 0.3, 0.6 });
+  std::vector<Real> hasenbusch({ 0.02, 0.2, 0.6 });
 
   auto GridPtr   = TheHMC.Resources.GetCartesian();
   auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
@@ -263,7 +134,7 @@ int main(int argc, char **argv) {
   FermionActionF::ImplParams ParamsF(boundary);
   
   double ActionStoppingCondition     = 1e-10;
-  double DerivativeStoppingCondition = 1e-6;
+  double DerivativeStoppingCondition = 1e-8;
   double MaxCGIterations = 30000;
 
   ////////////////////////////////////
@@ -302,40 +173,37 @@ int main(int argc, char **argv) {
   ConjugateGradient<FermionField>  DerivativeCG(DerivativeStoppingCondition,MaxCGIterations);
 #ifdef MIXED_PRECISION
   const int MX_inner = 1000;
+  const RealD MX_tol = 1.0e-6;
   // Mixed precision EOFA
   LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L);
   LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R);
   LinearOperatorEOFAF Strange_LinOp_LF(Strange_Op_LF);
   LinearOperatorEOFAF Strange_LinOp_RF(Strange_Op_RF);
 
-  MxPCG_EOFA ActionCGL(ActionStoppingCondition,
+  MxPCG_EOFA ActionCGL(ActionStoppingCondition,MX_tol,
 		       MX_inner,
 		       MaxCGIterations,
-		       GridPtrF,
 		       FrbGridF,
 		       Strange_Op_LF,Strange_Op_L,
 		       Strange_LinOp_LF,Strange_LinOp_L);
 
-  MxPCG_EOFA DerivativeCGL(DerivativeStoppingCondition,
+  MxPCG_EOFA DerivativeCGL(DerivativeStoppingCondition,MX_tol,
 			   MX_inner,
 			   MaxCGIterations,
-			   GridPtrF,
 			   FrbGridF,
 			   Strange_Op_LF,Strange_Op_L,
 			   Strange_LinOp_LF,Strange_LinOp_L);
   
-  MxPCG_EOFA ActionCGR(ActionStoppingCondition,
+  MxPCG_EOFA ActionCGR(ActionStoppingCondition,MX_tol,
 		       MX_inner,
 		       MaxCGIterations,
-		       GridPtrF,
 		       FrbGridF,
 		       Strange_Op_RF,Strange_Op_R,
 		       Strange_LinOp_RF,Strange_LinOp_R);
   
-  MxPCG_EOFA DerivativeCGR(DerivativeStoppingCondition,
+  MxPCG_EOFA DerivativeCGR(DerivativeStoppingCondition,MX_tol,
 			   MX_inner,
 			   MaxCGIterations,
-			   GridPtrF,
 			   FrbGridF,
 			   Strange_Op_RF,Strange_Op_R,
 			   Strange_LinOp_RF,Strange_LinOp_R);
@@ -401,18 +269,16 @@ int main(int argc, char **argv) {
     LinOpD.push_back(new LinearOperatorD(*Denominators[h]));
     LinOpF.push_back(new LinearOperatorF(*DenominatorsF[h]));
 
-    MPCG.push_back(new MxPCG(DerivativeStoppingCondition,
+    MPCG.push_back(new MxPCG(DerivativeStoppingCondition,MX_tol,
 			     MX_inner,
 			     MaxCGIterations,
-			     GridPtrF,
 			     FrbGridF,
 			     *DenominatorsF[h],*Denominators[h],
 			     *LinOpF[h], *LinOpD[h]) );
 
-    ActionMPCG.push_back(new MxPCG(ActionStoppingCondition,
+    ActionMPCG.push_back(new MxPCG(ActionStoppingCondition,MX_tol,
 				   MX_inner,
 				   MaxCGIterations,
-				   GridPtrF,
 				   FrbGridF,
 				   *DenominatorsF[h],*Denominators[h],
 				   *LinOpF[h], *LinOpD[h]) );

HMC/Mobius2p1fEOFA_4dPseudoFermion.cc

@@ -0,0 +1,338 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: 
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu
+Author: David Murphy
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/qcd/action/pseudofermion/TwoFlavourRatioEO4DPseudoFermion.h>
+#include <Grid/qcd/action/pseudofermion/TwoFlavourRatio4DPseudoFermion.h>
+
+#ifdef GRID_DEFAULT_PRECISION_DOUBLE
+#define MIXED_PRECISION
+#endif
+#include <Grid/qcd/utils/MixedPrecisionOperatorFunction.h>
+
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionR FermionAction;
+  typedef MobiusFermionF FermionActionF;
+  typedef MobiusEOFAFermionR FermionEOFAAction;
+  typedef MobiusEOFAFermionF FermionEOFAActionF;
+  typedef typename FermionAction::FermionField FermionField;
+  typedef typename FermionActionF::FermionField FermionFieldF;
+
+  typedef Grid::XmlReader       Serialiser;
+  
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper; 
+  //  MD.name    = std::string("Leap Frog");
+  //  typedef GenericHMCRunner<ForceGradient> HMCWrapper; 
+  //  MD.name    = std::string("Force Gradient");
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper; 
+  MD.name    = std::string("MinimumNorm2");
+  MD.MDsteps = 12;
+  MD.trajL   = 1.0;
+  
+  HMCparameters HMCparams;
+  HMCparams.StartTrajectory  = 211;
+  HMCparams.Trajectories     = 1000;
+  HMCparams.NoMetropolisUntil=  0;
+  //  "[HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+  //  HMCparams.StartingType     =std::string("ColdStart");
+  HMCparams.StartingType     =std::string("CheckpointStart");
+  HMCparams.MD = MD;
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+  
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_EOFA4D_lat";
+  CPparams.rng_prefix    = "ckpoint_EOFA4D_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection 
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  const int Ls      = 16;
+  Real beta         = 2.13;
+  Real light_mass   = 0.01;
+  Real strange_mass = 0.04;
+  Real pv_mass      = 1.0;
+  RealD M5  = 1.8;
+  RealD b   = 1.0; 
+  RealD c   = 0.0;
+
+  std::vector<Real> hasenbusch({ 0.1, 0.3, 0.6 });
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  Coordinate latt  = GridDefaultLatt();
+  Coordinate mpi   = GridDefaultMpi();
+  Coordinate simdF = GridDefaultSimd(Nd,vComplexF::Nsimd());
+  Coordinate simdD = GridDefaultSimd(Nd,vComplexD::Nsimd());
+  auto GridPtrF   = SpaceTimeGrid::makeFourDimGrid(latt,simdF,mpi);
+  auto GridRBPtrF = SpaceTimeGrid::makeFourDimRedBlackGrid(GridPtrF);
+  auto FGridF     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtrF);
+  auto FrbGridF   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtrF);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+  LatticeGaugeFieldF UF(GridPtrF);
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+  FermionActionF::ImplParams ParamsF(boundary);
+  
+  double ActionStoppingCondition     = 1e-10;
+  double DerivativeStoppingCondition = 1e-8;
+  double MaxCGIterations = 30000;
+
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(8);
+
+  ////////////////////////////////////
+  // Strange action
+  ////////////////////////////////////
+  typedef SchurDiagMooeeOperator<FermionActionF,FermionFieldF> LinearOperatorF;
+  typedef SchurDiagMooeeOperator<FermionAction ,FermionField > LinearOperatorD;
+
+  typedef SchurDiagMooeeDagOperator<FermionActionF,FermionFieldF> LinearOperatorDagF;
+  typedef SchurDiagMooeeDagOperator<FermionAction ,FermionField > LinearOperatorDagD;
+
+  typedef SchurDiagMooeeOperator<FermionEOFAActionF,FermionFieldF> LinearOperatorEOFAF;
+  typedef SchurDiagMooeeOperator<FermionEOFAAction ,FermionField > LinearOperatorEOFAD;
+
+  typedef MixedPrecisionConjugateGradientOperatorFunction<MobiusFermionD,MobiusFermionF,LinearOperatorD,LinearOperatorF> MxPCG;
+  typedef MixedPrecisionConjugateGradientOperatorFunction<MobiusFermionD,MobiusFermionF,LinearOperatorDagD,LinearOperatorDagF> MxDagPCG;
+  typedef MixedPrecisionConjugateGradientOperatorFunction<MobiusEOFAFermionD,MobiusEOFAFermionF,LinearOperatorEOFAD,LinearOperatorEOFAF> MxPCG_EOFA;
+
+  // DJM: setup for EOFA ratio (Mobius)
+  OneFlavourRationalParams OFRp;
+  OFRp.lo       = 0.1;
+  OFRp.hi       = 25.0;
+  OFRp.MaxIter  = 10000;
+  OFRp.tolerance= 1.0e-9;
+  OFRp.degree   = 14;
+  OFRp.precision= 50;
+
+  
+  MobiusEOFAFermionR Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionF Strange_Op_LF(UF, *FGridF, *FrbGridF, *GridPtrF, *GridRBPtrF, strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionR Strange_Op_R (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+  MobiusEOFAFermionF Strange_Op_RF(UF, *FGridF, *FrbGridF, *GridPtrF, *GridRBPtrF, pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+
+  ConjugateGradient<FermionField>      ActionCG(ActionStoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionField>  DerivativeCG(DerivativeStoppingCondition,MaxCGIterations);
+#ifdef MIXED_PRECISION
+  const int MX_inner = 1000;
+  const RealD MX_tol = 1.0e-4;
+  // Mixed precision EOFA
+  LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L);
+  LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R);
+  LinearOperatorEOFAF Strange_LinOp_LF(Strange_Op_LF);
+  LinearOperatorEOFAF Strange_LinOp_RF(Strange_Op_RF);
+
+  MxPCG_EOFA ActionCGL(ActionStoppingCondition,MX_tol,
+		       MX_inner,
+		       MaxCGIterations,
+		       FrbGridF,
+		       Strange_Op_LF,Strange_Op_L,
+		       Strange_LinOp_LF,Strange_LinOp_L);
+
+  MxPCG_EOFA DerivativeCGL(DerivativeStoppingCondition,MX_tol,
+			   MX_inner,
+			   MaxCGIterations,
+			   FrbGridF,
+			   Strange_Op_LF,Strange_Op_L,
+			   Strange_LinOp_LF,Strange_LinOp_L);
+  
+  MxPCG_EOFA ActionCGR(ActionStoppingCondition,MX_tol,
+		       MX_inner,
+		       MaxCGIterations,
+		       FrbGridF,
+		       Strange_Op_RF,Strange_Op_R,
+		       Strange_LinOp_RF,Strange_LinOp_R);
+  
+  MxPCG_EOFA DerivativeCGR(DerivativeStoppingCondition,MX_tol,
+			   MX_inner,
+			   MaxCGIterations,
+			   FrbGridF,
+			   Strange_Op_RF,Strange_Op_R,
+			   Strange_LinOp_RF,Strange_LinOp_R);
+
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 ActionCG, 
+	 ActionCGL, ActionCGR,
+	 DerivativeCGL, DerivativeCGR,
+	 OFRp, true);
+#else
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 ActionCG,
+	 ActionCG, ActionCG,
+	 DerivativeCG, DerivativeCG, 
+	 OFRp, true);
+#endif
+  Level1.push_back(&EOFA);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  //////////////////////////////////////////////////////////////
+  // Forced to replicate the MxPCG and DenominatorsF etc.. because
+  // there is no convenient way to "Clone" physics params from double op
+  // into single op for any operator pair.
+  // Same issue prevents using MxPCG in the Heatbath step
+  //////////////////////////////////////////////////////////////
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourRatioEO4DPseudoFermionAction<FermionImplPolicy> *> Quotients;
+  std::vector<MxPCG *> ActionMPCG;
+  std::vector<MxPCG *> MPCG;
+  std::vector<MxDagPCG *> MPCGdag;
+  std::vector<FermionActionF *> DenominatorsF;
+  std::vector<LinearOperatorD *> LinOpD;
+  std::vector<LinearOperatorF *> LinOpF; 
+  std::vector<LinearOperatorDagD *> LinOpDagD;
+  std::vector<LinearOperatorDagF *> LinOpDagF; 
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+
+    std::cout << GridLogMessage << " 2f quotient Action  "<< light_num[h] << " / " << light_den[h]<< std::endl;
+
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, Params));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, Params));
+
+#ifdef MIXED_PRECISION
+    ////////////////////////////////////////////////////////////////////////////
+    // Mixed precision CG for 2f force
+    ////////////////////////////////////////////////////////////////////////////
+
+    DenominatorsF.push_back(new FermionActionF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[h],M5,b,c, ParamsF));
+    LinOpD.push_back(new LinearOperatorD(*Denominators[h]));
+    LinOpF.push_back(new LinearOperatorF(*DenominatorsF[h]));
+    LinOpDagD.push_back(new LinearOperatorDagD(*Denominators[h]));
+    LinOpDagF.push_back(new LinearOperatorDagF(*DenominatorsF[h]));
+    
+    MPCG.push_back(new MxPCG(DerivativeStoppingCondition,MX_tol,
+			     MX_inner,
+			     MaxCGIterations,
+			     FrbGridF,
+			     *DenominatorsF[h],*Denominators[h],
+			     *LinOpF[h], *LinOpD[h]) );
+
+    MPCGdag.push_back(new MxDagPCG(DerivativeStoppingCondition,MX_tol,
+				   MX_inner,
+				   MaxCGIterations,
+				   FrbGridF,
+				   *DenominatorsF[h],*Denominators[h],
+				   *LinOpDagF[h], *LinOpDagD[h]) );
+    
+    ActionMPCG.push_back(new MxPCG(ActionStoppingCondition,MX_tol,
+				   MX_inner,
+				   MaxCGIterations,
+				   FrbGridF,
+				   *DenominatorsF[h],*Denominators[h],
+				   *LinOpF[h], *LinOpD[h]) );
+
+    // Heatbath not mixed yet. As inverts numerators not so important as raised mass.
+    Quotients.push_back (new TwoFlavourRatioEO4DPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],*MPCG[h],*MPCGdag[h],*ActionMPCG[h],ActionCG));
+#else
+    ////////////////////////////////////////////////////////////////////////////
+    // Standard CG for 2f force
+    ////////////////////////////////////////////////////////////////////////////
+    Quotients.push_back   (new TwoFlavourRatioEO4DPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],DerivativeCG,ActionCG));
+#endif
+
+  }
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    Level1.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  Level2.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+  std::cout << GridLogMessage << " Action complete "<< std::endl;
+
+  /////////////////////////////////////////////////////////////
+  // HMC parameters are serialisable
+
+  std::cout << GridLogMessage << " Running the HMC "<< std::endl;
+  TheHMC.Run();  // no smearing
+
+  Grid_finalize();
+} // main
+
+
+

HMC/Mobius2p1fEOFA_C1M.cc

@@ -0,0 +1,312 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: 
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu
+Author: David Murphy
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+#ifdef GRID_DEFAULT_PRECISION_DOUBLE
+#define MIXED_PRECISION
+#endif
+#include <Grid/qcd/utils/MixedPrecisionOperatorFunction.h>
+
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionR FermionAction;
+  typedef MobiusFermionF FermionActionF;
+  typedef MobiusEOFAFermionR FermionEOFAAction;
+  typedef MobiusEOFAFermionF FermionEOFAActionF;
+  typedef typename FermionAction::FermionField FermionField;
+  typedef typename FermionActionF::FermionField FermionFieldF;
+
+  typedef Grid::XmlReader       Serialiser;
+  
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  IntegratorParameters MD;
+  //  typedef GenericHMCRunner<LeapFrog> HMCWrapper; 
+  //  MD.name    = std::string("Leap Frog");
+  typedef GenericHMCRunner<ForceGradient> HMCWrapper; 
+  
+  HMCparameters HMCparams;
+  {
+    XmlReader  HMCrd("HMCparameters.xml");
+    read(HMCrd,"HMCparameters",HMCparams);
+    std::cout << GridLogMessage<< HMCparams <<std::endl;
+  }
+  HMCWrapper TheHMC(HMCparams);
+
+  // Grid from the command line arguments --grid and --mpi
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+  
+  CheckpointerParameters CPparams;
+  CPparams.config_prefix = "ckpoint_C1M_lat";
+  CPparams.rng_prefix    = "ckpoint_C1M_rng";
+  CPparams.saveInterval  = 1;
+  CPparams.format        = "IEEE64BIG";
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection 
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  const int Ls      = 24;
+  Real beta         = 2.13;
+  Real light_mass   = 0.005;
+  Real strange_mass = 0.0362;
+  Real pv_mass      = 1.0;
+  RealD M5  = 1.8;
+  RealD b   = 1.5; 
+  RealD c   = 0.5;
+
+  std::vector<Real> hasenbusch({ 0.02, 0.2, 0.6 });
+
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+  Coordinate latt  = GridDefaultLatt();
+  Coordinate mpi   = GridDefaultMpi();
+  Coordinate simdF = GridDefaultSimd(Nd,vComplexF::Nsimd());
+  Coordinate simdD = GridDefaultSimd(Nd,vComplexD::Nsimd());
+  auto GridPtrF   = SpaceTimeGrid::makeFourDimGrid(latt,simdF,mpi);
+  auto GridRBPtrF = SpaceTimeGrid::makeFourDimRedBlackGrid(GridPtrF);
+  auto FGridF     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtrF);
+  auto FrbGridF   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtrF);
+
+  IwasakiGaugeActionR GaugeAction(beta);
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+  LatticeGaugeFieldF UF(GridPtrF);
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+  FermionActionF::ImplParams ParamsF(boundary);
+  
+  double ActionStoppingCondition     = 1e-10;
+  double DerivativeStoppingCondition = 1e-8;
+  double MaxCGIterations = 30000;
+
+  ////////////////////////////////////
+  // Collect actions
+  ////////////////////////////////////
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  ActionLevel<HMCWrapper::Field> Level2(8);
+
+  ////////////////////////////////////
+  // Strange action
+  ////////////////////////////////////
+  typedef SchurDiagMooeeOperator<FermionActionF,FermionFieldF> LinearOperatorF;
+  typedef SchurDiagMooeeOperator<FermionAction ,FermionField > LinearOperatorD;
+  typedef SchurDiagMooeeOperator<FermionEOFAActionF,FermionFieldF> LinearOperatorEOFAF;
+  typedef SchurDiagMooeeOperator<FermionEOFAAction ,FermionField > LinearOperatorEOFAD;
+
+  typedef MixedPrecisionConjugateGradientOperatorFunction<MobiusFermionD,MobiusFermionF,LinearOperatorD,LinearOperatorF> MxPCG;
+  typedef MixedPrecisionConjugateGradientOperatorFunction<MobiusEOFAFermionD,MobiusEOFAFermionF,LinearOperatorEOFAD,LinearOperatorEOFAF> MxPCG_EOFA;
+
+  // DJM: setup for EOFA ratio (Mobius)
+  OneFlavourRationalParams OFRp;
+  OFRp.lo       = 0.1;
+  OFRp.hi       = 25.0;
+  OFRp.MaxIter  = 10000;
+  OFRp.tolerance= 1.0e-9;
+  OFRp.degree   = 14;
+  OFRp.precision= 50;
+
+  
+  MobiusEOFAFermionR Strange_Op_L (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionF Strange_Op_LF(UF, *FGridF, *FrbGridF, *GridPtrF, *GridRBPtrF, strange_mass, strange_mass, pv_mass, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionR Strange_Op_R (U , *FGrid , *FrbGrid , *GridPtr , *GridRBPtr , pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+  MobiusEOFAFermionF Strange_Op_RF(UF, *FGridF, *FrbGridF, *GridPtrF, *GridRBPtrF, pv_mass, strange_mass,      pv_mass, -1.0, 1, M5, b, c);
+
+  ConjugateGradient<FermionField>      ActionCG(ActionStoppingCondition,MaxCGIterations);
+  ConjugateGradient<FermionField>  DerivativeCG(DerivativeStoppingCondition,MaxCGIterations);
+#ifdef MIXED_PRECISION
+  const int MX_inner = 1000;
+  const RealD MX_tol = 1.0e-6;
+  // Mixed precision EOFA
+  LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L);
+  LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R);
+  LinearOperatorEOFAF Strange_LinOp_LF(Strange_Op_LF);
+  LinearOperatorEOFAF Strange_LinOp_RF(Strange_Op_RF);
+
+  MxPCG_EOFA ActionCGL(ActionStoppingCondition,MX_tol,
+		       MX_inner,
+		       MaxCGIterations,
+		       FrbGridF,
+		       Strange_Op_LF,Strange_Op_L,
+		       Strange_LinOp_LF,Strange_LinOp_L);
+
+  MxPCG_EOFA DerivativeCGL(DerivativeStoppingCondition,MX_tol,
+			   MX_inner,
+			   MaxCGIterations,
+			   FrbGridF,
+			   Strange_Op_LF,Strange_Op_L,
+			   Strange_LinOp_LF,Strange_LinOp_L);
+  
+  MxPCG_EOFA ActionCGR(ActionStoppingCondition,MX_tol,
+		       MX_inner,
+		       MaxCGIterations,
+		       FrbGridF,
+		       Strange_Op_RF,Strange_Op_R,
+		       Strange_LinOp_RF,Strange_LinOp_R);
+  
+  MxPCG_EOFA DerivativeCGR(DerivativeStoppingCondition,MX_tol,
+			   MX_inner,
+			   MaxCGIterations,
+			   FrbGridF,
+			   Strange_Op_RF,Strange_Op_R,
+			   Strange_LinOp_RF,Strange_LinOp_R);
+
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 ActionCG, 
+	 ActionCGL, ActionCGR,
+	 DerivativeCGL, DerivativeCGR,
+	 OFRp, true);
+#else
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> 
+    EOFA(Strange_Op_L, Strange_Op_R, 
+	 ActionCG,
+	 ActionCG, ActionCG,
+	 DerivativeCG, DerivativeCG, 
+	 OFRp, true);
+#endif
+  Level1.push_back(&EOFA);
+
+  ////////////////////////////////////
+  // up down action
+  ////////////////////////////////////
+  std::vector<Real> light_den;
+  std::vector<Real> light_num;
+
+  int n_hasenbusch = hasenbusch.size();
+  light_den.push_back(light_mass);
+  for(int h=0;h<n_hasenbusch;h++){
+    light_den.push_back(hasenbusch[h]);
+    light_num.push_back(hasenbusch[h]);
+  }
+  light_num.push_back(pv_mass);
+
+  //////////////////////////////////////////////////////////////
+  // Forced to replicate the MxPCG and DenominatorsF etc.. because
+  // there is no convenient way to "Clone" physics params from double op
+  // into single op for any operator pair.
+  // Same issue prevents using MxPCG in the Heatbath step
+  //////////////////////////////////////////////////////////////
+  std::vector<FermionAction *> Numerators;
+  std::vector<FermionAction *> Denominators;
+  std::vector<TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> *> Quotients;
+  std::vector<MxPCG *> ActionMPCG;
+  std::vector<MxPCG *> MPCG;
+  std::vector<FermionActionF *> DenominatorsF;
+  std::vector<LinearOperatorD *> LinOpD;
+  std::vector<LinearOperatorF *> LinOpF; 
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+
+    std::cout << GridLogMessage << " 2f quotient Action  "<< light_num[h] << " / " << light_den[h]<< std::endl;
+
+    Numerators.push_back  (new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[h],M5,b,c, Params));
+    Denominators.push_back(new FermionAction(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[h],M5,b,c, Params));
+
+#ifdef MIXED_PRECISION
+    ////////////////////////////////////////////////////////////////////////////
+    // Mixed precision CG for 2f force
+    ////////////////////////////////////////////////////////////////////////////
+
+    DenominatorsF.push_back(new FermionActionF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[h],M5,b,c, ParamsF));
+    LinOpD.push_back(new LinearOperatorD(*Denominators[h]));
+    LinOpF.push_back(new LinearOperatorF(*DenominatorsF[h]));
+
+    MPCG.push_back(new MxPCG(DerivativeStoppingCondition,MX_tol,
+			     MX_inner,
+			     MaxCGIterations,
+			     FrbGridF,
+			     *DenominatorsF[h],*Denominators[h],
+			     *LinOpF[h], *LinOpD[h]) );
+
+    ActionMPCG.push_back(new MxPCG(ActionStoppingCondition,MX_tol,
+				   MX_inner,
+				   MaxCGIterations,
+				   FrbGridF,
+				   *DenominatorsF[h],*Denominators[h],
+				   *LinOpF[h], *LinOpD[h]) );
+
+    // Heatbath not mixed yet. As inverts numerators not so important as raised mass.
+    Quotients.push_back (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],*MPCG[h],*ActionMPCG[h],ActionCG));
+#else
+    ////////////////////////////////////////////////////////////////////////////
+    // Standard CG for 2f force
+    ////////////////////////////////////////////////////////////////////////////
+    Quotients.push_back   (new TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy>(*Numerators[h],*Denominators[h],DerivativeCG,ActionCG));
+#endif
+
+  }
+
+  for(int h=0;h<n_hasenbusch+1;h++){
+    Level1.push_back(Quotients[h]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Gauge action
+  /////////////////////////////////////////////////////////////
+  Level2.push_back(&GaugeAction);
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+  std::cout << GridLogMessage << " Action complete "<< std::endl;
+
+  /////////////////////////////////////////////////////////////
+  // HMC parameters are serialisable
+
+  std::cout << GridLogMessage << " Running the HMC "<< std::endl;
+  TheHMC.Run();  // no smearing
+
+  Grid_finalize();
+} // main
+
+
+