diff --git a/Grid/qcd/action/fermion/SchurFactoredFermionOperator.h b/Grid/qcd/action/fermion/SchurFactoredFermionOperator.h index 1ccf0950..84fb0bd1 100644 --- a/Grid/qcd/action/fermion/SchurFactoredFermionOperator.h +++ b/Grid/qcd/action/fermion/SchurFactoredFermionOperator.h @@ -62,6 +62,7 @@ NAMESPACE_BEGIN(Grid); // - The Dirichlet ops can be passed to dOmega(Bar) solvers etc... // //////////////////////////////////////////////////////// + template class SchurFactoredFermionOperator : public Impl { @@ -71,17 +72,26 @@ public: FermionOperator & DirichletFermOp; FermionOperator & FermOp; - OperatorFunction &Solver; + OperatorFunction &OmegaSolver; + OperatorFunction &OmegaDagSolver; + OperatorFunction &DSolver; + OperatorFunction &DdagSolver; Coordinate Block; SchurFactoredFermionOperator(FermionOperator & _FermOp, FermionOperator & _DirichletFermOp, - OperatorFunction &_Solver, + OperatorFunction &_OmegaSolver, + OperatorFunction &_OmegaDagSolver, + OperatorFunction &_DSolver, + OperatorFunction &_DdagSolver, Coordinate &_Block) : Block(_Block), FermOp(_FermOp), DirichletFermOp(_DirichletFermOp), - Solver(_Solver) + OmegaSolver(_OmegaSolver), + OmegaDagSolver(_OmegaDagSolver), + DSolver(_DSolver), + DdagSolver(_DdagSolver) { // Pass in Dirichlet FermOp because we really need two dirac operators // as double stored gauge fields differ and they will otherwise overwrite @@ -274,12 +284,12 @@ public: }; void dOmegaInvAndOmegaBarInv(FermionField &in,FermionField &out) { - SchurRedBlackDiagMooeeSolve PrecSolve(Solver); + SchurRedBlackDiagMooeeSolve PrecSolve(OmegaSolver); PrecSolve(DirichletFermOp,in,out); }; void dOmegaDagInvAndOmegaBarDagInv(FermionField &in,FermionField &out) { - SchurRedBlackDiagMooeeDagSolve PrecSolve(Solver); + SchurRedBlackDiagMooeeDagSolve PrecSolve(OmegaDagSolver); PrecSolve(DirichletFermOp,in,out); }; @@ -335,12 +345,12 @@ public: // Non-dirichlet inverter using red-black preconditioning void Dinverse(FermionField &in,FermionField &out) { - SchurRedBlackDiagMooeeSolve Solve(Solver); + SchurRedBlackDiagMooeeSolve Solve(DSolver); Solve(FermOp,in,out); } void DinverseDag(FermionField &in,FermionField &out) { - SchurRedBlackDiagMooeeDagSolve Solve(Solver); + SchurRedBlackDiagMooeeDagSolve Solve(DdagSolver); Solve(FermOp,in,out); } }; diff --git a/Grid/qcd/action/momentum/MomentumFilter.h b/Grid/qcd/action/momentum/MomentumFilter.h index 0a84b7e2..8348930c 100644 --- a/Grid/qcd/action/momentum/MomentumFilter.h +++ b/Grid/qcd/action/momentum/MomentumFilter.h @@ -36,7 +36,7 @@ NAMESPACE_BEGIN(Grid); template struct MomentumFilterBase{ - virtual void applyFilter(MomentaField &P) const; + virtual void applyFilter(MomentaField &P) const = 0; }; //Do nothing diff --git a/Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion.h b/Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion.h index 5ae0b49b..e88df5f6 100644 --- a/Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion.h +++ b/Grid/qcd/action/pseudofermion/DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion.h @@ -41,22 +41,21 @@ private: SchurFactoredFermionOperator & NumOp;// the basic operator SchurFactoredFermionOperator & DenOp;// the basic operator - OperatorFunction &DerivativeSolver; - OperatorFunction &ActionSolver; + // OperatorFunction &DerivativeSolver; + // OperatorFunction &ActionSolver; FermionField Phi; // the pseudo fermion field for this trajectory public: DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion(SchurFactoredFermionOperator &_NumOp, - SchurFactoredFermionOperator &_DenOp, - OperatorFunction & DS, - OperatorFunction & AS - ) : NumOp(_NumOp), DenOp(_DenOp), - DerivativeSolver(DS), ActionSolver(AS), + SchurFactoredFermionOperator &_DenOp + // OperatorFunction & AS, + // OperatorFunction & DS + ) : NumOp(_NumOp), DenOp(_DenOp), + // DerivativeSolver(DS), ActionSolver(AS), Phi(_NumOp.FermOp.FermionGrid()) {}; virtual std::string action_name(){return "DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion";} - virtual std::string LogParameters(){ std::stringstream sstream; diff --git a/HMC/Mobius2f_DDHMC.cc b/HMC/Mobius2f_DDHMC.cc index 96e19bde..c44f3ef1 100644 --- a/HMC/Mobius2f_DDHMC.cc +++ b/HMC/Mobius2f_DDHMC.cc @@ -213,12 +213,16 @@ int main(int argc, char **argv) { BoundaryNumerators.push_back (new SchurFactoredFermionOperator (*PeriNumerators[h], *DirichletNumerators[h], - ActionCG,Block)); + ActionCG,ActionCG, + ActionCG,ActionCG, + Block)); BoundaryDenominators.push_back (new SchurFactoredFermionOperator (*PeriDenominators[h], *DirichletDenominators[h], - ActionCG,Block)); + ActionCG,ActionCG, + ActionCG,ActionCG, + Block)); //////////////////////////////////////////////////////////////////////////// // Standard CG for 2f force @@ -237,8 +241,7 @@ int main(int argc, char **argv) { BoundaryQuotients.push_back(new DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion (*BoundaryNumerators[h], - *BoundaryDenominators[h], - ActionCG,ActionCG)); + *BoundaryDenominators[h])); } diff --git a/HMC/Mobius2f_DDHMC_mixed.cc b/HMC/Mobius2f_DDHMC_mixed.cc index 28b5138f..e4a098ac 100644 --- a/HMC/Mobius2f_DDHMC_mixed.cc +++ b/HMC/Mobius2f_DDHMC_mixed.cc @@ -231,7 +231,7 @@ int main(int argc, char **argv) RealD b = 1.0; RealD c = 0.0; - std::vector hasenbusch({ 0.016, 0.04, 0.4 }); + std::vector hasenbusch({ 0.04, 0.3 }); auto GridPtr = TheHMC.Resources.GetCartesian(); auto GridRBPtr = TheHMC.Resources.GetRBCartesian(); @@ -342,11 +342,15 @@ int main(int argc, char **argv) BoundaryNumerators.push_back (new SchurFactoredFermionOperator (*PeriNumerators[h], *DirichletNumerators[h], - ActionCG,Block)); + ActionCG,ActionCG, + ActionCG,ActionCG, + Block)); BoundaryDenominators.push_back (new SchurFactoredFermionOperator (*PeriDenominators[h], *DirichletDenominators[h], - ActionCG,Block)); + ActionCG,ActionCG, + ActionCG,ActionCG, + Block)); // Dirichlet Schur even odd MpsDagMpc operators on local domains LinOpD.push_back(new LinearOperatorD(*DirichletDenominators[h])); @@ -385,8 +389,7 @@ int main(int argc, char **argv) BoundaryQuotients.push_back(new DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion (*BoundaryNumerators[h], - *BoundaryDenominators[h], - ActionCG,ActionCG)); + *BoundaryDenominators[h])); } diff --git a/HMC/Mobius2f_DDHMC_mixed_R.cc b/HMC/Mobius2f_DDHMC_mixed_R.cc new file mode 100644 index 00000000..604a3e27 --- /dev/null +++ b/HMC/Mobius2f_DDHMC_mixed_R.cc @@ -0,0 +1,560 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: + +Copyright (C) 2015-2016 + +Author: Peter Boyle + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution +directory +*************************************************************************************/ +/* END LEGAL */ +#include +#include +#include +#include +#include + +#include +#include +#include + + + +NAMESPACE_BEGIN(Grid); + +#define MIXED_PRECISION + /* + * Need a plan for gauge field update for mixed precision in HMC (2x speed up) + * -- Store the single prec action operator. + * -- Clone the gauge field from the operator function argument. + * -- Build the mixed precision operator dynamically from the passed operator and single prec clone. + */ + template + class MixedPrecisionConjugateGradientOperatorFunction : public OperatorFunction { + public: + typedef typename FermionOperatorD::FermionField FieldD; + typedef typename FermionOperatorF::FermionField FieldF; + + using OperatorFunction::operator(); + + RealD Tolerance; + RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed + Integer MaxInnerIterations; + Integer MaxOuterIterations; + GridBase* SinglePrecGrid4; //Grid for single-precision fields + GridBase* SinglePrecGrid5; //Grid for single-precision fields + RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance + + FermionOperatorF &FermOpF; + FermionOperatorD &FermOpD;; + SchurOperatorF &LinOpF; + SchurOperatorD &LinOpD; + + Integer TotalInnerIterations; //Number of inner CG iterations + Integer TotalOuterIterations; //Number of restarts + Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step + + MixedPrecisionConjugateGradientOperatorFunction(RealD tol, + Integer maxinnerit, + Integer maxouterit, + GridBase* _sp_grid4, + GridBase* _sp_grid5, + FermionOperatorF &_FermOpF, + FermionOperatorD &_FermOpD, + SchurOperatorF &_LinOpF, + SchurOperatorD &_LinOpD): + LinOpF(_LinOpF), + LinOpD(_LinOpD), + FermOpF(_FermOpF), + FermOpD(_FermOpD), + Tolerance(tol), + InnerTolerance(tol), + MaxInnerIterations(maxinnerit), + MaxOuterIterations(maxouterit), + SinglePrecGrid4(_sp_grid4), + SinglePrecGrid5(_sp_grid5), + OuterLoopNormMult(100.) + { }; + + void operator()(LinearOperatorBase &LinOpU, const FieldD &src, FieldD &psi) + { + + SchurOperatorD * SchurOpU = static_cast(&LinOpU); + + // Assumption made in code to extract gauge field + // We could avoid storing LinopD reference alltogether ? + assert(&(SchurOpU->_Mat)==&(LinOpD._Mat)); + + //////////////////////////////////////////////////////////////////////////////////// + // Must snarf a single precision copy of the gauge field in Linop_d argument + //////////////////////////////////////////////////////////////////////////////////// + typedef typename FermionOperatorF::GaugeField GaugeFieldF; + typedef typename FermionOperatorD::GaugeField GaugeFieldD; + typedef typename FermionOperatorF::GaugeLinkField GaugeLinkFieldF; + typedef typename FermionOperatorD::GaugeLinkField GaugeLinkFieldD; + + GridBase * GridPtrF = SinglePrecGrid4; + GridBase * GridPtrD = FermOpD.GaugeGrid(); + + //////////////////////////////////////////////////////////////////////////////////// + // Moving this to a Clone method of fermion operator would allow to duplicate the + // physics parameters and decrease gauge field copies + //////////////////////////////////////////////////////////////////////////////////// + auto &Umu_d = FermOpD.GetDoubledGaugeField(); + auto &Umu_f = FermOpF.GetDoubledGaugeField(); + auto &Umu_fe= FermOpF.GetDoubledGaugeFieldE(); + auto &Umu_fo= FermOpF.GetDoubledGaugeFieldO(); + precisionChange(Umu_f,Umu_d); + pickCheckerboard(Even,Umu_fe,Umu_f); + pickCheckerboard(Odd ,Umu_fo,Umu_f); + + //////////////////////////////////////////////////////////////////////////////////// + // Could test to make sure that LinOpF and LinOpD agree to single prec? + //////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////// + // Make a mixed precision conjugate gradient + //////////////////////////////////////////////////////////////////////////////////// + MixedPrecisionConjugateGradient MPCG(Tolerance,MaxInnerIterations,MaxOuterIterations,SinglePrecGrid5,LinOpF,LinOpD); + std::cout << GridLogMessage << "Calling mixed precision Conjugate Gradient" < DirichletFermion; + typedef DirichletFermionOperator DirichletFermionF; + + typedef MobiusEOFAFermionR FermionEOFAAction; + typedef typename FermionAction::FermionField FermionField; + typedef typename FermionActionF::FermionField FermionFieldF; + + typedef Grid::XmlReader Serialiser; + + //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + IntegratorParameters MD; + // typedef GenericHMCRunner HMCWrapper; + // MD.name = std::string("Leap Frog"); + // typedef GenericHMCRunner HMCWrapper; + // MD.name = std::string("Force Gradient"); + typedef GenericHMCRunner HMCWrapper; + MD.name = std::string("MinimumNorm2"); + MD.MDsteps = 12; + MD.trajL = 1.0; + + HMCparameters HMCparams; + HMCparams.StartTrajectory = 26; + HMCparams.Trajectories = 1000; + HMCparams.NoMetropolisUntil= 10; + // "[HotStart, ColdStart, TepidStart, CheckpointStart]\n"; + // HMCparams.StartingType =std::string("ColdStart"); + HMCparams.StartingType =std::string("CheckpointStart"); + HMCparams.MD = MD; + HMCWrapper TheHMC(HMCparams); + + // Grid from the command line arguments --grid and --mpi + TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition + + CheckpointerParameters CPparams; + CPparams.config_prefix = "ckpoint_EOFA4D_lat"; + CPparams.rng_prefix = "ckpoint_EOFA4D_rng"; + CPparams.saveInterval = 1; + CPparams.format = "IEEE64BIG"; + TheHMC.Resources.LoadNerscCheckpointer(CPparams); + + RNGModuleParameters RNGpar; + RNGpar.serial_seeds = "1 2 3 4 5"; + RNGpar.parallel_seeds = "6 7 8 9 10"; + TheHMC.Resources.SetRNGSeeds(RNGpar); + + // Momentum Dirichlet + Coordinate Block({16,16,16,16}); + + // TheHMC.Resources.SetMomentumFilter(new DirichletFilter(Block)); + TheHMC.Resources.SetMomentumFilter(new DDHMCFilter(Block,1)); + // Construct observables + // here there is too much indirection + typedef PlaquetteMod PlaqObs; + TheHMC.Resources.AddObservable(); + ////////////////////////////////////////////// + + const int Ls = 16; + Real beta = 2.13; + Real light_mass = 0.01; + Real strange_mass = 0.04; + Real pv_mass = 1.0; + RealD M5 = 1.8; + RealD b = 1.0; + RealD c = 0.0; + + std::vector hasenbusch({ 0.04, 0.4, 0.7 }); + + auto GridPtr = TheHMC.Resources.GetCartesian(); + auto GridRBPtr = TheHMC.Resources.GetRBCartesian(); + auto FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr); + auto FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr); + + Coordinate latt = GridDefaultLatt(); + Coordinate mpi = GridDefaultMpi(); + Coordinate simdF = GridDefaultSimd(Nd,vComplexF::Nsimd()); + Coordinate simdD = GridDefaultSimd(Nd,vComplexD::Nsimd()); + auto GridPtrF = SpaceTimeGrid::makeFourDimGrid(latt,simdF,mpi); + auto GridRBPtrF = SpaceTimeGrid::makeFourDimRedBlackGrid(GridPtrF); + auto FGridF = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtrF); + auto FrbGridF = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtrF); + + IwasakiGaugeActionR GaugeAction(beta); + + // temporarily need a gauge field + LatticeGaugeField U(GridPtr); + LatticeGaugeFieldF UF(GridPtrF); + + // These lines are unecessary if BC are all periodic + std::vector boundary = {1,1,1,-1}; + FermionAction::ImplParams Params(boundary); + FermionAction::ImplParams DirichletParams(boundary); + DirichletParams.locally_periodic=true; + + double ActionStoppingCondition = 1e-10; + double DerivativeStoppingCondition = 1e-8; + double MaxCGIterations = 30000; + + //////////////////////////////////// + // Collect actions + //////////////////////////////////// + ActionLevel Level1(1); + ActionLevel Level2(8); + + ConjugateGradient ActionCG(ActionStoppingCondition,MaxCGIterations); + ConjugateGradient DerivativeCG(DerivativeStoppingCondition,MaxCGIterations); + + //////////////////////////////////// + // up down action + //////////////////////////////////// + std::vector light_den; + std::vector light_num; + + int n_hasenbusch = hasenbusch.size(); + light_den.push_back(light_mass); + for(int h=0;h DirichletLinearOperatorF; + typedef SchurDiagMooeeOperator DirichletLinearOperatorD; + typedef SchurDiagMooeeDagOperator DirichletLinearOperatorDagF; + typedef SchurDiagMooeeDagOperator DirichletLinearOperatorDagD; + typedef SchurDiagMooeeOperator PeriodicLinearOperatorF; + typedef SchurDiagMooeeOperator PeriodicLinearOperatorD; + typedef SchurDiagMooeeDagOperator PeriodicLinearOperatorDagF; + typedef SchurDiagMooeeDagOperator PeriodicLinearOperatorDagD; + + typedef MixedPrecisionConjugateGradientOperatorFunction DirichletMxPCG; + typedef MixedPrecisionConjugateGradientOperatorFunction DirichletMxDagPCG; + typedef MixedPrecisionConjugateGradientOperatorFunction PeriodicMxPCG; + typedef MixedPrecisionConjugateGradientOperatorFunction PeriodicMxDagPCG; + + // std::vector DenominatorsF; + ///////////////////////////////////////////////// + // These are consumed/owned by the Dirichlet wrappers + ///////////////////////////////////////////////// + std::vector DNumerators; + std::vector DNumeratorsF; + std::vector DDenominators; + std::vector DDenominatorsF; + ///////////////////////////////////////////////// + // Dirichlet wrappers + ///////////////////////////////////////////////// + std::vector DirichletNumerators; + std::vector DirichletNumeratorsF; + std::vector DirichletDenominators; + std::vector DirichletDenominatorsF; + + std::vector *> Quotients; + + std::vector *> BoundaryQuotients; + + std::vector ActionMPCG; + std::vector MPCG; + std::vector LinOpD; + std::vector LinOpF; + + const int MX_inner = 1000; + + for(int h=0;h(*Numerators[h],*Denominators[h],*MPCG[h],*ActionMPCG[h],ActionCG)); + Quotients.push_back (new + TwoFlavourEvenOddRatioPseudoFermionAction + (*DirichletNumerators[h], + *DirichletDenominators[h], + *ActionMPCG[h], + *ActionMPCG[h], + ActionCG)); + + Level1.push_back(Quotients[h]); + } + + ///////////////////////////////////////////////////////////// + // Boundary action + ///////////////////////////////////////////////////////////// + int l_idx = 0; + int pv_idx = n_hasenbusch; + + std::cout << GridLogMessage<<" Boundary action masses " < DirichletLinearOperatorF; + typedef SchurDiagMooeeOperator DirichletLinearOperatorD; + typedef SchurDiagMooeeDagOperator DirichletLinearOperatorDagF; + typedef SchurDiagMooeeDagOperator DirichletLinearOperatorDagD; + typedef SchurDiagMooeeOperator PeriodicLinearOperatorF; + typedef SchurDiagMooeeOperator PeriodicLinearOperatorD; + typedef SchurDiagMooeeDagOperator PeriodicLinearOperatorDagF; + typedef SchurDiagMooeeDagOperator PeriodicOperatorDagD; + */ + + typedef MixedPrecisionConjugateGradientOperatorFunction DirichletMxPCG; + + typedef MixedPrecisionConjugateGradientOperatorFunction DirichletMxDagPCG; + + typedef MixedPrecisionConjugateGradientOperatorFunction PeriodicMxPCG; + + typedef MixedPrecisionConjugateGradientOperatorFunction PeriodicMxDagPCG; + + FermionAction PeriNumerator (U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_num[pv_idx],M5,b,c, Params); + FermionAction PeriDenominator (U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,light_den[l_idx] ,M5,b,c, Params); + FermionActionF PeriNumeratorF (UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_num[pv_idx],M5,b,c, Params); + FermionActionF PeriDenominatorF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[l_idx] ,M5,b,c, Params); + + DirichletFermion & DirichletNumerator = *DirichletNumerators[pv_idx]; + DirichletFermionF & DirichletNumeratorF = *DirichletNumeratorsF[pv_idx]; + DirichletFermion & DirichletDenominator = *DirichletDenominators[l_idx]; + DirichletFermionF & DirichletDenominatorF= *DirichletDenominatorsF[l_idx]; + + DirichletLinearOperatorD DirichletNumeratorLinOpD(DirichletNumerator); + DirichletLinearOperatorF DirichletNumeratorLinOpF(DirichletNumeratorF); + DirichletLinearOperatorDagD DirichletNumeratorLinOpDagD(DirichletNumerator); + DirichletLinearOperatorDagF DirichletNumeratorLinOpDagF(DirichletNumeratorF); + + DirichletLinearOperatorD DirichletDenominatorLinOpD(DirichletDenominator); + DirichletLinearOperatorF DirichletDenominatorLinOpF(DirichletDenominatorF); + DirichletLinearOperatorDagD DirichletDenominatorLinOpDagD(DirichletDenominator); + DirichletLinearOperatorDagF DirichletDenominatorLinOpDagF(DirichletDenominatorF); + + PeriodicLinearOperatorF PeriodicNumeratorLinOpF(PeriNumeratorF); + PeriodicLinearOperatorD PeriodicNumeratorLinOpD(PeriNumerator); + PeriodicLinearOperatorDagF PeriodicNumeratorLinOpDagF(PeriNumeratorF); + PeriodicLinearOperatorDagD PeriodicNumeratorLinOpDagD(PeriNumerator); + + PeriodicLinearOperatorF PeriodicDenominatorLinOpF(PeriDenominatorF); + PeriodicLinearOperatorD PeriodicDenominatorLinOpD(PeriDenominator); + PeriodicLinearOperatorDagF PeriodicDenominatorLinOpDagF(PeriDenominatorF); + PeriodicLinearOperatorDagD PeriodicDenominatorLinOpDagD(PeriDenominator); + + ConjugateGradient OmegaSolverCG (ActionStoppingCondition,MaxCGIterations); + ConjugateGradient OmegaDagSolverCG(ActionStoppingCondition,MaxCGIterations); + ConjugateGradient DSolverCG (ActionStoppingCondition,MaxCGIterations); + ConjugateGradient DdagSolverCG (ActionStoppingCondition,MaxCGIterations); + + DirichletMxPCG NumeratorOmegaSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + DirichletNumeratorF,DirichletNumerator, + DirichletNumeratorLinOpF, DirichletNumeratorLinOpD); + + DirichletMxDagPCG NumeratorOmegaDagSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + DirichletNumeratorF,DirichletNumerator, + DirichletNumeratorLinOpDagF,DirichletNumeratorLinOpDagD); + + PeriodicMxPCG NumeratorDSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + PeriNumeratorF,PeriNumerator, + PeriodicNumeratorLinOpF, PeriodicNumeratorLinOpD); + + PeriodicMxDagPCG NumeratorDdagSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + PeriNumeratorF,PeriNumerator, + PeriodicNumeratorLinOpDagF, PeriodicNumeratorLinOpDagD); + + SchurFactoredFermionOperator BoundaryNumerator(PeriNumerator, + DirichletNumerator, + NumeratorOmegaSolver,NumeratorOmegaDagSolver, + NumeratorDSolver,NumeratorDdagSolver, + // ActionCG,ActionCG, + // ActionCG,ActionCG, + Block); + + DirichletMxPCG DenominatorOmegaSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + DirichletDenominatorF,DirichletDenominator, + DirichletDenominatorLinOpF, DirichletDenominatorLinOpD); + + DirichletMxDagPCG DenominatorOmegaDagSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + DirichletDenominatorF,DirichletDenominator, + DirichletDenominatorLinOpDagF,DirichletDenominatorLinOpDagD); + + PeriodicMxPCG DenominatorDSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + PeriDenominatorF,PeriDenominator, + PeriodicDenominatorLinOpF, PeriodicDenominatorLinOpD); + + PeriodicMxDagPCG DenominatorDdagSolver(ActionStoppingCondition, + MX_inner, + MaxCGIterations, + GridPtrF, + FrbGridF, + PeriDenominatorF,PeriDenominator, + PeriodicDenominatorLinOpDagF, PeriodicDenominatorLinOpDagD); + + SchurFactoredFermionOperator BoundaryDenominator(PeriDenominator, + DirichletDenominator, + //ActionCG,ActionCG, + //ActionCG,ActionCG, + DenominatorOmegaSolver,DenominatorOmegaDagSolver, + DenominatorDSolver,DenominatorDdagSolver, + Block); + Level1.push_back(new + DomainDecomposedBoundaryTwoFlavourRatioPseudoFermion + (BoundaryNumerator, + BoundaryDenominator)); + + ///////////////////////////////////////////////////////////// + // Gauge action + ///////////////////////////////////////////////////////////// + Level2.push_back(&GaugeAction); + TheHMC.TheAction.push_back(Level1); + TheHMC.TheAction.push_back(Level2); + std::cout << GridLogMessage << " Action complete "<< std::endl; + + ///////////////////////////////////////////////////////////// + // HMC parameters are serialisable + std::cout << GridLogMessage << " Running the HMC "<< std::endl; + TheHMC.Run(); // no smearing + + Grid_finalize(); +} // main + + +