Adding HMC test file example for Mobius + smearing

benchmarks/Benchmark_dwf.cc

@@ -1,9 +1,6 @@
  /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./benchmarks/Benchmark_dwf.cc
-
     Copyright (C) 2015
 
     Author: Peter Boyle <paboyle@ph.ed.ac.uk>
@@ -13,16 +10,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -97,7 +91,6 @@ int main (int argc, char ** argv)
 
   LatticeFermion result(FGrid); result=zero;
   LatticeFermion    ref(FGrid);    ref=zero;
-  LatticeFermion    refDag(FGrid);    refDag=zero;
   LatticeFermion    tmp(FGrid);
   LatticeFermion    err(FGrid);
 
@@ -167,7 +160,6 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
 
   DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
-
   int ncall =1000;
   if (1) {
     FGrid->Barrier();
@@ -207,8 +199,6 @@ int main (int argc, char ** argv)
     Dw.Report();
   }
 
-
-
   DomainWallFermionRL DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
   if (1) {
     FGrid->Barrier();
@@ -275,7 +265,7 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "Called Dw s_inner "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
     std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
     std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
-    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
+    //    std::cout<<GridLogMessage<< "res norms "<< norm2(result)<<" " <<norm2(sresult)<<std::endl;
     sDw.Report();
     RealD sum=0;
 
@@ -288,6 +278,7 @@ int main (int argc, char ** argv)
       std::cout<< "sD REF\n " <<ref << std::endl;
       std::cout<< "sD ERR   \n " <<err  <<std::endl;
     }
+    //    assert(sum < 1.0e-4);
 
     err=zero;
     localConvert(sresult,err);
@@ -300,23 +291,6 @@ int main (int argc, char ** argv)
     }
     assert(sum < 1.0e-4);
     
-    
-    // Check Dag
-    std::cout << GridLogMessage << "Compare WilsonFermion5D<DomainWallVec5dImplR>::Dhop to naive wilson implementation Dag to verify correctness" << std::endl;
-    sDw.Dhop(ssrc,sresult,1);
-    err=zero;
-    localConvert(sresult,err);
-    err = err - refDag;
-    sum = norm2(err);
-    std::cout<<GridLogMessage<<" difference between normal dag ref and simd is "<<sum<<std::endl;
-    if(sum > 1.0e-4 ){
-      std::cout<< "sD REF\n " <<result << std::endl;
-      std::cout<< "sD ERR   \n " << err  <<std::endl;
-    }
-    assert(sum < 1.0e-4);
-    
-    
-    
     if(1){
       std::cout << GridLogMessage<< "*********************************************************" <<std::endl;
       std::cout << GridLogMessage<< "* Benchmarking WilsonFermion5D<DomainWallVec5dImplR>::DhopEO "<<std::endl;
@@ -371,17 +345,15 @@ int main (int argc, char ** argv)
       pickCheckerboard(Even,ssrc_e,sresult);
       pickCheckerboard(Odd ,ssrc_o,sresult);
 
-      // Check even part
       ssrc_e = ssrc_e - sr_e;
       RealD error = norm2(ssrc_e);
-      std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm: "<<norm2(sr_e) <<std::endl;
+      std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm"<<norm2(sr_e) <<std::endl;
 
-      // Check odd part
       ssrc_o = ssrc_o - sr_o;
       error+= norm2(ssrc_o);
-      std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm: "<<norm2(sr_o) <<std::endl;
+      std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm"<<norm2(sr_o) <<std::endl;
 
-      if(error>1.0e-4) { 
+      if(( error>1.0e-4) ) { 
 	setCheckerboard(ssrc,ssrc_o);
 	setCheckerboard(ssrc,ssrc_e);
 	std::cout<< "DIFF\n " <<ssrc << std::endl;
@@ -391,37 +363,14 @@ int main (int argc, char ** argv)
 	std::cout<< "RESULT\n " <<sresult<< std::endl;
       }
       assert(error<1.0e-4);
-      
-      // Check the dag
-      std::cout << GridLogMessage << "Compare WilsonFermion5D<DomainWallVec5dImplR>::DhopEO to Dhop to verify correctness" << std::endl;
-      pickCheckerboard(Even,ssrc_e,ssrc);
-      pickCheckerboard(Odd,ssrc_o,ssrc);
-      sDw.DhopEO(ssrc_o,sr_e,DaggerYes);
-      sDw.DhopOE(ssrc_e,sr_o,DaggerYes);
-      sDw.Dhop  (ssrc  ,sresult,DaggerYes);
-      
-      pickCheckerboard(Even,ssrc_e,sresult);
-      pickCheckerboard(Odd ,ssrc_o,sresult);
-
-      ssrc_e = ssrc_e - sr_e;
-      error = norm2(ssrc_e);
-      std::cout<<GridLogMessage << "sE norm diff   "<< norm2(ssrc_e)<< "  vec nrm: "<<norm2(sr_e) <<std::endl;
-      ssrc_o = ssrc_o - sr_o;
-      error+= norm2(ssrc_o);
-      std::cout<<GridLogMessage << "sO norm diff   "<< norm2(ssrc_o)<< "  vec nrm: "<<norm2(sr_o) <<std::endl;
-
-      if(error>1.0e-4) { 
-	setCheckerboard(ssrc,ssrc_o);
-	setCheckerboard(ssrc,ssrc_e);
-	std::cout<< ssrc << std::endl;
-      } 
-
     }
   }
 
-  if (1) { // Naive wilson dag implementation
+  if (1)
+  { // Naive wilson dag implementation
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
+
       //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu+1,1);
       for(int i=0;i<ref._odata.size();i++){
@@ -436,9 +385,9 @@ int main (int argc, char ** argv)
     }
     ref = -0.5*ref;
   }
-  
+  //  dump=1;
   Dw.Dhop(src,result,1);
-  std::cout << GridLogMessage << "Compare DomainWallFermionR::Dhop to naive wilson implementation Dag to verify correctness" << std::endl;
+  std::cout << GridLogMessage << "Compare to naive wilson implementation Dag to verify correctness" << std::endl;
   std::cout<<GridLogMessage << "Called DwDag"<<std::endl;
   std::cout<<GridLogMessage << "norm dag result "<< norm2(result)<<std::endl;
   std::cout<<GridLogMessage << "norm dag ref    "<< norm2(ref)<<std::endl;

lib/qcd/action/ActionParams.h

@@ -44,7 +44,12 @@ namespace QCD {
   
   struct WilsonImplParams {
     bool overlapCommsCompute;
-    WilsonImplParams() : overlapCommsCompute(false){};
+    std::vector<Complex> boundary_phases;
+    WilsonImplParams() : overlapCommsCompute(false) {
+      boundary_phases.resize(Nd, 1.0);
+    };
+    WilsonImplParams(const std::vector<Complex> phi)
+      : boundary_phases(phi), overlapCommsCompute(false) {}
   };
 
   struct StaggeredImplParams {

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -198,7 +198,9 @@ namespace QCD {
     
     ImplParams Params;
     
-    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
+    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){
+      assert(Params.boundary_phases.size() == Nd);
+    };
       
     bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
       
@@ -222,10 +224,16 @@ namespace QCD {
       conformable(Uds._grid, GaugeGrid);
       conformable(Umu._grid, GaugeGrid);
       GaugeLinkField U(GaugeGrid);
+      GaugeLinkField tmp(GaugeGrid);
+      Lattice<iScalar<vInteger> > coor(GaugeGrid);
       for (int mu = 0; mu < Nd; mu++) {
+        LatticeCoordinate(coor, mu);
+        int Lmu = GaugeGrid->GlobalDimensions()[mu] - 1;
         U = PeekIndex<LorentzIndex>(Umu, mu);
-        PokeIndex<LorentzIndex>(Uds, U, mu);
+        tmp = where(coor == Lmu, Params.boundary_phases[mu] * U, U);
+        PokeIndex<LorentzIndex>(Uds, tmp, mu);
         U = adj(Cshift(U, mu, -1));
+        U = where(coor == 0, Params.boundary_phases[mu] * U, U);
         PokeIndex<LorentzIndex>(Uds, U, mu + 4);
       }
     }

tests/hmc/Test_hmc_EOMobiusRatio.cc

@@ -0,0 +1,155 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_EODWFRatio.cc
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+  using namespace Grid::QCD;
+
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;  // Uses the default minimum norm
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionR FermionAction;
+  typedef typename FermionAction::FermionField FermionField;
+
+  // Apply smearing to the fermionic action
+  bool ApplySmearing = false;
+  
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  HMCWrapper TheHMC;
+
+  // Grid from the command line
+  TheHMC.Resources.AddFourDimGrid("gauge");
+  // Possibile to create the module by hand 
+  // hardcoding parameters or using a Reader
+
+
+  // Checkpointer definition
+  CheckpointerParameters CPparams;  
+  CPparams.config_prefix = "ckpoint_EODWF_lat";
+  CPparams.rng_prefix = "ckpoint_EODWF_rng";
+  CPparams.saveInterval = 5;
+  CPparams.format = "IEEE64BIG";
+  
+  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+
+  RNGModuleParameters RNGpar;
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection 
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  /////////////////////////////////////////////////////////////
+  // Collect actions, here use more encapsulation
+  // need wrappers of the fermionic classes 
+  // that have a complex construction
+  // standard
+  RealD beta = 5.6 ;
+  WilsonGaugeActionR Waction(beta);
+    
+
+  const int Ls = 8;
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+
+  Real mass = 0.04;
+  Real pv   = 1.0;
+  RealD M5  = 1.5;
+  // Note: IroIro and Grid notation for b and c differ
+  RealD b   = 3./2.;
+  RealD c   = 1./2.;
+
+  FermionAction DenOp(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,mass,M5,b,c);
+  FermionAction NumOp(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,pv,  M5,b,c);
+
+  double StoppingCondition = 1.0e-8;
+  double MaxCGIterations = 10000;
+  ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
+  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2(NumOp, DenOp,CG,CG);
+
+  // Set smearing (true/false), default: false
+  Nf2.is_smeared = ApplySmearing;
+  
+  // Collect actions
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  Level1.push_back(&Nf2);
+
+  ActionLevel<HMCWrapper::Field> Level2(4);
+  Level2.push_back(&Waction);
+
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+
+  /////////////////////////////////////////////////////////////
+  // HMC parameters are serialisable 
+  TheHMC.Parameters.MD.MDsteps = 20;
+  TheHMC.Parameters.MD.trajL   = 1.0;
+
+  TheHMC.ReadCommandLine(argc, argv); // these can be parameters from file
+  // Reset performance counters 
+  NumOp.ZeroCounters();
+  DenOp.ZeroCounters();
+
+  if (ApplySmearing){
+    double rho = 0.1;  // smearing parameter
+    int Nsmear = 3;    // number of smearing levels
+    Smear_Stout<HMCWrapper::ImplPolicy> Stout(rho);
+    SmearedConfiguration<HMCWrapper::ImplPolicy> SmearingPolicy(GridPtr, Nsmear, Stout);
+    TheHMC.Run(SmearingPolicy); // for smearing
+  } else {
+    TheHMC.Run();  // no smearing
+  }
+
+  std::cout << GridLogMessage << "Numerator report, Pauli-Villars term         : " << std::endl;
+  NumOp.Report();
+  std::cout << GridLogMessage << "Denominator report, Dw(m) term (includes CG) : " << std::endl;
+  DenOp.Report();
+
+  Grid_finalize();
+} // main
+