Merge branch 'develop' of https://github.com/paboyle/Grid into develop

.gitignore

@@ -92,6 +92,7 @@ build*/*
 #####################
 *.xcodeproj/*
 build.sh
+.vscode
 
 # Eigen source #
 ################
@@ -106,6 +107,10 @@ lib/fftw/*
 m4/lt*
 m4/libtool.m4
 
+# github pages #
+################
+gh-pages/
+
 # Buck files #
 ##############
 .buck*
@@ -117,3 +122,4 @@ make-bin-BUCK.sh
 #####################
 lib/qcd/spin/gamma-gen/*.h
 lib/qcd/spin/gamma-gen/*.cc
+

benchmarks/Benchmark_dwf.cc

@@ -1,9 +1,6 @@
  /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./benchmarks/Benchmark_dwf.cc
-
     Copyright (C) 2015
 
     Author: Peter Boyle <paboyle@ph.ed.ac.uk>
@@ -13,16 +10,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -498,3 +492,4 @@ int main (int argc, char ** argv)
 
   Grid_finalize();
 }
+

bootstrap.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+]#!/usr/bin/env bash
 
 EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
 

configure.ac

@@ -9,6 +9,9 @@ AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h],[sed -i 's|PACKAGE_|GRID_|' lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
+################ Get git info
+#AC_REVISION([m4_esyscmd_s([./scripts/configure.commit])])
+
 ############### Checks for programs
 CXXFLAGS="-O3 $CXXFLAGS"
 AC_PROG_CXX
@@ -67,6 +70,13 @@ AC_ARG_WITH([fftw],
             [AM_CXXFLAGS="-I$with_fftw/include $AM_CXXFLAGS"]
             [AM_LDFLAGS="-L$with_fftw/lib $AM_LDFLAGS"])
 
+############### LIME
+AC_ARG_WITH([lime],
+            [AS_HELP_STRING([--with-lime=prefix],
+            [try this for a non-standard install prefix of the LIME library])],
+            [AM_CXXFLAGS="-I$with_lime/include $AM_CXXFLAGS"]
+            [AM_LDFLAGS="-L$with_lime/lib $AM_LDFLAGS"])
+
 ############### lapack
 AC_ARG_ENABLE([lapack],
     [AC_HELP_STRING([--enable-lapack=yes|no|prefix], [enable LAPACK])],
@@ -164,6 +174,14 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
                [have_fftw=true])
 
+AC_SEARCH_LIBS([limeCreateReader], [lime],
+               [AC_DEFINE([HAVE_LIME], [1], [Define to 1 if you have the `LIME' library])]
+               [have_lime=true],
+	       [AC_MSG_WARN(C-LIME library was not found in your system.
+In order to use ILGG file format please install or provide the correct path to your installation
+Info at: http://usqcd.jlab.org/usqcd-docs/c-lime/)])
+
+
 AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
                [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
                [have_hdf5=true]
@@ -421,10 +439,13 @@ AC_SUBST([GRID_LIBS])
 AC_SUBST([GRID_SHA])
 AC_SUBST([GRID_BRANCH])
 
+git_commit=`cd $srcdir && ./scripts/configure.commit`
+
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Summary of configuration for $PACKAGE v$VERSION
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-git: $GRID_BRANCH $GRID_SHORT_SHA
+----- GIT VERSION -------------------------------------
+$git_commit
 ----- PLATFORM ----------------------------------------
 architecture (build)        : $build_cpu
 os (build)                  : $build_os
@@ -442,6 +463,7 @@ RNG choice                  : ${ac_RNG}
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 LAPACK                      : ${ac_LAPACK}
 FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+LIME (ILDG support)         : `if test "x$have_lime" = xtrue; then echo yes; else echo no; fi`
 HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
 build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
@@ -451,9 +473,9 @@ LDFLAGS:
 `echo ${AM_LDFLAGS} ${LDFLAGS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 LIBS:
 `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
--------------------------------------------------------" > config.summary
+-------------------------------------------------------" > grid.configure.summary
 
-GRID_SUMMARY="`cat config.summary`"
+GRID_SUMMARY="`cat grid.configure.summary`"
 AM_SUBST_NOTMAKE([GRID_SUMMARY])
 AC_SUBST([GRID_SUMMARY])
 
@@ -476,5 +498,6 @@ AC_CONFIG_FILES(extras/Hadrons/Makefile)
 AC_OUTPUT
 
 echo ""
-cat config.summary
+cat grid.configure.summary
 echo ""
+

lib/DisableWarnings.h

@@ -0,0 +1,37 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/DisableWarnings.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef DISABLE_WARNINGS_H
+#define DISABLE_WARNINGS_H
+
+ //disables and intel compiler specific warning (in json.hpp)
+#pragma warning disable 488  
+
+
+#endif

lib/algorithms/approx/Chebyshev.h

@@ -197,8 +197,9 @@ namespace Grid {
     void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
 
       GridBase *grid=in._grid;
+
       // std::cout << "Chevyshef(): in._grid="<<in._grid<<std::endl;
-//<<" Linop.Grid()="<<Linop.Grid()<<"Linop.RedBlackGrid()="<<Linop.RedBlackGrid()<<std::endl;
+      //std::cout <<" Linop.Grid()="<<Linop.Grid()<<"Linop.RedBlackGrid()="<<Linop.RedBlackGrid()<<std::endl;
 
       int vol=grid->gSites();
 

lib/algorithms/iterative/ConjugateGradient.h

@@ -123,8 +123,11 @@ class ConjugateGradient : public OperatorFunction<Field> {
       p = p * b + r;
 
       LinalgTimer.Stop();
+
       std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
                 << " residual " << cp << " target " << rsq << std::endl;
+      std::cout << GridLogDebug << "a = "<< a << " b_pred = "<< b_pred << "  b = "<< b << std::endl;
+      std::cout << GridLogDebug << "qq = "<< qq << " d = "<< d << "  c = "<< c << std::endl;
 
       // Stopping condition
       if (cp <= rsq) {
@@ -157,8 +160,10 @@ class ConjugateGradient : public OperatorFunction<Field> {
     }
     std::cout << GridLogMessage << "ConjugateGradient did NOT converge"
               << std::endl;
+
     if (ErrorOnNoConverge) assert(0);
     IterationsToComplete = k;
+
   }
 };
 }

lib/cartesian/Cartesian_base.h

@@ -8,6 +8,7 @@
 
     Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -121,6 +122,12 @@ public:
       Lexicographic::CoorFromIndex(coor,Oindex,_rdimensions);
     }
 
+    inline void InOutCoorToLocalCoor (std::vector<int> &ocoor, std::vector<int> &icoor, std::vector<int> &lcoor) {
+      lcoor.resize(_ndimension);
+      for (int d = 0; d < _ndimension; d++)
+        lcoor[d] = ocoor[d] + _rdimensions[d] * icoor[d];
+    }
+
     //////////////////////////////////////////////////////////
     // SIMD lane addressing
     //////////////////////////////////////////////////////////
@@ -128,6 +135,7 @@ public:
     {
       Lexicographic::CoorFromIndex(coor,lane,_simd_layout);
     }
+
     inline int PermuteDim(int dimension){
       return _simd_layout[dimension]>1;
     }
@@ -173,6 +181,24 @@ public:
     inline const std::vector<int> &LocalDimensions(void)        { return _ldimensions;};
     inline const std::vector<int> &VirtualLocalDimensions(void) { return _ldimensions;};
 
+    ////////////////////////////////////////////////////////////////
+    // Utility to print the full decomposition details 
+    ////////////////////////////////////////////////////////////////
+
+    void show_decomposition(){
+      std::cout << GridLogMessage << "Full Dimensions    : " << _fdimensions << std::endl;
+      std::cout << GridLogMessage << "Global Dimensions  : " << _gdimensions << std::endl;
+      std::cout << GridLogMessage << "Local Dimensions   : " << _ldimensions << std::endl;
+      std::cout << GridLogMessage << "Reduced Dimensions : " << _rdimensions << std::endl;
+      std::cout << GridLogMessage << "Outer strides      : " << _ostride << std::endl;
+      std::cout << GridLogMessage << "Inner strides      : " << _istride << std::endl;
+      std::cout << GridLogMessage << "iSites             : " << _isites << std::endl;
+      std::cout << GridLogMessage << "oSites             : " << _osites << std::endl;
+      std::cout << GridLogMessage << "lSites             : " << lSites() << std::endl;        
+      std::cout << GridLogMessage << "gSites             : " << gSites() << std::endl;
+      std::cout << GridLogMessage << "Nd                 : " << _ndimension << std::endl;             
+    } 
+
     ////////////////////////////////////////////////////////////////
     // Global addressing
     ////////////////////////////////////////////////////////////////
@@ -184,6 +210,9 @@ public:
       assert(lidx<lSites());
       Lexicographic::CoorFromIndex(lcoor,lidx,_ldimensions);
     }
+
+
+
     void GlobalCoorToGlobalIndex(const std::vector<int> & gcoor,int & gidx){
       gidx=0;
       int mult=1;

lib/lattice/Lattice_base.h

@@ -261,12 +261,22 @@ public:
   
   virtual ~Lattice(void) = default;
     
+  void reset(GridBase* grid) {
+    if (_grid != grid) {
+      _grid = grid;
+      _odata.resize(grid->oSites());
+      checkerboard = 0;
+    }
+  }
+  
+
   template<class sobj> strong_inline Lattice<vobj> & operator = (const sobj & r){
     parallel_for(int ss=0;ss<_grid->oSites();ss++){
       this->_odata[ss]=r;
     }
     return *this;
   }
+  
   template<class robj> strong_inline Lattice<vobj> & operator = (const Lattice<robj> & r){
     this->checkerboard = r.checkerboard;
     conformable(*this,r);

lib/lattice/Lattice_reduction.h

@@ -1,29 +1,21 @@
  /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./lib/lattice/Lattice_reduction.h
-
     Copyright (C) 2015
-
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
-
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -404,7 +396,6 @@ static void sliceMaddVectorSlow (Lattice<vobj> &R,std::vector<RealD> &a,const La
     InsertSlice(Rslice,R,i,Orthog);
   }
 };
-
 template<class vobj>
 static void sliceInnerProductVectorSlow( std::vector<ComplexD> & vec, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog) 
   {
@@ -418,11 +409,9 @@ static void sliceInnerProductVectorSlow( std::vector<ComplexD> & vec, const Latt
     typedef typename scalar::scalar_object  scomplex;
   
     int Nblock = lhs._grid->GlobalDimensions()[Orthog];
-
     vec.resize(Nblock);
     std::vector<scomplex> sip(Nblock);
     Lattice<scalar> IP(lhs._grid); 
-
     IP=localInnerProduct(lhs,rhs);
     sliceSum(IP,sip,Orthog);
   
@@ -528,3 +517,5 @@ static void sliceInnerProductMatrix(  Eigen::MatrixXcd &mat, const Lattice<vobj>
 } /*END NAMESPACE GRID*/
 #endif
 
+
+

lib/lattice/Lattice_rng.h

@@ -7,7 +7,7 @@
     Copyright (C) 2015
 
     Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -75,6 +75,55 @@ namespace Grid {
     return multiplicity;
   }
 
+  
+// merge of April 11 2017
+//<<<<<<< HEAD
+
+
+  // this function is necessary for the LS vectorised field
+  inline int RNGfillable_general(GridBase *coarse,GridBase *fine)
+  {
+    int rngdims = coarse->_ndimension;
+    
+    // trivially extended in higher dims, with locality guaranteeing RNG state is local to node
+    int lowerdims   = fine->_ndimension - coarse->_ndimension;  assert(lowerdims >= 0);
+    // assumes that the higher dimensions are not using more processors
+    // all further divisions are local
+    for(int d=0;d<lowerdims;d++) assert(fine->_processors[d]==1);
+    for(int d=0;d<rngdims;d++) assert(coarse->_processors[d] == fine->_processors[d+lowerdims]);
+    
+
+    // then divide the number of local sites
+    // check that the total number of sims agree, meanse the iSites are the same
+    assert(fine->Nsimd() == coarse->Nsimd());
+
+    // check that the two grids divide cleanly
+    assert( (fine->lSites() / coarse->lSites() ) * coarse->lSites() == fine->lSites() );
+
+    return fine->lSites() / coarse->lSites();
+  }
+
+  /*
+  // Wrap seed_seq to give common interface with random_device
+  class fixedSeed {
+  public:
+    typedef std::seed_seq::result_type result_type;
+    std::seed_seq src;
+    
+    fixedSeed(const std::vector<int> &seeds) : src(seeds.begin(),seeds.end()) {};
+
+    result_type operator () (void){
+      std::vector<result_type> list(1);
+      src.generate(list.begin(),list.end());
+      return list[0];
+    }
+
+  };
+
+=======
+>>>>>>> develop
+  */
+  
   // real scalars are one component
   template<class scalar,class distribution,class generator> 
   void fillScalar(scalar &s,distribution &dist,generator & gen)
@@ -284,10 +333,12 @@ namespace Grid {
   };
 
   class GridParallelRNG : public GridRNGbase {
+
+    double _time_counter;
+
   public:
     GridBase *_grid;
-    int _vol;
+    unsigned int _vol;
-  public:
 
     int generator_idx(int os,int is) {
       return is*_grid->oSites()+os;
@@ -310,32 +361,33 @@ namespace Grid {
       typedef typename vobj::scalar_type scalar_type;
       typedef typename vobj::vector_type vector_type;
 
-      int multiplicity = RNGfillable(_grid,l._grid);
+      double inner_time_counter = usecond();
 
-      int     Nsimd =_grid->Nsimd();
+      int multiplicity = RNGfillable_general(_grid, l._grid); // l has finer or same grid
-      int     osites=_grid->oSites();
+      int Nsimd  = _grid->Nsimd();  // guaranteed to be the same for l._grid too
+      int osites = _grid->oSites();  // guaranteed to be <= l._grid->oSites() by a factor multiplicity
       int words  = sizeof(scalar_object) / sizeof(scalar_type);
 
       parallel_for(int ss=0;ss<osites;ss++){
-
         std::vector<scalar_object> buf(Nsimd);
         for (int m = 0; m < multiplicity; m++) {  // Draw from same generator multiplicity times
 
           int sm = multiplicity * ss + m;  // Maps the generator site to the fine site
 
           for (int si = 0; si < Nsimd; si++) {
+            
             int gdx = generator_idx(ss, si);  // index of generator state
             scalar_type *pointer = (scalar_type *)&buf[si];
             dist[gdx].reset();
-	    for(int idx=0;idx<words;idx++){
+            for (int idx = 0; idx < words; idx++) 
               fillScalar(pointer[idx], dist[gdx], _generators[gdx]);
           }
-	  }
+          // merge into SIMD lanes, FIXME suboptimal implementation
-
-	  // merge into SIMD lanes
           merge(l._odata[sm], buf);
         }
       }
+
+      _time_counter += usecond()- inner_time_counter;
     };
 
     void SeedFixedIntegers(const std::vector<int> &seeds){
@@ -412,6 +464,12 @@ namespace Grid {
       }
 #endif
     }
+
+    void Report(){
+      std::cout << GridLogMessage << "Time spent in the fill() routine by GridParallelRNG: "<< _time_counter/1e3 << " ms" << std::endl;
+    }
+
+
     ////////////////////////////////////////////////////////////////////////
     // Support for rigorous test of RNG's
     // Return uniform random uint32_t from requested site generator
@@ -419,7 +477,6 @@ namespace Grid {
     uint32_t GlobalU01(int gsite){
 
       uint32_t the_number;
-
       // who
       std::vector<int> gcoor;
       int rank,o_idx,i_idx;

lib/parallelIO/BinaryIO.h

@@ -7,7 +7,7 @@
     Copyright (C) 2015
 
     Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu<guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -30,6 +30,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define GRID_BINARY_IO_H
 
 
+#include "IldgIOtypes.h"
+
 #ifdef HAVE_ENDIAN_H
 #include <endian.h>
 #endif
@@ -150,6 +152,47 @@ class BinaryIO {
     }
   }
 
+  // Simple classes for precision conversion
+  template <class fobj, class sobj>
+  struct BinarySimpleUnmunger {
+    typedef typename getPrecision<fobj>::real_scalar_type fobj_stype;
+    typedef typename getPrecision<sobj>::real_scalar_type sobj_stype;
+
+    void operator()(sobj &in, fobj &out, uint32_t &csum) {
+      // take word by word and transform accoding to the status
+      fobj_stype *out_buffer = (fobj_stype *)&out;
+      sobj_stype *in_buffer = (sobj_stype *)&in;
+      size_t fobj_words = sizeof(out) / sizeof(fobj_stype);
+      size_t sobj_words = sizeof(in) / sizeof(sobj_stype);
+      assert(fobj_words == sobj_words);
+
+      for (unsigned int word = 0; word < sobj_words; word++)
+        out_buffer[word] = in_buffer[word];  // type conversion on the fly
+
+      BinaryIO::Uint32Checksum((uint32_t *)&out, sizeof(out), csum);
+    }
+  };
+
+  template <class fobj, class sobj>
+  struct BinarySimpleMunger {
+    typedef typename getPrecision<fobj>::real_scalar_type fobj_stype;
+    typedef typename getPrecision<sobj>::real_scalar_type sobj_stype;
+
+    void operator()(fobj &in, sobj &out, uint32_t &csum) {
+      // take word by word and transform accoding to the status
+      fobj_stype *in_buffer = (fobj_stype *)&in;
+      sobj_stype *out_buffer = (sobj_stype *)&out;
+      size_t fobj_words = sizeof(in) / sizeof(fobj_stype);
+      size_t sobj_words = sizeof(out) / sizeof(sobj_stype);
+      assert(fobj_words == sobj_words);
+
+      for (unsigned int word = 0; word < sobj_words; word++)
+        out_buffer[word] = in_buffer[word];  // type conversion on the fly
+
+      BinaryIO::Uint32Checksum((uint32_t *)&in, sizeof(in), csum);
+    }
+  };
+
   template<class vobj,class fobj,class munger>
   static inline uint32_t readObjectSerial(Lattice<vobj> &Umu,std::string file,munger munge,int offset,const std::string &format)
   {
@@ -246,7 +289,6 @@ class BinaryIO {
 
       
       if ( grid->IsBoss() ) {
-  
 	if(ieee32big) htobe32_v((void *)&file_object,sizeof(file_object));
 	if(ieee32)    htole32_v((void *)&file_object,sizeof(file_object));
 	if(ieee64big) htobe64_v((void *)&file_object,sizeof(file_object));
@@ -270,24 +312,29 @@ class BinaryIO {
     typedef typename GridSerialRNG::RngStateType RngStateType;
     const int RngStateCount = GridSerialRNG::RngStateCount;
 
-
     GridBase *grid = parallel._grid;
     int gsites = grid->_gsites;
 
+    GridStopWatch timer; timer.Start();
     //////////////////////////////////////////////////
     // Serialise through node zero
     //////////////////////////////////////////////////
-    std::cout<< GridLogMessage<< "Serial RNG write I/O "<< file<<std::endl;
-
     std::ofstream fout;
     if (grid->IsBoss()) {
-      fout.open(file,std::ios::binary|std::ios::out|std::ios::in);
+      fout.open(file, std::ios::binary | std::ios::out);
+      if (!fout.is_open()) {
+        std::cout << GridLogMessage << "writeRNGSerial: Error opening file " << file << std::endl;
+        exit(0);// write better error handling
+      }
       fout.seekp(offset);
     }
 
+    std::cout << GridLogMessage << "Serial RNG write I/O on file " << file << std::endl;
     uint32_t csum = 0;
     std::vector<RngStateType> saved(RngStateCount);
     int bytes = sizeof(RngStateType) * saved.size();
+    std::cout << GridLogDebug << "RngStateCount: " << RngStateCount << std::endl;
+    std::cout << GridLogDebug << "Type has " << bytes << " bytes" << std::endl;
     std::vector<int> gcoor;
 
     for(int gidx=0;gidx<gsites;gidx++){
@@ -301,7 +348,6 @@ class BinaryIO {
 	//  std::cout << "rank" << rank<<" Getting state for index "<<l_idx<<std::endl;
 	parallel.GetState(saved,l_idx);
       }
-
       grid->Broadcast(rank, (void *)&saved[0], bytes);
 
       if ( grid->IsBoss() ) {
@@ -316,9 +362,20 @@ class BinaryIO {
       Uint32Checksum((uint32_t *)&saved[0],bytes,csum);
       fout.write((char *)&saved[0],bytes);assert( fout.fail()==0);
     }
+
     grid->Broadcast(0, (void *)&csum, sizeof(csum));
+
+    if (grid->IsBoss()) 
+      fout.close();
+
+    timer.Stop();
+
+    std::cout << GridLogMessage   << "RNG file checksum " << std::hex << csum << std::dec << std::endl;
+    std::cout << GridLogMessage << "RNG state saved in " << timer.Elapsed() << std::endl;
     return csum;
   }
+
+
   static inline uint32_t readRNGSerial(GridSerialRNG &serial,GridParallelRNG &parallel,std::string file,int offset)
   {
     typedef typename GridSerialRNG::RngStateType RngStateType;
@@ -330,22 +387,35 @@ class BinaryIO {
     //////////////////////////////////////////////////
     // Serialise through node zero
     //////////////////////////////////////////////////
-    std::cout<< GridLogMessage<< "Serial RNG read I/O "<< file<<std::endl;
+    std::cout<< GridLogMessage<< "Serial RNG read I/O of file "<<file<<std::endl;
 
-    std::ifstream fin(file,std::ios::binary|std::ios::in);
+    std::ifstream fin;
+    if (grid->IsBoss()) {
+      fin.open(file, std::ios::binary | std::ios::in);
+      if (!fin.is_open()) {
+        std::cout << GridLogMessage << "readRNGSerial: Error opening file " << file << std::endl;
+        exit(0);// write better error handling
+      }
       fin.seekg(offset);
+    }
+
     
     uint32_t csum=0;
     std::vector<RngStateType> saved(RngStateCount);
     int bytes = sizeof(RngStateType)*saved.size();
+    std::cout << GridLogDebug << "RngStateCount: " << RngStateCount << std::endl;
+    std::cout << GridLogDebug << "Type has " << bytes << " bytes" << std::endl;
     std::vector<int> gcoor;
 
+    std::cout << GridLogDebug << "gsites: " << gsites << " loop" << std::endl;
     for(int gidx=0;gidx<gsites;gidx++){
 
       int rank,o_idx,i_idx;
       grid->GlobalIndexToGlobalCoor(gidx,gcoor);
       grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor);
       int l_idx=parallel.generator_idx(o_idx,i_idx);
+      //std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx
+      //          << " i_idx " << i_idx << " rank " << rank << std::endl;
 
       if ( grid->IsBoss() ) {
 	fin.read((char *)&saved[0],bytes);assert( fin.fail()==0);
@@ -357,7 +427,6 @@ class BinaryIO {
       if( rank == grid->ThisRank() ){
         parallel.SetState(saved,l_idx);
       }
-
     }
 
     if ( grid->IsBoss() ) {
@@ -366,6 +435,8 @@ class BinaryIO {
       Uint32Checksum((uint32_t *)&saved[0],bytes,csum);
     }
 
+    std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl;
+
     grid->Broadcast(0,(void *)&csum,sizeof(csum));
 
     return csum;
@@ -373,9 +444,12 @@ class BinaryIO {
 
 
   template <class vobj, class fobj, class munger>
-  static inline uint32_t readObjectParallel(Lattice<vobj> &Umu,std::string file,munger munge,int offset,
+  static inline uint32_t readObjectParallel(Lattice<vobj> &Umu,
-					    const std::string &format)
+                                            std::string file, 
-  {
+                                            munger munge,
+                                            int offset,
+                                            const std::string &format,
+                                            ILDGtype ILDG = ILDGtype()) {
     typedef typename vobj::scalar_object sobj;
 
     GridBase *grid = Umu._grid;
@@ -441,6 +515,7 @@ class BinaryIO {
     int myrank = grid->ThisRank();
     int iorank = grid->RankFromProcessorCoor(ioproc);
 
+    if (!ILDG.is_ILDG)
     	if ( IOnode ) { 
     		fin.open(file,std::ios::binary|std::ios::in);
     	}
@@ -451,7 +526,8 @@ class BinaryIO {
     // available (how short sighted is that?)
     //////////////////////////////////////////////////////////
     Umu = zero;
-    static uint32_t csum; csum=0;
+    static uint32_t csum; csum=0;//static for SHMEM
+
     fobj fileObj;
     static sobj siteObj; // Static to place in symmetric region for SHMEM
 
@@ -470,6 +546,7 @@ class BinaryIO {
 	gsite[d] = tsite[d]+start[d];               // global site
       }
 
+
       /////////////////////////
       // Get the rank of owner of data
       /////////////////////////
@@ -482,8 +559,18 @@ class BinaryIO {
       ////////////////////////////////
       if (myrank == iorank) {
 
+
+      	if (ILDG.is_ILDG){
+      		// use C-LIME to populate the record
+          #ifdef HAVE_LIME
+          uint64_t sizeFO = sizeof(fileObj);
+          limeReaderSeek(ILDG.LR, g_idx*sizeFO, SEEK_SET);
+          int status = limeReaderReadData((void *)&fileObj, &sizeFO, ILDG.LR);
+          #endif
+        } else{
           fin.seekg(offset+g_idx*sizeof(fileObj));
-	fin.read((char *)&fileObj,sizeof(fileObj));assert( fin.fail()==0);
+          fin.read((char *)&fileObj,sizeof(fileObj));
+        }
         bytes+=sizeof(fileObj);
 
         if(ieee32big) be32toh_v((void *)&fileObj,sizeof(fileObj));
@@ -515,7 +602,6 @@ class BinaryIO {
     timer.Stop();
     std::cout<<GridLogPerformance<<"readObjectParallel: read "<< bytes <<" bytes in "<<timer.Elapsed() <<" "
 	     << (double)bytes/timer.useconds() <<" MB/s "  <<std::endl;
-    
     return csum;
   }
 
@@ -523,9 +609,11 @@ class BinaryIO {
   // Parallel writer
   //////////////////////////////////////////////////////////
   template <class vobj, class fobj, class munger>
-  static inline uint32_t writeObjectParallel(Lattice<vobj> &Umu,std::string file,munger munge,int offset,
+  static inline uint32_t writeObjectParallel(Lattice<vobj> &Umu,
-					     const std::string & format)
+                                             std::string file, munger munge,
-  {
+                                             int offset,
+                                             const std::string &format,
+                                             ILDGtype ILDG = ILDGtype()) {
     typedef typename vobj::scalar_object sobj;
     GridBase *grid = Umu._grid;
 
@@ -534,6 +622,15 @@ class BinaryIO {
     int ieee64big = (format == std::string("IEEE64BIG"));
     int ieee64 = (format == std::string("IEEE64"));
 
+    if (!(ieee32big || ieee32 || ieee64big || ieee64)) {
+      std::cout << GridLogError << "Unrecognized file format " << format
+                << std::endl;
+      std::cout << GridLogError
+                << "Allowed: IEEE32BIG | IEEE32 | IEEE64BIG | IEEE64"
+                << std::endl;
+      exit(0);
+    }
+
     int nd = grid->_ndimension;
     for (int d = 0; d < nd; d++) {
       assert(grid->CheckerBoarded(d) == 0);
@@ -549,7 +646,6 @@ class BinaryIO {
     int IOnode = 1;
 
     for (int d = 0; d < grid->_ndimension; d++) {
-
       if (d != grid->_ndimension - 1) parallel[d] = 0;
 
       if (parallel[d]) {
@@ -570,7 +666,8 @@ class BinaryIO {
     {
       uint32_t tmp = IOnode;
       grid->GlobalSum(tmp);
-      std::cout<< GridLogMessage<< "Parallel write I/O from "<< file << " with " <<tmp<< " IOnodes for subslice ";
+      std::cout<< GridLogMessage<< "Parallel write I/O from "<< file
+	       << " with " <<tmp<< " IOnodes for subslice ";
       for(int d=0;d<grid->_ndimension;d++){
 	std::cout<< range[d];
 	if( d< grid->_ndimension-1 ) 
@@ -579,7 +676,8 @@ class BinaryIO {
       std::cout << std::endl;
     }
     
-    GridStopWatch timer; timer.Start();
+    GridStopWatch timer;
+    timer.Start();
     uint64_t bytes=0;
 
     int myrank = grid->ThisRank();
@@ -590,22 +688,33 @@ class BinaryIO {
     // Ideally one reader/writer per xy plane and read these contiguously
     // with comms from nominated I/O nodes.
     std::ofstream fout;
-    if ( IOnode ) fout.open(file,std::ios::binary|std::ios::in|std::ios::out);
+    if (!ILDG.is_ILDG)
+    	if (IOnode){
+    		fout.open(file, std::ios::binary | std::ios::in | std::ios::out);
+    		if (!fout.is_open()) {
+    			std::cout << GridLogMessage << "writeObjectParallel: Error opening file " << file
+    			<< std::endl;
+    			exit(0);
+    		}
+    	}
+
 
     //////////////////////////////////////////////////////////
     // Find the location of each site and send to primary node
-    // Take loop order from Chroma; defines loop order now that NERSC doc no longer
+    // Take loop order from Chroma; defines loop order now that NERSC doc no
+    // longer
     // available (how short sighted is that?)
     //////////////////////////////////////////////////////////
 
     uint32_t csum = 0;
     fobj fileObj;
-    static sobj siteObj; // static for SHMEM target; otherwise dynamic allocate with AlignedAllocator
+    static sobj siteObj;  // static for SHMEM target; otherwise dynamic allocate
+                          // with AlignedAllocator
 
     // should aggregate a whole chunk and then write.
-    // need to implement these loops in Nd independent way with a lexico conversion
+    // need to implement these loops in Nd independent way with a lexico
+    // conversion
     for (int tlex = 0; tlex < slice_vol; tlex++) {
-  
       std::vector<int> tsite(nd);  // temporary mixed up site
       std::vector<int> gsite(nd);
       std::vector<int> lsite(nd);
@@ -618,7 +727,6 @@ class BinaryIO {
 	gsite[d] = tsite[d] + start[d];               // global site
       }
 
-
       /////////////////////////
       // Get the rank of owner of data
       /////////////////////////
@@ -644,7 +752,6 @@ class BinaryIO {
       grid->Barrier();  // necessary?
 
       if (myrank == iorank) {
-  
         munge(siteObj, fileObj, csum);
 
         if (ieee32big) htobe32_v((void *)&fileObj, sizeof(fileObj));
@@ -652,8 +759,19 @@ class BinaryIO {
         if (ieee64big) htobe64_v((void *)&fileObj, sizeof(fileObj));
         if (ieee64) htole64_v((void *)&fileObj, sizeof(fileObj));
 
+
+        if (ILDG.is_ILDG) {
+          #ifdef HAVE_LIME
+          uint64_t sizeFO = sizeof(fileObj);
+ 					limeWriterSeek(ILDG.LW, g_idx*sizeFO, SEEK_SET);
+          int status = limeWriteRecordData((void *)&fileObj, &sizeFO, ILDG.LW);
+          #endif
+        } 
+
+        else {
           fout.seekp(offset + g_idx * sizeof(fileObj));
           fout.write((char *)&fileObj, sizeof(fileObj));assert( fout.fail()==0);
+        }
         bytes += sizeof(fileObj);
       }
     }
@@ -662,14 +780,20 @@ class BinaryIO {
     grid->GlobalSum(bytes);
     
     timer.Stop();
-    std::cout<<GridLogPerformance<<"writeObjectParallel: wrote "<< bytes <<" bytes in "<<timer.Elapsed() <<" "
+    std::cout << GridLogPerformance << "writeObjectParallel: wrote " << bytes
+              << " bytes in " << timer.Elapsed() << " "
               << (double)bytes / timer.useconds() << " MB/s " << std::endl;
 
+
+
+     grid->Barrier();  // necessary?
+     if (IOnode) 
+      fout.close();
+
+
     return csum;
   }
-
 };
-
 }
 
 #endif

lib/parallelIO/IldgIO.h

@@ -0,0 +1,251 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/parallelIO/IldgIO.h
+
+Copyright (C) 2015
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_ILDG_IO_H
+#define GRID_ILDG_IO_H
+
+#include <algorithm>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <map>
+
+#include <pwd.h>
+#include <sys/utsname.h>
+#include <unistd.h>
+
+#ifdef HAVE_LIME
+
+extern "C" {  // for linkage
+#include "lime.h"
+}
+
+namespace Grid {
+namespace QCD {
+
+inline void ILDGGrid(GridBase *grid, ILDGField &header) {
+  assert(grid->_ndimension == 4);  // emit error if not
+  header.dimension.resize(4);
+  header.boundary.resize(4);
+  for (int d = 0; d < 4; d++) {
+    header.dimension[d] = grid->_fdimensions[d];
+    // Read boundary conditions from ... ?
+    header.boundary[d] = std::string("periodic");
+  }
+}
+
+inline void ILDGChecksum(uint32_t *buf, uint32_t buf_size_bytes,
+                         uint32_t &csum) {
+  BinaryIO::Uint32Checksum(buf, buf_size_bytes, csum);
+}
+
+//////////////////////////////////////////////////////////////////////
+// Utilities ; these are QCD aware
+//////////////////////////////////////////////////////////////////////
+template <class GaugeField>
+inline void ILDGStatistics(GaugeField &data, ILDGField &header) {
+  // How to convert data precision etc...
+  header.link_trace = Grid::QCD::WilsonLoops<PeriodicGimplR>::linkTrace(data);
+  header.plaquette = Grid::QCD::WilsonLoops<PeriodicGimplR>::avgPlaquette(data);
+  // header.polyakov =
+}
+
+// Forcing QCD here
+template <class fobj, class sobj>
+struct ILDGMunger {
+  void operator()(fobj &in, sobj &out, uint32_t &csum) {
+    for (int mu = 0; mu < 4; mu++) {
+      for (int i = 0; i < 3; i++) {
+        for (int j = 0; j < 3; j++) {
+          out(mu)()(i, j) = in(mu)()(i, j);
+        }
+      }
+    }
+    ILDGChecksum((uint32_t *)&in, sizeof(in), csum);
+  };
+};
+
+template <class fobj, class sobj>
+struct ILDGUnmunger {
+  void operator()(sobj &in, fobj &out, uint32_t &csum) {
+    for (int mu = 0; mu < 4; mu++) {
+      for (int i = 0; i < 3; i++) {
+        for (int j = 0; j < 3; j++) {
+          out(mu)()(i, j) = in(mu)()(i, j);
+        }
+      }
+    }
+    ILDGChecksum((uint32_t *)&out, sizeof(out), csum);
+  };
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Write and read from fstream; compute header offset for payload
+////////////////////////////////////////////////////////////////////////////////
+enum ILDGstate {ILDGread, ILDGwrite};
+
+class ILDGIO : public BinaryIO {
+  FILE *File;
+  LimeWriter *LimeW;
+  LimeRecordHeader *LimeHeader;
+  LimeReader *LimeR;
+  std::string filename;
+
+
+ public:
+  ILDGIO(std::string file, ILDGstate RW) {
+      filename = file;
+    if (RW == ILDGwrite){
+      File = fopen(file.c_str(), "w");
+      // check if opened correctly
+
+      LimeW = limeCreateWriter(File);
+    } else {
+      File = fopen(file.c_str(), "r");
+      // check if opened correctly
+
+      LimeR = limeCreateReader(File);
+    }
+  }
+
+  ~ILDGIO() { fclose(File); }
+
+  int createHeader(std::string message, int MB, int ME, size_t PayloadSize, LimeWriter* L){
+    LimeRecordHeader *h;
+    h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
+    int status = limeWriteRecordHeader(h, L);
+    if (status < 0) {
+      std::cerr << "ILDG Header error\n";
+      return status;
+    }
+    limeDestroyHeader(h);
+    return LIME_SUCCESS;
+  }
+
+  unsigned int writeHeader(ILDGField &header) {
+    // write header in LIME
+    n_uint64_t nbytes;
+    int MB_flag = 1, ME_flag = 0;
+
+    char message[] = "ildg-format";
+    nbytes = strlen(message);
+    LimeHeader = limeCreateHeader(MB_flag, ME_flag, message, nbytes);
+    limeWriteRecordHeader(LimeHeader, LimeW);
+    limeDestroyHeader(LimeHeader);
+    // save the xml header here
+    // use the xml_writer to c++ streams in pugixml
+    // and convert to char message
+    limeWriteRecordData(message, &nbytes, LimeW);
+    limeWriterCloseRecord(LimeW);
+
+    return 0;
+  }
+
+  unsigned int readHeader(ILDGField &header) {
+    return 0;
+  }
+
+  template <class vsimd>
+  uint32_t readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu) {
+    typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
+    typedef LorentzColourMatrixD sobjd;
+    typedef LorentzColourMatrixF sobjf;
+    typedef iLorentzColourMatrix<vsimd> itype;
+    typedef LorentzColourMatrix sobj;
+    GridBase *grid = Umu._grid;
+
+    ILDGField header;
+    readHeader(header);
+
+    // now just the conf, ignore the header
+    std::string format = std::string("IEEE64BIG");
+    do {limeReaderNextRecord(LimeR);}
+    while (strncmp(limeReaderType(LimeR), "ildg-binary-data",16));
+
+    n_uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration)
+
+
+    ILDGtype ILDGt(true, LimeR);
+    // this is special for double prec data, just for the moment
+    uint32_t csum = BinaryIO::readObjectParallel< itype, sobjd >(
+       Umu, filename, ILDGMunger<sobjd, sobj>(), 0, format, ILDGt);
+
+    // Check configuration 
+    // todo
+
+    return csum;
+  }
+
+  template <class vsimd>
+  uint32_t writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu, std::string format) {
+    typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
+    typedef iLorentzColourMatrix<vsimd> vobj;
+    typedef typename vobj::scalar_object sobj;
+    typedef LorentzColourMatrixD fobj;
+
+    ILDGField header;
+    // fill the header
+    header.floating_point = format;
+
+    ILDGUnmunger<fobj, sobj> munge;
+    unsigned int offset = writeHeader(header);
+
+    BinaryIO::Uint32Checksum<vobj, fobj>(Umu, munge, header.checksum);
+
+    // Write data record header
+    n_uint64_t PayloadSize = sizeof(fobj) * Umu._grid->_gsites;
+    createHeader("ildg-binary-data", 0, 1, PayloadSize, LimeW);
+
+    ILDGtype ILDGt(true, LimeW);
+    uint32_t csum = BinaryIO::writeObjectParallel<vobj, fobj>(
+       Umu, filename, munge, 0, header.floating_point, ILDGt);
+
+    limeWriterCloseRecord(LimeW);
+
+    // Last record
+    // the logical file name LNF
+    // look into documentation on how to generate this string
+    std::string LNF = "empty"; 
+
+
+    PayloadSize = sizeof(LNF);
+    createHeader("ildg-binary-lfn", 1 , 1, PayloadSize, LimeW);
+    limeWriteRecordData(const_cast<char*>(LNF.c_str()), &PayloadSize, LimeW);
+
+    limeWriterCloseRecord(LimeW);
+
+    return csum;
+  }
+
+  // format for RNG? Now just binary out
+};
+}
+}
+
+//HAVE_LIME
+#endif
+
+#endif

lib/parallelIO/IldgIOtypes.h

@@ -0,0 +1,80 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/parallelIO/IldgIO.h
+
+Copyright (C) 2015
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_ILDGTYPES_IO_H
+#define GRID_ILDGTYPES_IO_H
+
+#ifdef HAVE_LIME
+extern "C" { // for linkage
+#include "lime.h"
+}
+
+namespace Grid {
+
+struct ILDGtype {
+  bool is_ILDG;
+  LimeWriter* LW;
+  LimeReader* LR;
+
+  ILDGtype(bool is, LimeWriter* L) : is_ILDG(is), LW(L), LR(NULL) {}
+  ILDGtype(bool is, LimeReader* L) : is_ILDG(is), LW(NULL), LR(L) {}
+  ILDGtype() : is_ILDG(false), LW(NULL), LR(NULL) {}
+};
+
+class ILDGField {
+ public:
+  // header strings (not in order)
+  std::vector<int> dimension;
+  std::vector<std::string> boundary;
+  int data_start;
+  std::string hdr_version;
+  std::string storage_format;
+  // Checks on data
+  double link_trace;
+  double plaquette;
+  uint32_t checksum;
+  unsigned int sequence_number;
+  std::string data_type;
+  std::string ensemble_id;
+  std::string ensemble_label;
+  std::string creator;
+  std::string creator_hardware;
+  std::string creation_date;
+  std::string archive_date;
+  std::string floating_point;
+};
+}
+#else
+namespace Grid {
+
+struct ILDGtype {
+  bool is_ILDG;
+  ILDGtype() : is_ILDG(false) {}
+};
+}
+
+#endif
+#endif

lib/parallelIO/NerscIO.h

@@ -145,14 +145,14 @@ inline void NerscMachineCharacteristics(NerscField &header)
 
     template<class fobj,class sobj>
     struct NerscSimpleMunger{
-
       void operator()(fobj &in, sobj &out, uint32_t &csum) {
-
+        for (int mu = 0; mu < Nd; mu++) {
-      for(int mu=0;mu<4;mu++){
+          for (int i = 0; i < Nc; i++) {
-      for(int i=0;i<3;i++){
+            for (int j = 0; j < Nc; j++) {
-      for(int j=0;j<3;j++){
               out(mu)()(i, j) = in(mu)()(i, j);
-      }}}
+            }
+          }
+        }
         NerscChecksum((uint32_t *)&in, sizeof(in), csum);
       };
     };
@@ -164,7 +164,9 @@ inline void NerscMachineCharacteristics(NerscField &header)
           for (int i = 0; i < Nc; i++) {
             for (int j = 0; j < Nc; j++) {
               out(mu)()(i, j) = in(mu)()(i, j);
-	}}}
+            }
+          }
+        }
         NerscChecksum((uint32_t *)&out, sizeof(out), csum);
       };
     };
@@ -420,8 +422,6 @@ static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu
 
 	typedef LorentzColourMatrixD fobj3D;
 	typedef LorentzColour2x3D    fobj2D;
-  typedef LorentzColourMatrixF fobj3f;
-  typedef LorentzColour2x3F    fobj2f;
   
 	GridBase *grid = Umu._grid;
 
@@ -446,7 +446,6 @@ static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu
 #else
 	  csum=BinaryIO::writeObjectSerial<vobj,fobj2D>(Umu,file,munge,offset,header.floating_point);
 #endif
-
 	} else { 
 	  header.floating_point = std::string("IEEE64BIG");
 	  header.data_type      = std::string("4D_SU3_GAUGE_3x3");
@@ -460,7 +459,7 @@ static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu
 #endif
 	}
 
-  std::cout<<GridLogMessage <<"Written NERSC Configuration "<<file<< " checksum "<<std::hex<<csum<< std::dec<<" plaq "<< header.plaquette <<std::endl;
+	std::cout<<GridLogMessage <<"Written NERSC Configuration on "<< file << " checksum "<<std::hex<<csum<< std::dec<<" plaq "<< header.plaquette <<std::endl;
 
       }
 

lib/qcd/LatticeTheories.h

@@ -0,0 +1,124 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/QCD.h
+
+Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: neo <cossu@post.kek.jp>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_LT_H
+#define GRID_LT_H
+namespace Grid{
+
+// First steps in the complete generalization of the Physics part
+// Design not final
+namespace LatticeTheories {
+
+template <int Dimensions>
+struct LatticeTheory {
+  static const int Nd = Dimensions;
+  static const int Nds = Dimensions * 2;  // double stored field
+  template <typename vtype>
+  using iSinglet = iScalar<iScalar<iScalar<vtype> > >;
+};
+
+template <int Dimensions, int Colours>
+struct LatticeGaugeTheory : public LatticeTheory<Dimensions> {
+  static const int Nds = Dimensions * 2;
+  static const int Nd = Dimensions;
+  static const int Nc = Colours;
+
+  template <typename vtype> 
+  using iColourMatrix = iScalar<iScalar<iMatrix<vtype, Nc> > >;
+  template <typename vtype>
+  using iLorentzColourMatrix = iVector<iScalar<iMatrix<vtype, Nc> >, Nd>;
+  template <typename vtype>
+  using iDoubleStoredColourMatrix = iVector<iScalar<iMatrix<vtype, Nc> >, Nds>;
+  template <typename vtype>
+  using iColourVector = iScalar<iScalar<iVector<vtype, Nc> > >;
+};
+
+template <int Dimensions, int Colours, int Spin>
+struct FermionicLatticeGaugeTheory
+    : public LatticeGaugeTheory<Dimensions, Colours> {
+  static const int Nd = Dimensions;
+  static const int Nds = Dimensions * 2;
+  static const int Nc = Colours;
+  static const int Ns = Spin;
+
+  template <typename vtype>
+  using iSpinMatrix = iScalar<iMatrix<iScalar<vtype>, Ns> >;
+  template <typename vtype>
+  using iSpinColourMatrix = iScalar<iMatrix<iMatrix<vtype, Nc>, Ns> >;
+  template <typename vtype>
+  using iSpinVector = iScalar<iVector<iScalar<vtype>, Ns> >;
+  template <typename vtype>
+  using iSpinColourVector = iScalar<iVector<iVector<vtype, Nc>, Ns> >;
+  // These 2 only if Spin is a multiple of 2
+  static const int Nhs = Spin / 2;
+  template <typename vtype>
+  using iHalfSpinVector = iScalar<iVector<iScalar<vtype>, Nhs> >;
+  template <typename vtype>
+  using iHalfSpinColourVector = iScalar<iVector<iVector<vtype, Nc>, Nhs> >;
+
+  //tests
+  typedef iColourMatrix<Complex> ColourMatrix;
+  typedef iColourMatrix<ComplexF> ColourMatrixF;
+  typedef iColourMatrix<ComplexD> ColourMatrixD;
+
+
+};
+
+// Examples, not complete now.
+struct QCD : public FermionicLatticeGaugeTheory<4, 3, 4> {
+    static const int Xp = 0;
+    static const int Yp = 1;
+    static const int Zp = 2;
+    static const int Tp = 3;
+    static const int Xm = 4;
+    static const int Ym = 5;
+    static const int Zm = 6;
+    static const int Tm = 7;
+
+    typedef FermionicLatticeGaugeTheory FLGT;
+
+    typedef FLGT::iSpinMatrix<Complex  >          SpinMatrix;
+    typedef FLGT::iSpinMatrix<ComplexF >          SpinMatrixF;
+    typedef FLGT::iSpinMatrix<ComplexD >          SpinMatrixD;
+
+};
+struct QED : public FermionicLatticeGaugeTheory<4, 1, 4> {//fill
+};
+
+template <int Dimensions>
+struct Scalar : public LatticeTheory<Dimensions> {};
+
+};  // LatticeTheories
+
+} // Grid
+
+#endif

lib/qcd/QCD.h

@@ -32,9 +32,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_QCD_BASE_H
 #define GRID_QCD_BASE_H
 namespace Grid{
-
 namespace QCD {
 
+    static const int Xdir = 0;
+    static const int Ydir = 1;
+    static const int Zdir = 2;
+    static const int Tdir = 3;
 
     static const int Xp = 0;
     static const int Yp = 1;
@@ -492,6 +495,38 @@ namespace QCD {
 }   //namespace QCD
 } // Grid
 
+/*
+<<<<<<< HEAD
+#include <Grid/qcd/utils/SpaceTimeGrid.h>
+#include <Grid/qcd/spin/Dirac.h>
+#include <Grid/qcd/spin/TwoSpinor.h>
+#include <Grid/qcd/utils/LinalgUtils.h>
+#include <Grid/qcd/utils/CovariantCshift.h>
+
+// Include representations  
+#include <Grid/qcd/utils/SUn.h>
+#include <Grid/qcd/utils/SUnAdjoint.h>
+#include <Grid/qcd/utils/SUnTwoIndex.h>
+#include <Grid/qcd/representations/hmc_types.h>
+
+// Scalar field
+#include <Grid/qcd/utils/ScalarObjs.h>
+
+#include <Grid/qcd/action/Actions.h>
+
+#include <Grid/qcd/smearing/Smearing.h>
+
+#include <Grid/qcd/hmc/integrators/Integrator.h>
+#include <Grid/qcd/hmc/integrators/Integrator_algorithm.h>
+#include <Grid/qcd/observables/hmc_observable.h>
+#include <Grid/qcd/hmc/HMC.h>
+
+
+//#include <Grid/qcd/modules/mods.h>
+=======
+
+>>>>>>> develop
+*/
 
 
 #endif

lib/qcd/action/ActionBase.h

@@ -4,10 +4,11 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: ./lib/qcd/action/ActionBase.h
 
-Copyright (C) 2015
+Copyright (C) 2015-2016
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: neo <cossu@post.kek.jp>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -27,128 +28,29 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#ifndef QCD_ACTION_BASE
+
-#define QCD_ACTION_BASE
+#ifndef ACTION_BASE_H
+#define ACTION_BASE_H
+
 namespace Grid {
 namespace QCD {
 
 template <class GaugeField >
-class Action {
+class Action 
+{
+
  public:
   bool is_smeared = false;
-  // Boundary conditions? // Heatbath?
+  // Heatbath?
-  virtual void refresh(const GaugeField& U,
+  virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG) = 0; // refresh pseudofermions
-                       GridParallelRNG& pRNG) = 0;  // refresh pseudofermions
   virtual RealD S(const GaugeField& U) = 0;                             // evaluate the action
-  virtual void deriv(const GaugeField& U,
+  virtual void deriv(const GaugeField& U, GaugeField& dSdU) = 0;        // evaluate the action derivative
-                     GaugeField& dSdU) = 0;  // evaluate the action derivative
+  virtual std::string action_name()    = 0;                             // return the action name
-  virtual ~Action(){};
+  virtual std::string LogParameters()  = 0;                             // prints action parameters
+  virtual ~Action(){}
 };
 
-// Indexing of tuple types
-template <class T, class Tuple>
-struct Index;
-
-template <class T, class... Types>
-struct Index<T, std::tuple<T, Types...>> {
-  static const std::size_t value = 0;
-};
-
-template <class T, class U, class... Types>
-struct Index<T, std::tuple<U, Types...>> {
-  static const std::size_t value = 1 + Index<T, std::tuple<Types...>>::value;
-};
-
-/*
-template <class GaugeField>
-struct ActionLevel {
- public:
-  typedef Action<GaugeField>*
-      ActPtr;  // now force the same colours as the rest of the code
-
-  //Add supported representations here
-
-
-  unsigned int multiplier;
-
-  std::vector<ActPtr> actions;
-
-  ActionLevel(unsigned int mul = 1) : actions(0), multiplier(mul) {
-    assert(mul >= 1);
-  };
-
-  void push_back(ActPtr ptr) { actions.push_back(ptr); }
-};
-*/
-
-template <class GaugeField, class Repr = NoHirep >
-struct ActionLevel {
- public:
-  unsigned int multiplier; 
-
-  // Fundamental repr actions separated because of the smearing
-  typedef Action<GaugeField>* ActPtr;
-
-  // construct a tuple of vectors of the actions for the corresponding higher
-  // representation fields
-  typedef typename AccessTypes<Action, Repr>::VectorCollection action_collection;
-  action_collection actions_hirep;
-  typedef typename  AccessTypes<Action, Repr>::FieldTypeCollection action_hirep_types;
-
-  std::vector<ActPtr>& actions;
-
-  // Temporary conversion between ActionLevel and ActionLevelHirep
-  //ActionLevelHirep(ActionLevel<GaugeField>& AL ):actions(AL.actions), multiplier(AL.multiplier){}
-
-  ActionLevel(unsigned int mul = 1) : actions(std::get<0>(actions_hirep)), multiplier(mul) {
-    // initialize the hirep vectors to zero.
-    //apply(this->resize, actions_hirep, 0); //need a working resize
-    assert(mul >= 1);
-  };
-
-  //void push_back(ActPtr ptr) { actions.push_back(ptr); }
-
-
-
-  template < class Field >
-  void push_back(Action<Field>* ptr) {
-    // insert only in the correct vector
-    std::get< Index < Field, action_hirep_types>::value >(actions_hirep).push_back(ptr);
-  };
-
- 
-
-  template < class ActPtr>
-  static void resize(ActPtr ap, unsigned int n){
-    ap->resize(n);
-
-  }
-
-  //template <std::size_t I>
-  //auto getRepresentation(Repr& R)->decltype(std::get<I>(R).U)  {return std::get<I>(R).U;}
-
-  // Loop on tuple for a callable function
-  template <std::size_t I = 1, typename Callable, typename ...Args>
-  inline typename std::enable_if<I == std::tuple_size<action_collection>::value, void>::type apply(
-      Callable, Repr& R,Args&...) const {}
-
-  template <std::size_t I = 1, typename Callable, typename ...Args>
-  inline typename std::enable_if<I < std::tuple_size<action_collection>::value, void>::type apply(
-      Callable fn, Repr& R, Args&... arguments) const {
-    fn(std::get<I>(actions_hirep), std::get<I>(R.rep), arguments...);
-    apply<I + 1>(fn, R, arguments...);
-  }  
-
-};
-
-
-//template <class GaugeField>
-//using ActionSet = std::vector<ActionLevel<GaugeField> >;
-
-template <class GaugeField, class R>
-using ActionSet = std::vector<ActionLevel<GaugeField, R> >;
-
 }
 }
 
-#endif
+#endif // ACTION_BASE_H

lib/qcd/action/ActionCore.h

@@ -31,15 +31,31 @@ directory
 #define QCD_ACTION_CORE
 
 #include <Grid/qcd/action/ActionBase.h>
+#include <Grid/qcd/action/ActionSet.h>
 #include <Grid/qcd/action/ActionParams.h>
 
 ////////////////////////////////////////////
 // Gauge Actions
 ////////////////////////////////////////////
 #include <Grid/qcd/action/gauge/Gauge.h>
+
 ////////////////////////////////////////////
 // Fermion prereqs
 ////////////////////////////////////////////
 #include <Grid/qcd/action/fermion/FermionCore.h>
 
+////////////////////////////////////////////
+// Scalar Actions
+////////////////////////////////////////////
+#include <Grid/qcd/action/scalar/Scalar.h>
+
+////////////////////////////////////////////
+// Utility functions
+////////////////////////////////////////////
+#include <Grid/qcd/utils/Metric.h>
+#include <Grid/qcd/utils/CovariantLaplacian.h>
+
+
+
+
 #endif

lib/qcd/action/ActionParams.h

@@ -8,6 +8,7 @@
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -23,9 +24,11 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
+
 #ifndef GRID_QCD_ACTION_PARAMS_H
 #define GRID_QCD_ACTION_PARAMS_H
 
@@ -37,31 +40,53 @@ namespace QCD {
     bool overlapCommsCompute;
     std::vector<int> twists;
     GparityWilsonImplParams() : twists(Nd, 0), overlapCommsCompute(false){};
-
   };
   
   struct WilsonImplParams {
     bool overlapCommsCompute;
-      WilsonImplParams() : overlapCommsCompute(false) {};
+    std::vector<Complex> boundary_phases;
+    WilsonImplParams() : overlapCommsCompute(false) {
+      boundary_phases.resize(Nd, 1.0);
+    };
+    WilsonImplParams(const std::vector<Complex> phi)
+      : boundary_phases(phi), overlapCommsCompute(false) {}
   };
 
   struct StaggeredImplParams {
     StaggeredImplParams()  {};
   };
   
-    struct OneFlavourRationalParams { 
+  struct OneFlavourRationalParams : Serializable {
-      RealD  lo;
+    GRID_SERIALIZABLE_CLASS_MEMBERS(OneFlavourRationalParams, 
-      RealD  hi;
+				    RealD, lo, 
-      int MaxIter;   // Vector?
+				    RealD, hi, 
-      RealD tolerance; // Vector? 
+				    int,   MaxIter, 
-      int    degree=10;
+				    RealD, tolerance, 
-      int precision=64;
+				    int,   degree, 
+				    int,   precision);
     
-      OneFlavourRationalParams (RealD _lo,RealD _hi,int _maxit,RealD tol=1.0e-8,int _degree = 10,int _precision=64) :
+    // MaxIter and tolerance, vectors??
-        lo(_lo), hi(_hi), MaxIter(_maxit), tolerance(tol), degree(_degree), precision(_precision)
+    
-      {};
+    // constructor 
+    OneFlavourRationalParams(	RealD _lo      = 0.0, 
+				RealD _hi      = 1.0, 
+				int _maxit     = 1000,
+				RealD tol      = 1.0e-8, 
+                           	int _degree    = 10,
+				int _precision = 64)
+      : lo(_lo),
+	hi(_hi),
+	MaxIter(_maxit),
+	tolerance(tol),
+	degree(_degree),
+	precision(_precision){};
   };
   
-}}
+  
+}
+}
+
+
+
 
 #endif

lib/qcd/action/ActionSet.h

@@ -0,0 +1,116 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/ActionSet.h
+
+Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef ACTION_SET_H
+#define ACTION_SET_H
+
+namespace Grid {
+
+// Should drop this namespace here
+namespace QCD {
+
+//////////////////////////////////
+// Indexing of tuple types
+//////////////////////////////////
+
+template <class T, class Tuple>
+struct Index;
+
+template <class T, class... Types>
+struct Index<T, std::tuple<T, Types...>> {
+  static const std::size_t value = 0;
+};
+
+template <class T, class U, class... Types>
+struct Index<T, std::tuple<U, Types...>> {
+  static const std::size_t value = 1 + Index<T, std::tuple<Types...>>::value;
+};
+
+
+////////////////////////////////////////////
+// Action Level
+// Action collection 
+// in a integration level
+// (for multilevel integration schemes)
+////////////////////////////////////////////
+
+template <class Field, class Repr = NoHirep >
+struct ActionLevel {
+ public:
+  unsigned int multiplier;
+
+  // Fundamental repr actions separated because of the smearing
+  typedef Action<Field>* ActPtr;
+
+  // construct a tuple of vectors of the actions for the corresponding higher
+  // representation fields
+  typedef typename AccessTypes<Action, Repr>::VectorCollection action_collection;
+  typedef typename  AccessTypes<Action, Repr>::FieldTypeCollection action_hirep_types;
+
+  action_collection actions_hirep;
+  std::vector<ActPtr>& actions;
+
+  explicit ActionLevel(unsigned int mul = 1) : 
+  actions(std::get<0>(actions_hirep)), multiplier(mul) {
+    // initialize the hirep vectors to zero.
+    // apply(this->resize, actions_hirep, 0); //need a working resize
+    assert(mul >= 1);
+  }
+
+  template < class GenField >
+  void push_back(Action<GenField>* ptr) {
+    // insert only in the correct vector
+    std::get< Index < GenField, action_hirep_types>::value >(actions_hirep).push_back(ptr);
+  };
+
+  template <class ActPtr>
+  static void resize(ActPtr ap, unsigned int n) {
+    ap->resize(n);
+  }
+
+  // Loop on tuple for a callable function
+  template <std::size_t I = 1, typename Callable, typename ...Args>
+  inline typename std::enable_if<I == std::tuple_size<action_collection>::value, void>::type apply(Callable, Repr& R,Args&...) const {}
+
+  template <std::size_t I = 1, typename Callable, typename ...Args>
+  inline typename std::enable_if<I < std::tuple_size<action_collection>::value, void>::type apply(Callable fn, Repr& R, Args&... arguments) const {
+    fn(std::get<I>(actions_hirep), std::get<I>(R.rep), arguments...);
+    apply<I + 1>(fn, R, arguments...);
+  }  
+
+};
+
+// Define the ActionSet
+template <class GaugeField, class R>
+using ActionSet = std::vector<ActionLevel<GaugeField, R> >;
+
+} // QCD
+} // Grid
+
+#endif  // ACTION_SET_H

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -198,7 +198,9 @@ namespace QCD {
     
     ImplParams Params;
     
-    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
+    WilsonImpl(const ImplParams &p = ImplParams()) : Params(p){
+      assert(Params.boundary_phases.size() == Nd);
+    };
       
     bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
       
@@ -222,10 +224,16 @@ namespace QCD {
       conformable(Uds._grid, GaugeGrid);
       conformable(Umu._grid, GaugeGrid);
       GaugeLinkField U(GaugeGrid);
+      GaugeLinkField tmp(GaugeGrid);
+      Lattice<iScalar<vInteger> > coor(GaugeGrid);
       for (int mu = 0; mu < Nd; mu++) {
+        LatticeCoordinate(coor, mu);
+        int Lmu = GaugeGrid->GlobalDimensions()[mu] - 1;
         U = PeekIndex<LorentzIndex>(Umu, mu);
-	PokeIndex<LorentzIndex>(Uds, U, mu);
+        tmp = where(coor == Lmu, Params.boundary_phases[mu] * U, U);
+        PokeIndex<LorentzIndex>(Uds, tmp, mu);
         U = adj(Cshift(U, mu, -1));
+        U = where(coor == 0, Params.boundary_phases[mu] * U, U);
         PokeIndex<LorentzIndex>(Uds, U, mu + 4);
       }
     }
@@ -353,9 +361,52 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
     assert(0);
   }
 
-  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde,FermionField &Atilde, int mu) 
+  inline void InsertForce5D(GaugeField &mat, FermionField &Btilde, FermionField &Atilde, int mu) {
-  {
+
     assert(0);
+    // Following lines to be revised after Peter's addition of half prec
+    // missing put lane...
+    /*
+    typedef decltype(traceIndex<SpinIndex>(outerProduct(Btilde[0], Atilde[0]))) result_type;
+    unsigned int LLs = Btilde._grid->_rdimensions[0];
+    conformable(Atilde._grid,Btilde._grid);
+    GridBase* grid = mat._grid;
+    GridBase* Bgrid = Btilde._grid;
+    unsigned int dimU = grid->Nd();
+    unsigned int dimF = Bgrid->Nd();
+    GaugeLinkField tmp(grid); 
+    tmp = zero;
+    
+    // FIXME 
+    // Current implementation works, thread safe, probably suboptimal
+    // Passing through the local coordinate for grid transformation
+    // the force grid is in general very different from the Ls vectorized grid
+
+    PARALLEL_FOR_LOOP
+    for (int so = 0; so < grid->oSites(); so++) {
+      std::vector<typename result_type::scalar_object> vres(Bgrid->Nsimd());
+      std::vector<int> ocoor;  grid->oCoorFromOindex(ocoor,so); 
+      for (int si = 0; si < tmp._grid->iSites(); si++){
+        typename result_type::scalar_object scalar_object; scalar_object = zero;
+        std::vector<int> local_coor;      
+        std::vector<int> icoor; grid->iCoorFromIindex(icoor,si);
+        grid->InOutCoorToLocalCoor(ocoor, icoor, local_coor);
+        for (int s = 0; s < LLs; s++) {
+          std::vector<int> slocal_coor(dimF);
+          slocal_coor[0] = s;
+          for (int s4d = 1; s4d< dimF; s4d++) slocal_coor[s4d] = local_coor[s4d-1];
+          int sF = Bgrid->oIndexReduced(slocal_coor);  
+          assert(sF < Bgrid->oSites());
+
+          extract(traceIndex<SpinIndex>(outerProduct(Btilde[sF], Atilde[sF])), vres); 
+          // sum across the 5d dimension
+          for (auto v : vres) scalar_object += v;  
+        }
+        tmp._odata[so].putlane(scalar_object, si);
+      }
+    }
+    PokeIndex<LorentzIndex>(mat, tmp, mu);
+    */
   }
 };
     
@@ -512,6 +563,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
        Utmp = where(coor==0,Uconj,Utmp);
      }
 
+	  
      parallel_for(auto ss=U.begin();ss<U.end();ss++){
        Uds[ss](0)(mu+4) = Utmp[ss]();
      }

lib/qcd/action/fermion/WilsonFermion.cc

@@ -230,8 +230,7 @@ void WilsonFermion<Impl>::DerivInternal(StencilImpl &st, DoubledGaugeField &U,
 }
 
 template <class Impl>
-void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U,
+void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U, const FermionField &V, int dag) {
-                                    const FermionField &V, int dag) {
   conformable(U._grid, _grid);
   conformable(U._grid, V._grid);
   conformable(U._grid, mat._grid);
@@ -242,11 +241,11 @@ void WilsonFermion<Impl>::DhopDeriv(GaugeField &mat, const FermionField &U,
 }
 
 template <class Impl>
-void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U,
+void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U, const FermionField &V, int dag) {
-                                      const FermionField &V, int dag) {
   conformable(U._grid, _cbgrid);
   conformable(U._grid, V._grid);
-  conformable(U._grid, mat._grid);
+  //conformable(U._grid, mat._grid); not general, leaving as a comment (Guido)
+  // Motivation: look at the SchurDiff operator
   
   assert(V.checkerboard == Even);
   assert(U.checkerboard == Odd);
@@ -256,11 +255,10 @@ void WilsonFermion<Impl>::DhopDerivOE(GaugeField &mat, const FermionField &U,
 }
 
 template <class Impl>
-void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U,
+void WilsonFermion<Impl>::DhopDerivEO(GaugeField &mat, const FermionField &U, const FermionField &V, int dag) {
-                                      const FermionField &V, int dag) {
   conformable(U._grid, _cbgrid);
   conformable(U._grid, V._grid);
-  conformable(U._grid, mat._grid);
+  //conformable(U._grid, mat._grid);
 
   assert(V.checkerboard == Odd);
   assert(U.checkerboard == Even);

lib/qcd/action/fermion/WilsonFermion5D.cc

@@ -11,6 +11,7 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -117,7 +118,6 @@ WilsonFermion5D<Impl>::WilsonFermion5D(GaugeField &_Umu,
     
   // Allocate the required comms buffer
   ImportGauge(_Umu);
-
   // Build lists of exterior only nodes
   int LLs = FiveDimGrid._rdimensions[0];
   int vol4;
@@ -267,6 +267,8 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
   DerivCommTime+=usecond();
 
   Atilde=A;
+  int LLs = B._grid->_rdimensions[0];
+
 
   DerivComputeTime-=usecond();
   for (int mu = 0; mu < Nd; mu++) {
@@ -296,6 +298,9 @@ void WilsonFermion5D<Impl>::DerivInternal(StencilImpl & st,
         ////////////////////////////
       }
     }
+    ////////////////////////////
+    // spin trace outer product
+    ////////////////////////////
     DerivDhopComputeTime += usecond();
     Impl::InsertForce5D(mat, Btilde, Atilde, mu);
   }
@@ -310,7 +315,8 @@ void WilsonFermion5D<Impl>::DhopDeriv(GaugeField &mat,
 {
   conformable(A._grid,FermionGrid());  
   conformable(A._grid,B._grid);
-  conformable(GaugeGrid(),mat._grid);
+
+  //conformable(GaugeGrid(),mat._grid);// this is not general! leaving as a comment
 
   mat.checkerboard = A.checkerboard;
 
@@ -324,7 +330,6 @@ void WilsonFermion5D<Impl>::DhopDerivEO(GaugeField &mat,
                                         int dag)
 {
   conformable(A._grid,FermionRedBlackGrid());
-  conformable(GaugeRedBlackGrid(),mat._grid);
   conformable(A._grid,B._grid);
 
   assert(B.checkerboard==Odd);
@@ -342,7 +347,6 @@ void WilsonFermion5D<Impl>::DhopDerivOE(GaugeField &mat,
                                         int dag)
 {
   conformable(A._grid,FermionRedBlackGrid());
-  conformable(GaugeRedBlackGrid(),mat._grid);
   conformable(A._grid,B._grid);
 
   assert(B.checkerboard==Even);

lib/qcd/action/gauge/Gauge.h

@@ -29,7 +29,7 @@ directory
 #ifndef GRID_QCD_GAUGE_H
 #define GRID_QCD_GAUGE_H
 
-#include <Grid/qcd/action/gauge/GaugeImpl.h>
+#include <Grid/qcd/action/gauge/GaugeImplementations.h>
 #include <Grid/qcd/utils/WilsonLoops.h>
 #include <Grid/qcd/action/gauge/WilsonGaugeAction.h>
 #include <Grid/qcd/action/gauge/PlaqPlusRectangleAction.h>

lib/qcd/action/gauge/GaugeImplTypes.h

@@ -0,0 +1,142 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/gauge/GaugeImpl.h
+
+Copyright (C) 2015
+
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_GAUGE_IMPL_TYPES_H
+#define GRID_GAUGE_IMPL_TYPES_H
+
+namespace Grid {
+namespace QCD {
+
+////////////////////////////////////////////////////////////////////////
+// Implementation dependent gauge types
+////////////////////////////////////////////////////////////////////////
+
+#define INHERIT_GIMPL_TYPES(GImpl)                  \
+  typedef typename GImpl::Simd Simd;                \
+  typedef typename GImpl::LinkField GaugeLinkField; \
+  typedef typename GImpl::Field GaugeField;         \
+  typedef typename GImpl::SiteField SiteGaugeField; \
+  typedef typename GImpl::SiteLink SiteGaugeLink;
+
+#define INHERIT_FIELD_TYPES(Impl)             \
+  typedef typename Impl::Simd Simd;           \
+  typedef typename Impl::SiteField SiteField; \
+  typedef typename Impl::Field Field;
+
+// hardcodes the exponential approximation in the template
+template <class S, int Nrepresentation = Nc, int Nexp = 12 > class GaugeImplTypes {
+public:
+  typedef S Simd;
+
+  template <typename vtype> using iImplGaugeLink  = iScalar<iScalar<iMatrix<vtype, Nrepresentation>>>;
+  template <typename vtype> using iImplGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation>>, Nd>;
+
+  typedef iImplGaugeLink<Simd>  SiteLink;
+  typedef iImplGaugeField<Simd> SiteField;
+
+  typedef Lattice<SiteLink>  LinkField; 
+  typedef Lattice<SiteField> Field;
+
+  // Guido: we can probably separate the types from the HMC functions
+  // this will create 2 kind of implementations
+  // probably confusing the users
+  // Now keeping only one class
+
+
+  // Move this elsewhere? FIXME
+  static inline void AddLink(Field &U, LinkField &W,
+                                  int mu) { // U[mu] += W
+    PARALLEL_FOR_LOOP
+    for (auto ss = 0; ss < U._grid->oSites(); ss++) {
+      U._odata[ss]._internal[mu] =
+          U._odata[ss]._internal[mu] + W._odata[ss]._internal;
+    }
+  }
+
+  ///////////////////////////////////////////////////////////
+  // Move these to another class
+  // HMC auxiliary functions 
+  static inline void generate_momenta(Field &P, GridParallelRNG &pRNG) {
+    // specific for SU gauge fields
+    LinkField Pmu(P._grid);
+    Pmu = zero;
+    for (int mu = 0; mu < Nd; mu++) {
+      SU<Nrepresentation>::GaussianFundamentalLieAlgebraMatrix(pRNG, Pmu);
+      PokeIndex<LorentzIndex>(P, Pmu, mu);
+    }
+  }
+
+  static inline Field projectForce(Field &P) { return Ta(P); }
+  
+  static inline void update_field(Field& P, Field& U, double ep){
+    for (int mu = 0; mu < Nd; mu++) {
+      auto Umu = PeekIndex<LorentzIndex>(U, mu);
+      auto Pmu = PeekIndex<LorentzIndex>(P, mu);
+      Umu = expMat(Pmu, ep, Nexp) * Umu;
+      PokeIndex<LorentzIndex>(U, ProjectOnGroup(Umu), mu);
+    }
+  }
+
+  static inline RealD FieldSquareNorm(Field& U){
+    LatticeComplex Hloc(U._grid);
+    Hloc = zero;
+    for (int mu = 0; mu < Nd; mu++) {
+      auto Umu = PeekIndex<LorentzIndex>(U, mu);
+      Hloc += trace(Umu * Umu);
+    }
+    Complex Hsum = sum(Hloc);
+    return Hsum.real();
+  }
+
+  static inline void HotConfiguration(GridParallelRNG &pRNG, Field &U) {
+    SU<Nc>::HotConfiguration(pRNG, U);
+  }
+
+  static inline void TepidConfiguration(GridParallelRNG &pRNG, Field &U) {
+    SU<Nc>::TepidConfiguration(pRNG, U);
+  }
+
+  static inline void ColdConfiguration(GridParallelRNG &pRNG, Field &U) {
+    SU<Nc>::ColdConfiguration(pRNG, U);
+  }
+};
+
+
+typedef GaugeImplTypes<vComplex, Nc> GimplTypesR;
+typedef GaugeImplTypes<vComplexF, Nc> GimplTypesF;
+typedef GaugeImplTypes<vComplexD, Nc> GimplTypesD;
+
+typedef GaugeImplTypes<vComplex, SU<Nc>::AdjointDimension> GimplAdjointTypesR;
+typedef GaugeImplTypes<vComplexF, SU<Nc>::AdjointDimension> GimplAdjointTypesF;
+typedef GaugeImplTypes<vComplexD, SU<Nc>::AdjointDimension> GimplAdjointTypesD;
+
+
+} // QCD
+} // Grid
+
+#endif // GRID_GAUGE_IMPL_TYPES_H

lib/qcd/action/gauge/GaugeImplementations.h

@@ -2,7 +2,7 @@
 
 Grid physics library, www.github.com/paboyle/Grid
 
-Source file: ./lib/qcd/action/gauge/GaugeImpl.h
+Source file: ./lib/qcd/action/gauge/GaugeImplementations.h
 
 Copyright (C) 2015
 
@@ -26,53 +26,14 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#ifndef GRID_QCD_GAUGE_IMPL_H
+#ifndef GRID_QCD_GAUGE_IMPLEMENTATIONS_H
-#define GRID_QCD_GAUGE_IMPL_H
+#define GRID_QCD_GAUGE_IMPLEMENTATIONS_H
+
+#include "GaugeImplTypes.h"
 
 namespace Grid {
 namespace QCD {
 
-////////////////////////////////////////////////////////////////////////
-// Implementation dependent gauge types
-////////////////////////////////////////////////////////////////////////
-
-template <class Gimpl> class WilsonLoops;
-
-#define INHERIT_GIMPL_TYPES(GImpl)                                             \
-  typedef typename GImpl::Simd Simd;                                           \
-  typedef typename GImpl::GaugeLinkField GaugeLinkField;                       \
-  typedef typename GImpl::GaugeField GaugeField;                               \
-  typedef typename GImpl::SiteGaugeField SiteGaugeField;                       \
-  typedef typename GImpl::SiteGaugeLink SiteGaugeLink;
-
-//
-template <class S, int Nrepresentation = Nc> class GaugeImplTypes {
-public:
-  typedef S Simd;
-
-  template <typename vtype>
-  using iImplGaugeLink  = iScalar<iScalar<iMatrix<vtype, Nrepresentation>>>;
-  template <typename vtype>
-  using iImplGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation>>, Nd>;
-
-  typedef iImplGaugeLink<Simd> SiteGaugeLink;
-  typedef iImplGaugeField<Simd> SiteGaugeField;
-
-  typedef Lattice<SiteGaugeLink> GaugeLinkField; // bit ugly naming; polarised
-                                                 // gauge field, lorentz... all
-                                                 // ugly
-  typedef Lattice<SiteGaugeField> GaugeField;
-
-  // Move this elsewhere? FIXME
-  static inline void AddGaugeLink(GaugeField &U, GaugeLinkField &W,
-                                  int mu) { // U[mu] += W
-    parallel_for (auto ss = 0; ss < U._grid->oSites(); ss++) {
-      U._odata[ss]._internal[mu] =
-          U._odata[ss]._internal[mu] + W._odata[ss]._internal;
-    }
-  }
-};
-
 // Composition with smeared link, bc's etc.. probably need multiple inheritance
 // Variable precision "S" and variable Nc
 template <class GimplTypes> class PeriodicGaugeImpl : public GimplTypes {
@@ -168,14 +129,6 @@ public:
   static inline bool isPeriodicGaugeField(void) { return false; }
 };
 
-typedef GaugeImplTypes<vComplex, Nc> GimplTypesR;
-typedef GaugeImplTypes<vComplexF, Nc> GimplTypesF;
-typedef GaugeImplTypes<vComplexD, Nc> GimplTypesD;
-
-typedef GaugeImplTypes<vComplex, SU<Nc>::AdjointDimension> GimplAdjointTypesR;
-typedef GaugeImplTypes<vComplexF, SU<Nc>::AdjointDimension> GimplAdjointTypesF;
-typedef GaugeImplTypes<vComplexD, SU<Nc>::AdjointDimension> GimplAdjointTypesD;
-
 typedef PeriodicGaugeImpl<GimplTypesR> PeriodicGimplR; // Real.. whichever prec
 typedef PeriodicGaugeImpl<GimplTypesF> PeriodicGimplF; // Float
 typedef PeriodicGaugeImpl<GimplTypesD> PeriodicGimplD; // Double
@@ -187,6 +140,8 @@ typedef PeriodicGaugeImpl<GimplAdjointTypesD> PeriodicGimplAdjD; // Double
 typedef ConjugateGaugeImpl<GimplTypesR> ConjugateGimplR; // Real.. whichever prec
 typedef ConjugateGaugeImpl<GimplTypesF> ConjugateGimplF; // Float
 typedef ConjugateGaugeImpl<GimplTypesD> ConjugateGimplD; // Double
+
+
 }
 }
 

lib/qcd/action/gauge/PlaqPlusRectangleAction.h

@@ -48,8 +48,18 @@ namespace Grid{
     public:
     PlaqPlusRectangleAction(RealD b,RealD c): c_plaq(b),c_rect(c){};
 
+      virtual std::string action_name(){return "PlaqPlusRectangleAction";}
+      
       virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {}; // noop as no pseudoferms
       
+      virtual std::string LogParameters(){
+      	std::stringstream sstream;
+      	sstream << GridLogMessage << "["<<action_name() <<"] c_plaq: " << c_plaq << std::endl;
+      	sstream << GridLogMessage << "["<<action_name() <<"] c_rect: " << c_rect << std::endl;
+      	return sstream.str();
+      }
+
+
       virtual RealD S(const GaugeField &U) {
 	RealD vol = U._grid->gSites();
 
@@ -108,32 +118,32 @@ namespace Grid{
     class RBCGaugeAction : public PlaqPlusRectangleAction<Gimpl> {
     public:
       INHERIT_GIMPL_TYPES(Gimpl);
-      RBCGaugeAction(RealD beta,RealD c1) : PlaqPlusRectangleAction<Gimpl>(beta*(1.0-8.0*c1), beta*c1) {
+      RBCGaugeAction(RealD beta,RealD c1) : PlaqPlusRectangleAction<Gimpl>(beta*(1.0-8.0*c1), beta*c1) {};
-      };
+      virtual std::string action_name(){return "RBCGaugeAction";}
     };
 
     template<class Gimpl>
     class IwasakiGaugeAction : public RBCGaugeAction<Gimpl> {
     public:
       INHERIT_GIMPL_TYPES(Gimpl);
-      IwasakiGaugeAction(RealD beta) : RBCGaugeAction<Gimpl>(beta,-0.331) {
+      IwasakiGaugeAction(RealD beta) : RBCGaugeAction<Gimpl>(beta,-0.331) {};
-      };
+      virtual std::string action_name(){return "IwasakiGaugeAction";}
     };
 
     template<class Gimpl>
     class SymanzikGaugeAction : public RBCGaugeAction<Gimpl> {
     public:
       INHERIT_GIMPL_TYPES(Gimpl);
-      SymanzikGaugeAction(RealD beta) : RBCGaugeAction<Gimpl>(beta,-1.0/12.0) {
+      SymanzikGaugeAction(RealD beta) : RBCGaugeAction<Gimpl>(beta,-1.0/12.0) {};
-      };
+      virtual std::string action_name(){return "SymanzikGaugeAction";}
     };
 
     template<class Gimpl>
     class DBW2GaugeAction : public RBCGaugeAction<Gimpl> {
     public:
       INHERIT_GIMPL_TYPES(Gimpl);
-      DBW2GaugeAction(RealD beta) : RBCGaugeAction<Gimpl>(beta,-1.4067) {
+      DBW2GaugeAction(RealD beta) : RBCGaugeAction<Gimpl>(beta,-1.4067) {};
-      };
+      virtual std::string action_name(){return "DBW2GaugeAction";}
     };
 
   }

lib/qcd/action/gauge/WilsonGaugeAction.h

@@ -10,6 +10,7 @@ Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: neo <cossu@post.kek.jp>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -25,7 +26,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 #ifndef QCD_WILSON_GAUGE_ACTION_H
@@ -40,17 +42,21 @@ namespace Grid{
 template <class Gimpl>
 class WilsonGaugeAction : public Action<typename Gimpl::GaugeField> {
  public:  
-
   INHERIT_GIMPL_TYPES(Gimpl);
 
-      //      typedef LorentzScalar<GaugeField> GaugeLinkField;
+  /////////////////////////// constructors
+  explicit WilsonGaugeAction(RealD beta_):beta(beta_){};
 
-    private:
+  virtual std::string action_name() {return "WilsonGaugeAction";}
-      RealD beta;
-    public:
-    WilsonGaugeAction(RealD b):beta(b){};
 
-      virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {}; // noop as no pseudoferms
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    sstream << GridLogMessage << "[WilsonGaugeAction] Beta: " << beta << std::endl;
+    return sstream.str();
+  }
+
+  virtual void refresh(const GaugeField &U,
+                       GridParallelRNG &pRNG){};  // noop as no pseudoferms
 
   virtual RealD S(const GaugeField &U) {
     RealD plaq = WilsonLoops<Gimpl>::avgPlaquette(U);
@@ -63,22 +69,25 @@ namespace Grid{
     // not optimal implementation FIXME
     // extend Ta to include Lorentz indexes
 
-	//RealD factor = 0.5*beta/RealD(Nc);
     RealD factor = 0.5 * beta / RealD(Nc);
 
     GaugeLinkField Umu(U._grid);
     GaugeLinkField dSdU_mu(U._grid);
     for (int mu = 0; mu < Nd; mu++) {
-
       Umu = PeekIndex<LorentzIndex>(U, mu);
 
       // Staple in direction mu
       WilsonLoops<Gimpl>::Staple(dSdU_mu, U, mu);
       dSdU_mu = Ta(Umu * dSdU_mu) * factor;
+
       PokeIndex<LorentzIndex>(dSdU, dSdU_mu, mu);
     }
+  }
+private:
+  RealD beta;  
 };
-    };
+
+
 
 }
 }

lib/qcd/action/pseudofermion/EvenOddSchurDifferentiable.h

@@ -7,6 +7,7 @@
     Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -53,10 +54,6 @@ namespace Grid{
         
           GridBase *fgrid   = this->_Mat.FermionGrid();
           GridBase *fcbgrid = this->_Mat.FermionRedBlackGrid();
-	  GridBase *ugrid   = this->_Mat.GaugeGrid();
-	  GridBase *ucbgrid = this->_Mat.GaugeRedBlackGrid();
-
-	  Real coeff = 1.0;
 
           FermionField tmp1(fcbgrid);
           FermionField tmp2(fcbgrid);
@@ -68,15 +65,20 @@ namespace Grid{
           assert(U.checkerboard==Odd);
           assert(V.checkerboard==U.checkerboard);
 
-	  GaugeField ForceO(ucbgrid);
+          // NOTE Guido: WE DO NOT WANT TO USE THE ucbgrid GRID FOR THE FORCE
-	  GaugeField ForceE(ucbgrid);
+          // it is not conformable with the HMC force field
+	  // Case: Ls vectorised fields
+          // INHERIT FROM THE Force field instead
+          GridRedBlackCartesian* forcecb = new GridRedBlackCartesian(Force._grid);
+          GaugeField ForceO(forcecb);
+          GaugeField ForceE(forcecb);
+
 
           //  X^dag Der_oe MeeInv Meo Y
           // Use Mooee as nontrivial but gauge field indept
           this->_Mat.Meooe   (V,tmp1);      // odd->even -- implicit -0.5 factor to be applied
 	  this->_Mat.MooeeInv(tmp1,tmp2);   // even->even 
           this->_Mat.MoeDeriv(ForceO,U,tmp2,DaggerNo);
-	  
           //  Accumulate X^dag M_oe MeeInv Der_eo Y
           this->_Mat.MeooeDag   (U,tmp1);    // even->odd -- implicit -0.5 factor to be applied
           this->_Mat.MooeeInvDag(tmp1,tmp2); // even->even 
@@ -88,6 +90,8 @@ namespace Grid{
           setCheckerboard(Force,ForceE); 
           setCheckerboard(Force,ForceO);
           Force=-Force;
+
+          delete forcecb;
         }
 
 
@@ -95,10 +99,6 @@ namespace Grid{
         
           GridBase *fgrid   = this->_Mat.FermionGrid();
           GridBase *fcbgrid = this->_Mat.FermionRedBlackGrid();
-	  GridBase *ugrid   = this->_Mat.GaugeGrid();
-	  GridBase *ucbgrid = this->_Mat.GaugeRedBlackGrid();
-
-	  Real coeff = 1.0;
 
           FermionField tmp1(fcbgrid);
           FermionField tmp2(fcbgrid);
@@ -110,8 +110,12 @@ namespace Grid{
           assert(V.checkerboard==Odd);
           assert(V.checkerboard==V.checkerboard);
 
-	  GaugeField ForceO(ucbgrid);
+          // NOTE Guido: WE DO NOT WANT TO USE THE ucbgrid GRID FOR THE FORCE
-	  GaugeField ForceE(ucbgrid);
+          // it is not conformable with the HMC force field
+          // INHERIT FROM THE Force field instead
+	  GridRedBlackCartesian* forcecb = new GridRedBlackCartesian(Force._grid);
+          GaugeField ForceO(forcecb);
+          GaugeField ForceE(forcecb);
 
           //  X^dag Der_oe MeeInv Meo Y
           // Use Mooee as nontrivial but gauge field indept
@@ -130,6 +134,8 @@ namespace Grid{
           setCheckerboard(Force,ForceE); 
           setCheckerboard(Force,ForceO);
           Force=-Force;
+
+          delete forcecb;
         }
 
     };

lib/qcd/action/pseudofermion/OneFlavourEvenOddRational.h

@@ -1,3 +1,4 @@
+
 /*************************************************************************************
 
 Grid physics library, www.github.com/paboyle/Grid
@@ -90,6 +91,19 @@ class OneFlavourEvenOddRationalPseudoFermionAction
     PowerNegQuarter.Init(remez, param.tolerance, true);
   };
 
+  virtual std::string action_name(){return "OneFlavourEvenOddRationalPseudoFermionAction";}
+
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    sstream << GridLogMessage << "["<<action_name()<<"] Low            :" << param.lo <<  std::endl;
+    sstream << GridLogMessage << "["<<action_name()<<"] High           :" << param.hi <<  std::endl;
+    sstream << GridLogMessage << "["<<action_name()<<"] Max iterations :" << param.MaxIter <<  std::endl;
+    sstream << GridLogMessage << "["<<action_name()<<"] Tolerance      :" << param.tolerance <<  std::endl;
+    sstream << GridLogMessage << "["<<action_name()<<"] Degree         :" << param.degree <<  std::endl;
+    sstream << GridLogMessage << "["<<action_name()<<"] Precision      :" << param.precision <<  std::endl;
+    return sstream.str();
+  }
+  
   virtual void refresh(const GaugeField &U, GridParallelRNG &pRNG) {
     // P(phi) = e^{- phi^dag (MpcdagMpc)^-1/2 phi}
     //        = e^{- phi^dag (MpcdagMpc)^-1/4 (MpcdagMpc)^-1/4 phi}

lib/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h

@@ -88,6 +88,20 @@ namespace Grid{
 	PowerNegQuarter.Init(remez,param.tolerance,true);
       };
 
+      virtual std::string action_name(){return "OneFlavourEvenOddRatioRationalPseudoFermionAction";}
+
+      virtual std::string LogParameters(){
+	std::stringstream sstream;
+	sstream << GridLogMessage << "["<<action_name()<<"] Low            :" << param.lo <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] High           :" << param.hi <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Max iterations :" << param.MaxIter <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Tolerance      :" << param.tolerance <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Degree         :" << param.degree <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Precision      :" << param.precision <<  std::endl;
+	return sstream.str();
+      }
+      
+      
       virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
 
 	// S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi       

lib/qcd/action/pseudofermion/OneFlavourRational.h

@@ -84,8 +84,24 @@ namespace Grid{
 	PowerNegQuarter.Init(remez,param.tolerance,true);
       };
 
+      virtual std::string action_name(){return "OneFlavourRationalPseudoFermionAction";}
+
+      virtual std::string LogParameters(){
+	std::stringstream sstream;
+	sstream << GridLogMessage << "["<<action_name()<<"] Low            :" << param.lo <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] High           :" << param.hi <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Max iterations :" << param.MaxIter <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Tolerance      :" << param.tolerance <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Degree         :" << param.degree <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Precision      :" << param.precision <<  std::endl;
+	return sstream.str();
+      }  
+
+
+      
       virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
 
+	
 	// P(phi) = e^{- phi^dag (MdagM)^-1/2 phi}
 	//        = e^{- phi^dag (MdagM)^-1/4 (MdagM)^-1/4 phi}
 	// Phi = Mdag^{1/4} eta 

lib/qcd/action/pseudofermion/OneFlavourRationalRatio.h

@@ -82,6 +82,20 @@ namespace Grid{
 	PowerNegQuarter.Init(remez,param.tolerance,true);
       };
 
+      virtual std::string action_name(){return "OneFlavourRatioRationalPseudoFermionAction";}
+      
+      virtual std::string LogParameters(){
+	std::stringstream sstream;
+	sstream << GridLogMessage << "["<<action_name()<<"] Low            :" << param.lo <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] High           :" << param.hi <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Max iterations :" << param.MaxIter <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Tolerance      :" << param.tolerance <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Degree         :" << param.degree <<  std::endl;
+	sstream << GridLogMessage << "["<<action_name()<<"] Precision      :" << param.precision <<  std::endl;
+	return sstream.str();
+      }
+      
+
       virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
 
 	// S_f = chi^dag* P(V^dag*V)/Q(V^dag*V)* N(M^dag*M)/D(M^dag*M)* P(V^dag*V)/Q(V^dag*V)* chi       

lib/qcd/action/pseudofermion/TwoFlavour.h

@@ -62,6 +62,15 @@ class TwoFlavourPseudoFermionAction : public Action<typename Impl::GaugeField> {
         ActionSolver(AS),
         Phi(Op.FermionGrid()){};
 
+
+  virtual std::string action_name(){return "TwoFlavourPseudoFermionAction";}
+
+  virtual std::string LogParameters(){
+    std::stringstream sstream;
+    sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+    return sstream.str();
+  }  
+  
   //////////////////////////////////////////////////////////////////////////////////////
   // Push the gauge field in to the dops. Assume any BC's and smearing already applied
   //////////////////////////////////////////////////////////////////////////////////////
@@ -80,7 +89,9 @@ class TwoFlavourPseudoFermionAction : public Action<typename Impl::GaugeField> {
     //         in the Phi integral, and thus is only an irrelevant prefactor for
     //         the partition function.
     //
+
     RealD scale = std::sqrt(0.5);
+
     FermionField eta(FermOp.FermionGrid());
 
     gaussian(pRNG, eta);

lib/qcd/action/pseudofermion/TwoFlavourEvenOdd.h

@@ -65,6 +65,15 @@ class TwoFlavourEvenOddPseudoFermionAction
 	  PhiOdd(Op.FermionRedBlackGrid())
       {};
   
+      virtual std::string action_name(){return "TwoFlavourEvenOddPseudoFermionAction";}
+      
+      virtual std::string LogParameters(){
+	std::stringstream sstream;
+	sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+	return sstream.str();
+      }  
+
+
       //////////////////////////////////////////////////////////////////////////////////////
       // Push the gauge field in to the dops. Assume any BC's and smearing already applied
       //////////////////////////////////////////////////////////////////////////////////////
@@ -135,7 +144,6 @@ class TwoFlavourEvenOddPseudoFermionAction
       //
       //////////////////////////////////////////////////////
       virtual void deriv(const GaugeField &U,GaugeField & dSdU) {
-
 	FermOp.ImportGauge(U);
 
 	FermionField X(FermOp.FermionRedBlackGrid());

lib/qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h

@@ -68,6 +68,15 @@ namespace Grid{
           conformable(_NumOp.GaugeRedBlackGrid(), _DenOp.GaugeRedBlackGrid());
         };
 
+      virtual std::string action_name(){return "TwoFlavourEvenOddRatioPseudoFermionAction";}
+
+      virtual std::string LogParameters(){
+	std::stringstream sstream;
+	sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+	return sstream.str();
+      } 
+
+      
       virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
 
         // P(phi) = e^{- phi^dag Vpc (MpcdagMpc)^-1 Vpcdag phi}
@@ -164,7 +173,8 @@ namespace Grid{
         FermionField  X(NumOp.FermionRedBlackGrid());
         FermionField  Y(NumOp.FermionRedBlackGrid());
 
-	GaugeField   force(NumOp.GaugeGrid());	
+        // This assignment is necessary to be compliant with the HMC grids
+	GaugeField force(dSdU._grid);
 
         //Y=Vdag phi
         //X = (Mdag M)^-1 V^dag phi
@@ -190,7 +200,6 @@ namespace Grid{
         assert(NumOp.ConstEE() == 1);
         assert(DenOp.ConstEE() == 1);
 
-	//dSdU = -Ta(dSdU);
         dSdU = -dSdU;
         
       };

lib/qcd/action/pseudofermion/TwoFlavourRatio.h

@@ -57,6 +57,14 @@ namespace Grid{
 					 OperatorFunction<FermionField> & AS
 					 ) : NumOp(_NumOp), DenOp(_DenOp), DerivativeSolver(DS), ActionSolver(AS), Phi(_NumOp.FermionGrid()) {};
       
+      virtual std::string action_name(){return "TwoFlavourRatioPseudoFermionAction";}
+
+      virtual std::string LogParameters(){
+	std::stringstream sstream;
+	sstream << GridLogMessage << "["<<action_name()<<"] has no parameters" << std::endl;
+	return sstream.str();
+      }  
+      
       virtual void refresh(const GaugeField &U, GridParallelRNG& pRNG) {
 
 	// P(phi) = e^{- phi^dag V (MdagM)^-1 Vdag phi}

lib/qcd/action/scalar/Scalar.h

@@ -0,0 +1,45 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/gauge/Scalar.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_QCD_SCALAR_H
+#define GRID_QCD_SCALAR_H
+
+#include <Grid/qcd/action/scalar/ScalarImpl.h>
+#include <Grid/qcd/action/scalar/ScalarAction.h>
+
+namespace Grid {
+namespace QCD {
+
+  typedef ScalarAction<ScalarImplR>                 ScalarActionR;
+  typedef ScalarAction<ScalarImplF>                 ScalarActionF;
+  typedef ScalarAction<ScalarImplD>                 ScalarActionD;
+
+}
+}
+
+#endif  // GRID_QCD_SCALAR_H

lib/qcd/action/scalar/ScalarAction.h

@@ -0,0 +1,84 @@
+/*************************************************************************************
+
+  Grid physics library, www.github.com/paboyle/Grid
+
+  Source file: ./lib/qcd/action/gauge/WilsonGaugeAction.h
+
+  Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+  See the full license in the file "LICENSE" in the top level distribution
+directory
+  *************************************************************************************/
+/*  END LEGAL */
+
+#ifndef SCALAR_ACTION_H
+#define SCALAR_ACTION_H
+
+namespace Grid {
+  // FIXME drop the QCD namespace everywhere here
+  
+  template <class Impl>
+  class ScalarAction : public QCD::Action<typename Impl::Field> {
+  public:
+    INHERIT_FIELD_TYPES(Impl);
+    
+  private:
+    RealD mass_square;
+    RealD lambda;
+    
+  public:
+    ScalarAction(RealD ms, RealD l) : mass_square(ms), lambda(l){};
+
+    virtual std::string LogParameters(){
+      std::stringstream sstream;
+      sstream << GridLogMessage << "[ScalarAction] lambda      : " << lambda      << std::endl;
+      sstream << GridLogMessage << "[ScalarAction] mass_square : " << mass_square << std::endl;
+      return sstream.str();
+      
+    }
+    
+    virtual std::string action_name(){return "ScalarAction";}
+    
+    virtual void refresh(const Field &U,
+			 GridParallelRNG &pRNG){};  // noop as no pseudoferms
+    
+    virtual RealD S(const Field &p) {
+      return (mass_square * 0.5 + QCD::Nd) * ScalarObs<Impl>::sumphisquared(p) +
+	(lambda / 24.) * ScalarObs<Impl>::sumphifourth(p) +
+	ScalarObs<Impl>::sumphider(p);
+    };
+    
+    virtual void deriv(const Field &p,
+		       Field &force) {
+      Field tmp(p._grid);
+      Field p2(p._grid);
+      ScalarObs<Impl>::phisquared(p2, p);
+      tmp = -(Cshift(p, 0, -1) + Cshift(p, 0, 1));
+      for (int mu = 1; mu < QCD::Nd; mu++) tmp -= Cshift(p, mu, -1) + Cshift(p, mu, 1);
+      
+      force=+(mass_square + 2. * QCD::Nd) * p + (lambda / 6.) * p2 * p + tmp;
+    };
+  };
+  
+} // Grid
+
+#endif // SCALAR_ACTION_H

lib/qcd/action/scalar/ScalarImpl.h

@@ -0,0 +1,100 @@
+#ifndef SCALAR_IMPL
+#define SCALAR_IMPL
+
+
+namespace Grid {
+  //namespace QCD {
+
+  template <class S>
+  class ScalarImplTypes {
+  public:
+    typedef S Simd;
+    
+    template <typename vtype>
+    using iImplField = iScalar<iScalar<iScalar<vtype> > >;
+    
+    typedef iImplField<Simd> SiteField;
+    
+    
+    typedef Lattice<SiteField> Field;
+    
+    static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){
+      gaussian(pRNG, P);
+    }
+    
+    static inline Field projectForce(Field& P){return P;}
+    
+    static inline void update_field(Field& P, Field& U, double ep){
+      U += P*ep;
+    }
+    
+    static inline RealD FieldSquareNorm(Field& U){
+      return (- sum(trace(U*U))/2.0);
+    }
+    
+    static inline void HotConfiguration(GridParallelRNG &pRNG, Field &U) {
+      gaussian(pRNG, U);
+    }
+    
+    static inline void TepidConfiguration(GridParallelRNG &pRNG, Field &U) {
+      gaussian(pRNG, U);
+    }
+    
+    static inline void ColdConfiguration(GridParallelRNG &pRNG, Field &U) {
+      U = 1.0;
+    }
+    
+  };
+
+  template <class S, unsigned int N>
+  class ScalarMatrixImplTypes {
+  public:
+    typedef S Simd;
+    
+    template <typename vtype>
+    using iImplField = iScalar<iScalar<iMatrix<vtype, N> > >;
+    
+    typedef iImplField<Simd> SiteField;
+    
+    
+    typedef Lattice<SiteField> Field;
+    
+    static inline void generate_momenta(Field& P, GridParallelRNG& pRNG){
+      gaussian(pRNG, P);
+    }
+    
+    static inline Field projectForce(Field& P){return P;}
+    
+    static inline void update_field(Field& P, Field& U, double ep){
+      U += P*ep;
+    }
+    
+    static inline RealD FieldSquareNorm(Field& U){
+      return (TensorRemove(- sum(trace(U*U))*0.5).real());
+    }
+    
+    static inline void HotConfiguration(GridParallelRNG &pRNG, Field &U) {
+      gaussian(pRNG, U);
+    }
+    
+    static inline void TepidConfiguration(GridParallelRNG &pRNG, Field &U) {
+      gaussian(pRNG, U);
+    }
+    
+    static inline void ColdConfiguration(GridParallelRNG &pRNG, Field &U) {
+      U = 1.0;
+    }
+    
+  };
+
+
+  
+  
+  typedef ScalarImplTypes<vReal> ScalarImplR;
+  typedef ScalarImplTypes<vRealF> ScalarImplF;
+  typedef ScalarImplTypes<vRealD> ScalarImplD;
+  
+  //} 
+} 
+
+#endif

lib/qcd/action/scalar/ScalarInteractionAction.h

@@ -0,0 +1,84 @@
+/*************************************************************************************
+
+  Grid physics library, www.github.com/paboyle/Grid
+
+  Source file: ./lib/qcd/action/gauge/WilsonGaugeAction.h
+
+  Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: neo <cossu@post.kek.jp>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+  See the full license in the file "LICENSE" in the top level distribution
+directory
+  *************************************************************************************/
+/*  END LEGAL */
+
+#ifndef SCALAR_ACTION_H
+#define SCALAR_ACTION_H
+
+namespace Grid {
+  // FIXME drop the QCD namespace everywhere here
+  
+  template <class Impl>
+  class ScalarInteractionAction : public QCD::Action<typename Impl::Field> {
+  public:
+    INHERIT_FIELD_TYPES(Impl);
+    
+  private:
+    RealD mass_square;
+    RealD lambda;
+    
+  public:
+    ScalarAction(RealD ms, RealD l) : mass_square(ms), lambda(l){};
+
+    virtual std::string LogParameters(){
+      std::stringstream sstream;
+      sstream << GridLogMessage << "[ScalarAction] lambda      : " << lambda      << std::endl;
+      sstream << GridLogMessage << "[ScalarAction] mass_square : " << mass_square << std::endl;
+      return sstream.str();
+      
+    }
+    
+    virtual std::string action_name(){return "ScalarAction";}
+    
+    virtual void refresh(const Field &U,
+			 GridParallelRNG &pRNG){};  // noop as no pseudoferms
+    
+    virtual RealD S(const Field &p) {
+      return (mass_square * 0.5 + QCD::Nd) * ScalarObs<Impl>::sumphisquared(p) +
+	(lambda / 24.) * ScalarObs<Impl>::sumphifourth(p) +
+	ScalarObs<Impl>::sumphider(p);
+    };
+    
+    virtual void deriv(const Field &p,
+		       Field &force) {
+      Field tmp(p._grid);
+      Field p2(p._grid);
+      ScalarObs<Impl>::phisquared(p2, p);
+      tmp = -(Cshift(p, 0, -1) + Cshift(p, 0, 1));
+      for (int mu = 1; mu < QCD::Nd; mu++) tmp -= Cshift(p, mu, -1) + Cshift(p, mu, 1);
+      
+      force=+(mass_square + 2. * QCD::Nd) * p + (lambda / 6.) * p2 * p + tmp;
+    };
+  };
+  
+} // Grid
+
+#endif // SCALAR_ACTION_H

lib/qcd/hmc/GenericHMCrunner.h

@@ -0,0 +1,213 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/GenericHmcRunner.h
+
+Copyright (C) 2015
+
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+  See the full license in the file "LICENSE" in the top level distribution
+  directory
+  *************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_GENERIC_HMC_RUNNER
+#define GRID_GENERIC_HMC_RUNNER
+
+#include <unordered_map>
+
+namespace Grid {
+namespace QCD {
+
+
+// very ugly here but possibly resolved if we had a base Reader class
+template < class ReaderClass >
+class HMCRunnerBase {
+public:
+  virtual void Run() = 0;
+  virtual void initialize(ReaderClass& ) = 0;
+};
+
+
+template <class Implementation,
+          template <typename, typename, typename> class Integrator,
+          class RepresentationsPolicy = NoHirep, class ReaderClass = XmlReader>
+class HMCWrapperTemplate: public HMCRunnerBase<ReaderClass> {
+ public:
+  INHERIT_FIELD_TYPES(Implementation);
+  typedef Implementation ImplPolicy;  // visible from outside
+  template <typename S = NoSmearing<Implementation> >
+  using IntegratorType = Integrator<Implementation, S, RepresentationsPolicy>;
+
+  HMCparameters Parameters;
+  std::string ParameterFile;
+  HMCResourceManager<Implementation> Resources;
+
+  // The set of actions (keep here for lower level users, for now)
+  ActionSet<Field, RepresentationsPolicy> TheAction;
+
+  HMCWrapperTemplate() = default;
+
+  HMCWrapperTemplate(HMCparameters Par){
+    Parameters = Par;
+  }
+
+  void initialize(ReaderClass & TheReader){
+    std::cout  << "Initialization of the HMC" << std::endl;
+    Resources.initialize(TheReader);
+
+    // eventually add smearing
+
+    Resources.GetActionSet(TheAction);    
+  }
+
+
+  void ReadCommandLine(int argc, char **argv) {
+    std::string arg;
+
+    if (GridCmdOptionExists(argv, argv + argc, "--StartingType")) {
+      arg = GridCmdOptionPayload(argv, argv + argc, "--StartingType");
+
+      if (arg != "HotStart" && arg != "ColdStart" && arg != "TepidStart" &&
+          arg != "CheckpointStart") {
+        std::cout << GridLogError << "Unrecognized option in --StartingType\n";
+        std::cout
+            << GridLogError
+            << "Valid [HotStart, ColdStart, TepidStart, CheckpointStart]\n";
+        exit(1);
+      }
+      Parameters.StartingType = arg;
+    }
+
+    if (GridCmdOptionExists(argv, argv + argc, "--StartingTrajectory")) {
+      arg = GridCmdOptionPayload(argv, argv + argc, "--StartingTrajectory");
+      std::vector<int> ivec(0);
+      GridCmdOptionIntVector(arg, ivec);
+      Parameters.StartTrajectory = ivec[0];
+    }
+
+    if (GridCmdOptionExists(argv, argv + argc, "--Trajectories")) {
+      arg = GridCmdOptionPayload(argv, argv + argc, "--Trajectories");
+      std::vector<int> ivec(0);
+      GridCmdOptionIntVector(arg, ivec);
+      Parameters.Trajectories = ivec[0];
+    }
+
+    if (GridCmdOptionExists(argv, argv + argc, "--Thermalizations")) {
+      arg = GridCmdOptionPayload(argv, argv + argc, "--Thermalizations");
+      std::vector<int> ivec(0);
+      GridCmdOptionIntVector(arg, ivec);
+      Parameters.NoMetropolisUntil = ivec[0];
+    }
+    if (GridCmdOptionExists(argv, argv + argc, "--ParameterFile")) {
+      arg = GridCmdOptionPayload(argv, argv + argc, "--ParameterFile");
+      ParameterFile = arg;
+    }
+  }
+
+
+  template <class SmearingPolicy>
+  void Run(SmearingPolicy &S) {
+    Runner(S);
+  }
+
+  void Run(){
+    NoSmearing<Implementation> S;
+    Runner(S);
+  }
+
+  //////////////////////////////////////////////////////////////////
+
+ private:
+  template <class SmearingPolicy>
+  void Runner(SmearingPolicy &Smearing) {
+    auto UGrid = Resources.GetCartesian();
+    Resources.AddRNGs();
+    Field U(UGrid);
+
+    // Can move this outside?
+    typedef IntegratorType<SmearingPolicy> TheIntegrator;
+    TheIntegrator MDynamics(UGrid, Parameters.MD, TheAction, Smearing);
+
+    if (Parameters.StartingType == "HotStart") {
+      // Hot start
+      Resources.SeedFixedIntegers();
+      Implementation::HotConfiguration(Resources.GetParallelRNG(), U);
+    } else if (Parameters.StartingType == "ColdStart") {
+      // Cold start
+      Resources.SeedFixedIntegers();
+      Implementation::ColdConfiguration(Resources.GetParallelRNG(), U);
+    } else if (Parameters.StartingType == "TepidStart") {
+      // Tepid start
+      Resources.SeedFixedIntegers();
+      Implementation::TepidConfiguration(Resources.GetParallelRNG(), U);
+    } else if (Parameters.StartingType == "CheckpointStart") {
+      // CheckpointRestart
+      Resources.GetCheckPointer()->CheckpointRestore(Parameters.StartTrajectory, U,
+                                   Resources.GetSerialRNG(),
+                                   Resources.GetParallelRNG());
+    }
+
+    Smearing.set_Field(U);
+
+    HybridMonteCarlo<TheIntegrator> HMC(Parameters, MDynamics,
+                                        Resources.GetSerialRNG(),
+                                        Resources.GetParallelRNG(), 
+                                        Resources.GetObservables(), U);
+
+    // Run it
+    HMC.evolve();
+  }
+};
+
+// These are for gauge fields, default integrator MinimumNorm2
+template <template <typename, typename, typename> class Integrator>
+using GenericHMCRunner = HMCWrapperTemplate<PeriodicGimplR, Integrator>;
+template <template <typename, typename, typename> class Integrator>
+using GenericHMCRunnerF = HMCWrapperTemplate<PeriodicGimplF, Integrator>;
+template <template <typename, typename, typename> class Integrator>
+using GenericHMCRunnerD = HMCWrapperTemplate<PeriodicGimplD, Integrator>;
+
+
+// These are for gauge fields, default integrator MinimumNorm2
+template <template <typename, typename, typename> class Integrator>
+using ConjugateHMCRunner = HMCWrapperTemplate<ConjugateGimplR, Integrator>;
+template <template <typename, typename, typename> class Integrator>
+using ConjugateHMCRunnerF = HMCWrapperTemplate<ConjugateGimplF, Integrator>;
+template <template <typename, typename, typename> class Integrator>
+using ConjugateHMCRunnerD = HMCWrapperTemplate<ConjugateGimplD, Integrator>;
+
+
+
+template <class RepresentationsPolicy,
+          template <typename, typename, typename> class Integrator>
+using GenericHMCRunnerHirep =
+    HMCWrapperTemplate<PeriodicGimplR, Integrator, RepresentationsPolicy>;
+
+template <class Implementation, class RepresentationsPolicy, 
+          template <typename, typename, typename> class Integrator>
+using GenericHMCRunnerTemplate = HMCWrapperTemplate<Implementation, Integrator, RepresentationsPolicy>;
+
+typedef HMCWrapperTemplate<ScalarImplR, MinimumNorm2, ScalarFields>
+    ScalarGenericHMCRunner;
+
+}  // namespace QCD
+}  // namespace Grid
+
+#endif  // GRID_GENERIC_HMC_RUNNER

lib/qcd/hmc/HMC.h

@@ -34,13 +34,15 @@ directory
  * @brief Classes for Hybrid Monte Carlo update
  *
  * @author Guido Cossu
- * Time-stamp: <2015-07-30 16:58:26 neo>
  */
 //--------------------------------------------------------------------
 #ifndef HMC_INCLUDED
 #define HMC_INCLUDED
 
 #include <string>
+#include <list>
+
+
 
 #include <Grid/qcd/hmc/integrators/Integrator.h>
 #include <Grid/qcd/hmc/integrators/Integrator_algorithm.h>
@@ -48,91 +50,64 @@ directory
 namespace Grid {
 namespace QCD {
 
-struct HMCparameters {
+struct HMCparameters: Serializable {
-  Integer StartTrajectory;
+	GRID_SERIALIZABLE_CLASS_MEMBERS(HMCparameters,
-  Integer Trajectories; /* @brief Number of sweeps in this run */
+                                  Integer, StartTrajectory,
-  bool MetropolisTest;
+                                  Integer, Trajectories, /* @brief Number of sweeps in this run */
-  Integer NoMetropolisUntil;
+                                  bool, MetropolisTest,
+                                  Integer, NoMetropolisUntil,
+                                  std::string, StartingType,
+                                  IntegratorParameters, MD)
 
   HMCparameters() {
     ////////////////////////////// Default values
     MetropolisTest    = true;
     NoMetropolisUntil = 10;
     StartTrajectory   = 0;
-    Trajectories = 200;
+    Trajectories      = 10;
+    StartingType      = "HotStart";
     /////////////////////////////////
   }
 
-  void print() const {
+  template <class ReaderClass >
-    std::cout << GridLogMessage << "[HMC parameter] Trajectories            : " << Trajectories << "\n";
+  HMCparameters(Reader<ReaderClass> & TheReader){
-    std::cout << GridLogMessage << "[HMC parameter] Start trajectory        : " << StartTrajectory << "\n";
+  	initialize(TheReader);
-    std::cout << GridLogMessage << "[HMC parameter] Metropolis test (on/off): " << MetropolisTest << "\n";
+  }
-    std::cout << GridLogMessage << "[HMC parameter] Thermalization trajs    : " << NoMetropolisUntil << "\n";
+
+  template < class ReaderClass > 
+  void initialize(Reader<ReaderClass> &TheReader){
+  	std::cout << "Reading HMC\n";
+  	read(TheReader, "HMC", *this);
+  }
+
+
+  void print_parameters() const {
+    std::cout << GridLogMessage << "[HMC parameters] Trajectories            : " << Trajectories << "\n";
+    std::cout << GridLogMessage << "[HMC parameters] Start trajectory        : " << StartTrajectory << "\n";
+    std::cout << GridLogMessage << "[HMC parameters] Metropolis test (on/off): " << std::boolalpha << MetropolisTest << "\n";
+    std::cout << GridLogMessage << "[HMC parameters] Thermalization trajs    : " << NoMetropolisUntil << "\n";
+    std::cout << GridLogMessage << "[HMC parameters] Starting type           : " << StartingType << "\n";
+    MD.print_parameters();
   }
   
 };
 	
-template <class GaugeField>
+template <class IntegratorType>
-class HmcObservable {
- public:
-  virtual void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG,
-                                  GridParallelRNG &pRNG) = 0;
-};
-
-template <class Gimpl>
-class PlaquetteLogger : public HmcObservable<typename Gimpl::GaugeField> {
- private:
-  std::string Stem;
-
- public:
-  INHERIT_GIMPL_TYPES(Gimpl);
-  PlaquetteLogger(std::string cf) { Stem = cf; };
-
-  void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG,
-                          GridParallelRNG &pRNG) {
-    std::string file;
-    {
-      std::ostringstream os;
-      os << Stem << "." << traj;
-      file = os.str();
-    }
-    std::ofstream of(file);
-
-    RealD peri_plaq = WilsonLoops<PeriodicGimplR>::avgPlaquette(U);
-    RealD peri_rect = WilsonLoops<PeriodicGimplR>::avgRectangle(U);
-
-    RealD impl_plaq = WilsonLoops<Gimpl>::avgPlaquette(U);
-    RealD impl_rect = WilsonLoops<Gimpl>::avgRectangle(U);
-
-    of << traj << " " << impl_plaq << " " << impl_rect << "  " << peri_plaq
-       << " " << peri_rect << std::endl;
-    std::cout << GridLogMessage << "traj"
-              << " "
-              << "plaq "
-              << " "
-              << " rect  "
-              << "  "
-              << "peri_plaq"
-              << " "
-              << "peri_rect" << std::endl;
-    std::cout << GridLogMessage << traj << " " << impl_plaq << " " << impl_rect
-              << "  " << peri_plaq << " " << peri_rect << std::endl;
-  }
-};
-
-//    template <class GaugeField, class Integrator, class Smearer, class
-//    Boundary>
-template <class GaugeField, class IntegratorType>
 class HybridMonteCarlo {
  private:
   const HMCparameters Params;
 
-  GridSerialRNG &sRNG;    // Fixme: need a RNG management strategy.
+  typedef typename IntegratorType::Field Field;
-  GridParallelRNG &pRNG;  // Fixme: need a RNG management strategy.
+  typedef std::vector< HmcObservable<Field> * > ObsListType;
-  GaugeField &Ucur;
+  
+  	//pass these from the resource manager
+  GridSerialRNG &sRNG;   
+  GridParallelRNG &pRNG; 
+
+  Field &Ucur;
   
   IntegratorType &TheIntegrator;
-  std::vector<HmcObservable<GaugeField> *> Observables;
+	ObsListType Observables;
 
   /////////////////////////////////////////////////////////
   // Metropolis step
@@ -167,13 +142,13 @@ class HybridMonteCarlo {
   /////////////////////////////////////////////////////////
   // Evolution
   /////////////////////////////////////////////////////////
-  RealD evolve_step(GaugeField &U) {
+  RealD evolve_hmc_step(Field &U) {
     TheIntegrator.refresh(U, pRNG);  // set U and initialize P and phi's
 
     RealD H0 = TheIntegrator.S(U);  // initial state action
 
     std::streamsize current_precision = std::cout.precision();
-    std::cout.precision(17);
+    std::cout.precision(15);
     std::cout << GridLogMessage << "Total H before trajectory = " << H0 << "\n";
     std::cout.precision(current_precision);
 
@@ -181,7 +156,19 @@ class HybridMonteCarlo {
 
     RealD H1 = TheIntegrator.S(U);  // updated state action
 
-    std::cout.precision(17);
+    ///////////////////////////////////////////////////////////
+    if(0){
+      std::cout << "------------------------- Reversibility test" << std::endl;
+      TheIntegrator.reverse_momenta();
+      TheIntegrator.integrate(U);
+
+      H1 = TheIntegrator.S(U);  // updated state action
+      std::cout << "--------------------------------------------" << std::endl;
+    }
+    ///////////////////////////////////////////////////////////
+
+
+    std::cout.precision(15);
     std::cout << GridLogMessage << "Total H after trajectory  = " << H1
 	      << "  dH = " << H1 - H0 << "\n";
     std::cout.precision(current_precision);
@@ -189,56 +176,76 @@ class HybridMonteCarlo {
     return (H1 - H0);
   }
   
+
+  
+
  public:
   /////////////////////////////////////////
   // Constructor
   /////////////////////////////////////////
-  HybridMonteCarlo(HMCparameters Pams, IntegratorType &_Int,
+  HybridMonteCarlo(HMCparameters _Pams, IntegratorType &_Int,
-                   GridSerialRNG &_sRNG, GridParallelRNG &_pRNG, GaugeField &_U)
+                   GridSerialRNG &_sRNG, GridParallelRNG &_pRNG, 
-      : Params(Pams), TheIntegrator(_Int), sRNG(_sRNG), pRNG(_pRNG), Ucur(_U) {}
+                   ObsListType _Obs, Field &_U)
+    : Params(_Pams), TheIntegrator(_Int), sRNG(_sRNG), pRNG(_pRNG), Observables(_Obs), Ucur(_U) {}
   ~HybridMonteCarlo(){};
 
-  void AddObservable(HmcObservable<GaugeField> *obs) {
-    Observables.push_back(obs);
-  }
-
   void evolve(void) {
     Real DeltaH;
 
-    GaugeField Ucopy(Ucur._grid);
+    Field Ucopy(Ucur._grid);
 
-    Params.print();
+    Params.print_parameters();
+    TheIntegrator.print_actions();
 
     // Actual updates (evolve a copy Ucopy then copy back eventually)
-    for (int traj = Params.StartTrajectory;
+    unsigned int FinalTrajectory = Params.Trajectories + Params.NoMetropolisUntil + Params.StartTrajectory;
-         traj < Params.Trajectories + Params.StartTrajectory; ++traj) {
+    for (int traj = Params.StartTrajectory; traj < FinalTrajectory; ++traj) {
       std::cout << GridLogMessage << "-- # Trajectory = " << traj << "\n";
+      if (traj < Params.StartTrajectory + Params.NoMetropolisUntil) {
+      	std::cout << GridLogMessage << "-- Thermalization" << std::endl;
+      }
+      
+      double t0=usecond();
       Ucopy = Ucur;
 
-      DeltaH = evolve_step(Ucopy);
+      DeltaH = evolve_hmc_step(Ucopy);
-
+      // Metropolis-Hastings test
       bool accept = true;
-      if (traj >= Params.NoMetropolisUntil) {
+      if (traj >= Params.StartTrajectory + Params.NoMetropolisUntil) {
         accept = metropolis_test(DeltaH);
+      } else {
+      	std::cout << GridLogMessage << "Skipping Metropolis test" << std::endl;
       }
 
-      if (accept) {
+      if (accept)
         Ucur = Ucopy; 
-      }
+      
+     
+      
+      double t1=usecond();
+      std::cout << GridLogMessage << "Total time for trajectory (s): " << (t1-t0)/1e6 << std::endl;
+
 
       for (int obs = 0; obs < Observables.size(); obs++) {
+      	std::cout << GridLogDebug << "Observables # " << obs << std::endl;
+      	std::cout << GridLogDebug << "Observables total " << Observables.size() << std::endl;
+      	std::cout << GridLogDebug << "Observables pointer " << Observables[obs] << std::endl;
         Observables[obs]->TrajectoryComplete(traj + 1, Ucur, sRNG, pRNG);
       }
+      std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::::" << std::endl;
     }
   }
+
 };
 
 
 }  // QCD
 }  // Grid
 
-#include <Grid/parallelIO/NerscIO.h>
+
-#include <Grid/qcd/hmc/NerscCheckpointer.h>
+// april 11 2017 merge, Guido, commenting out
-#include <Grid/qcd/hmc/HmcRunner.h>
+//#include <Grid/parallelIO/NerscIO.h>
+//#include <Grid/qcd/hmc/NerscCheckpointer.h>
+//#include <Grid/qcd/hmc/HmcRunner.h>
 
 #endif

lib/qcd/hmc/HMCModules.h

@@ -0,0 +1,111 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/GenericHmcRunner.h
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_HMC_MODULES
+#define GRID_HMC_MODULES
+
+
+#include "HMC_GridModules.h"
+
+namespace Grid {
+namespace QCD {
+
+////////////////////////////////////////////////////////////////////
+struct RNGModuleParameters: Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(RNGModuleParameters,
+  std::string, serial_seeds,
+  std::string, parallel_seeds,);
+
+  std::vector<int> getSerialSeeds(){return strToVec<int>(serial_seeds);}
+  std::vector<int> getParallelSeeds(){return strToVec<int>(parallel_seeds);}
+
+  RNGModuleParameters(): serial_seeds("1"), parallel_seeds("1"){}
+
+  template <class ReaderClass >
+  RNGModuleParameters(Reader<ReaderClass>& Reader){
+    read(Reader, "RandomNumberGenerator", *this); 
+  }
+  
+};
+
+// Random number generators module
+class RNGModule{
+   GridSerialRNG sRNG_;
+   std::unique_ptr<GridParallelRNG> pRNG_;
+   RNGModuleParameters Params_;
+
+public:
+
+  RNGModule(){};
+
+  void set_pRNG(GridParallelRNG* pRNG){
+    pRNG_.reset(pRNG);
+  }
+
+  void set_RNGSeeds(RNGModuleParameters& Params) {
+    Params_ = Params;
+  }
+
+  GridSerialRNG& get_sRNG() { return sRNG_; }
+  GridParallelRNG& get_pRNG() { return *pRNG_.get(); }
+
+  void seed() {
+    auto SerialSeeds   = Params_.getSerialSeeds();
+    auto ParallelSeeds = Params_.getParallelSeeds();
+    if (SerialSeeds.size() == 0 && ParallelSeeds.size() == 0) {
+      std::cout << GridLogError << "Seeds not initialized" << std::endl;
+      exit(1);
+    }
+    sRNG_.SeedFixedIntegers(SerialSeeds);
+    pRNG_->SeedFixedIntegers(ParallelSeeds);
+  }
+};
+
+
+/*
+///////////////////////////////////////////////////////////////////
+/// Smearing module
+template <class ImplementationPolicy>
+class SmearingModule{
+   virtual void get_smearing();
+};
+
+template <class ImplementationPolicy>
+class StoutSmearingModule: public SmearingModule<ImplementationPolicy>{
+   SmearedConfiguration<ImplementationPolicy> SmearingPolicy;
+};
+
+*/
+
+
+
+}  // namespace QCD
+}  // namespace Grid
+
+#endif  // GRID_HMC_MODULES

lib/qcd/hmc/HMCResourceManager.h

@@ -0,0 +1,300 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/GenericHmcRunner.h
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+  See the full license in the file "LICENSE" in the top level distribution
+  directory
+  *************************************************************************************/
+/*  END LEGAL */
+#ifndef HMC_RESOURCE_MANAGER_H
+#define HMC_RESOURCE_MANAGER_H
+
+#include <unordered_map>
+
+// One function per Checkpointer, use a macro to simplify
+#define RegisterLoadCheckPointerFunction(NAME)                           \
+  void Load##NAME##Checkpointer(const CheckpointerParameters& Params_) { \
+    if (!have_CheckPointer) {                                            \
+      std::cout << GridLogDebug << "Loading Checkpointer " << #NAME      \
+                << std::endl;                                            \
+      CP = std::unique_ptr<CheckpointerBaseModule>(                      \
+        new NAME##CPModule<ImplementationPolicy>(Params_));              \
+      have_CheckPointer = true;                                          \
+    } else {                                                             \
+      std::cout << GridLogError << "Checkpointer already loaded "        \
+                << std::endl;                                            \
+      exit(1);                                                           \
+    }                                                                    \
+  }
+
+namespace Grid {
+namespace QCD {
+
+// HMC Resource manager
+template <class ImplementationPolicy>
+class HMCResourceManager {
+  typedef HMCModuleBase< QCD::BaseHmcCheckpointer<ImplementationPolicy> > CheckpointerBaseModule;
+  typedef HMCModuleBase< QCD::HmcObservable<typename ImplementationPolicy::Field> > ObservableBaseModule;
+  typedef ActionModuleBase< QCD::Action<typename ImplementationPolicy::Field>, GridModule > ActionBaseModule;
+
+  // Named storage for grid pairs (std + red-black)
+  std::unordered_map<std::string, GridModule> Grids;
+  RNGModule RNGs;
+
+  // SmearingModule<ImplementationPolicy> Smearing;
+  std::unique_ptr<CheckpointerBaseModule> CP;
+
+  // A vector of HmcObservable modules
+  std::vector<std::unique_ptr<ObservableBaseModule> > ObservablesList;
+
+
+  // A vector of HmcObservable modules
+  std::multimap<int, std::unique_ptr<ActionBaseModule> > ActionsList;
+  std::vector<int> multipliers;
+
+  bool have_RNG;
+  bool have_CheckPointer;
+
+  // NOTE: operator << is not overloaded for std::vector<string> 
+  // so thsi function is necessary
+  void output_vector_string(const std::vector<std::string> &vs){
+    for (auto &i: vs)
+      std::cout << i << " ";
+    std::cout << std::endl;
+  }
+
+
+ public:
+  HMCResourceManager() : have_RNG(false), have_CheckPointer(false) {}
+
+  template <class ReaderClass, class vector_type = vComplex >
+  void initialize(ReaderClass &Read){
+    // assumes we are starting from the main node
+
+    // Geometry
+    GridModuleParameters GridPar(Read);
+    GridFourDimModule<vector_type> GridMod( GridPar) ;
+    AddGrid("gauge", GridMod);
+
+    // Checkpointer
+    auto &CPfactory = HMC_CPModuleFactory<cp_string, ImplementationPolicy, ReaderClass >::getInstance();
+    Read.push("Checkpointer");
+    std::string cp_type;
+    read(Read,"name", cp_type);
+    std::cout << "Registered types " << std::endl;
+    output_vector_string(CPfactory.getBuilderList());
+
+
+    CP = CPfactory.create(cp_type, Read);
+    CP->print_parameters();
+    Read.pop();    
+    have_CheckPointer = true;  
+
+    RNGModuleParameters RNGpar(Read);
+    SetRNGSeeds(RNGpar);
+
+    // Observables
+    auto &ObsFactory = HMC_ObservablesModuleFactory<observable_string, typename ImplementationPolicy::Field, ReaderClass>::getInstance(); 
+    Read.push(observable_string);// here must check if existing...
+    do {
+      std::string obs_type;
+      read(Read,"name", obs_type);
+      std::cout << "Registered types " << std::endl;
+      output_vector_string(ObsFactory.getBuilderList() );
+
+      ObservablesList.emplace_back(ObsFactory.create(obs_type, Read));
+      ObservablesList[ObservablesList.size() - 1]->print_parameters();
+    } while (Read.nextElement(observable_string));
+    Read.pop();
+
+    // Loop on levels
+    if(!Read.push("Actions")){
+      std::cout << "Actions not found" << std::endl; 
+      exit(1);
+    }
+
+    if(!Read.push("Level")){// push must check if the node exist
+         std::cout << "Level not found" << std::endl; 
+      exit(1);
+    }
+    do
+    {
+      fill_ActionsLevel(Read); 
+    }
+    while(Read.push("Level"));
+
+    Read.pop();
+  }
+
+
+ 
+  template <class RepresentationPolicy>
+  void GetActionSet(ActionSet<typename ImplementationPolicy::Field, RepresentationPolicy>& Aset){
+    Aset.resize(multipliers.size());
+ 
+    for(auto it = ActionsList.begin(); it != ActionsList.end(); it++){
+      (*it).second->acquireResource(Grids["gauge"]);
+      Aset[(*it).first-1].push_back((*it).second->getPtr());
+    }
+  }
+
+
+
+  //////////////////////////////////////////////////////////////
+  // Grids
+  //////////////////////////////////////////////////////////////
+
+  void AddGrid(std::string s, GridModule& M) {
+    // Check for name clashes
+    auto search = Grids.find(s);
+    if (search != Grids.end()) {
+      std::cout << GridLogError << "Grid with name \"" << search->first
+                << "\" already present. Terminating\n";
+      exit(1);
+    }
+    Grids[s] = std::move(M);
+  }
+
+  // Add a named grid set, 4d shortcut
+  void AddFourDimGrid(std::string s) {
+    GridFourDimModule<vComplex> Mod;
+    AddGrid(s, Mod);
+  }
+
+
+
+  GridCartesian* GetCartesian(std::string s = "") {
+    if (s.empty()) s = Grids.begin()->first;
+    std::cout << GridLogDebug << "Getting cartesian grid from: " << s
+              << std::endl;
+    return Grids[s].get_full();
+  }
+
+  GridRedBlackCartesian* GetRBCartesian(std::string s = "") {
+    if (s.empty()) s = Grids.begin()->first;
+    std::cout << GridLogDebug << "Getting rb-cartesian grid from: " << s
+              << std::endl;
+    return Grids[s].get_rb();
+  }
+
+  //////////////////////////////////////////////////////
+  // Random number generators
+  //////////////////////////////////////////////////////
+
+  void AddRNGs(std::string s = "") {
+    // Couple the RNGs to the GridModule tagged by s
+    // the default is the first grid registered
+    assert(Grids.size() > 0 && !have_RNG);
+    if (s.empty()) s = Grids.begin()->first;
+    std::cout << GridLogDebug << "Adding RNG to grid: " << s << std::endl;
+    RNGs.set_pRNG(new GridParallelRNG(GetCartesian(s)));
+    have_RNG = true;
+  }
+
+  void SetRNGSeeds(RNGModuleParameters& Params) { RNGs.set_RNGSeeds(Params); }
+
+  GridSerialRNG& GetSerialRNG() { return RNGs.get_sRNG(); }
+
+  GridParallelRNG& GetParallelRNG() {
+    assert(have_RNG);
+    return RNGs.get_pRNG();
+  }
+
+  void SeedFixedIntegers() {
+    assert(have_RNG);
+    RNGs.seed();
+  }
+
+  //////////////////////////////////////////////////////
+  // Checkpointers
+  //////////////////////////////////////////////////////
+
+  BaseHmcCheckpointer<ImplementationPolicy>* GetCheckPointer() {
+    if (have_CheckPointer)
+      return CP->getPtr();
+    else {
+      std::cout << GridLogError << "Error: no checkpointer defined"
+                << std::endl;
+      exit(1);
+    }
+  }
+
+  RegisterLoadCheckPointerFunction(Binary);
+  RegisterLoadCheckPointerFunction(Nersc);
+  #ifdef HAVE_LIME
+  RegisterLoadCheckPointerFunction(ILDG);
+  #endif
+
+  ////////////////////////////////////////////////////////
+  // Observables
+  ////////////////////////////////////////////////////////
+
+  template<class T, class... Types>
+  void AddObservable(Types&&... Args){
+    ObservablesList.push_back(std::unique_ptr<T>(new T(std::forward<Types>(Args)...)));
+  }
+
+  std::vector<HmcObservable<typename ImplementationPolicy::Field>* > GetObservables(){
+    std::vector<HmcObservable<typename ImplementationPolicy::Field>* > out;
+    for (auto &i : ObservablesList){
+      out.push_back(i->getPtr());
+    }
+
+    // Add the checkpointer to the observables
+    out.push_back(GetCheckPointer());
+    return out;
+  }
+
+
+
+private:
+   // this private
+  template <class ReaderClass >
+  void fill_ActionsLevel(ReaderClass &Read){
+    // Actions set
+    int m;
+    Read.readDefault("multiplier",m);
+    multipliers.push_back(m);
+    std::cout << "Level : " << multipliers.size()  << " with multiplier : " << m << std::endl; 
+    // here gauge
+    Read.push("Action");
+    do{
+      auto &ActionFactory = HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, ReaderClass>::getInstance(); 
+      std::string action_type;
+      Read.readDefault("name", action_type); 
+      output_vector_string(ActionFactory.getBuilderList() );
+      ActionsList.emplace(m, ActionFactory.create(action_type, Read));
+    } while (Read.nextElement("Action"));
+    ActionsList.find(m)->second->print_parameters();    
+    Read.pop();
+
+  }
+
+
+
+};
+}
+}
+
+#endif  // HMC_RESOURCE_MANAGER_H

lib/qcd/hmc/HMCRunnerModule.h

@@ -0,0 +1,137 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/GenericHmcRunner.h
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef HMC_RUNNER_MODULE
+#define HMC_RUNNER_MODULE
+
+namespace Grid {
+
+// the reader class is necessary here for the automatic initialization of the resources
+// if we had a virtual reader would have been unecessary
+template <class HMCType, class ReaderClass >
+class HMCModule
+    : public Parametrized< QCD::HMCparameters >,
+      public HMCModuleBase< QCD::HMCRunnerBase<ReaderClass> > {
+ public:
+  typedef HMCModuleBase< QCD::HMCRunnerBase<ReaderClass> > Base;
+  typedef typename Base::Product Product;
+
+  std::unique_ptr<HMCType> HMCPtr;
+
+  HMCModule(QCD::HMCparameters Par) : Parametrized<QCD::HMCparameters>(Par) {}
+
+  template <class ReaderCl>
+  HMCModule(Reader<ReaderCl>& R) : Parametrized<QCD::HMCparameters>(R, "HMC"){};
+
+  Product* getPtr() {
+    if (!HMCPtr) initialize();
+ 
+    return HMCPtr.get();
+  }
+
+ private:
+  virtual void initialize() = 0;
+};
+
+// Factory
+template <char const *str, class ReaderClass >
+class HMCRunnerModuleFactory
+    : public Factory < HMCModuleBase< QCD::HMCRunnerBase<ReaderClass> > ,	Reader<ReaderClass> > {
+ public:
+ 	typedef Reader<ReaderClass> TheReader; 
+ 	// use SINGLETON FUNCTOR MACRO HERE
+  HMCRunnerModuleFactory(const HMCRunnerModuleFactory& e) = delete;
+  void operator=(const HMCRunnerModuleFactory& e) = delete;
+  static HMCRunnerModuleFactory& getInstance(void) {
+    static HMCRunnerModuleFactory e;
+    return e;
+  }
+
+ private:
+  HMCRunnerModuleFactory(void) = default;
+  std::string obj_type() const {
+  	return std::string(str);
+  }
+};
+
+
+
+
+
+///////////////
+// macro for these
+
+template < class ImplementationPolicy, class RepresentationPolicy, class ReaderClass >
+class HMCLeapFrog: public HMCModule< QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::LeapFrog>, ReaderClass >{
+  typedef HMCModule< QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::LeapFrog>, ReaderClass  > HMCBaseMod;
+  using HMCBaseMod::HMCBaseMod;
+
+  // aquire resource
+  virtual void initialize(){
+    this->HMCPtr.reset(new QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::LeapFrog>(this->Par_) );
+  }
+};
+
+template < class ImplementationPolicy, class RepresentationPolicy, class ReaderClass >
+class HMCMinimumNorm2: public HMCModule< QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::MinimumNorm2>, ReaderClass  >{
+  typedef HMCModule< QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::MinimumNorm2>, ReaderClass  > HMCBaseMod;
+  using HMCBaseMod::HMCBaseMod;
+
+  // aquire resource
+  virtual void initialize(){
+    this->HMCPtr.reset(new QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::MinimumNorm2>(this->Par_));
+  }
+};
+
+
+template < class ImplementationPolicy, class RepresentationPolicy, class ReaderClass >
+class HMCForceGradient: public HMCModule< QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::ForceGradient>, ReaderClass  >{
+  typedef HMCModule< QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::ForceGradient>, ReaderClass   > HMCBaseMod;
+  using HMCBaseMod::HMCBaseMod;
+
+  // aquire resource
+  virtual void initialize(){
+    this->HMCPtr.reset(new QCD::GenericHMCRunnerTemplate<ImplementationPolicy, RepresentationPolicy, QCD::ForceGradient>(this->Par_) );
+  }
+};
+
+extern char hmc_string[];
+
+
+
+
+
+
+//////////////////////////////////////////////////////////////
+
+
+
+}
+
+#endif

lib/qcd/hmc/HMC_GridModules.h

@@ -0,0 +1,133 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/HMC_GridModules.h
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef HMC_GRID_MODULES
+#define HMC_GRID_MODULES
+
+namespace Grid {
+
+// Resources
+// Modules for grids 
+
+// Introduce another namespace HMCModules?
+
+class GridModuleParameters: Serializable{   
+public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(GridModuleParameters,
+  std::string, lattice,
+  std::string, mpi);
+
+  std::vector<int> getLattice(){return strToVec<int>(lattice);}
+  std::vector<int> getMpi()    {return strToVec<int>(mpi);}
+
+  void check(){
+    if (getLattice().size() != getMpi().size()) {
+      std::cout << GridLogError 
+                << "Error in GridModuleParameters: lattice and mpi dimensions "
+                   "do not match"
+                << std::endl;
+      exit(1);
+    }
+  }    
+
+  template <class ReaderClass>
+  GridModuleParameters(Reader<ReaderClass>& Reader, std::string n = "LatticeGrid"):name(n) {
+    read(Reader, name, *this);
+    check();
+  }
+
+  // Save on file
+  template< class WriterClass>
+  void save(Writer<WriterClass>& Writer){
+    check();
+    write(Writer, name, *this);
+  }
+private:
+    std::string name;
+};
+
+// Lower level class
+class GridModule {
+ public:
+  GridCartesian* get_full() { 
+    std::cout << GridLogDebug << "Getting cartesian in module"<< std::endl;
+    return grid_.get(); }
+  GridRedBlackCartesian* get_rb() { 
+    std::cout << GridLogDebug << "Getting rb-cartesian in module"<< std::endl;
+    return rbgrid_.get(); }
+
+  void set_full(GridCartesian* grid) { grid_.reset(grid); }
+  void set_rb(GridRedBlackCartesian* rbgrid) { rbgrid_.reset(rbgrid); }
+
+ protected:
+  std::unique_ptr<GridCartesian> grid_;
+  std::unique_ptr<GridRedBlackCartesian> rbgrid_;
+  
+};
+
+////////////////////////////////////
+// Classes for the user
+////////////////////////////////////
+// Note: the space time grid should be out of the QCD namespace
+template< class vector_type>
+class GridFourDimModule : public GridModule {
+ public:
+  GridFourDimModule() {
+    using namespace QCD;
+    set_full(SpaceTimeGrid::makeFourDimGrid(
+        GridDefaultLatt(), GridDefaultSimd(4, vector_type::Nsimd()),
+        GridDefaultMpi()));
+    set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get()));
+  }
+
+  GridFourDimModule(GridModuleParameters Params) {
+    using namespace QCD;
+    Params.check();
+    std::vector<int> lattice_v = Params.getLattice();
+    std::vector<int> mpi_v = Params.getMpi();
+    if (lattice_v.size() == 4) {
+      set_full(SpaceTimeGrid::makeFourDimGrid(
+          lattice_v, GridDefaultSimd(4, vector_type::Nsimd()),
+          mpi_v));
+      set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get()));
+    } else {
+      std::cout << GridLogError 
+          << "Error in GridFourDimModule: lattice dimension different from 4"
+          << std::endl;
+      exit(1);
+    }
+  }
+};
+
+typedef GridFourDimModule<vComplex> GridDefaultFourDimModule;
+
+
+}  // namespace Grid
+
+#endif  // HMC_GRID_MODULES

lib/qcd/hmc/HMC_aggregate.h

@@ -33,10 +33,20 @@ directory
 
 #include <string>
 
+#include <Grid/qcd/observables/hmc_observable.h>
 #include <Grid/qcd/hmc/HMC.h>
+
+
 // annoying location; should move this ?
+#include <Grid/parallelIO/IldgIOtypes.h>
+#include <Grid/parallelIO/IldgIO.h>
 #include <Grid/parallelIO/NerscIO.h>
-#include <Grid/qcd/hmc/NerscCheckpointer.h>
+
-#include <Grid/qcd/hmc/HmcRunner.h>
+#include <Grid/qcd/hmc/checkpointers/CheckPointers.h>
+#include <Grid/qcd/hmc/HMCModules.h>
+#include <Grid/qcd/modules/mods.h>
+#include <Grid/qcd/hmc/HMCResourceManager.h>
+#include <Grid/qcd/hmc/GenericHMCrunner.h>
+#include <Grid/qcd/hmc/HMCRunnerModule.h>
 
 #endif

lib/qcd/hmc/HmcRunner.h

@@ -1,191 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid
-
-Source file: ./lib/qcd/hmc/HmcRunner.h
-
-Copyright (C) 2015
-
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution
-directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef HMC_RUNNER
-#define HMC_RUNNER
-
-namespace Grid {
-namespace QCD {
-
-template <class Gimpl, class RepresentationsPolicy = NoHirep >
-class NerscHmcRunnerTemplate {
- public:
-  INHERIT_GIMPL_TYPES(Gimpl);
-
-  enum StartType_t { ColdStart, HotStart, TepidStart, CheckpointStart };
-
-  ActionSet<GaugeField, RepresentationsPolicy> TheAction;
-
-  GridCartesian *UGrid;
-  GridCartesian *FGrid;
-  GridRedBlackCartesian *UrbGrid;
-  GridRedBlackCartesian *FrbGrid;
-
-  virtual void BuildTheAction(int argc, char **argv) = 0;  // necessary?
-
-  void Run(int argc, char **argv) {
-    StartType_t StartType = HotStart;
-
-    std::string arg;
-
-    if (GridCmdOptionExists(argv, argv + argc, "--StartType")) {
-      arg = GridCmdOptionPayload(argv, argv + argc, "--StartType");
-      if (arg == "HotStart") {
-        StartType = HotStart;
-      } else if (arg == "ColdStart") {
-        StartType = ColdStart;
-      } else if (arg == "TepidStart") {
-        StartType = TepidStart;
-      } else if (arg == "CheckpointStart") {
-        StartType = CheckpointStart;
-      } else {
-        std::cout << GridLogError << "Unrecognized option in --StartType\n";
-        std::cout << GridLogError << "Valid [HotStart, ColdStart, TepidStart, CheckpointStart]\n";
-        assert(0);
-      }
-    }
-
-    int StartTraj = 0;
-    if (GridCmdOptionExists(argv, argv + argc, "--StartTrajectory")) {
-      arg = GridCmdOptionPayload(argv, argv + argc, "--StartTrajectory");
-      std::vector<int> ivec(0);
-      GridCmdOptionIntVector(arg, ivec);
-      StartTraj = ivec[0];
-    }
-
-    int NumTraj = 1;
-    if (GridCmdOptionExists(argv, argv + argc, "--Trajectories")) {
-      arg = GridCmdOptionPayload(argv, argv + argc, "--Trajectories");
-      std::vector<int> ivec(0);
-      GridCmdOptionIntVector(arg, ivec);
-      NumTraj = ivec[0];
-    }
-
-    int NumThermalizations = 10;
-    if (GridCmdOptionExists(argv, argv + argc, "--Thermalizations")) {
-      arg = GridCmdOptionPayload(argv, argv + argc, "--Thermalizations");
-      std::vector<int> ivec(0);
-      GridCmdOptionIntVector(arg, ivec);
-      NumThermalizations = ivec[0];
-    }
-
-    GridSerialRNG sRNG;
-    GridParallelRNG pRNG(UGrid);
-    LatticeGaugeField U(UGrid);  // change this to an extended field (smearing class)?
-
-    std::vector<int> SerSeed({1, 2, 3, 4, 5});
-    std::vector<int> ParSeed({6, 7, 8, 9, 10});
-
-    // Create integrator, including the smearing policy
-    // Smearing policy, only defined for Nc=3
-    /*
-    std::cout << GridLogDebug << " Creating the Stout class\n";
-    double rho = 0.1;  // smearing parameter, now hardcoded
-    int Nsmear = 1;    // number of smearing levels
-    Smear_Stout<Gimpl> Stout(rho);
-    std::cout << GridLogDebug << " Creating the SmearedConfiguration class\n";
-    //SmearedConfiguration<Gimpl> SmearingPolicy(UGrid, Nsmear, Stout);
-    std::cout << GridLogDebug << " done\n";
-    */
-    //////////////
-    NoSmearing<Gimpl> SmearingPolicy;
-    // change here to change the algorithm
-    typedef MinimumNorm2<GaugeField, NoSmearing<Gimpl>, RepresentationsPolicy >  IntegratorType;  
-    IntegratorParameters MDpar(40, 1.0);
-    IntegratorType MDynamics(UGrid, MDpar, TheAction, SmearingPolicy);
-
-    // Checkpoint strategy
-    NerscHmcCheckpointer<Gimpl> Checkpoint(std::string("ckpoint_lat"),
-                                           std::string("ckpoint_rng"), 1);
-    PlaquetteLogger<Gimpl> PlaqLog(std::string("plaq"));
-
-    HMCparameters HMCpar;
-    HMCpar.StartTrajectory = StartTraj;
-    HMCpar.Trajectories = NumTraj;
-    HMCpar.NoMetropolisUntil = NumThermalizations;
-
-    if (StartType == HotStart) {
-      // Hot start
-      HMCpar.MetropolisTest = true;
-      sRNG.SeedFixedIntegers(SerSeed);
-      pRNG.SeedFixedIntegers(ParSeed);
-      SU<Nc>::HotConfiguration(pRNG, U);
-    } else if (StartType == ColdStart) {
-      // Cold start
-      HMCpar.MetropolisTest = true;
-      sRNG.SeedFixedIntegers(SerSeed);
-      pRNG.SeedFixedIntegers(ParSeed);
-      SU<Nc>::ColdConfiguration(pRNG, U);
-    } else if (StartType == TepidStart) {
-      // Tepid start
-      HMCpar.MetropolisTest = true;
-      sRNG.SeedFixedIntegers(SerSeed);
-      pRNG.SeedFixedIntegers(ParSeed);
-      SU<Nc>::TepidConfiguration(pRNG, U);
-    } else if (StartType == CheckpointStart) {
-      HMCpar.MetropolisTest = true;
-      // CheckpointRestart
-      Checkpoint.CheckpointRestore(StartTraj, U, sRNG, pRNG);
-    }
-
-    // Attach the gauge field to the smearing Policy and create the fill the
-    // smeared set
-    // notice that the unit configuration is singular in this procedure
-    std::cout << GridLogMessage << "Filling the smeared set\n";
-    SmearingPolicy.set_GaugeField(U);
-
-    HybridMonteCarlo<GaugeField, IntegratorType> HMC(HMCpar, MDynamics, sRNG,
-                                                     pRNG, U);
-    HMC.AddObservable(&Checkpoint);
-    HMC.AddObservable(&PlaqLog);
-
-    // Run it
-    HMC.evolve();
-  }
-};
-
-typedef NerscHmcRunnerTemplate<PeriodicGimplR> NerscHmcRunner;
-typedef NerscHmcRunnerTemplate<PeriodicGimplF> NerscHmcRunnerF;
-typedef NerscHmcRunnerTemplate<PeriodicGimplD> NerscHmcRunnerD;
-
-typedef NerscHmcRunnerTemplate<PeriodicGimplR> PeriodicNerscHmcRunner;
-typedef NerscHmcRunnerTemplate<PeriodicGimplF> PeriodicNerscHmcRunnerF;
-typedef NerscHmcRunnerTemplate<PeriodicGimplD> PeriodicNerscHmcRunnerD;
-
-typedef NerscHmcRunnerTemplate<ConjugateGimplR> ConjugateNerscHmcRunner;
-typedef NerscHmcRunnerTemplate<ConjugateGimplF> ConjugateNerscHmcRunnerF;
-typedef NerscHmcRunnerTemplate<ConjugateGimplD> ConjugateNerscHmcRunnerD;
-
-template <class RepresentationsPolicy>
-using NerscHmcRunnerHirep = NerscHmcRunnerTemplate<PeriodicGimplR, RepresentationsPolicy>;
-
-
-
-}
-}
-#endif

lib/qcd/hmc/NerscCheckpointer.h

@@ -1,80 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/qcd/hmc/NerscCheckpointer.h
-
-    Copyright (C) 2015
-
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef NERSC_CHECKPOINTER
-#define NERSC_CHECKPOINTER
-
-#include <string>
-#include <iostream>
-#include <sstream>
-
-
-namespace Grid{
-  namespace QCD{
-    
-
-    template<class Gimpl> 
-    class NerscHmcCheckpointer : public HmcObservable<typename Gimpl::GaugeField> {
-    private:
-      std::string configStem;
-      std::string rngStem;
-      int SaveInterval;
-    public:
-      INHERIT_GIMPL_TYPES(Gimpl);
-
-      NerscHmcCheckpointer(std::string cf, std::string rn,int savemodulo) {
-        configStem  = cf;
-        rngStem     = rn;
-        SaveInterval= savemodulo;
-      };
-
-      void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG & pRNG )
-      {
-	if ( (traj % SaveInterval)== 0 ) {
-	  std::string rng;   { std::ostringstream os; os << rngStem     <<"."<< traj; rng = os.str(); }
-	  std::string config;{ std::ostringstream os; os << configStem  <<"."<< traj; config = os.str();}
-
-	  int precision32=1;
-	  int tworow     =0;
-	  NerscIO::writeRNGState(sRNG,pRNG,rng);
-	  NerscIO::writeConfiguration(U,config,tworow,precision32);
-	}
-      };
-
-      void CheckpointRestore(int traj, GaugeField &U, GridSerialRNG &sRNG, GridParallelRNG & pRNG ){
-
-	std::string rng;   { std::ostringstream os; os << rngStem     <<"."<< traj; rng = os.str(); }
-	std::string config;{ std::ostringstream os; os << configStem  <<"."<< traj; config = os.str();}
-
-	NerscField header;
-	NerscIO::readRNGState(sRNG,pRNG,header,rng);
-	NerscIO::readConfiguration(U,header,config);
-      };
-
-    };
-}}
-#endif

lib/qcd/hmc/UsingHMC.md

@@ -0,0 +1,107 @@
+Using HMC in Grid version 0.5.1
+
+These are the instructions to use the Generalised HMC on Grid version 0.5.1.
+Disclaimer: GRID is still under active development so any information here can be changed in future releases.
+
+
+Command line options
+===================
+(relevant file GenericHMCrunner.h)
+The initial configuration can be changed at the command line using 
+--StartType <your choice>
+valid choices, one among these
+HotStart, ColdStart, TepidStart, CheckpointStart
+default: HotStart
+
+example
+./My_hmc_exec  --StartType HotStart
+
+The CheckpointStart option uses the prefix for the configurations and rng seed files defined in your executable and the initial configuration is specified by
+--StartTrajectory <integer>
+default: 0
+
+The number of trajectories for a specific run are specified at command line by
+--Trajectories <integer>
+default: 1
+
+The number of thermalization steps (i.e. steps when the Metropolis acceptance check is turned off) is specified by
+--Thermalizations <integer>
+default: 10
+
+
+Any other parameter is defined in the source for the executable.
+
+HMC controls
+===========
+
+The lines 
+
+  std::vector<int> SerSeed({1, 2, 3, 4, 5});
+  std::vector<int> ParSeed({6, 7, 8, 9, 10});
+
+define the seeds for the serial and the parallel RNG.
+
+The line 
+
+  TheHMC.MDparameters.set(20, 1.0);// MDsteps, traj length
+
+declares the number of molecular dynamics steps and the total trajectory length.
+
+
+Actions
+======
+
+Action names are defined in the file
+lib/qcd/Actions.h
+
+Gauge actions list:
+
+WilsonGaugeActionR;
+WilsonGaugeActionF;
+WilsonGaugeActionD;
+PlaqPlusRectangleActionR;
+PlaqPlusRectangleActionF;
+PlaqPlusRectangleActionD;
+IwasakiGaugeActionR;
+IwasakiGaugeActionF;
+IwasakiGaugeActionD;
+SymanzikGaugeActionR;
+SymanzikGaugeActionF;
+SymanzikGaugeActionD;
+
+
+ConjugateWilsonGaugeActionR;
+ConjugateWilsonGaugeActionF;
+ConjugateWilsonGaugeActionD;
+ConjugatePlaqPlusRectangleActionR;
+ConjugatePlaqPlusRectangleActionF;
+ConjugatePlaqPlusRectangleActionD;
+ConjugateIwasakiGaugeActionR;
+ConjugateIwasakiGaugeActionF;
+ConjugateIwasakiGaugeActionD;
+ConjugateSymanzikGaugeActionR;
+ConjugateSymanzikGaugeActionF;
+ConjugateSymanzikGaugeActionD;
+
+
+ScalarActionR;
+ScalarActionF;
+ScalarActionD;
+
+
+each of these action accept one single parameter at creation time (beta).
+Example for creating a Symanzik action with beta=4.0
+
+	SymanzikGaugeActionR(4.0)
+
+The suffixes R,F,D in the action names refer to the Real
+(the precision is defined at compile time by the --enable-precision flag in the configure),
+Float and Double, that force the precision of the action to be 32, 64 bit respectively.
+
+
+
+
+
+
+
+

lib/qcd/hmc/checkpointers/BaseCheckpointer.h

@@ -0,0 +1,89 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/BaseCheckpointer.h
+
+Copyright (C) 2015
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef BASE_CHECKPOINTER
+#define BASE_CHECKPOINTER
+
+namespace Grid {
+namespace QCD {
+
+class CheckpointerParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(CheckpointerParameters, 
+  	std::string, config_prefix, 
+  	std::string, rng_prefix, 
+  	int, saveInterval, 
+  	std::string, format, );
+
+  CheckpointerParameters(std::string cf = "cfg", std::string rn = "rng",
+   		      int savemodulo = 1, const std::string &f = "IEEE64BIG")
+      : config_prefix(cf),
+        rng_prefix(rn),
+        saveInterval(savemodulo),
+        format(f){};
+
+
+  template <class ReaderClass >
+  CheckpointerParameters(Reader<ReaderClass> &Reader) {
+    read(Reader, "Checkpointer", *this);
+  }
+ 
+
+};
+
+//////////////////////////////////////////////////////////////////////////////
+// Base class for checkpointers
+template <class Impl>
+class BaseHmcCheckpointer : public HmcObservable<typename Impl::Field> {
+ public:
+  void build_filenames(int traj, CheckpointerParameters &Params,
+                       std::string &conf_file, std::string &rng_file) {
+    {
+      std::ostringstream os;
+      os << Params.rng_prefix << "." << traj;
+      rng_file = os.str();
+    }
+
+    {
+      std::ostringstream os;
+      os << Params.config_prefix << "." << traj;
+      conf_file = os.str();
+    }
+ 	} 
+
+  virtual void initialize(const CheckpointerParameters &Params) = 0;
+
+  virtual void CheckpointRestore(int traj, typename Impl::Field &U,
+                                 GridSerialRNG &sRNG,
+                                 GridParallelRNG &pRNG) = 0;
+
+};  // class BaseHmcCheckpointer
+///////////////////////////////////////////////////////////////////////////////
+}
+}
+#endif

lib/qcd/hmc/checkpointers/BinaryCheckpointer.h

@@ -0,0 +1,99 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/BinaryCheckpointer.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef BINARY_CHECKPOINTER
+#define BINARY_CHECKPOINTER
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+namespace Grid {
+namespace QCD {
+
+// Simple checkpointer, only binary file
+template <class Impl>
+class BinaryHmcCheckpointer : public BaseHmcCheckpointer<Impl> {
+ private:
+  CheckpointerParameters Params;
+
+ public:
+  INHERIT_FIELD_TYPES(Impl);  // Gets the Field type, a Lattice object
+
+  // Extract types from the Field
+  typedef typename Field::vector_object vobj;
+  typedef typename vobj::scalar_object sobj;
+  typedef typename getPrecision<sobj>::real_scalar_type sobj_stype;
+  typedef typename sobj::DoublePrecision sobj_double;
+
+  BinaryHmcCheckpointer(const CheckpointerParameters &Params_) {
+    initialize(Params_);
+  }
+
+  void initialize(const CheckpointerParameters &Params_) { Params = Params_; }
+
+  void truncate(std::string file) {
+    std::ofstream fout(file, std::ios::out);
+    fout.close();
+  }
+
+  void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG,
+                          GridParallelRNG &pRNG) {
+    if ((traj % Params.saveInterval) == 0) {
+      std::string config, rng;
+      this->build_filenames(traj, Params, config, rng);
+
+      BinaryIO::BinarySimpleUnmunger<sobj_double, sobj> munge;
+      truncate(rng);
+      BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0);
+      truncate(config);
+      uint32_t csum = BinaryIO::writeObjectParallel<vobj, sobj_double>(
+          U, config, munge, 0, Params.format);
+
+      std::cout << GridLogMessage << "Written Binary Configuration " << config
+                << " checksum " << std::hex << csum << std::dec << std::endl;
+    }
+  };
+
+  void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG,
+                         GridParallelRNG &pRNG) {
+    std::string config, rng;
+    this->build_filenames(traj, Params, config, rng);
+
+    BinaryIO::BinarySimpleMunger<sobj_double, sobj> munge;
+    BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0);
+    uint32_t csum = BinaryIO::readObjectParallel<vobj, sobj_double>(
+        U, config, munge, 0, Params.format);
+
+    std::cout << GridLogMessage << "Read Binary Configuration " << config
+              << " checksum " << std::hex << csum << std::dec << std::endl;
+  };
+};
+}
+}
+#endif

lib/qcd/hmc/checkpointers/CheckPointerModules.h

@@ -0,0 +1,158 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/gauge/WilsonGaugeAction.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef CP_MODULES_H
+#define CP_MODULES_H
+
+
+// FIXME  Reorganize QCD namespace 
+
+
+namespace Grid {
+
+////////////////////////////////////////////////////////////////////////
+// Checkpoint module, owns the Checkpointer
+////////////////////////////////////////////////////////////////////////
+
+template <class ImplementationPolicy>
+class CheckPointerModule: public Parametrized<QCD::CheckpointerParameters>, public HMCModuleBase< QCD::BaseHmcCheckpointer<ImplementationPolicy> >  {
+ public:
+ 	std::unique_ptr<QCD::BaseHmcCheckpointer<ImplementationPolicy> > CheckPointPtr;
+ 	typedef QCD::CheckpointerParameters APar;
+  typedef HMCModuleBase< QCD::BaseHmcCheckpointer<ImplementationPolicy> > Base;
+  typedef typename Base::Product Product;
+
+  CheckPointerModule(APar Par): Parametrized<APar>(Par) {}
+  template <class ReaderClass>
+  CheckPointerModule(Reader<ReaderClass>& Reader) : Parametrized<APar>(Reader){};
+
+  virtual void print_parameters(){
+  	std::cout << this->Par_ << std::endl;
+  }
+
+  Product* getPtr() {
+    if (!CheckPointPtr) initialize();
+
+    return CheckPointPtr.get();
+  }
+
+ private:
+  virtual void initialize() = 0;
+
+};
+
+
+
+template <char const *str, class ImplementationPolicy, class ReaderClass >
+class HMC_CPModuleFactory
+    : public Factory < HMCModuleBase< QCD::BaseHmcCheckpointer<ImplementationPolicy> > ,	Reader<ReaderClass> > {
+ public:
+ 	typedef Reader<ReaderClass> TheReader; 
+ 	// use SINGLETON FUNCTOR MACRO HERE
+  HMC_CPModuleFactory(const HMC_CPModuleFactory& e) = delete;
+  void operator=(const HMC_CPModuleFactory& e) = delete;
+  static HMC_CPModuleFactory& getInstance(void) {
+    static HMC_CPModuleFactory e;
+    return e;
+  }
+
+ private:
+  HMC_CPModuleFactory(void) = default;
+  std::string obj_type() const {
+  	return std::string(str);
+  }
+};
+
+
+
+/////////////////////////////////////////////////////////////////////
+// Concrete classes
+/////////////////////////////////////////////////////////////////////
+namespace QCD{
+
+template<class ImplementationPolicy>
+class BinaryCPModule: public CheckPointerModule< ImplementationPolicy> {
+  typedef CheckPointerModule< ImplementationPolicy> CPBase;
+  using CPBase::CPBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->CheckPointPtr.reset(new BinaryHmcCheckpointer<ImplementationPolicy>(this->Par_));
+  }
+
+};
+
+
+template<class ImplementationPolicy>
+class NerscCPModule: public CheckPointerModule< ImplementationPolicy> {
+  typedef CheckPointerModule< ImplementationPolicy> CPBase;
+  using CPBase::CPBase; // for constructors inheritance
+
+  // acquire resource
+  virtual void initialize(){
+     this->CheckPointPtr.reset(new NerscHmcCheckpointer<ImplementationPolicy>(this->Par_));
+  }
+
+};
+
+
+#ifdef HAVE_LIME
+  
+template<class ImplementationPolicy>
+class ILDGCPModule: public CheckPointerModule< ImplementationPolicy> {
+  typedef CheckPointerModule< ImplementationPolicy> CPBase;
+  using CPBase::CPBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+     this->CheckPointPtr.reset(new ILDGHmcCheckpointer<ImplementationPolicy>(this->Par_));
+  }
+
+};
+
+#endif
+
+
+}// QCD temporarily here
+
+
+extern char cp_string[];
+
+/*
+// use macros?
+static Registrar<QCD::BinaryCPModule<QCD::PeriodicGimplR>, HMC_CPModuleFactory<cp_string, QCD::PeriodicGimplR, XmlReader> > __CPBinarymodXMLInit("Binary");
+static Registrar<QCD::NerscCPModule<QCD::PeriodicGimplR> , HMC_CPModuleFactory<cp_string, QCD::PeriodicGimplR, XmlReader> > __CPNerscmodXMLInit("Nersc");
+
+#ifdef HAVE_LIME
+static Registrar<QCD::ILDGCPModule<QCD::PeriodicGimplR>  , HMC_CPModuleFactory<cp_string, QCD::PeriodicGimplR, XmlReader> > __CPILDGmodXMLInit("ILDG");
+#endif
+*/
+
+}// Grid
+#endif //CP_MODULES_H

lib/qcd/hmc/checkpointers/CheckPointers.h

@@ -0,0 +1,40 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/gauge/WilsonGaugeAction.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef CHECKPOINTERS_H
+#define CHECKPOINTERS_H
+
+#include <Grid/qcd/hmc/checkpointers/BaseCheckpointer.h>
+#include <Grid/qcd/hmc/checkpointers/NerscCheckpointer.h>
+#include <Grid/qcd/hmc/checkpointers/BinaryCheckpointer.h>
+#include <Grid/qcd/hmc/checkpointers/ILDGCheckpointer.h>
+//#include <Grid/qcd/hmc/checkpointers/CheckPointerModules.h>
+
+
+#endif // CHECKPOINTERS_H

lib/qcd/hmc/checkpointers/ILDGCheckpointer.h

@@ -0,0 +1,104 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/ILDGCheckpointer.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef ILDG_CHECKPOINTER
+#define ILDG_CHECKPOINTER
+
+#ifdef HAVE_LIME
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+namespace Grid {
+namespace QCD {
+
+// Only for Gauge fields
+template <class Implementation>
+class ILDGHmcCheckpointer : public BaseHmcCheckpointer<Implementation> {
+ private:
+  CheckpointerParameters Params;
+
+ public:
+  INHERIT_GIMPL_TYPES(Implementation);
+
+  ILDGHmcCheckpointer(const CheckpointerParameters &Params_) { initialize(Params_); }
+
+  void initialize(const CheckpointerParameters &Params_) {
+    Params = Params_;
+
+    // check here that the format is valid
+    int ieee32big = (Params.format == std::string("IEEE32BIG"));
+    int ieee32 = (Params.format == std::string("IEEE32"));
+    int ieee64big = (Params.format == std::string("IEEE64BIG"));
+    int ieee64 = (Params.format == std::string("IEEE64"));
+
+    if (!(ieee64big || ieee32 || ieee32big || ieee64)) {
+      std::cout << GridLogError << "Unrecognized file format " << Params.format
+                << std::endl;
+      std::cout << GridLogError
+                << "Allowed: IEEE32BIG | IEEE32 | IEEE64BIG | IEEE64"
+                << std::endl;
+
+      exit(1);
+    }
+  }
+
+  void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG,
+                          GridParallelRNG &pRNG) {
+    if ((traj % Params.saveInterval) == 0) {
+      std::string config, rng;
+      this->build_filenames(traj, Params, config, rng);
+
+      ILDGIO IO(config, ILDGwrite);
+      BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0);
+      uint32_t csum = IO.writeConfiguration(U, Params.format);
+
+      std::cout << GridLogMessage << "Written ILDG Configuration on " << config
+                << " checksum " << std::hex << csum << std::dec << std::endl;
+    }
+  };
+
+  void CheckpointRestore(int traj, GaugeField &U, GridSerialRNG &sRNG,
+                         GridParallelRNG &pRNG) {
+    std::string config, rng;
+    this->build_filenames(traj, Params, config, rng);
+
+    ILDGIO IO(config, ILDGread);
+    BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0);
+    uint32_t csum = IO.readConfiguration(U);  // format from the header
+
+    std::cout << GridLogMessage << "Read ILDG Configuration from " << config
+              << " checksum " << std::hex << csum << std::dec << std::endl;
+  };
+};
+}
+}
+
+#endif  // HAVE_LIME
+#endif  // ILDG_CHECKPOINTER

lib/qcd/hmc/checkpointers/NerscCheckpointer.h

@@ -0,0 +1,80 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/NerscCheckpointer.h
+
+Copyright (C) 2015
+
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef NERSC_CHECKPOINTER
+#define NERSC_CHECKPOINTER
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+namespace Grid {
+namespace QCD {
+
+// Only for Gauge fields
+template <class Gimpl>
+class NerscHmcCheckpointer : public BaseHmcCheckpointer<Gimpl> {
+ private:
+  CheckpointerParameters Params;
+
+ public:
+  INHERIT_GIMPL_TYPES(Gimpl);  // only for gauge configurations
+
+  NerscHmcCheckpointer(const CheckpointerParameters &Params_) { initialize(Params_); }
+
+  void initialize(const CheckpointerParameters &Params_) {
+    Params = Params_;
+    Params.format = "IEEE64BIG";  // fixed, overwrite any other choice
+  }
+
+  void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG,
+                          GridParallelRNG &pRNG) {
+    if ((traj % Params.saveInterval) == 0) {
+      std::string config, rng;
+      this->build_filenames(traj, Params, config, rng);
+
+      int precision32 = 1;
+      int tworow = 0;
+      NerscIO::writeRNGState(sRNG, pRNG, rng);
+      NerscIO::writeConfiguration(U, config, tworow, precision32);
+    }
+  };
+
+  void CheckpointRestore(int traj, GaugeField &U, GridSerialRNG &sRNG,
+                         GridParallelRNG &pRNG) {
+    std::string config, rng;
+    this->build_filenames(traj, Params, config, rng);
+
+    NerscField header;
+    NerscIO::readRNGState(sRNG, pRNG, header, rng);
+    NerscIO::readConfiguration(U, header, config);
+  };
+};
+}
+}
+#endif

lib/qcd/hmc/integrators/Integrator.h

@@ -8,8 +8,7 @@ Copyright (C) 2015
 
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: neo <cossu@post.kek.jp>
+Author: Guido Cossu <cossu@post.kek.jp>
-Author: paboyle <paboyle@ph.ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -30,79 +29,62 @@ directory
 *************************************************************************************/
 /*  END LEGAL */
 //--------------------------------------------------------------------
-/*! @file Integrator.h
- * @brief Classes for the Molecular Dynamics integrator
- *
- * @author Guido Cossu
- * Time-stamp: <2015-07-30 16:21:29 neo>
- */
-//--------------------------------------------------------------------
-
 #ifndef INTEGRATOR_INCLUDED
 #define INTEGRATOR_INCLUDED
 
-// class Observer;
-
 #include <memory>
 
 namespace Grid {
 namespace QCD {
 
-struct IntegratorParameters {
+class IntegratorParameters: Serializable {
-  int Nexp;
+public:
-  int MDsteps;  // number of outer steps
+        GRID_SERIALIZABLE_CLASS_MEMBERS(IntegratorParameters,
-  RealD trajL;  // trajectory length
+                std::string, name,      // name of the integrator
-  RealD stepsize;
+    unsigned int, MDsteps,  // number of outer steps
+    RealD, trajL,           // trajectory length
+  )
 
-  IntegratorParameters(int MDsteps_, RealD trajL_ = 1.0, int Nexp_ = 12)
+  IntegratorParameters(int MDsteps_ = 10, RealD trajL_ = 1.0)
-      : Nexp(Nexp_),
+      : MDsteps(MDsteps_),
-        MDsteps(MDsteps_),
+        trajL(trajL_){
-        trajL(trajL_),
-        stepsize(trajL / MDsteps){
         // empty body constructor
   };
+
+
+  template <class ReaderClass, typename std::enable_if<isReader<ReaderClass>::value, int >::type = 0 >
+  IntegratorParameters(ReaderClass & Reader){
+    std::cout << "Reading integrator\n";
+        read(Reader, "Integrator", *this);
+  }
+
+  void print_parameters() const {
+    std::cout << GridLogMessage << "[Integrator] Type               : " << name << std::endl;
+    std::cout << GridLogMessage << "[Integrator] Trajectory length  : " << trajL << std::endl;
+    std::cout << GridLogMessage << "[Integrator] Number of MD steps : " << MDsteps << std::endl;
+    std::cout << GridLogMessage << "[Integrator] Step size          : " << trajL/MDsteps << std::endl;
+  }
 };
 
 /*! @brief Class for Molecular Dynamics management */
-template <class GaugeField, class SmearingPolicy, class RepresentationPolicy>
+template <class FieldImplementation, class SmearingPolicy, class RepresentationPolicy>
 class Integrator {
  protected:
-  typedef IntegratorParameters ParameterType;
+  typedef typename FieldImplementation::Field MomentaField;  //for readability
+  typedef typename FieldImplementation::Field Field;
 
+  int levels;  // number of integration levels
+  double t_U;  // Track time passing on each level and for U and for P
+  std::vector<double> t_P;  
+
+  MomentaField P;
+  SmearingPolicy& Smearer;
+  RepresentationPolicy Representations;
   IntegratorParameters Params;
 
-  const ActionSet<GaugeField, RepresentationPolicy> as;
+  const ActionSet<Field, RepresentationPolicy> as;
 
-  int levels;  //
+  void update_P(Field& U, int level, double ep) {
-  double t_U;  // Track time passing on each level and for U and for P
-  std::vector<double> t_P;  //
-
-  GaugeField P;
-
-  SmearingPolicy& Smearer;
-
-  RepresentationPolicy Representations;
-
-  // Should match any legal (SU(n)) gauge field
-  // Need to use this template to match Ncol to pass to SU<N> class
-  template <int Ncol, class vec>
-  void generate_momenta(Lattice<iVector<iScalar<iMatrix<vec, Ncol> >, Nd> >& P,
-                        GridParallelRNG& pRNG) {
-    typedef Lattice<iScalar<iScalar<iMatrix<vec, Ncol> > > > GaugeLinkField;
-    GaugeLinkField Pmu(P._grid);
-    Pmu = zero;
-    for (int mu = 0; mu < Nd; mu++) {
-      SU<Ncol>::GaussianFundamentalLieAlgebraMatrix(pRNG, Pmu);
-      PokeIndex<LorentzIndex>(P, Pmu, mu);
-    }
-  }
-
-  // ObserverList observers; // not yet
-  //      typedef std::vector<Observer*> ObserverList;
-  //      void register_observers();
-  //      void notify_observers();
-
-  void update_P(GaugeField& U, int level, double ep) {
     t_P[level] += ep;
     update_P(P, U, level, ep);
 
@@ -120,31 +102,29 @@ class Integrator {
         FieldType forceR(U._grid);
         // Implement smearing only for the fundamental representation now
         repr_set.at(a)->deriv(Rep.U, forceR);
-        GF force =
+        GF force = Rep.RtoFundamentalProject(forceR);  // Ta for the fundamental rep
-            Rep.RtoFundamentalProject(forceR);  // Ta for the fundamental rep
+        Real force_abs = std::sqrt(norm2(force)/(U._grid->gSites()));
-        std::cout << GridLogIntegrator << "Hirep Force average: "
+        std::cout << GridLogIntegrator << "Hirep Force average: " << force_abs << std::endl;
-                  << norm2(force) / (U._grid->gSites()) << std::endl;
         Mom -= force * ep ;
       }
     }
   } update_P_hireps{};
 
-  void update_P(GaugeField& Mom, GaugeField& U, int level, double ep) {
+  void update_P(MomentaField& Mom, Field& U, int level, double ep) {
     // input U actually not used in the fundamental case
     // Fundamental updates, include smearing
+
    for (int a = 0; a < as[level].actions.size(); ++a) {
-      GaugeField force(U._grid);
+      Field force(U._grid);
-      GaugeField& Us = Smearer.get_U(as[level].actions.at(a)->is_smeared);
+      conformable(U._grid, Mom._grid);
+      Field& Us = Smearer.get_U(as[level].actions.at(a)->is_smeared);
       as[level].actions.at(a)->deriv(Us, force);  // deriv should NOT include Ta
 
-      std::cout << GridLogIntegrator
+      std::cout << GridLogIntegrator << "Smearing (on/off): " << as[level].actions.at(a)->is_smeared << std::endl;
-                << "Smearing (on/off): " << as[level].actions.at(a)->is_smeared
-                << std::endl;
       if (as[level].actions.at(a)->is_smeared) Smearer.smeared_force(force);
-      force = Ta(force);
+      force = FieldImplementation::projectForce(force); // Ta for gauge fields
-      std::cout << GridLogIntegrator
+      Real force_abs = std::sqrt(norm2(force)/U._grid->gSites());
-                << "Force average: " << norm2(force) / (U._grid->gSites())
+      std::cout << GridLogIntegrator << "Force average: " << force_abs << std::endl;
-                << std::endl;
       Mom -= force * ep; 
     }
 
@@ -152,35 +132,30 @@ class Integrator {
     as[level].apply(update_P_hireps, Representations, Mom, U, ep);
   }
 
-  void update_U(GaugeField& U, double ep) {
+  void update_U(Field& U, double ep) {
     update_U(P, U, ep);
 
     t_U += ep;
     int fl = levels - 1;
-    std::cout << GridLogIntegrator << "   "
+    std::cout << GridLogIntegrator << "   " << "[" << fl << "] U " << " dt " << ep << " : t_U " << t_U << std::endl;
-              << "[" << fl << "] U "
-              << " dt " << ep << " : t_U " << t_U << std::endl;
-  }
-  void update_U(GaugeField& Mom, GaugeField& U, double ep) {
-    // rewrite exponential to deal internally with the lorentz index?
-    for (int mu = 0; mu < Nd; mu++) {
-      auto Umu = PeekIndex<LorentzIndex>(U, mu);
-      auto Pmu = PeekIndex<LorentzIndex>(Mom, mu);
-      Umu = expMat(Pmu, ep, Params.Nexp) * Umu;
-      PokeIndex<LorentzIndex>(U, ProjectOnGroup(Umu), mu);
   }
   
+  void update_U(MomentaField& Mom, Field& U, double ep) {
+    // exponential of Mom*U in the gauge fields case
+    FieldImplementation::update_field(Mom, U, ep);
+
     // Update the smeared fields, can be implemented as observer
-    Smearer.set_GaugeField(U);
+    Smearer.set_Field(U);
+
     // Update the higher representations fields
     Representations.update(U);  // void functions if fundamental representation
   }
 
-  virtual void step(GaugeField& U, int level, int first, int last) = 0;
+  virtual void step(Field& U, int level, int first, int last) = 0;
 
  public:
   Integrator(GridBase* grid, IntegratorParameters Par,
-             ActionSet<GaugeField, RepresentationPolicy>& Aset,
+             ActionSet<Field, RepresentationPolicy>& Aset,
              SmearingPolicy& Sm)
       : Params(Par),
         as(Aset),
@@ -195,6 +170,31 @@ class Integrator {
 
   virtual ~Integrator() {}
 
+  virtual std::string integrator_name() = 0;
+
+  void print_parameters(){
+    std::cout << GridLogMessage << "[Integrator] Name : "<< integrator_name() << std::endl;
+    Params.print_parameters();
+  }
+
+  void print_actions(){
+        std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::" << std::endl;
+        std::cout << GridLogMessage << "[Integrator] Action summary: "<<std::endl;
+        for (int level = 0; level < as.size(); ++level) {
+                std::cout << GridLogMessage << "[Integrator] ---- Level: "<< level << std::endl;
+                for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
+                        std::cout << GridLogMessage << "["<< as[level].actions.at(actionID)->action_name() << "] ID: " << actionID << std::endl;
+                        std::cout << as[level].actions.at(actionID)->LogParameters();
+                }
+        }
+        std::cout << GridLogMessage << ":::::::::::::::::::::::::::::::::::::::::"<< std::endl;
+
+  }
+
+  void reverse_momenta(){
+    P *= -1.0;
+  }
+
   // to be used by the actionlevel class to iterate
   // over the representations
   struct _refresh {
@@ -210,14 +210,16 @@ class Integrator {
   } refresh_hireps{};
 
   // Initialization of momenta and actions
-  void refresh(GaugeField& U, GridParallelRNG& pRNG) {
+  void refresh(Field& U, GridParallelRNG& pRNG) {
+    assert(P._grid == U._grid);
     std::cout << GridLogIntegrator << "Integrator refresh\n";
-    generate_momenta(P, pRNG);
+
+    FieldImplementation::generate_momenta(P, pRNG);
 
     // Update the smeared fields, can be implemented as observer
     // necessary to keep the fields updated even after a reject
     // of the Metropolis
-    Smearer.set_GaugeField(U);
+    Smearer.set_Field(U);
     // Set the (eventual) representations gauge fields
     Representations.update(U);
 
@@ -228,7 +230,7 @@ class Integrator {
       for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
         // get gauge field from the SmearingPolicy and
         // based on the boolean is_smeared in actionID
-        GaugeField& Us =
+        Field& Us =
             Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
         as[level].actions.at(actionID)->refresh(Us, pRNG);
       }
@@ -256,18 +258,9 @@ class Integrator {
   } S_hireps{};
 
   // Calculate action
-  RealD S(GaugeField& U) {  // here also U not used
+  RealD S(Field& U) {  // here also U not used
 
-    LatticeComplex Hloc(U._grid);
+    RealD H = - FieldImplementation::FieldSquareNorm(P); // - trace (P*P)
-    Hloc = zero;
-    // Momenta
-    for (int mu = 0; mu < Nd; mu++) {
-      auto Pmu = PeekIndex<LorentzIndex>(P, mu);
-      Hloc -= trace(Pmu * Pmu);
-    }
-    Complex Hsum = sum(Hloc);
-
-    RealD H = Hsum.real();
     RealD Hterm;
     std::cout << GridLogMessage << "Momentum action H_p = " << H << "\n";
 
@@ -276,7 +269,7 @@ class Integrator {
       for (int actionID = 0; actionID < as[level].actions.size(); ++actionID) {
         // get gauge field from the SmearingPolicy and
         // based on the boolean is_smeared in actionID
-        GaugeField& Us =
+        Field& Us =
             Smearer.get_U(as[level].actions.at(actionID)->is_smeared);
         Hterm = as[level].actions.at(actionID)->S(Us);
         std::cout << GridLogMessage << "S Level " << level << " term "
@@ -289,7 +282,7 @@ class Integrator {
     return H;
   }
 
-  void integrate(GaugeField& U) {
+  void integrate(Field& U) {
     // reset the clocks
     t_U = 0;
     for (int level = 0; level < as.size(); ++level) {
@@ -311,8 +304,14 @@ class Integrator {
 
     // and that we indeed got to the end of the trajectory
     assert(fabs(t_U - Params.trajL) < 1.0e-6);
+
   }
+
+
+  
+
 };
 }
 }
 #endif  // INTEGRATOR_INCLUDED
+

lib/qcd/hmc/integrators/Integrator_algorithm.h

@@ -23,14 +23,16 @@ Author: neo <cossu@post.kek.jp>
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-    See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
 //--------------------------------------------------------------------
+
+
 /*! @file Integrator_algorithm.h
  * @brief Declaration of classes for the Molecular Dynamics algorithms
  *
- * @author Guido Cossu
  */
 //--------------------------------------------------------------------
 
@@ -91,35 +93,35 @@ namespace Grid{
  *  P 1/2                            P 1/2
  */
 
-    template<class GaugeField,
+template <class FieldImplementation, class SmearingPolicy,
-	     class SmearingPolicy,
+          class RepresentationPolicy =
-	     class RepresentationPolicy = Representations< FundamentalRepresentation > > class LeapFrog :
+              Representations<FundamentalRepresentation> >
-      public Integrator<GaugeField, SmearingPolicy, RepresentationPolicy> {
+class LeapFrog : public Integrator<FieldImplementation, SmearingPolicy,
+                                   RepresentationPolicy> {
  public:
+  typedef LeapFrog<FieldImplementation, SmearingPolicy, RepresentationPolicy>
+      Algorithm;
+  INHERIT_FIELD_TYPES(FieldImplementation);
 
-      typedef LeapFrog<GaugeField, SmearingPolicy, RepresentationPolicy> Algorithm;
+  std::string integrator_name(){return "LeapFrog";}
 
-      LeapFrog(GridBase* grid, 
+  LeapFrog(GridBase* grid, IntegratorParameters Par,
-	       IntegratorParameters Par,
+           ActionSet<Field, RepresentationPolicy>& Aset, SmearingPolicy& Sm)
-	       ActionSet<GaugeField, RepresentationPolicy> & Aset,
+      : Integrator<FieldImplementation, SmearingPolicy, RepresentationPolicy>(
-	       SmearingPolicy & Sm):
+            grid, Par, Aset, Sm){};
-	Integrator<GaugeField, SmearingPolicy, RepresentationPolicy>(grid,Par,Aset,Sm) {};
-
-
-      void step (GaugeField& U, int level,int _first, int _last){
 
+  void step(Field& U, int level, int _first, int _last) {
     int fl = this->as.size() - 1;
     // level  : current level
     // fl     : final level
     // eps    : current step size
 
     // Get current level step size
-        RealD eps = this->Params.stepsize;
+    RealD eps = this->Params.trajL/this->Params.MDsteps;
     for (int l = 0; l <= level; ++l) eps /= this->as[l].multiplier;
 
     int multiplier = this->as[level].multiplier;
     for (int e = 0; e < multiplier; ++e) {
-
       int first_step = _first && (e == 0);
       int last_step = _last && (e == multiplier - 1);
 
@@ -135,35 +137,36 @@ namespace Grid{
 
       int mm = last_step ? 1 : 2;
       this->update_P(U, level, mm * eps / 2.0);
-
     }
   }
 };
 
-    template<class GaugeField,
+template <class FieldImplementation, class SmearingPolicy,
-	     class SmearingPolicy,
+          class RepresentationPolicy =
-	     class RepresentationPolicy = Representations < FundamentalRepresentation > > class MinimumNorm2 :
+              Representations<FundamentalRepresentation> >
-      public Integrator<GaugeField, SmearingPolicy, RepresentationPolicy> {
+class MinimumNorm2 : public Integrator<FieldImplementation, SmearingPolicy,
+                                       RepresentationPolicy> {
  private:
   const RealD lambda = 0.1931833275037836;
 
  public:
+  INHERIT_FIELD_TYPES(FieldImplementation);
 
-      MinimumNorm2(GridBase* grid, 
+  MinimumNorm2(GridBase* grid, IntegratorParameters Par,
-		   IntegratorParameters Par,
+               ActionSet<Field, RepresentationPolicy>& Aset, SmearingPolicy& Sm)
-		   ActionSet<GaugeField, RepresentationPolicy> & Aset,
+      : Integrator<FieldImplementation, SmearingPolicy, RepresentationPolicy>(
-		   SmearingPolicy& Sm):
+            grid, Par, Aset, Sm){};
-	Integrator<GaugeField, SmearingPolicy, RepresentationPolicy>(grid,Par,Aset,Sm) {};
 
-      void step (GaugeField& U, int level, int _first,int _last){
+  std::string integrator_name(){return "MininumNorm2";}
 
+  void step(Field& U, int level, int _first, int _last) {
     // level  : current level
     // fl     : final level
     // eps    : current step size
 
     int fl = this->as.size() - 1;
 
-	RealD eps = this->Params.stepsize*2.0;                              
+    RealD eps = this->Params.trajL/this->Params.MDsteps * 2.0;
     for (int l = 0; l <= level; ++l) eps /= 2.0 * this->as[l].multiplier;
 
     // Nesting:  2xupdate_U of size eps/2
@@ -195,37 +198,41 @@ namespace Grid{
 
       int mm = (last_step) ? 1 : 2;
       this->update_P(U, level, lambda * eps * mm);
-
     }
   }
 };
 
-
+template <class FieldImplementation, class SmearingPolicy,
-    template<class GaugeField,
+          class RepresentationPolicy =
-	     class SmearingPolicy,
+              Representations<FundamentalRepresentation> >
-	     class RepresentationPolicy = Representations< FundamentalRepresentation > > class ForceGradient :
+class ForceGradient : public Integrator<FieldImplementation, SmearingPolicy,
-      public Integrator<GaugeField, SmearingPolicy, RepresentationPolicy> {
+                                        RepresentationPolicy> {
  private:
-      const RealD lambda = 1.0/6.0;;
+  const RealD lambda = 1.0 / 6.0;
+  ;
   const RealD chi = 1.0 / 72.0;
   const RealD xi = 0.0;
   const RealD theta = 0.0;
+
  public:
+  INHERIT_FIELD_TYPES(FieldImplementation);
 
   // Looks like dH scales as dt^4. tested wilson/wilson 2 level.
-    ForceGradient(GridBase* grid, 
+  ForceGradient(GridBase* grid, IntegratorParameters Par,
-		  IntegratorParameters Par,
+                ActionSet<Field, RepresentationPolicy>& Aset,
-		  ActionSet<GaugeField, RepresentationPolicy> & Aset,
+                SmearingPolicy& Sm)
-		  SmearingPolicy &Sm):
+      : Integrator<FieldImplementation, SmearingPolicy, RepresentationPolicy>(
-      Integrator<GaugeField, SmearingPolicy, RepresentationPolicy>(grid,Par,Aset, Sm) {};
+            grid, Par, Aset, Sm){};
 
+  std::string integrator_name(){return "ForceGradient";}
   
-      void FG_update_P(GaugeField&U, int level,double fg_dt,double ep){
+  void FG_update_P(Field& U, int level, double fg_dt, double ep) {
-	GaugeField Ufg(U._grid);
+    Field Ufg(U._grid);
-	GaugeField Pfg(U._grid);
+    Field Pfg(U._grid);
     Ufg = U;
     Pfg = zero;
-	std::cout << GridLogMessage << "FG update "<<fg_dt<<" "<<ep<<std::endl;
+    std::cout << GridLogMessage << "FG update " << fg_dt << " " << ep
+              << std::endl;
     // prepare_fg; no prediction/result cache for now
     // could relax CG stopping conditions for the
     // derivatives in the small step since the force gets multiplied by
@@ -239,9 +246,8 @@ namespace Grid{
     this->update_P(Ufg, level, ep);
   }
 
-      void step (GaugeField& U, int level, int _first,int _last){
+  void step(Field& U, int level, int _first, int _last) {
-
+    RealD eps = this->Params.trajL/this->Params.MDsteps * 2.0;
-	RealD eps = this->Params.stepsize*2.0;                              
     for (int l = 0; l <= level; ++l) eps /= 2.0 * this->as[l].multiplier;
 
     RealD Chi = chi * eps * eps * eps;
@@ -252,7 +258,6 @@ namespace Grid{
 
     for (int e = 0; e < multiplier; ++e) {  // steps per step
 
-
       int first_step = _first && (e == 0);
       int last_step = _last && (e == multiplier - 1);
 
@@ -266,7 +271,8 @@ namespace Grid{
         this->step(U, level + 1, first_step, 0);
       }
 
-	  this->FG_update_P(U,level,2*Chi/((1.0-2.0*lambda)*eps),(1.0-2.0*lambda)*eps);
+      this->FG_update_P(U, level, 2 * Chi / ((1.0 - 2.0 * lambda) * eps),
+                        (1.0 - 2.0 * lambda) * eps);
 
       if (level == fl) {  // lowest level
         this->update_U(U, 0.5 * eps);
@@ -276,11 +282,13 @@ namespace Grid{
 
       int mm = (last_step) ? 1 : 2;
       this->update_P(U, level, lambda * eps * mm);
-
     }
   }
 };
 
+
+
+
 }
 }
 

lib/qcd/modules/ActionModules.h

@@ -0,0 +1,517 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/ActionModules.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef ACTION_MODULES_H
+#define ACTION_MODULES_H
+
+/*
+Define loadable, serializable modules
+for the HMC execution
+*/
+
+namespace Grid {
+
+//////////////////////////////////////////////
+//              Actions
+//////////////////////////////////////////////
+
+template <class Product, class R>
+class ActionModuleBase: public HMCModuleBase<Product>{
+public:
+  typedef R Resource;
+  virtual void acquireResource(R& ){};
+
+};
+
+
+template <class ActionType, class APar>
+class ActionModule
+    : public Parametrized<APar>,
+      public ActionModuleBase< QCD::Action<typename ActionType::GaugeField> , QCD::GridModule > {
+ public:
+  typedef ActionModuleBase< QCD::Action<typename ActionType::GaugeField>, QCD::GridModule > Base;
+  typedef typename Base::Product Product;
+  typedef APar Parameters;
+
+  std::unique_ptr<ActionType> ActionPtr;
+
+  ActionModule(APar Par) : Parametrized<APar>(Par) {}
+
+  template <class ReaderClass>
+  ActionModule(Reader<ReaderClass>& Reader) : Parametrized<APar>(Reader){};
+
+
+  virtual void print_parameters(){
+    Parametrized<APar>::print_parameters();
+  }
+
+  Product* getPtr() {
+    if (!ActionPtr) initialize();
+
+    return ActionPtr.get();
+  }
+
+ private:
+  virtual void initialize() = 0;
+
+};
+
+//////////////////////////
+// Modules
+//////////////////////////
+
+namespace QCD{
+
+class PlaqPlusRectangleGaugeActionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(PlaqPlusRectangleGaugeActionParameters, 
+    RealD, c_plaq,
+    RealD, c_rect);
+
+};
+
+class RBCGaugeActionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(RBCGaugeActionParameters, 
+    RealD, beta,
+    RealD, c1);
+
+};
+
+class BetaGaugeActionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(BetaGaugeActionParameters, 
+    RealD, beta);
+};
+
+
+
+
+template <class Impl >
+class WilsonGModule: public ActionModule<WilsonGaugeAction<Impl>, BetaGaugeActionParameters> {
+  typedef ActionModule<WilsonGaugeAction<Impl>, BetaGaugeActionParameters> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->ActionPtr.reset(new WilsonGaugeAction<Impl>(this->Par_.beta));
+  }
+
+};
+
+template <class Impl >
+class PlaqPlusRectangleGModule: public ActionModule<PlaqPlusRectangleAction<Impl>, PlaqPlusRectangleGaugeActionParameters> {
+  typedef ActionModule<PlaqPlusRectangleAction<Impl>, PlaqPlusRectangleGaugeActionParameters> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->ActionPtr.reset(new PlaqPlusRectangleAction<Impl>(this->Par_.c_plaq, this->Par_.c_rect));
+  }
+
+};
+
+template <class Impl >
+class RBCGModule: public ActionModule<RBCGaugeAction<Impl>, RBCGaugeActionParameters> {
+  typedef ActionModule<RBCGaugeAction<Impl>, RBCGaugeActionParameters> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->ActionPtr.reset(new RBCGaugeAction<Impl>(this->Par_.beta, this->Par_.c1));
+  }
+
+};
+
+
+
+
+template <class Impl >
+class SymanzikGModule: public ActionModule<SymanzikGaugeAction<Impl>, BetaGaugeActionParameters> {
+  typedef ActionModule<SymanzikGaugeAction<Impl>, BetaGaugeActionParameters> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->ActionPtr.reset(new SymanzikGaugeAction<Impl>(this->Par_.beta));
+  }
+
+};
+
+template <class Impl >
+class IwasakiGModule: public ActionModule<IwasakiGaugeAction<Impl>, BetaGaugeActionParameters> {
+  typedef ActionModule<IwasakiGaugeAction<Impl>, BetaGaugeActionParameters> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->ActionPtr.reset(new IwasakiGaugeAction<Impl>(this->Par_.beta));
+  }
+
+};
+
+
+template <class Impl >
+class DBW2GModule: public ActionModule<DBW2GaugeAction<Impl>, BetaGaugeActionParameters> {
+  typedef ActionModule<DBW2GaugeAction<Impl>, BetaGaugeActionParameters> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->ActionPtr.reset(new DBW2GaugeAction<Impl>(this->Par_.beta));
+  }
+
+};
+
+/////////////////////////////////////////
+// Fermion Actions
+/////////////////////////////////////////
+
+
+template <class Impl, template <typename> class FermionA, class Params = NoParameters >
+class PseudoFermionModuleBase: public ActionModule<FermionA<Impl>, Params> {
+protected:
+  typedef ActionModule<FermionA<Impl>, Params> ActionBase;
+  using ActionBase::ActionBase; // for constructors
+
+  typedef std::unique_ptr<FermionOperatorModuleBase<FermionOperator<Impl>> > operator_type;
+  typedef std::unique_ptr<HMCModuleBase<OperatorFunction<typename Impl::FermionField> > > solver_type;
+
+  template <class ReaderClass>
+  void getFermionOperator(Reader<ReaderClass>& Reader, operator_type &fo, std::string section_name){
+    auto &FOFactory = HMC_FermionOperatorModuleFactory<fermionop_string, Impl, ReaderClass>::getInstance();
+    Reader.push(section_name);
+    std::string op_name;
+    read(Reader,"name", op_name);
+    fo = FOFactory.create(op_name, Reader);
+    Reader.pop();  
+  }
+
+  template <class ReaderClass>
+  void getSolverOperator(Reader<ReaderClass>& Reader, solver_type &so, std::string section_name){
+    auto& SolverFactory = HMC_SolverModuleFactory<solver_string, typename Impl::FermionField, ReaderClass>::getInstance();
+    Reader.push(section_name);
+    std::string solv_name;
+    read(Reader,"name", solv_name);
+    so = SolverFactory.create(solv_name, Reader);
+    Reader.pop();    
+  }
+};
+
+
+template <class Impl >
+class TwoFlavourFModule: public PseudoFermionModuleBase<Impl, TwoFlavourPseudoFermionAction>{
+  typedef PseudoFermionModuleBase<Impl, TwoFlavourPseudoFermionAction> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_mod;
+  typename Base::solver_type   solver_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   TwoFlavourFModule(Reader<ReaderClass>& R): Base(R) {
+    this->getSolverOperator(R, solver_mod, "Solver");
+    this->getFermionOperator(R, fop_mod, "Operator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    // here temporarily assuming that the force and action solver are the same
+    this->ActionPtr.reset(new TwoFlavourPseudoFermionAction<Impl>(*(this->fop_mod->getPtr()), *(this->solver_mod->getPtr()), *(this->solver_mod->getPtr())));
+  }
+
+};
+
+// very similar, I could have templated this but it is overkilling
+template <class Impl >
+class TwoFlavourEOFModule: public PseudoFermionModuleBase<Impl, TwoFlavourEvenOddPseudoFermionAction>{
+  typedef PseudoFermionModuleBase<Impl, TwoFlavourEvenOddPseudoFermionAction> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_mod;
+  typename Base::solver_type   solver_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   TwoFlavourEOFModule(Reader<ReaderClass>& R): PseudoFermionModuleBase<Impl, TwoFlavourEvenOddPseudoFermionAction>(R) {
+    this->getSolverOperator(R, solver_mod, "Solver");
+    this->getFermionOperator(R, fop_mod, "Operator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    // here temporarily assuming that the force and action solver are the same
+    this->ActionPtr.reset(new TwoFlavourEvenOddPseudoFermionAction<Impl>(*(this->fop_mod->getPtr()), *(this->solver_mod->getPtr()), *(this->solver_mod->getPtr())));
+  }
+
+};
+
+
+template <class Impl >
+class TwoFlavourRatioFModule: public PseudoFermionModuleBase<Impl, TwoFlavourRatioPseudoFermionAction>{
+  typedef PseudoFermionModuleBase<Impl, TwoFlavourRatioPseudoFermionAction> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_numerator_mod;
+  typename Base::operator_type fop_denominator_mod;
+  typename Base::solver_type   solver_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_numerator_mod->AddGridPair(GridMod);
+    fop_denominator_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   TwoFlavourRatioFModule(Reader<ReaderClass>& R): PseudoFermionModuleBase<Impl, TwoFlavourRatioPseudoFermionAction>(R) {
+    this->getSolverOperator(R, solver_mod, "Solver");
+    this->getFermionOperator(R, fop_numerator_mod, "Numerator");
+    this->getFermionOperator(R, fop_denominator_mod, "Denominator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    // here temporarily assuming that the force and action solver are the same
+    this->ActionPtr.reset(new TwoFlavourRatioPseudoFermionAction<Impl>(*(this->fop_numerator_mod->getPtr()), 
+      *(this->fop_denominator_mod->getPtr()), *(this->solver_mod->getPtr()), *(this->solver_mod->getPtr())));
+  }
+
+};
+
+template <class Impl >
+class TwoFlavourRatioEOFModule: public PseudoFermionModuleBase<Impl, TwoFlavourEvenOddRatioPseudoFermionAction>{
+  typedef PseudoFermionModuleBase<Impl, TwoFlavourEvenOddRatioPseudoFermionAction> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_numerator_mod;
+  typename Base::operator_type fop_denominator_mod;
+  typename Base::solver_type   solver_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_numerator_mod->AddGridPair(GridMod);
+    fop_denominator_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   TwoFlavourRatioEOFModule(Reader<ReaderClass>& R): Base(R) {
+    this->getSolverOperator(R, solver_mod, "Solver");
+    this->getFermionOperator(R, fop_numerator_mod, "Numerator");
+    this->getFermionOperator(R, fop_denominator_mod, "Denominator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    // here temporarily assuming that the force and action solver are the same
+    this->ActionPtr.reset(new TwoFlavourEvenOddRatioPseudoFermionAction<Impl>(*(this->fop_numerator_mod->getPtr()), 
+      *(this->fop_denominator_mod->getPtr()), *(this->solver_mod->getPtr()), *(this->solver_mod->getPtr())));
+  }
+
+};
+
+
+template <class Impl >
+class OneFlavourFModule: public PseudoFermionModuleBase<Impl, OneFlavourRationalPseudoFermionAction, OneFlavourRationalParams>{
+  typedef PseudoFermionModuleBase<Impl, OneFlavourRationalPseudoFermionAction, OneFlavourRationalParams> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   OneFlavourFModule(Reader<ReaderClass>& R): Base(R) {
+    this->getFermionOperator(R, fop_mod, "Operator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    this->ActionPtr.reset(new OneFlavourRationalPseudoFermionAction<Impl>(*(this->fop_mod->getPtr()), this->Par_ ));
+  }
+
+};
+
+template <class Impl >
+class OneFlavourEOFModule: 
+  public PseudoFermionModuleBase<Impl, OneFlavourEvenOddRationalPseudoFermionAction, OneFlavourRationalParams>
+  {
+  typedef PseudoFermionModuleBase<Impl, OneFlavourEvenOddRationalPseudoFermionAction, OneFlavourRationalParams> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   OneFlavourEOFModule(Reader<ReaderClass>& R): Base(R) {
+    this->getFermionOperator(R, fop_mod, "Operator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    this->ActionPtr.reset(new OneFlavourEvenOddRationalPseudoFermionAction<Impl>(*(this->fop_mod->getPtr()), this->Par_ ));
+  }
+
+};
+
+
+template <class Impl >
+class OneFlavourRatioFModule: 
+  public PseudoFermionModuleBase<Impl, OneFlavourRatioRationalPseudoFermionAction, OneFlavourRationalParams>
+  {
+
+  typedef PseudoFermionModuleBase<Impl, OneFlavourRatioRationalPseudoFermionAction, OneFlavourRationalParams> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_numerator_mod;
+  typename Base::operator_type fop_denominator_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_numerator_mod->AddGridPair(GridMod);
+    fop_denominator_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   OneFlavourRatioFModule(Reader<ReaderClass>& R): Base(R) {
+    this->getFermionOperator(R, fop_numerator_mod, "Numerator");
+    this->getFermionOperator(R, fop_denominator_mod, "Denominator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    this->ActionPtr.reset(new OneFlavourRatioRationalPseudoFermionAction<Impl>( *(this->fop_numerator_mod->getPtr()), 
+                                                                                *(this->fop_denominator_mod->getPtr()), 
+                                                                                this->Par_ ));
+  }
+
+};
+
+
+template <class Impl >
+class OneFlavourRatioEOFModule: 
+  public PseudoFermionModuleBase<Impl, OneFlavourEvenOddRatioRationalPseudoFermionAction, OneFlavourRationalParams>
+  {
+
+  typedef PseudoFermionModuleBase<Impl, OneFlavourEvenOddRatioRationalPseudoFermionAction, OneFlavourRationalParams> Base;
+  using Base::Base;
+
+  typename Base::operator_type fop_numerator_mod;
+  typename Base::operator_type fop_denominator_mod;
+
+ public:
+  virtual void acquireResource(typename Base::Resource& GridMod){
+    fop_numerator_mod->AddGridPair(GridMod);
+    fop_denominator_mod->AddGridPair(GridMod);
+  }
+
+   // constructor
+   template <class ReaderClass>
+   OneFlavourRatioEOFModule(Reader<ReaderClass>& R): Base(R) {
+    this->getFermionOperator(R, fop_numerator_mod, "Numerator");
+    this->getFermionOperator(R, fop_denominator_mod, "Denominator");
+   } 
+
+  // acquire resource
+  virtual void initialize() {
+    this->ActionPtr.reset(new OneFlavourEvenOddRatioRationalPseudoFermionAction<Impl>(*(this->fop_numerator_mod->getPtr()), 
+                                                                                      *(this->fop_denominator_mod->getPtr()), 
+                                                                                      this->Par_ ));
+  }
+
+};
+
+}// QCD temporarily here
+
+
+
+
+
+
+
+////////////////////////////////////////
+// Factories specialisations
+////////////////////////////////////////
+
+
+
+// use the same classed defined by Antonin, does not make sense to rewrite
+// Factory is perfectly fine
+// Registar must be changed because I do not want to use the ModuleFactory
+
+// explicit ref to LatticeGaugeField must be changed or put in the factory
+//typedef ActionModuleBase< QCD::Action< QCD::LatticeGaugeField >, QCD::GridModule > HMC_LGTActionModBase;
+//typedef ActionModuleBase< QCD::Action< QCD::LatticeReal >, QCD::GridModule > HMC_ScalarActionModBase;
+
+template <char const *str, class Field, class ReaderClass >
+class HMC_ActionModuleFactory
+    : public Factory < ActionModuleBase< QCD::Action< Field >, QCD::GridModule > , Reader<ReaderClass> > {
+ public:
+  typedef Reader<ReaderClass> TheReader; 
+  // use SINGLETON FUNCTOR MACRO HERE
+  HMC_ActionModuleFactory(const HMC_ActionModuleFactory& e) = delete;
+  void operator=(const HMC_ActionModuleFactory& e) = delete;
+  static HMC_ActionModuleFactory& getInstance(void) {
+    static HMC_ActionModuleFactory e;
+    return e;
+  }
+
+ private:
+  HMC_ActionModuleFactory(void) = default;
+    std::string obj_type() const {
+        return std::string(str);
+  }
+};
+
+
+extern char gauge_string[];
+} // Grid
+
+
+#endif //HMC_MODULES_H

lib/qcd/modules/Factory.h

@@ -0,0 +1,109 @@
+/*************************************************************************************
+Grid physics library, www.github.com/paboyle/Grid 
+Source file: Factory.h
+
+Copyright (C) 2015
+Copyright (C) 2016
+
+Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef Factory_hpp_
+#define Factory_hpp_
+
+
+namespace Grid{
+
+
+
+
+/******************************************************************************
+ *                        abstract factory class                              *
+ ******************************************************************************/
+template <typename T, typename CreatorInput>
+class Factory
+{
+public:
+    typedef std::function< std::unique_ptr<T> (const CreatorInput&) > Func;
+
+    // constructor
+    Factory(void) = default;
+    // destructor
+    virtual ~Factory(void) = default;
+    // registration
+    void registerBuilder(const std::string type, const Func &f);
+    // get builder list
+    std::vector<std::string> getBuilderList(void) const;
+    // factory
+    std::unique_ptr<T> create(const std::string type,
+              								const CreatorInput& input) const;
+private:
+    std::map<std::string, Func> builder_;
+    virtual std::string obj_type() const = 0;
+};
+
+/******************************************************************************
+ *                         template implementation                            *
+ ******************************************************************************/
+// registration ////////////////////////////////////////////////////////////////
+template <typename T, typename CreatorInput>
+void Factory<T, CreatorInput>::registerBuilder(const std::string type, const Func &f)
+{
+    builder_[type] = f;
+}
+
+// get module list /////////////////////////////////////////////////////////////
+template <typename T, typename CreatorInput>
+std::vector<std::string> Factory<T, CreatorInput>::getBuilderList(void) const
+{
+    std::vector<std::string> list;
+    
+    for (auto &b: builder_)
+    {
+        list.push_back(b.first);
+    }
+    
+    return list;
+}
+
+// factory /////////////////////////////////////////////////////////////////////
+template <typename T, typename CreatorInput>
+std::unique_ptr<T> Factory<T, CreatorInput>::create(const std::string type,
+                                      							const CreatorInput& input) const
+{
+    Func func;
+    
+    std::cout << GridLogDebug << "Creating object of type "<< type << std::endl;
+    try
+    {
+        func = builder_.at(type);
+    }
+    catch (std::out_of_range &)
+    {
+      //HADRON_ERROR("object of type '" + type + "' unknown");
+    	std::cout << GridLogError << "Error" << std::endl;
+    	std::cout << GridLogError << obj_type() << " object of name [" << type << "] unknown" << std::endl;
+    	exit(1);
+    }
+    
+    return func(input);
+}
+
+}
+
+#endif // Factory_hpp_

lib/qcd/modules/FermionOperatorModules.h

@@ -0,0 +1,210 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/FermionOperatorModules.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef FERMIONOPERATOR_MODULES_H
+#define FERMIONOPERATOR_MODULES_H
+
+namespace Grid {
+
+////////////////////////////////////
+//  Fermion operators
+/////////////////////////////////////
+template < class Product>
+class FermionOperatorModuleBase : public HMCModuleBase<Product>{
+public:
+  virtual void AddGridPair(QCD::GridModule&) = 0;
+};
+
+template <template <typename> class FOType, class FermionImpl, class FOPar>
+class FermionOperatorModule
+    : public Parametrized<FOPar>,
+      public FermionOperatorModuleBase<QCD::FermionOperator<FermionImpl> > {
+
+protected:
+  std::unique_ptr< FOType<FermionImpl> > FOPtr;
+  std::vector< QCD::GridModule* >    GridRefs;
+ public:
+  typedef HMCModuleBase< QCD::FermionOperator<FermionImpl> > Base;
+  typedef typename Base::Product Product;
+
+  FermionOperatorModule(FOPar Par) : Parametrized<FOPar>(Par) {}
+
+  template <class ReaderClass>
+  FermionOperatorModule(Reader<ReaderClass>& Reader) : Parametrized<FOPar>(Reader){};
+
+  void AddGridPair(QCD::GridModule &Mod){
+    if (GridRefs.size()>1){
+      std::cout << GridLogError << "Adding too many Grids to the FermionOperatorModule" << std::endl;
+      exit(1);
+    }
+    GridRefs.push_back(&Mod);
+
+    if (Ls()){
+      GridRefs.push_back(new QCD::GridModule());
+      GridRefs[1]->set_full(QCD::SpaceTimeGrid::makeFiveDimGrid(Ls(),GridRefs[0]->get_full()));
+      GridRefs[1]->set_rb(QCD::SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls(),GridRefs[0]->get_full()));
+    }
+  }
+
+  virtual unsigned int Ls(){
+    return 0;  
+  }
+
+  virtual void print_parameters(){
+    std::cout << this->Par_ << std::endl;
+  }
+
+  Product* getPtr() {
+    if (!FOPtr) initialize();
+
+    return FOPtr.get();
+  }
+
+ private:
+  virtual void initialize() = 0;
+};
+
+
+
+// Factory
+template <char const *str, class FermionImpl, class ReaderClass >
+class HMC_FermionOperatorModuleFactory
+    : public Factory < FermionOperatorModuleBase<QCD::FermionOperator<FermionImpl> > ,  Reader<ReaderClass> > {
+ public:
+  // use SINGLETON FUNCTOR MACRO HERE
+  typedef Reader<ReaderClass> TheReader; 
+
+  HMC_FermionOperatorModuleFactory(const HMC_FermionOperatorModuleFactory& e) = delete;
+  void operator=(const HMC_FermionOperatorModuleFactory& e) = delete;
+  static HMC_FermionOperatorModuleFactory& getInstance(void) {
+    static HMC_FermionOperatorModuleFactory e;
+    return e;
+  }
+
+ private:
+  HMC_FermionOperatorModuleFactory(void) = default;
+    std::string obj_type() const {
+        return std::string(str);
+  }
+};
+
+
+
+
+extern char fermionop_string[];
+namespace QCD{
+
+// Modules
+class WilsonFermionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonFermionParameters, 
+    RealD, mass);
+};
+
+
+template <class FermionImpl >
+class WilsonFermionModule: public FermionOperatorModule<WilsonFermion, FermionImpl, WilsonFermionParameters> {
+  typedef FermionOperatorModule<WilsonFermion, FermionImpl, WilsonFermionParameters> FermBase;
+  using FermBase::FermBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    auto GridMod = this->GridRefs[0];
+    typename FermionImpl::GaugeField U(GridMod->get_full());
+    this->FOPtr.reset(new WilsonFermion<FermionImpl>(U, *(GridMod->get_full()), *(GridMod->get_rb()), this->Par_.mass));
+  }
+};
+
+
+
+class MobiusFermionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MobiusFermionParameters, 
+    RealD, mass,
+    RealD, M5,
+    RealD, b,
+    RealD, c,
+    unsigned int, Ls);
+};
+
+template <class FermionImpl >
+class MobiusFermionModule: public FermionOperatorModule<MobiusFermion, FermionImpl, MobiusFermionParameters> {
+  typedef FermionOperatorModule<MobiusFermion, FermionImpl, MobiusFermionParameters> FermBase;
+  using FermBase::FermBase; // for constructors
+
+  virtual unsigned int Ls(){
+    return this->Par_.Ls;
+  }
+
+  // acquire resource
+  virtual void initialize(){
+    auto GridMod = this->GridRefs[0];
+    auto GridMod5d = this->GridRefs[1];
+    typename FermionImpl::GaugeField U(GridMod->get_full());
+    this->FOPtr.reset(new MobiusFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()), 
+                                                      *(GridMod5d->get_full()), *(GridMod5d->get_rb()),
+                                                      this->Par_.mass, this->Par_.M5, this->Par_.b, this->Par_.c));
+  }
+};
+
+
+class DomainWallFermionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(DomainWallFermionParameters, 
+    RealD, mass,
+    RealD, M5,
+    unsigned int, Ls);
+};
+
+template <class FermionImpl >
+class DomainWallFermionModule: public FermionOperatorModule<DomainWallFermion, FermionImpl, DomainWallFermionParameters> {
+  typedef FermionOperatorModule<DomainWallFermion, FermionImpl, DomainWallFermionParameters> FermBase;
+  using FermBase::FermBase; // for constructors
+
+  virtual unsigned int Ls(){
+    return this->Par_.Ls;
+  }
+
+  // acquire resource
+  virtual void initialize(){
+    auto GridMod = this->GridRefs[0];
+    auto GridMod5d = this->GridRefs[1];
+    typename FermionImpl::GaugeField U(GridMod->get_full());
+    this->FOPtr.reset(new DomainWallFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()), 
+                                                      *(GridMod5d->get_full()), *(GridMod5d->get_rb()),
+                                                      this->Par_.mass, this->Par_.M5));
+  }
+};
+
+
+
+} // QCD
+} // Grid
+
+
+#endif //FERMIONOPERATOR_MODULES_H

lib/qcd/modules/Modules.cc

@@ -0,0 +1,39 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/Modules.cc
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+namespace Grid{
+
+char gauge_string[]      = "gauge";
+char cp_string[]         = "CheckPointer";
+char hmc_string[]        = "HMC";
+char observable_string[] = "Observable";
+char solver_string[]     = "Solver";
+char fermionop_string[]  = "FermionOperator";
+
+}

lib/qcd/modules/Modules.h

@@ -0,0 +1,130 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/gauge/WilsonGaugeAction.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef HMC_MODULES_H
+#define HMC_MODULES_H
+
+/*
+Define loadable, serializable modules
+for the HMC execution
+*/
+
+namespace Grid {
+
+// Empty class for no parameters
+class NoParameters{};
+
+
+/*
+Base class for modules with parameters
+*/
+template < class P >
+class Parametrized{
+public:
+  typedef P Parameters;
+
+  Parametrized(Parameters Par):Par_(Par){};
+
+  template <class ReaderClass>
+  Parametrized(Reader<ReaderClass> & R, std::string section_name = "parameters"){
+    read(R, section_name, Par_);
+  }
+
+  void set_parameters(Parameters Par){
+        Par_ = Par;
+  }
+
+  void print_parameters(){
+    std::cout << Par_ << std::endl;
+  }
+
+protected:
+  Parameters Par_;
+private:
+  std::string section_name;
+};
+
+
+template <>
+class Parametrized<NoParameters>{
+        public:
+  typedef NoParameters Parameters;
+
+  Parametrized(Parameters Par){};
+
+  template <class ReaderClass>
+  Parametrized(Reader<ReaderClass> & Reader){};
+
+  void set_parameters(Parameters Par){}
+
+  void print_parameters(){}
+
+};
+
+
+
+////////////////////////////////////////
+// Lowest level abstract module class
+////////////////////////////////////////
+template <class Prod>
+class HMCModuleBase {
+ public:
+  typedef Prod Product;
+
+  virtual Prod* getPtr() = 0;
+
+  // add a getReference? 
+  
+  virtual void print_parameters(){};  // default to nothing
+};
+
+
+/////////////////////////////////////////////
+// Registration class
+/////////////////////////////////////////////
+
+template <class T, class TheFactory>
+class Registrar {
+ public:
+  Registrar(std::string className) {
+    // register the class factory function
+    TheFactory::getInstance().registerBuilder(className, 
+        [&](typename TheFactory::TheReader Reader)
+        { 
+          return std::unique_ptr<T>(new T(Reader));
+        }
+        );
+  }
+};
+
+
+
+}
+
+
+#endif //HMC_MODULES_H

lib/qcd/modules/ObservableModules.h

@@ -0,0 +1,148 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/ObservableModules.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef HMC_OBSERVABLE_MODULES_H
+#define HMC_OBSERVABLE_MODULES_H
+
+namespace Grid {
+
+/////////////////////////////
+// Observables
+/////////////////////////////
+template <class ObservableType, class OPar>
+class ObservableModule
+    : public Parametrized<OPar>,
+      public HMCModuleBase< QCD::HmcObservable<typename ObservableType::Field> > {
+ public:
+  typedef HMCModuleBase< QCD::HmcObservable< typename ObservableType::Field> > Base;
+  typedef typename Base::Product Product;
+  typedef OPar Parameters;
+
+  std::unique_ptr<ObservableType> ObservablePtr;
+
+  ObservableModule(OPar Par) : Parametrized<OPar>(Par) {}
+
+  virtual void print_parameters(){
+    Parametrized<OPar>::print_parameters();
+  }
+
+  template <class ReaderClass>
+  ObservableModule(Reader<ReaderClass>& Reader) : Parametrized<OPar>(Reader){};
+
+  Product* getPtr() {
+    if (!ObservablePtr) initialize();
+
+    return ObservablePtr.get();
+  }
+
+ private:
+  virtual void initialize() = 0;
+};
+
+
+
+////////////////
+// Modules
+////////////////
+
+namespace QCD{
+
+//// Observables module
+class PlaquetteObsParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(PlaquetteObsParameters, 
+    std::string, output_prefix);
+};
+
+template < class Impl >
+class PlaquetteMod: public ObservableModule<PlaquetteLogger<Impl>, NoParameters>{
+  typedef ObservableModule<PlaquetteLogger<Impl>, NoParameters> ObsBase;
+  using ObsBase::ObsBase; // for constructors
+
+
+
+  // acquire resource
+  virtual void initialize(){
+    this->ObservablePtr.reset(new PlaquetteLogger<Impl>());
+  }
+  public:
+  PlaquetteMod(): ObsBase(NoParameters()){}
+};
+
+template < class Impl >
+class TopologicalChargeMod: public ObservableModule<TopologicalCharge<Impl>, NoParameters>{
+  typedef ObservableModule<TopologicalCharge<Impl>, NoParameters> ObsBase;
+  using ObsBase::ObsBase; // for constructors
+
+
+
+  // acquire resource
+  virtual void initialize(){
+    this->ObservablePtr.reset(new TopologicalCharge<Impl>());
+  }
+  public:
+  TopologicalChargeMod(): ObsBase(NoParameters()){}
+};
+
+
+
+}// QCD temporarily here
+
+
+////////////////////////////////////////
+// Factories specialisations
+////////////////////////////////////////
+// explicit ref to LatticeGaugeField must be changed or put in the factory
+//typedef HMCModuleBase< QCD::HmcObservable<QCD::LatticeGaugeField> > HMC_ObsModBase;
+
+template <char const *str, class Field, class ReaderClass >
+class HMC_ObservablesModuleFactory
+    : public Factory < HMCModuleBase< QCD::HmcObservable<Field> >, Reader<ReaderClass> > {
+ public:
+  typedef Reader<ReaderClass> TheReader; 
+  // use SINGLETON FUNCTOR MACRO HERE
+  HMC_ObservablesModuleFactory(const HMC_ObservablesModuleFactory& e) = delete;
+  void operator=(const HMC_ObservablesModuleFactory& e) = delete;
+  static HMC_ObservablesModuleFactory& getInstance(void) {
+    static HMC_ObservablesModuleFactory e;
+    return e;
+  }
+
+ private:
+  HMC_ObservablesModuleFactory(void) = default;
+    std::string obj_type() const {
+    return std::string(str);
+  }
+};
+
+extern char observable_string[];
+
+}
+
+
+#endif //HMC_OBSERVABLE_MODULES_H

lib/qcd/modules/Registration.h

@@ -0,0 +1,129 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/Registration.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef MODULES_REGISTRATION_H
+#define MODULES_REGISTRATION_H
+
+// simplify with macros
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Actions
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+typedef QCD::WilsonGModule<ImplementationPolicy> WilsonGMod;
+typedef QCD::SymanzikGModule<ImplementationPolicy> SymanzikGMod;
+typedef QCD::IwasakiGModule<ImplementationPolicy> IwasakiGMod;
+typedef QCD::DBW2GModule<ImplementationPolicy> DBW2GMod;
+typedef QCD::RBCGModule<ImplementationPolicy> RBCGMod;
+typedef QCD::PlaqPlusRectangleGModule<ImplementationPolicy> PlaqPlusRectangleGMod;
+
+static Registrar<QCD::WilsonGMod,            HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __WGmodXMLInit("Wilson"); 
+static Registrar<QCD::SymanzikGMod,          HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __SymGmodXMLInit("Symanzik"); 
+static Registrar<QCD::IwasakiGMod,           HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __IwGmodXMLInit("Iwasaki"); 
+static Registrar<QCD::DBW2GMod,              HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __DBW2GmodXMLInit("DBW2"); 
+static Registrar<QCD::RBCGMod,               HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __RBCGmodXMLInit("RBC"); 
+static Registrar<QCD::PlaqPlusRectangleGMod, HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __PPRectGmodXMLInit("PlaqPlusRect"); 
+
+
+// FIXME more general implementation
+static Registrar<QCD::TwoFlavourFModule<FermionImplementationPolicy> ,      
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __TwoFlavourFmodXMLInit("TwoFlavours"); 
+static Registrar<QCD::TwoFlavourRatioFModule<FermionImplementationPolicy> , 
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __TwoFlavourRatioFmodXMLInit("TwoFlavoursRatio"); 
+static Registrar<QCD::TwoFlavourEOFModule<FermionImplementationPolicy> ,    
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __TwoFlavourEOFmodXMLInit("TwoFlavoursEvenOdd"); 
+static Registrar<QCD::TwoFlavourRatioEOFModule<FermionImplementationPolicy>,
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __TwoFlavourRatioEOFmodXMLInit("TwoFlavoursEvenOddRatio"); 
+static Registrar<QCD::OneFlavourFModule<FermionImplementationPolicy> ,      
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __OneFlavourFmodXMLInit("OneFlavour"); 
+static Registrar<QCD::OneFlavourEOFModule<FermionImplementationPolicy> ,    
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __OneFlavourEOFmodXMLInit("OneFlavourEvenOdd"); 
+static Registrar<QCD::OneFlavourRatioFModule<FermionImplementationPolicy> , 
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __OneFlavourRatioFmodXMLInit("OneFlavourRatio"); 
+static Registrar<QCD::OneFlavourRatioEOFModule<FermionImplementationPolicy>,
+	HMC_ActionModuleFactory<gauge_string, typename ImplementationPolicy::Field, Serialiser> > __OneFlavourRatioEOFmodXMLInit("OneFlavourEvenOddRatio"); 
+
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Solvers
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+// Now a specific registration with a fermion field
+// here must instantiate CG and CR for every new fermion field type (macro!!)
+
+static Registrar< ConjugateGradientModule<QCD::WilsonFermionR::FermionField>,   
+                  HMC_SolverModuleFactory<solver_string, QCD::WilsonFermionR::FermionField, Serialiser> > __CGWFmodXMLInit("ConjugateGradient"); 
+static Registrar< ConjugateResidualModule<QCD::WilsonFermionR::FermionField>,   
+                  HMC_SolverModuleFactory<solver_string, QCD::WilsonFermionR::FermionField, Serialiser> > __CRWFmodXMLInit("ConjugateResidual"); 
+
+// add the staggered, scalar versions here
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Fermion operators
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static Registrar< QCD::WilsonFermionModule<FermionImplementationPolicy>,   
+                  HMC_FermionOperatorModuleFactory<fermionop_string, FermionImplementationPolicy, Serialiser> > __WilsonFOPmodXMLInit("Wilson"); 
+static Registrar< QCD::MobiusFermionModule<FermionImplementationPolicy>,   
+                  HMC_FermionOperatorModuleFactory<fermionop_string, FermionImplementationPolicy, Serialiser> > __MobiusFOPmodXMLInit("Mobius");
+static Registrar< QCD::DomainWallFermionModule<FermionImplementationPolicy>,   
+                  HMC_FermionOperatorModuleFactory<fermionop_string, FermionImplementationPolicy, Serialiser> > __DWFOPmodXMLInit("DomainWall");
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Observables
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static Registrar<QCD::PlaquetteMod<ImplementationPolicy>, HMC_ObservablesModuleFactory<observable_string, typename ImplementationPolicy::Field, Serialiser> > __OBSPLmodXMLInit("Plaquette"); 
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Checkpointers
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static Registrar<QCD::BinaryCPModule<ImplementationPolicy>, HMC_CPModuleFactory<cp_string, ImplementationPolicy, Serialiser> > __CPBinarymodXMLInit("Binary");
+static Registrar<QCD::NerscCPModule<ImplementationPolicy> , HMC_CPModuleFactory<cp_string, ImplementationPolicy, Serialiser> > __CPNerscmodXMLInit("Nersc");
+
+#ifdef HAVE_LIME
+static Registrar<QCD::ILDGCPModule<ImplementationPolicy>  , HMC_CPModuleFactory<cp_string, ImplementationPolicy, Serialiser> > __CPILDGmodXMLInit("ILDG");
+#endif
+
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Integrators
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+static Registrar< HMCLeapFrog<ImplementationPolicy, RepresentationPolicy, Serialiser>      , HMCRunnerModuleFactory<hmc_string, Serialiser> > __HMCLFmodXMLInit("LeapFrog");
+static Registrar< HMCMinimumNorm2<ImplementationPolicy, RepresentationPolicy, Serialiser>  , HMCRunnerModuleFactory<hmc_string, Serialiser> > __HMCMN2modXMLInit("MinimumNorm2");
+static Registrar< HMCForceGradient<ImplementationPolicy, RepresentationPolicy, Serialiser> , HMCRunnerModuleFactory<hmc_string, Serialiser> > __HMCFGmodXMLInit("ForceGradient");
+
+typedef HMCRunnerModuleFactory<hmc_string, Serialiser > HMCModuleFactory;
+
+#endif

lib/qcd/modules/SolverModules.h

@@ -0,0 +1,138 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/SolverModules.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef SOLVER_MODULES_H
+#define SOLVER_MODULES_H
+
+namespace Grid {
+
+//////////////////////////////////////////////
+//       Operator Functions (Solvers)
+//////////////////////////////////////////////
+
+template <template <typename> class SolverType, class Field, class SPar>
+class SolverModule
+    : public Parametrized<SPar>,
+      public HMCModuleBase<OperatorFunction<Field> > {
+ public:
+  typedef HMCModuleBase< OperatorFunction<Field> > Base;
+  typedef typename Base::Product Product;
+
+  std::unique_ptr< SolverType<Field> > SolverPtr;
+
+  SolverModule(SPar Par) : Parametrized<SPar>(Par) {}
+
+  template <class ReaderClass>
+  SolverModule(Reader<ReaderClass>& Reader) : Parametrized<SPar>(Reader){};
+
+  virtual void print_parameters(){
+    std::cout << this->Par_ << std::endl;
+  }
+
+  Product* getPtr() {
+    if (!SolverPtr) initialize();
+
+    return SolverPtr.get();
+  }
+
+ private:
+  virtual void initialize() = 0;
+};
+
+
+// Factory
+template <char const *str, class Field, class ReaderClass >
+class HMC_SolverModuleFactory
+    : public Factory < HMCModuleBase<OperatorFunction<Field> > ,  Reader<ReaderClass> > {
+ public:
+  // use SINGLETON FUNCTOR MACRO HERE
+  typedef Reader<ReaderClass> TheReader; 
+
+  HMC_SolverModuleFactory(const HMC_SolverModuleFactory& e) = delete;
+  void operator=(const HMC_SolverModuleFactory& e) = delete;
+  static HMC_SolverModuleFactory& getInstance(void) {
+    static HMC_SolverModuleFactory e;
+    return e;
+  }
+
+ private:
+  HMC_SolverModuleFactory(void) = default;
+    std::string obj_type() const {
+        return std::string(str);
+  }
+};
+
+
+
+class SolverParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(SolverParameters,
+    RealD, tolerance,
+    RealD, max_iterations);
+  // add error on no convergence?
+};
+
+
+class SolverObjName: Serializable {
+public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(SolverObjName, 
+  std::string, name,
+  SolverParameters, parameters);
+
+};
+
+
+
+template <class Field >
+class ConjugateGradientModule: public SolverModule<ConjugateGradient, Field, SolverParameters> {
+  typedef SolverModule<ConjugateGradient, Field, SolverParameters> SolverBase;
+  using SolverBase::SolverBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->SolverPtr.reset(new ConjugateGradient<Field>(this->Par_.tolerance, this->Par_.max_iterations, true));
+  }
+};
+
+template <class Field >
+class ConjugateResidualModule: public SolverModule<ConjugateResidual, Field, SolverParameters> {
+  typedef SolverModule<ConjugateResidual, Field, SolverParameters> SolverBase;
+  using SolverBase::SolverBase; // for constructors
+
+  // acquire resource
+  virtual void initialize(){
+    this->SolverPtr.reset(new ConjugateResidual<Field>(this->Par_.tolerance, this->Par_.max_iterations));
+  }
+
+};
+
+extern char solver_string[];
+} // Grid
+
+
+#endif //SOLVER_MODULES_H

lib/qcd/modules/mods.h

@@ -0,0 +1,46 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/gauge/WilsonGaugeAction.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef MODS_H
+#define MODS_H
+
+// Modules files
+
+#include <Grid/qcd/modules/Factory.h>
+#include <Grid/qcd/modules/Modules.h>
+
+
+#include <Grid/qcd/hmc/checkpointers/CheckPointerModules.h>
+#include <Grid/qcd/modules/SolverModules.h>
+#include <Grid/qcd/modules/FermionOperatorModules.h>
+#include <Grid/qcd/modules/ActionModules.h>
+#include <Grid/qcd/modules/ObservableModules.h>
+
+
+
+#endif //MODS_H

lib/qcd/observables/hmc_observable.h

@@ -0,0 +1,49 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/observables/hmc_observable.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef HMC_OBSERVABLE_H
+#define HMC_OBSERVABLE_H
+
+namespace Grid{
+
+template <class Field>
+class HmcObservable {
+ public:
+  virtual void TrajectoryComplete(int traj,
+                                  Field &U,
+                                  GridSerialRNG &sRNG,
+                                  GridParallelRNG &pRNG) = 0;
+};
+
+}  // namespace Grid
+
+#include "plaquette.h"
+#include "topological_charge.h"
+
+#endif  //  HMC_OBSERVABLE_H

lib/qcd/observables/plaquette.h

@@ -0,0 +1,68 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/plaquette.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef HMC_PLAQUETTE_H
+#define HMC_PLAQUETTE_H
+
+namespace Grid {
+namespace QCD {
+
+// this is only defined for a gauge theory
+template <class Impl>
+class PlaquetteLogger : public HmcObservable<typename Impl::Field> {
+ public:
+  // here forces the Impl to be of gauge fields
+  // if not the compiler will complain
+  INHERIT_GIMPL_TYPES(Impl);
+
+  // necessary for HmcObservable compatibility
+  typedef typename Impl::Field Field;
+
+  void TrajectoryComplete(int traj,
+                          Field &U,
+                          GridSerialRNG &sRNG,
+                          GridParallelRNG &pRNG) {
+
+    RealD plaq = WilsonLoops<Impl>::avgPlaquette(U);
+
+    int def_prec = std::cout.precision();
+
+    std::cout << GridLogMessage
+        << std::setprecision(std::numeric_limits<Real>::digits10 + 1)
+        << "Plaquette: [ " << traj << " ] "<< plaq << std::endl;
+
+    std::cout.precision(def_prec);
+
+  }
+};
+
+}  // namespace QCD
+}  // namespace Grid
+
+#endif  // HMC_PLAQUETTE_H

lib/qcd/observables/topological_charge.h

@@ -0,0 +1,67 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/topological_charge.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef HMC_TOP_CHARGE_H
+#define HMC_TOP_CHARGE_H
+
+namespace Grid {
+namespace QCD {
+
+// this is only defined for a gauge theory
+template <class Impl>
+class TopologicalCharge : public HmcObservable<typename Impl::Field> {
+ public:
+    // here forces the Impl to be of gauge fields
+    // if not the compiler will complain
+    INHERIT_GIMPL_TYPES(Impl);
+
+    // necessary for HmcObservable compatibility
+    typedef typename Impl::Field Field;
+
+    void TrajectoryComplete(int traj,
+                            Field &U,
+                            GridSerialRNG &sRNG,
+                            GridParallelRNG &pRNG) {
+
+    Real q = WilsonLoops<Impl>::TopologicalCharge(U);
+
+    int def_prec = std::cout.precision();
+
+    std::cout << GridLogMessage
+        << std::setprecision(std::numeric_limits<Real>::digits10 + 1)
+        << "Topological Charge: [ " << traj << " ] "<< q << std::endl;
+
+    std::cout.precision(def_prec);
+    }
+
+};
+}
+}
+
+#endif  //  HMC_TOP_CHARGE_H

lib/qcd/representations/fundamental.h

@@ -34,8 +34,21 @@ class FundamentalRep {
 
 };
 
+  template<class Field> 
+  class EmptyRep {
+  public:
+    typedef Field LatticeField;
+
+    explicit EmptyRep(GridBase* grid) {} //do nothing
+    void update_representation(const LatticeField& Uin) {} // do nothing
+    LatticeField RtoFundamentalProject(const LatticeField& in, Real scale = 1.0) const{}// do nothing
+  };
+    
+
+  
 typedef	 FundamentalRep<Nc> FundamentalRepresentation;
 
+  
 }
 }
 

lib/qcd/representations/hmc_types.h

@@ -4,6 +4,8 @@
 #include <Grid/qcd/representations/adjoint.h>
 #include <Grid/qcd/representations/two_index.h>
 #include <Grid/qcd/representations/fundamental.h>
+#include <Grid/qcd/action/scalar/ScalarImpl.h>
+
 #include <tuple>
 #include <utility>
 
@@ -39,13 +41,17 @@ class Representations {
   int size() { return tuple_size; }
 
   // update the fields
+  // fields in the main representation always the first in the list
+  // get the field type
+  typedef typename std::tuple_element<0,Representation_Fields>::type LatticeSourceField;
+  
   template <std::size_t I = 0>
   inline typename std::enable_if<(I == tuple_size), void>::type update(
-      LatticeGaugeField& U) {}
+      LatticeSourceField& U) {}
 
   template <std::size_t I = 0>
   inline typename std::enable_if<(I < tuple_size), void>::type update(
-      LatticeGaugeField& U) {
+      LatticeSourceField& U) {
     std::get<I>(rep).update_representation(U);
     update<I + 1>(U);
   }
@@ -55,6 +61,8 @@ class Representations {
 };
 
 typedef Representations<FundamentalRepresentation> NoHirep;
+typedef Representations<EmptyRep<typename ScalarImplR::Field> > ScalarFields;
+  //typedef Representations<EmptyRep<typename ScalarMatrixImplR::Field> > ScalarMatrixFields;
 
 // Helper classes to access the elements
 // Strips the first N parameters from the tuple

lib/qcd/smearing/APEsmearing.h

@@ -1,3 +1,31 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/plaquette.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
 /*!
   @brief Declaration of Smear_APE class for APE smearing
 */
@@ -92,19 +120,19 @@
 
 	  						temp_Sigma = -rho_numu*staple*iLambda_nu;  //ok
 	        				//-r_numu*U_nu(x+mu)*Udag_mu(x+nu)*Udag_nu(x)*Lambda_nu(x)
-	  						Gimpl::AddGaugeLink(SigmaTerm, temp_Sigma, mu);
+	  						Gimpl::AddLink(SigmaTerm, temp_Sigma, mu);
 
 	    					sh_field = Cshift(iLambda_nu, mu, 1);// general also for Gparity?
 
 	    					temp_Sigma = rho_numu*sh_field*staple; //ok
 	    					//r_numu*Lambda_nu(mu)*U_nu(x+mu)*Udag_mu(x+nu)*Udag_nu(x)
-	    					Gimpl::AddGaugeLink(SigmaTerm, temp_Sigma, mu);
+	    					Gimpl::AddLink(SigmaTerm, temp_Sigma, mu);
 
 	    					sh_field = Cshift(iLambda_mu, nu, 1);
 
 	    					temp_Sigma = -rho_munu*staple*U_nu*sh_field*adj(U_nu); //ok
 	    					//-r_munu*U_nu(x+mu)*Udag_mu(x+nu)*Lambda_mu(x+nu)*Udag_nu(x)
-	    					Gimpl::AddGaugeLink(SigmaTerm, temp_Sigma, mu);
+	    					Gimpl::AddLink(SigmaTerm, temp_Sigma, mu);
 
 	    					staple = zero;
 	    					sh_field = Cshift(U_nu, mu, 1);
@@ -116,7 +144,7 @@
 	    					sh_field = Cshift(u_tmp, mu, 1);
 	    					temp_Sigma += -rho_numu*sh_field*adj(U_mu)*U_nu;
 	    					sh_field = Cshift(temp_Sigma, nu, -1);
-	    					Gimpl::AddGaugeLink(SigmaTerm, sh_field, mu);
+	    					Gimpl::AddLink(SigmaTerm, sh_field, mu);
 
 	    				}
 	    			}

lib/qcd/smearing/BaseSmearing.h

@@ -1,3 +1,31 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/plaquette.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
 /*
   @brief Declares base smearing class Smear
  */
@@ -11,7 +39,6 @@ public:
 
   virtual ~Smear(){}
   virtual void smear     (GaugeField&,const GaugeField&)const = 0;
-  virtual void derivative(GaugeField&,
+  virtual void derivative(GaugeField&, const GaugeField&,const GaugeField&) const = 0;
-			  const GaugeField&,const GaugeField&) const = 0;
 };
 #endif

lib/qcd/smearing/GaugeConfiguration.h

@@ -11,24 +11,23 @@ namespace Grid {
 namespace QCD {
 
   //trivial class for no smearing
-  template< class Gimpl >
+  template< class Impl >
 class NoSmearing {
 public:
-  INHERIT_GIMPL_TYPES(Gimpl);
+  INHERIT_FIELD_TYPES(Impl);
 
-  GaugeField*
+  Field* ThinField;
-      ThinLinks;
 
-  NoSmearing(): ThinLinks(NULL) {}
+  NoSmearing(): ThinField(NULL) {}
 
-  void set_GaugeField(GaugeField& U) { ThinLinks = &U; }
+  void set_Field(Field& U) { ThinField = &U; }
 
-  void smeared_force(GaugeField& SigmaTilde) const {}
+  void smeared_force(Field&) const {}
 
-  GaugeField& get_SmearedU() { return *ThinLinks; }
+  Field& get_SmearedU() { return *ThinField; }
 
-  GaugeField& get_U(bool smeared = false) {
+  Field& get_U(bool smeared = false) {
-    return *ThinLinks;
+    return *ThinField;
   }
 
 };
@@ -227,7 +226,7 @@ class SmearedConfiguration {
 
   
   // attach the smeared routines to the thin links U and fill the smeared set
-  void set_GaugeField(GaugeField& U) { fill_smearedSet(U); }
+  void set_Field(GaugeField& U) { fill_smearedSet(U); }
 
   //====================================================================
   void smeared_force(GaugeField& SigmaTilde) const {

lib/qcd/smearing/Smearing.h

@@ -5,5 +5,6 @@
 #include <Grid/qcd/smearing/APEsmearing.h>
 #include <Grid/qcd/smearing/StoutSmearing.h>
 #include <Grid/qcd/smearing/GaugeConfiguration.h>
+#include <Grid/qcd/smearing/WilsonFlow.h>
 
 #endif

lib/qcd/smearing/WilsonFlow.h

@@ -0,0 +1,121 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/modules/plaquette.h
+
+Copyright (C) 2017
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef WILSONFLOW_H
+#define WILSONFLOW_H
+
+namespace Grid {
+namespace QCD {
+
+template <class Gimpl>
+class WilsonFlow: public Smear<Gimpl>{
+    unsigned int Nstep;
+    RealD epsilon;
+
+    mutable WilsonGaugeAction<Gimpl> SG;
+
+    void evolve_step(typename Gimpl::GaugeField&) const;
+    RealD tau(unsigned int t)const {return epsilon*(t+1.0); }
+
+
+ public:
+    INHERIT_GIMPL_TYPES(Gimpl)
+
+    explicit WilsonFlow(unsigned int Nstep, RealD epsilon):
+        Nstep(Nstep),
+        epsilon(epsilon),
+        SG(WilsonGaugeAction<Gimpl>(3.0)) {
+            // WilsonGaugeAction with beta 3.0
+            assert(epsilon > 0.0);
+            LogMessage();
+    }
+
+    void LogMessage() {
+        std::cout << GridLogMessage
+            << "[WilsonFlow] Nstep   : " << Nstep << std::endl;
+        std::cout << GridLogMessage
+            << "[WilsonFlow] epsilon : " << epsilon << std::endl;
+        std::cout << GridLogMessage
+            << "[WilsonFlow] full trajectory : " << Nstep * epsilon << std::endl;
+    }
+
+    virtual void smear(GaugeField&, const GaugeField&) const;
+
+    virtual void derivative(GaugeField&, const GaugeField&, const GaugeField&) const {
+        assert(0);
+        // undefined for WilsonFlow
+    }
+
+    RealD energyDensityPlaquette(unsigned int step, const GaugeField& U) const;
+};
+
+
+////////////////////////////////////////////////////////////////////////////////
+// Implementations
+////////////////////////////////////////////////////////////////////////////////
+template <class Gimpl>
+void WilsonFlow<Gimpl>::evolve_step(typename Gimpl::GaugeField &U) const{
+    GaugeField Z(U._grid);
+    GaugeField tmp(U._grid);
+    SG.deriv(U, Z);
+    Z *= 0.25;                                  // Z0 = 1/4 * F(U)
+    Gimpl::update_field(Z, U, -2.0*epsilon);    // U = W1 = exp(ep*Z0)*W0
+
+    Z *= -17.0/8.0;
+    SG.deriv(U, tmp); Z += tmp;                 // -17/32*Z0 +Z1
+    Z *= 8.0/9.0;                               // Z = -17/36*Z0 +8/9*Z1
+    Gimpl::update_field(Z, U, -2.0*epsilon);    // U_= W2 = exp(ep*Z)*W1
+
+    Z *= -4.0/3.0;
+    SG.deriv(U, tmp); Z += tmp;                 // 4/3*(17/36*Z0 -8/9*Z1) +Z2
+    Z *= 3.0/4.0;                               // Z = 17/36*Z0 -8/9*Z1 +3/4*Z2
+    Gimpl::update_field(Z, U, -2.0*epsilon);    // V(t+e) = exp(ep*Z)*W2
+}
+
+template <class Gimpl>
+RealD WilsonFlow<Gimpl>::energyDensityPlaquette(unsigned int step, const GaugeField& U) const {
+    RealD td = tau(step);
+    return 2.0 * td * td * SG.S(U)/U._grid->gSites();
+}
+
+template <class Gimpl>
+void WilsonFlow<Gimpl>::smear(GaugeField& out, const GaugeField& in) const {
+    out = in;
+    for (unsigned int step = 0; step < Nstep; step++) {
+        evolve_step(out);
+        std::cout << GridLogMessage << "[WilsonFlow] Energy density (plaq) : "
+            << step << " "
+            << energyDensityPlaquette(step,out) << std::endl;
+    }
+}
+
+}  // namespace QCD
+}  // namespace Grid
+
+#endif   // WILSONFLOW_H

lib/qcd/utils/CovariantLaplacian.h

@@ -0,0 +1,197 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/scalar/CovariantLaplacian.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef COVARIANT_LAPLACIAN_H
+#define COVARIANT_LAPLACIAN_H
+
+namespace Grid {
+namespace QCD {
+
+struct LaplacianParams : Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(LaplacianParams, 
+                                  RealD, lo, 
+                                  RealD, hi, 
+                                  int,   MaxIter, 
+                                  RealD, tolerance, 
+                                  int,   degree, 
+                                  int,   precision);
+  
+  // constructor 
+  LaplacianParams(RealD lo      = 0.0, 
+                  RealD hi      = 1.0, 
+                  int maxit     = 1000,
+                  RealD tol     = 1.0e-8, 
+                  int degree    = 10,
+                  int precision = 64)
+    : lo(lo),
+      hi(hi),
+      MaxIter(maxit),
+      tolerance(tol),
+      degree(degree),
+      precision(precision){};
+};
+
+
+
+////////////////////////////////////////////////////////////
+// Laplacian operator L on adjoint fields
+//
+// phi: adjoint field
+// L: D_mu^dag D_mu
+//
+// L phi(x) = Sum_mu [ U_mu(x)phi(x+mu)U_mu(x)^dag + 
+//                     U_mu(x-mu)^dag phi(x-mu)U_mu(x-mu)
+//                     -2phi(x)]
+//
+// Operator designed to be encapsulated by
+// an HermitianLinearOperator<.. , ..>
+////////////////////////////////////////////////////////////
+
+template <class Impl>
+class LaplacianAdjointField: public Metric<typename Impl::Field> {
+  OperatorFunction<typename Impl::Field> &Solver;
+  LaplacianParams param;
+  MultiShiftFunction PowerHalf;    
+  MultiShiftFunction PowerInvHalf;    
+
+ public:
+  INHERIT_GIMPL_TYPES(Impl);
+
+  LaplacianAdjointField(GridBase* grid, OperatorFunction<GaugeField>& S, LaplacianParams& p, const RealD k = 1.0)
+      : U(Nd, grid), Solver(S), param(p), kappa(k){
+        AlgRemez remez(param.lo,param.hi,param.precision);
+        std::cout<<GridLogMessage << "Generating degree "<<param.degree<<" for x^(1/2)"<<std::endl;
+        remez.generateApprox(param.degree,1,2);
+        PowerHalf.Init(remez,param.tolerance,false);
+        PowerInvHalf.Init(remez,param.tolerance,true);
+        
+
+      };
+
+  void Mdir(const GaugeField&, GaugeField&, int, int){ assert(0);}
+  void Mdiag(const GaugeField&, GaugeField&){ assert(0);}
+
+  void ImportGauge(const GaugeField& _U) {
+    for (int mu = 0; mu < Nd; mu++) {
+      U[mu] = PeekIndex<LorentzIndex>(_U, mu);
+    }
+  }
+
+  void M(const GaugeField& in, GaugeField& out) {
+    // in is an antihermitian matrix
+    // test
+    //GaugeField herm = in + adj(in);
+    //std::cout << "AHermiticity: " << norm2(herm) << std::endl;
+
+    GaugeLinkField tmp(in._grid);
+    GaugeLinkField tmp2(in._grid);
+    GaugeLinkField sum(in._grid);
+
+    for (int nu = 0; nu < Nd; nu++) {
+      sum = zero;
+      GaugeLinkField in_nu = PeekIndex<LorentzIndex>(in, nu);
+      GaugeLinkField out_nu(out._grid);
+      for (int mu = 0; mu < Nd; mu++) {
+        tmp = U[mu] * Cshift(in_nu, mu, +1) * adj(U[mu]);
+        tmp2 = adj(U[mu]) * in_nu * U[mu];
+        sum += tmp + Cshift(tmp2, mu, -1) - 2.0 * in_nu;
+      }
+      out_nu = (1.0 - kappa) * in_nu - kappa / (double(4 * Nd)) * sum;
+      PokeIndex<LorentzIndex>(out, out_nu, nu);
+    }
+  }
+
+  void MDeriv(const GaugeField& in, GaugeField& der) {
+    // in is anti-hermitian
+    RealD factor = -kappa / (double(4 * Nd));
+    
+    for (int mu = 0; mu < Nd; mu++){
+      GaugeLinkField der_mu(der._grid);
+      der_mu = zero;
+      for (int nu = 0; nu < Nd; nu++){
+        GaugeLinkField in_nu = PeekIndex<LorentzIndex>(in, nu);
+        der_mu += U[mu] * Cshift(in_nu, mu, 1) * adj(U[mu]) * in_nu;
+      }
+      // the minus sign comes by using the in_nu instead of the
+      // adjoint in the last multiplication
+      PokeIndex<LorentzIndex>(der,  -2.0 * factor * der_mu, mu);
+    } 
+  }
+
+  // separating this temporarily
+  void MDeriv(const GaugeField& left, const GaugeField& right,
+              GaugeField& der) {
+    // in is anti-hermitian
+    RealD factor = -kappa / (double(4 * Nd));
+
+    for (int mu = 0; mu < Nd; mu++) {
+      GaugeLinkField der_mu(der._grid);
+      der_mu = zero;
+      for (int nu = 0; nu < Nd; nu++) {
+        GaugeLinkField left_nu = PeekIndex<LorentzIndex>(left, nu);
+        GaugeLinkField right_nu = PeekIndex<LorentzIndex>(right, nu);
+        der_mu += U[mu] * Cshift(left_nu, mu, 1) * adj(U[mu]) * right_nu;
+        der_mu += U[mu] * Cshift(right_nu, mu, 1) * adj(U[mu]) * left_nu;
+      }
+      PokeIndex<LorentzIndex>(der, -factor * der_mu, mu);
+    }
+  }
+
+  void Minv(const GaugeField& in, GaugeField& inverted){
+    HermitianLinearOperator<LaplacianAdjointField<Impl>,GaugeField> HermOp(*this);
+    Solver(HermOp, in, inverted);
+  }
+
+  void MSquareRoot(GaugeField& P){
+    GaugeField Gp(P._grid);
+    HermitianLinearOperator<LaplacianAdjointField<Impl>,GaugeField> HermOp(*this);
+    ConjugateGradientMultiShift<GaugeField> msCG(param.MaxIter,PowerHalf);
+    msCG(HermOp,P,Gp);
+    P = Gp; 
+  }
+
+  void MInvSquareRoot(GaugeField& P){
+    GaugeField Gp(P._grid);
+    HermitianLinearOperator<LaplacianAdjointField<Impl>,GaugeField> HermOp(*this);
+    ConjugateGradientMultiShift<GaugeField> msCG(param.MaxIter,PowerInvHalf);
+    msCG(HermOp,P,Gp);
+    P = Gp; 
+  }
+
+
+
+ private:
+  RealD kappa;
+  std::vector<GaugeLinkField> U;
+};
+
+}
+}
+
+#endif

lib/qcd/utils/Metric.h

@@ -0,0 +1,226 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/integrators/Integrator.h
+
+Copyright (C) 2015
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+//--------------------------------------------------------------------
+#ifndef METRIC_H
+#define METRIC_H
+
+namespace Grid{
+namespace QCD{
+
+template <typename Field> 
+class Metric{
+public:
+  virtual void ImportGauge(const Field&)   = 0;
+  virtual void M(const Field&, Field&)     = 0;
+  virtual void Minv(const Field&, Field&)  = 0;
+  virtual void MSquareRoot(Field&) = 0;
+  virtual void MInvSquareRoot(Field&) = 0;
+  virtual void MDeriv(const Field&, Field&) = 0;
+  virtual void MDeriv(const Field&, const Field&, Field&) = 0;
+};
+
+
+// Need a trivial operator
+template <typename Field>
+class TrivialMetric : public Metric<Field>{
+public:
+  virtual void ImportGauge(const Field&){};
+  virtual void M(const Field& in, Field& out){
+    out = in;
+  }
+  virtual void Minv(const Field& in, Field& out){
+    out = in;
+  }
+  virtual void MSquareRoot(Field& P){
+    // do nothing
+  }
+  virtual void MInvSquareRoot(Field& P){
+    // do nothing
+  }
+  virtual void MDeriv(const Field& in, Field& out){
+    out = zero;
+  }
+  virtual void MDeriv(const Field& left, const Field& right, Field& out){
+    out = zero;
+  }
+
+};
+
+///////////////////////////////
+// Generalised momenta
+///////////////////////////////
+
+template <typename Implementation>
+class GeneralisedMomenta{
+public:
+  typedef typename Implementation::Field MomentaField;  //for readability
+  typedef typename Implementation::GaugeLinkField MomentaLinkField;  //for readability
+  Metric<MomentaField>& M;
+  MomentaField Mom;
+
+  // Auxiliary fields
+  // not hard coded but inherit the type from the metric
+  // created Nd new fields
+  // hide these in the metric?
+  //typedef Lattice<iVector<iScalar<iMatrix<vComplex, Nc> >, Nd/2 > > AuxiliaryMomentaType;
+  MomentaField AuxMom;
+  MomentaField AuxField;
+
+  GeneralisedMomenta(GridBase* grid, Metric<MomentaField>& M): M(M), Mom(grid), AuxMom(grid), AuxField(grid){}
+
+  // Correct
+  void MomentaDistribution(GridParallelRNG& pRNG){
+    // Generate a distribution for
+    // P^dag G P
+    // where G = M^-1
+
+    // Generate gaussian momenta
+    Implementation::generate_momenta(Mom, pRNG);
+    // Modify the distribution with the metric
+    M.MSquareRoot(Mom);
+
+    if (1) {
+      // Auxiliary momenta
+      // do nothing if trivial, so hide in the metric
+      MomentaField AuxMomTemp(Mom._grid);
+      Implementation::generate_momenta(AuxMom, pRNG);
+      Implementation::generate_momenta(AuxField, pRNG);
+      // Modify the distribution with the metric
+      // Aux^dag M Aux
+      M.MInvSquareRoot(AuxMom);  // AuxMom = M^{-1/2} AuxMomTemp
+    }
+  }
+
+  // Correct
+  RealD MomentaAction(){
+    MomentaField inv(Mom._grid);
+    inv = zero;
+    M.Minv(Mom, inv);
+    LatticeComplex Hloc(Mom._grid);
+    Hloc = zero;
+    for (int mu = 0; mu < Nd; mu++) {
+      // This is not very general
+      // hide in the metric
+      auto Mom_mu = PeekIndex<LorentzIndex>(Mom, mu);
+      auto inv_mu = PeekIndex<LorentzIndex>(inv, mu);
+      Hloc += trace(Mom_mu * inv_mu);
+    }
+
+    if (1) {
+      // Auxiliary Fields
+      // hide in the metric
+      M.M(AuxMom, inv);
+      for (int mu = 0; mu < Nd; mu++) {
+        // This is not very general
+        // hide in the operators
+        auto inv_mu = PeekIndex<LorentzIndex>(inv, mu);
+        auto am_mu = PeekIndex<LorentzIndex>(AuxMom, mu);
+        auto af_mu = PeekIndex<LorentzIndex>(AuxField, mu);
+        Hloc += trace(am_mu * inv_mu);// p M p
+        Hloc += trace(af_mu * af_mu);
+      }
+    }
+
+    Complex Hsum = sum(Hloc);
+    return Hsum.real();
+  }
+
+  // Correct
+  void DerivativeU(MomentaField& in, MomentaField& der){
+
+    // Compute the derivative of the kinetic term
+    // with respect to the gauge field
+    MomentaField MDer(in._grid);
+    MomentaField X(in._grid);
+    X = zero;
+    M.Minv(in, X);  // X = G in
+    M.MDeriv(X, MDer);  // MDer = U * dS/dU
+    der = Implementation::projectForce(MDer);  // Ta if gauge fields
+    
+  }
+
+  void AuxiliaryFieldsDerivative(MomentaField& der){
+    der = zero;
+    if (1){
+    // Auxiliary fields
+    MomentaField der_temp(der._grid);
+    MomentaField X(der._grid);
+    X=zero;
+    //M.M(AuxMom, X); // X = M Aux
+    // Two derivative terms
+    // the Mderiv need separation of left and right terms
+    M.MDeriv(AuxMom, der); 
+
+
+    // this one should not be necessary (identical to the previous one)
+    //M.MDeriv(X, AuxMom, der_temp); der += der_temp;
+
+    der = -1.0*Implementation::projectForce(der);
+    }
+  }
+
+  void DerivativeP(MomentaField& der){
+    der = zero;
+    M.Minv(Mom, der);
+    // is the projection necessary here?
+    // no for fields in the algebra
+    der = Implementation::projectForce(der); 
+  }
+
+  void update_auxiliary_momenta(RealD ep){
+    if(1){
+      AuxMom -= ep * AuxField;
+    }
+  }
+
+  void update_auxiliary_fields(RealD ep){
+    if (1) {
+      MomentaField tmp(AuxMom._grid);
+      MomentaField tmp2(AuxMom._grid);
+      M.M(AuxMom, tmp);
+      // M.M(tmp, tmp2);
+      AuxField += ep * tmp;  // M^2 AuxMom
+      // factor of 2?
+    }
+  }
+
+};
+
+
+
+
+
+
+
+
+}
+}
+
+
+#endif //METRIC_H

lib/qcd/utils/SUn.h

@@ -170,6 +170,7 @@ class SU {
     ta()()(i2, i1) = 1.0;
     ta = ta * 0.5;
   }
+  
   template <class cplx>
   static void generatorSigmaX(int su2Index, iSUnMatrix<cplx> &ta) {
     ta = zero;
@@ -194,6 +195,8 @@ class SU {
     ta = ta * nrm;
   }
 
+
+
   ////////////////////////////////////////////////////////////////////////
   // Map a su2 subgroup number to the pair of rows that are non zero
   ////////////////////////////////////////////////////////////////////////

lib/qcd/utils/ScalarObjs.h

@@ -0,0 +1,96 @@
+/*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./lib/qcd/utils/WilsonLoops.h
+
+    Copyright (C) 2015
+
+Author: neo <cossu@post.kek.jp>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef SCALAR_OBJS_H
+#define SCALAR_OBJS_H
+namespace Grid {
+
+  // FIXME drop the QCD namespace in Nd
+  
+
+// Scalar field obs
+template <class Impl>
+class ScalarObs {
+ public:
+  //////////////////////////////////////////////////
+  // squared field
+  //////////////////////////////////////////////////
+
+  static void phisquared(typename Impl::Field &fsq,
+                         const typename Impl::Field &f) {
+    fsq = f * f;
+  }
+  //////////////////////////////////////////////////
+  // phi^4 interaction term
+  //////////////////////////////////////////////////
+
+  static void phifourth(typename Impl::Field &fsq,
+                        const typename Impl::Field &f) {
+    fsq = f * f * f * f;
+  }
+
+  //////////////////////////////////////////////////
+  // phi(x)phi(x+mu)
+  //////////////////////////////////////////////////
+
+  static void phider(typename Impl::Field &fsq,
+                     const typename Impl::Field &f) {
+    fsq = Cshift(f, 0, -1) * f;
+    for (int mu = 1; mu < QCD::Nd; mu++) fsq += Cshift(f, mu, -1) * f;
+  }
+
+  //////////////////////////////////////////////////
+  // Vol sum of the previous obs.
+  //////////////////////////////////////////////////
+
+  static RealD sumphider(const typename Impl::Field &f) {
+    typename Impl::Field tmp(f._grid);
+    tmp = Cshift(f, 0, -1) * f;
+    for (int mu = 1; mu < QCD::Nd; mu++) {
+      tmp += Cshift(f, mu, -1) * f;
+    }
+    return -sum(trace(tmp));
+  }
+
+  static RealD sumphisquared(const typename Impl::Field &f) {
+    typename Impl::Field tmp(f._grid);
+    tmp = f * f;
+    return sum(trace(tmp));
+  }
+
+  static RealD sumphifourth(const typename Impl::Field &f) {
+    typename Impl::Field tmp(f._grid);
+    phifourth(tmp, f);
+    return sum(trace(tmp));
+  }
+};
+
+
+}
+
+#endif

lib/qcd/utils/Utils.h

@@ -3,7 +3,16 @@
 #include <Grid/qcd/utils/SpaceTimeGrid.h>
 #include <Grid/qcd/utils/LinalgUtils.h>
 #include <Grid/qcd/utils/CovariantCshift.h>
+
+// Scalar field
+#include <Grid/qcd/utils/ScalarObjs.h>
+
+// Include representations
 #include <Grid/qcd/utils/SUn.h>
 #include <Grid/qcd/utils/SUnAdjoint.h>
 #include <Grid/qcd/utils/SUnTwoIndex.h>
+
+
+
+
 #endif

lib/qcd/utils/WilsonLoops.h

@@ -54,9 +54,21 @@ public:
     // resolution throughout the usage in this file, and rather defeats the
     // purpose of deriving
     // from Gimpl.
+    /*
     plaq = Gimpl::CovShiftBackward(
         U[mu], mu, Gimpl::CovShiftBackward(
                        U[nu], nu, Gimpl::CovShiftForward(U[mu], mu, U[nu])));
+                       */
+    // _
+    //|< _|
+    plaq = Gimpl::CovShiftForward(U[mu],mu,
+           Gimpl::CovShiftForward(U[nu],nu,
+           Gimpl::CovShiftBackward(U[mu],mu,
+           Gimpl::CovShiftIdentityBackward(U[nu], nu))));
+
+
+
+
   }
   //////////////////////////////////////////////////
   // trace of directed plaquette oriented in mu,nu plane
@@ -86,8 +98,8 @@ public:
   // sum over all x,y,z,t and over all planes of plaquette
   //////////////////////////////////////////////////
   static RealD sumPlaquette(const GaugeLorentz &Umu) {
-    std::vector<GaugeMat> U(4, Umu._grid);
+    std::vector<GaugeMat> U(Nd, Umu._grid);
-
+    // inefficient here
     for (int mu = 0; mu < Nd; mu++) {
       U[mu] = PeekIndex<LorentzIndex>(Umu, mu);
     }
@@ -95,11 +107,12 @@ public:
     LatticeComplex Plaq(Umu._grid);
 
     sitePlaquette(Plaq, U);
-
     TComplex Tp = sum(Plaq);
     Complex p = TensorRemove(Tp);
     return p.real();
   }
+
+
   //////////////////////////////////////////////////
   // average over all x,y,z,t and over all planes of plaquette
   //////////////////////////////////////////////////
@@ -114,7 +127,7 @@ public:
   // average over traced single links
   //////////////////////////////////////////////////
   static RealD linkTrace(const GaugeLorentz &Umu) {
-    std::vector<GaugeMat> U(4, Umu._grid);
+    std::vector<GaugeMat> U(Nd, Umu._grid);
 
     LatticeComplex Tr(Umu._grid);
     Tr = zero;
@@ -139,7 +152,7 @@ public:
 
     GridBase *grid = Umu._grid;
 
-    std::vector<GaugeMat> U(4, grid);
+    std::vector<GaugeMat> U(Nd, grid);
     for (int d = 0; d < Nd; d++) {
       U[d] = PeekIndex<LorentzIndex>(Umu, d);
     }
@@ -216,8 +229,7 @@ public:
         //
         staple += Gimpl::ShiftStaple(
             Gimpl::CovShiftBackward(U[nu], nu,
-                                    Gimpl::CovShiftBackward(U[mu], mu, U[nu])),
+                                    Gimpl::CovShiftBackward(U[mu], mu, U[nu])), mu);
-            mu);
       }
     }
   }
@@ -227,15 +239,12 @@ public:
   //////////////////////////////////////////////////
   static void StapleUpper(GaugeMat &staple, const GaugeLorentz &Umu, int mu,
                           int nu) {
-
-    staple = zero;
-
     if (nu != mu) {
       GridBase *grid = Umu._grid;
 
-      std::vector<GaugeMat> U(4, grid);
+      std::vector<GaugeMat> U(Nd, grid);
       for (int d = 0; d < Nd; d++) {
-        U[d] = PeekIndex<LorentzIndex>(Umu, d);
+        U[d] = PeekIndex<LorentzIndex>(Umu, d);// some redundant copies
       }
 
       // mu
@@ -247,7 +256,7 @@ public:
       //    __|
       //
 
-      staple += Gimpl::ShiftStaple(
+      staple = Gimpl::ShiftStaple(
           Gimpl::CovShiftForward(
               U[nu], nu,
               Gimpl::CovShiftBackward(
@@ -256,6 +265,76 @@ public:
     }
   }
 
+  //////////////////////////////////////////////////
+  // the sum over all staples on each site in direction mu,nu, lower part
+  //////////////////////////////////////////////////
+  static void StapleLower(GaugeMat &staple, const GaugeLorentz &Umu, int mu,
+                          int nu) {
+    if (nu != mu) {
+      GridBase *grid = Umu._grid;
+
+      std::vector<GaugeMat> U(Nd, grid);
+      for (int d = 0; d < Nd; d++) {
+        U[d] = PeekIndex<LorentzIndex>(Umu, d);// some redundant copies
+      }
+
+      // mu
+      // ^
+      // |__>  nu
+
+      //  __
+      // |
+      // |__
+      //
+      //
+      staple = Gimpl::ShiftStaple(
+          Gimpl::CovShiftBackward(U[nu], nu,
+                                  Gimpl::CovShiftBackward(U[mu], mu, U[nu])),
+          mu);
+    }
+  }
+
+  //////////////////////////////////////////////////////
+  //  Field Strength
+  //////////////////////////////////////////////////////
+  static void FieldStrength(GaugeMat &FS, const GaugeLorentz &Umu, int mu, int nu){
+      // Fmn +--<--+  Ut +--<--+
+      //     |     |     |     |
+      //  (x)+-->--+     +-->--+(x)
+      //     |     |     |     |
+      //     +--<--+     +--<--+
+
+      GaugeMat Vup(Umu._grid), Vdn(Umu._grid);
+      StapleUpper(Vup, Umu, mu, nu);
+      StapleLower(Vdn, Umu, mu, nu);
+      GaugeMat v = adj(Vup) - adj(Vdn);
+      GaugeMat u = PeekIndex<LorentzIndex>(Umu, mu);  // some redundant copies
+      GaugeMat vu = v*u;
+      FS = 0.25*Ta(u*v + Cshift(vu, mu, +1));
+  }
+
+  static Real TopologicalCharge(GaugeLorentz &U){
+    // 4d topological charge
+    assert(Nd==4);
+    // Bx = -iF(y,z), By = -iF(z,y), Bz = -iF(x,y)
+    GaugeMat Bx(U._grid), By(U._grid), Bz(U._grid);
+    FieldStrength(Bx, U, Ydir, Zdir);
+    FieldStrength(By, U, Zdir, Xdir);
+    FieldStrength(Bz, U, Xdir, Ydir);
+
+    // Ex = -iF(t,x), Ey = -iF(t,y), Ez = -iF(t,z)
+    GaugeMat Ex(U._grid), Ey(U._grid), Ez(U._grid);
+    FieldStrength(Ex, U, Tdir, Xdir);
+    FieldStrength(Ey, U, Tdir, Ydir);
+    FieldStrength(Ez, U, Tdir, Zdir);
+
+    double coeff = 8.0/(32.0*M_PI*M_PI);
+
+    LatticeComplex qfield = coeff*trace(Bx*Ex + By*Ey + Bz*Ez);
+    TComplex Tq = sum(qfield);
+    return TensorRemove(Tq).real();
+  }
+
   //////////////////////////////////////////////////////
   // Similar to above for rectangle is required
   //////////////////////////////////////////////////////

lib/serialisation/BaseIO.h

@@ -8,6 +8,7 @@
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -110,7 +111,6 @@ namespace Grid {
     std::vector<size_t>  dim_;
   };
   
-  
   // Class to reconstruct a multidimensional std::vector
   template <typename V>
   class Reconstruct
@@ -216,7 +216,7 @@ namespace Grid {
   public:
     Reader(void);
     virtual ~Reader(void) = default;
-    void push(const std::string &s);
+    bool push(const std::string &s);
     void pop(void);
     template <typename U>
     typename std::enable_if<std::is_base_of<Serializable, U>::value, void>::type
@@ -231,6 +231,17 @@ namespace Grid {
     T *upcast;
   };
 
+   // What is the vtype
+  template<typename T> struct isReader {
+    static const bool value = false;
+  };
+  template<typename T> struct isWriter {
+    static const bool value = false;
+  }; 
+
+
+
+  // Generic writer interface
   // serializable base class
   class Serializable
   {
@@ -375,8 +386,7 @@ namespace Grid {
   
   // Generic writer interface //////////////////////////////////////////////////
   template <typename T>
-  inline void push(Writer<T> &w, const std::string &s)
+  inline void push(Writer<T> &w, const std::string &s) {
-  {
     w.push(s);
   }
   
@@ -400,15 +410,15 @@ namespace Grid {
   
   // Generic reader interface
   template <typename T>
-  inline void push(Reader<T> &r, const std::string &s)
+  inline bool push(Reader<T> &r, const std::string &s)
   {
-    r.push(s);
+    return r.push(s);
   }
   
   template <typename T>
-  inline void push(Reader<T> &r, const char *s)
+  inline bool push(Reader<T> &r, const char *s)
   {
-    r.push(std::string(s));
+    return r.push(std::string(s));
   }
   
   template <typename T>
@@ -466,9 +476,9 @@ namespace Grid {
   }
   
   template <typename T>
-  void Reader<T>::push(const std::string &s)
+  bool Reader<T>::push(const std::string &s)
   {
-    upcast->push(s);
+    return upcast->push(s);
   }
   
   template <typename T>

lib/serialisation/BinaryIO.h

@@ -60,7 +60,7 @@ namespace Grid {
   public:
     BinaryReader(const std::string &fileName);
     virtual ~BinaryReader(void) = default;
-    void push(const std::string &s) {};
+    bool push(const std::string &s) {return true;}
     void pop(void) {};
     template <typename U>
     void readDefault(const std::string &s, U &output);

lib/serialisation/JSON_IO.cc

@@ -0,0 +1,170 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/serialisation/JSON_IO.cc
+
+    Copyright (C) 2016
+
+    Author: Guido Cossu<guido.cossu@ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid.h>
+
+using namespace Grid;
+using namespace std;
+
+// Writer implementation ///////////////////////////////////////////////////////
+JSONWriter::JSONWriter(const string &fileName)
+: fileName_(fileName), ss_("{ ", ostringstream::ate){}
+
+JSONWriter::~JSONWriter(void)
+{
+  // close
+  delete_comma();
+  ss_ << "}";  
+
+  cout  << ss_.str() << endl;
+  // write prettified JSON to file
+  std::ofstream os(fileName_);
+  os << std::setw(2) << json::parse(ss_.str()) << std::endl;
+}
+
+void JSONWriter::push(const string &s)
+{
+  // adding a nested object
+  if (s.size())
+    ss_ << " \""<<s<<"\" : {";
+  else
+    ss_ << " {";
+}
+
+void JSONWriter::pop(void)
+{
+  delete_comma();
+  ss_ << "},";
+}
+
+void JSONWriter::delete_comma()
+{
+  std::string dlast = ss_.str();
+  dlast.pop_back(); // deletes the last comma
+  ss_.str(dlast);
+}
+
+// here we are hitting a g++ bug (Bug 56480)
+// compiles fine with clang
+// have to wrap in the Grid namespace
+// annoying, but necessary for TravisCI
+namespace Grid
+{
+  template<>
+  void JSONWriter::writeDefault(const std::string &s,
+				const std::string &x)
+  {
+    if (s.size())
+      ss_ << "\""<< s << "\" : \"" << x << "\" ," ; 
+    else
+      ss_ << "\"" << x << "\" ," ;
+  }
+}// namespace Grid 
+
+
+// Reader implementation ///////////////////////////////////////////////////////
+JSONReader::JSONReader(const string &fileName)
+: fileName_(fileName)
+{
+  std::ifstream file(fileName_);
+  file >> jobject_;
+
+  // test
+  // serialize to standard output
+  std::cout << "JSONReader::JSONReader : " << jobject_ << endl; 
+  jcur_ = jobject_;
+}
+
+bool JSONReader::push(const string &s)
+{
+  if (s.size()){
+    jold_.push_back(jcur_);
+    do_pop.push_back(true);
+    try
+    {
+      jcur_ = jcur_[s]; 
+    }
+    catch (std::out_of_range& e)
+    {
+      std::cout << "out of range: " << e.what() << '\n';
+      return false;
+    }
+    cout << "JSONReader::push : " << s << " : "<< jcur_ << endl;
+  }
+  else
+  {
+    do_pop.push_back(false);
+  }
+
+
+  return true;
+}
+
+void JSONReader::pop(void)
+{
+  if (do_pop.back()){
+    jcur_ = jold_.back();
+    jold_.pop_back();
+    do_pop.pop_back();
+  }
+  else
+    do_pop.pop_back();
+
+  cout << "JSONReader::pop : " << jcur_ << endl;
+}
+
+bool JSONReader::nextElement(const std::string &s)
+{
+  // JSON dictionaries do not support multiple names 
+  // Same name objects must be packed in vectors
+  ++it_;
+  
+  //if (it_ == it_end_){
+  //  return false;
+  //}
+
+  jcur_ = *it_; 
+  //cout << "JSONReader::nextElement(string) : " << s << " : "<< jcur_ << endl;
+  //return true;
+
+    return false;
+}
+
+template <>
+void JSONReader::readDefault(const string &s, string &output)
+{
+  cout << "JSONReader::readDefault(string) : " << s<< " " << jcur_ << endl;
+  if (s.size()){
+    std::cout << "String: "<< jcur_[s] << std::endl;
+    output = jcur_[s];
+  }
+  else
+  {
+    std::cout << "String: "<< jcur_ << std::endl;
+    output = jcur_;    
+  }
+}

lib/serialisation/JSON_IO.h

@@ -0,0 +1,182 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/serialisation/JSON_IO.h
+
+    Copyright (C) 2015
+
+		Author: Guido Cossu<guido.cossu@ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef GRID_SERIALISATION_JSON_IO_H
+#define GRID_SERIALISATION_JSON_IO_H
+
+#include <iostream>
+#include <iomanip>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+#include <cassert>
+
+#include <Grid/json/json.hpp>
+
+// for convenience
+using json = nlohmann::json;
+
+namespace Grid
+{
+  
+  class JSONWriter: public Writer<JSONWriter>
+  {
+    
+  public:
+    JSONWriter(const std::string &fileName);
+    virtual ~JSONWriter(void);
+    void push(const std::string &s);
+    void pop(void);
+    template <typename U>
+    void writeDefault(const std::string &s, const U &x);
+    template <typename U>
+    void writeDefault(const std::string &s, const std::vector<U> &x);
+
+    template<std::size_t N>
+    void writeDefault(const std::string &s, const char(&x)[N]);
+
+  private:
+    void delete_comma();
+    std::string         fileName_;
+    std::ostringstream  ss_;
+  };
+  
+  class JSONReader: public Reader<JSONReader>
+  {
+  public:
+    JSONReader(const std::string &fileName);
+    virtual ~JSONReader(void) = default;
+    bool push(const std::string &s);
+    void pop(void);
+    bool nextElement(const std::string &s);
+    template <typename U>
+    void readDefault(const std::string &s, U &output);
+    template <typename U>
+    void readDefault(const std::string &s, std::vector<U> &output);
+  private:
+    json                jobject_; // main object
+    json                jcur_;  // current json object
+    std::vector<json>   jold_;  // previous json object
+    std::string         fileName_;
+    std::vector<bool>   do_pop;
+    json::iterator      it_;
+    json::iterator      it_end_;
+  };
+
+  template <>
+  struct isReader< JSONReader > {
+    static const bool value = true;
+  };
+
+  template <>
+  struct isWriter< JSONWriter > {
+    static const bool value = true;
+  };
+  
+  // Writer template implementation ////////////////////////////////////////////
+  template <typename U>
+  void JSONWriter::writeDefault(const std::string &s, const U &x)
+  {
+    std::ostringstream os;
+    os << std::boolalpha << x;
+    if (s.size())
+      ss_ << "\""<< s << "\" : " << os.str() << " ," ;
+    else
+     ss_ << os.str() << " ," ;
+  }
+
+  template <typename U>
+  void JSONWriter::writeDefault(const std::string &s, const std::vector<U> &x)
+  {
+    if (s.size())
+      ss_ << " \""<<s<<"\" : [";
+    else
+      ss_ << " [";
+    for (auto &x_i: x)
+    {
+      write("", x_i);
+    }
+    delete_comma();
+    ss_<< "],";
+  }
+  
+  template<std::size_t N>
+  void JSONWriter::writeDefault(const std::string &s, const char(&x)[N]){
+    if (s.size())
+    ss_ << "\""<< s << "\" : \"" << x << "\" ," ;
+    else
+    ss_ << "\"" << x << "\" ," ; 
+  }
+
+  // Reader template implementation ////////////////////////////////////////////
+  template <typename U>
+  void JSONReader::readDefault(const std::string &s, U &output)
+  {
+    std::cout << "JSONReader::readDefault(U) : " << s << "  :  "<< jcur_ << std::endl;
+  
+    if (s.size()){
+      std::cout << "String: "<< jcur_[s] << std::endl;
+      output = jcur_[s];
+    }
+    else
+    {
+      std::cout << "String: "<< jcur_ << std::endl;
+      output = jcur_;    
+    }
+
+
+  }
+  
+  template <>
+  void JSONReader::readDefault(const std::string &s, std::string &output);
+  
+  template <typename U>
+  void JSONReader::readDefault(const std::string &s, std::vector<U> &output)
+  {
+    std::string    buf;
+    unsigned int   i = 0;
+    std::cout << "JSONReader::readDefault(vec) : " << jcur_ << std::endl;
+    if (s.size())
+      push(s);
+    
+    json j = jcur_;
+    for (json::iterator it = j.begin(); it != j.end(); ++it) {
+      jcur_ = *it;
+      std::cout << "Value: " << it.value() << "\n";
+      output.resize(i + 1);
+      read("", output[i++]);
+    }
+    
+
+    jcur_ = j;
+    if (s.size())
+      pop();
+  }
+  
+}
+#endif

lib/serialisation/Serialisation.h

@@ -36,13 +36,15 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include "BinaryIO.h"
 #include "TextIO.h"
 #include "XmlIO.h"
+#include "JSON_IO.h"
+
 #ifdef HAVE_HDF5
 #include "Hdf5IO.h"
 #endif
+
 //////////////////////////////////////////
 // Todo:
 //////////////////////////////////////////
-//#include "JsonIO.h"
 //#include "YamlIO.h"
 
 //////////////////////////////////////////

lib/serialisation/TextIO.cc

@@ -8,6 +8,7 @@
 
     Author: Antonin Portelli <antonin.portelli@me.com>
     Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu<guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -31,6 +32,9 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 using namespace Grid;
 using namespace std;
 
+#define GRID_TEXT_INDENT 2      //number of spaces for indentation of levels
+
+
 // Writer implementation ///////////////////////////////////////////////////////
 TextWriter::TextWriter(const string &fileName)
 : file_(fileName, ios::out)
@@ -49,19 +53,24 @@ void TextWriter::pop(void)
 void TextWriter::indent(void)
 {
   for (int i = 0; i < level_; ++i)
-  {
+    for (int t = 0; t < GRID_TEXT_INDENT; t++)
-    file_ << '\t';
+      file_ << ' ';
-  }
 };
 
 // Reader implementation ///////////////////////////////////////////////////////
 TextReader::TextReader(const string &fileName) 
-: file_(fileName, ios::in)
+{
-{}
+    file_.open(fileName, ios::in);
+    if (!file_.is_open()) {
+        std::cout << GridLogMessage << "TextReader: Error opening file " << fileName << std::endl;
+        exit(1);// write better error handling
+    } 
+}
 
-void TextReader::push(const string &s)
+bool TextReader::push(const string &s)
 {
   level_++;
+  return true;
 };
 
 void TextReader::pop(void)
@@ -75,12 +84,15 @@ void TextReader::checkIndent(void)
   
   for (int i = 0; i < level_; ++i)
   {
+    bool check = true;
+    for (int t = 0; t< GRID_TEXT_INDENT; t++){
     file_.get(c);
-    if (c != '\t')
+    check = check && isspace(c);
+  }
+    if (!check)
     {
-      cerr << "mismatch on tab " << c << " level " << level_;
+      cerr << "TextReader: mismatch on level " << level_ << std::endl;
-      cerr << " i "<< i << endl;
+      exit(1);
-      abort();
     }
   }
 }

lib/serialisation/TextIO.h

@@ -63,7 +63,7 @@ namespace Grid
   public:
     TextReader(const std::string &fileName);
     virtual ~TextReader(void) = default;
-    void push(const std::string &s);
+    bool push(const std::string &s);
     void pop(void);
     template <typename U>
     void readDefault(const std::string &s, U &output);

lib/serialisation/XmlIO.cc

@@ -70,9 +70,15 @@ XmlReader::XmlReader(const string &fileName)
   node_ = doc_.child("grid");
 }
 
-void XmlReader::push(const string &s)
+bool XmlReader::push(const string &s)
 {
+
+  if (node_.child(s.c_str()) == NULL )
+    return false;
+
   node_ = node_.child(s.c_str());
+  return true;
+    
 }
 
 void XmlReader::pop(void)
@@ -92,6 +98,7 @@ bool XmlReader::nextElement(const std::string &s)
     {
       return false;
     }
+
 }
 
 template <>

lib/serialisation/XmlIO.h

@@ -66,7 +66,7 @@ namespace Grid
   public:
     XmlReader(const std::string &fileName);
     virtual ~XmlReader(void) = default;
-    void push(const std::string &s);
+    bool push(const std::string &s);
     void pop(void);
     bool nextElement(const std::string &s);
     template <typename U>
@@ -79,6 +79,16 @@ namespace Grid
     std::string        fileName_;
   };
 
+  template <>
+  struct isReader< XmlReader > {
+    static const bool value = true;
+  };
+
+  template <>
+  struct isWriter<XmlWriter > {
+    static const bool value = true;
+  };
+  
   // Writer template implementation ////////////////////////////////////////////
   template <typename U>
   void XmlWriter::writeDefault(const std::string &s, const U &x)
@@ -108,6 +118,7 @@ namespace Grid
     std::string buf;
     
     readDefault(s, buf);
+    std::cout << s << "   " << buf << std::endl;
     fromString(output, buf);
   }
   

lib/simd/Grid_avx.h

@@ -29,15 +29,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-//----------------------------------------------------------------------
-/*! @file Grid_avx.h
-  @brief Optimization libraries for AVX1/2 instructions set
-
-  Using intrinsics
-*/
-// Time-stamp: <2015-06-16 23:30:41 neo>
-//----------------------------------------------------------------------
-
 #include <immintrin.h>
 #ifdef AVXFMA4
 #include <x86intrin.h>
@@ -642,7 +633,7 @@ namespace Optimization {
           _mm256_alignr_epi32_grid(ret,tmp,in,n);
       }
       return ret;
-    };
+    }
 
     template<int n>
     static inline __m256d tRotate(__m256d in){
@@ -756,4 +747,4 @@ namespace Optimization {
   typedef Optimization::TimesMinusI TimesMinusISIMD;
   typedef Optimization::TimesI      TimesISIMD;
 
-}
+}  // namespace Grid

lib/simd/Grid_vector_types.h

@@ -402,6 +402,19 @@ class Grid_simd {
     else if(perm==0) permute0(y, b);
   }
 
+  ///////////////////////////////
+  // Getting single lanes
+  ///////////////////////////////
+  inline Scalar_type getlane(int lane) {
+    return ((Scalar_type*)&v)[lane];
+  }
+
+  inline void putlane(const Scalar_type &S, int lane){
+    ((Scalar_type*)&v)[lane] = S;
+  }
+
+
+  
 };  // end of Grid_simd class definition
 
 

lib/tensors/Tensor_class.h

@@ -1,9 +1,6 @@
 /*************************************************************************************
-
 Grid physics library, www.github.com/paboyle/Grid
-
 Source file: ./lib/tensors/Tensor_class.h
-
 Copyright (C) 2015
 
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
@@ -13,16 +10,13 @@ This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.
-
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
-
 You should have received a copy of the GNU General Public License along
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
 See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
@@ -65,6 +59,9 @@ class iScalar {
   typedef iScalar<typename GridTypeMapper<vtype>::Complexified> Complexified;
   typedef iScalar<typename GridTypeMapper<vtype>::Realified> Realified;
 
+  // get double precision version
+  typedef iScalar<typename GridTypeMapper<vtype>::DoublePrecision> DoublePrecision;
+  
   enum { TensorLevel = GridTypeMapper<vtype>::TensorLevel + 1 };
 
   // Scalar no action
@@ -203,6 +200,9 @@ class iVector {
   typedef iVector<typename GridTypeMapper<vtype>::Complexified, N> Complexified;
   typedef iVector<typename GridTypeMapper<vtype>::Realified, N> Realified;
 
+  // get double precision version
+  typedef iVector<typename GridTypeMapper<vtype>::DoublePrecision, N> DoublePrecision;
+  
   template <class T, typename std::enable_if<!isGridTensor<T>::value, T>::type
                          * = nullptr>
   strong_inline auto operator=(T arg) -> iVector<vtype, N> {
@@ -314,7 +314,10 @@ class iMatrix {
   typedef iMatrix<typename GridTypeMapper<vtype>::Complexified, N> Complexified;
   typedef iMatrix<typename GridTypeMapper<vtype>::Realified, N> Realified;
 
-  // Tensure removal
+  // get double precision version
+  typedef iMatrix<typename GridTypeMapper<vtype>::DoublePrecision, N> DoublePrecision;
+  
+  // Tensor removal
   typedef iScalar<tensor_reduced_v> tensor_reduced;
   typedef iMatrix<recurse_scalar_object, N> scalar_object;
 
@@ -470,3 +473,6 @@ void vprefetch(const iMatrix<v, N> &vv) {
 }
 }
 #endif
+
+
+

lib/tensors/Tensor_outer.h

@@ -44,6 +44,8 @@ auto outerProduct (const iVector<l,N>& lhs,const iVector<r,N>& rhs) -> iMatrix<d
     }}
     return ret;
 }
+
+
 template<class l,class r> inline
 auto outerProduct (const iScalar<l>& lhs,const iScalar<r>& rhs) -> iScalar<decltype(outerProduct(lhs._internal,rhs._internal))>
 {
@@ -53,6 +55,7 @@ auto outerProduct (const iScalar<l>& lhs,const iScalar<r>& rhs) -> iScalar<declt
     return ret;
 }
 
+  
 inline ComplexF outerProduct(const ComplexF &l, const ComplexF& r)
 {
   std::cout << "outer product taking conj "<<r<<" "<<conj(r)<<std::endl;

lib/tensors/Tensor_traits.h

@@ -1,29 +1,21 @@
    /*************************************************************************************
-
     Grid physics library, www.github.com/paboyle/Grid 
-
     Source file: ./lib/tensors/Tensor_traits.h
-
     Copyright (C) 2015
-
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: Christopher Kelly <ckelly@phys.columbia.edu>
-
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
     the Free Software Foundation; either version 2 of the License, or
     (at your option) any later version.
-
     This program is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     GNU General Public License for more details.
-
     You should have received a copy of the GNU General Public License along
     with this program; if not, write to the Free Software Foundation, Inc.,
     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
@@ -58,6 +50,7 @@ namespace Grid {
     typedef typename T::scalar_object scalar_object;
     typedef typename T::Complexified Complexified;
     typedef typename T::Realified Realified;
+    typedef typename T::DoublePrecision DoublePrecision;
     enum { TensorLevel = T::TensorLevel };
   };
 
@@ -73,6 +66,7 @@ namespace Grid {
     typedef RealF scalar_object;
     typedef ComplexF Complexified;
     typedef RealF Realified;
+    typedef RealD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<RealD> {
@@ -84,6 +78,7 @@ namespace Grid {
     typedef RealD scalar_object;
     typedef ComplexD Complexified;
     typedef RealD Realified;
+    typedef RealD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<ComplexF> {
@@ -95,6 +90,7 @@ namespace Grid {
     typedef ComplexF scalar_object;
     typedef ComplexF Complexified;
     typedef RealF Realified;
+    typedef ComplexD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<ComplexD> {
@@ -106,6 +102,7 @@ namespace Grid {
     typedef ComplexD scalar_object;
     typedef ComplexD Complexified;
     typedef RealD Realified;
+    typedef ComplexD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<Integer> {
@@ -117,6 +114,7 @@ namespace Grid {
     typedef Integer scalar_object;
     typedef void Complexified;
     typedef void Realified;
+    typedef void DoublePrecision;
     enum { TensorLevel = 0 };
   };
 
@@ -129,6 +127,7 @@ namespace Grid {
     typedef RealF  scalar_object;
     typedef vComplexF Complexified;
     typedef vRealF Realified;
+    typedef vRealD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<vRealD> {
@@ -140,6 +139,7 @@ namespace Grid {
     typedef RealD  scalar_object;
     typedef vComplexD Complexified;
     typedef vRealD Realified;
+    typedef vRealD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<vComplexH> {
@@ -151,6 +151,7 @@ namespace Grid {
     typedef ComplexF  scalar_object;
     typedef vComplexH Complexified;
     typedef vRealH Realified;
+    typedef vComplexD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<vComplexF> {
@@ -162,6 +163,7 @@ namespace Grid {
     typedef ComplexF  scalar_object;
     typedef vComplexF Complexified;
     typedef vRealF Realified;
+    typedef vComplexD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<vComplexD> {
@@ -173,6 +175,7 @@ namespace Grid {
     typedef ComplexD  scalar_object;
     typedef vComplexD Complexified;
     typedef vRealD Realified;
+    typedef vComplexD DoublePrecision;
     enum { TensorLevel = 0 };
   };
   template<> class GridTypeMapper<vInteger> {
@@ -184,6 +187,7 @@ namespace Grid {
     typedef  Integer scalar_object;
     typedef void Complexified;
     typedef void Realified;
+    typedef void DoublePrecision;
     enum { TensorLevel = 0 };
   };
 
@@ -276,13 +280,15 @@ namespace Grid {
   //Get the precision of a Lattice, tensor or scalar type in units of sizeof(float)
   template<typename T>
   class getPrecision{
+  public:
     typedef typename getVectorType<T>::type vector_obj; //get the vector_obj (i.e. a grid Tensor) if its a Lattice<vobj>, do nothing otherwise (i.e. if fundamental or grid Tensor)
   
     typedef typename GridTypeMapper<vector_obj>::scalar_type scalar_type; //get the associated scalar type. Works on fundamental and tensor types
     typedef typename GridTypeMapper<scalar_type>::Realified real_scalar_type; //remove any std::complex wrapper, should get us to the fundamental type
-  public:
+
     enum { value = sizeof(real_scalar_type)/sizeof(float) };
   };
 }
 
 #endif
+