Merge branch 'master' into feature/scalar_adjointFT

.travis.yml

@@ -10,6 +10,7 @@ matrix:
       osx_image: xcode8.3
       compiler: clang
     - compiler: gcc
+      dist: trusty
       sudo: required
       addons:
         apt:
@@ -25,6 +26,7 @@ matrix:
             - binutils-dev
       env: VERSION=-4.9
     - compiler: gcc
+      dist: trusty
       sudo: required
       addons:
         apt:
@@ -40,6 +42,7 @@ matrix:
             - binutils-dev
       env: VERSION=-5
     - compiler: clang
+      dist: trusty
       addons:
         apt:
           sources:
@@ -54,6 +57,7 @@ matrix:
             - binutils-dev
       env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
     - compiler: clang
+      dist: trusty
       addons:
         apt:
           sources:
@@ -80,6 +84,10 @@ install:
     - export CC=$CC$VERSION
     - export CXX=$CXX$VERSION
     - echo $PATH
+    - which autoconf
+    - autoconf  --version
+    - which automake
+    - automake  --version
     - which $CC
     - $CC  --version
     - which $CXX
@@ -97,9 +105,10 @@ script:
     - ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
     - make -j4
     - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
+    - make check
     - echo make clean
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto CXXFLAGS='-DMPI_UINT32_T=MPI_UNSIGNED -DMPI_UINT64_T=MPI_UNSIGNED_LONG'; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto ; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make -j4; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then make -j4; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
+    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
 
 

Makefile.am

@@ -5,10 +5,13 @@ include $(top_srcdir)/doxygen.inc
 
 bin_SCRIPTS=grid-config
 
-tests: all
-	$(MAKE) -C tests tests
 
-.PHONY: tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
+.PHONY: bench check tests doxygen-run doxygen-doc $(DX_PS_GOAL) $(DX_PDF_GOAL)
+
+tests-local: all
+bench-local: all
+check-local: all
 
 AM_CXXFLAGS += -I$(top_builddir)/include
+
 ACLOCAL_AMFLAGS = -I m4

README.md

@@ -22,6 +22,26 @@ Last update Nov 2016.
 
 _Please do not send pull requests to the `master` branch which is reserved for releases._
 
+### Compilers
+
+Intel ICPC v16.0.3 and later
+
+Clang v3.5 and later (need 3.8 and later for OpenMP)
+
+GCC   v4.9.x (recommended)
+
+GCC   v6.3 and later
+
+### Important: 
+
+Some versions of GCC appear to have a bug under high optimisation (-O2, -O3).
+
+The safety of these compiler versions cannot be guaranteed at this time. Follow Issue 100 for details and updates.
+
+GCC   v5.x
+
+GCC   v6.1, v6.2
+
 ### Bug report
 
 _To help us tracking and solving more efficiently issues with Grid, please report problems using the issue system of GitHub rather than sending emails to Grid developers._
@@ -32,7 +52,7 @@ When you file an issue, please go though the following checklist:
 2. Give a description of the target platform (CPU, network, compiler). Please give the full CPU part description, using for example `cat /proc/cpuinfo | grep 'model name' | uniq` (Linux) or `sysctl machdep.cpu.brand_string` (macOS) and the full output the `--version` option of your compiler.
 3. Give the exact `configure` command used.
 4. Attach `config.log`.
-5. Attach `config.summary`.
+5. Attach `grid.config.summary`.
 6. Attach the output of `make V=1`.
 7. Describe the issue and any previous attempt to solve it. If relevant, show how to reproduce the issue using a minimal working example.
 
@@ -95,10 +115,10 @@ install Grid. Other options are detailed in the next section, you can also use `
 `CXX`, `CXXFLAGS`, `LDFLAGS`, ... environment variables can be modified to
 customise the build.
 
-Finally, you can build and install Grid:
+Finally, you can build, check, and install Grid:
 
 ``` bash
-make; make install
+make; make check; make install
 ```
 
 To minimise the build time, only the tests at the root of the `tests` directory are built by default. If you want to build tests in the sub-directory `<subdir>` you can execute:
@@ -121,7 +141,7 @@ If you want to build all the tests at once just use `make tests`.
 - `--enable-gen-simd-width=<size>`: select the size (in bytes) of the generic SIMD vector type (default: 32 bytes).
 - `--enable-precision={single|double}`: set the default precision (default: `double`).
 - `--enable-precision=<comm>`: Use `<comm>` for message passing (default: `none`). A list of possible SIMD targets is detailed in a section below.
-- `--enable-rng={ranlux48|mt19937}`: choose the RNG (default: `ranlux48 `).
+- `--enable-rng={sitmo|ranlux48|mt19937}`: choose the RNG (default: `sitmo `).
 - `--disable-timers`: disable system dependent high-resolution timers.
 - `--enable-chroma`: enable Chroma regression tests.
 - `--enable-doxygen-doc`: enable the Doxygen documentation generation (build with `make doxygen-doc`)
@@ -159,7 +179,6 @@ Alternatively, some CPU codenames can be directly used:
 
 | `<code>`    | Description                            |
 | ----------- | -------------------------------------- |
-| `KNC`       | [Intel Xeon Phi codename Knights Corner](http://ark.intel.com/products/codename/57721/Knights-Corner) |
 | `KNL`       | [Intel Xeon Phi codename Knights Landing](http://ark.intel.com/products/codename/48999/Knights-Landing) |
 | `BGQ`       | Blue Gene/Q                            |
 

VERSION

@@ -1,6 +1,5 @@
-Version : 0.6.0
+Version : 0.7.0
 
-- AVX512, AVX2, AVX, SSE good
+- Clang 3.5 and above, ICPC v16 and above, GCC 6.3 and above recommended
-- Clang 3.5 and above, ICPC v16 and above, GCC 4.9 and above
+- MPI and MPI3 comms optimisations for KNL and OPA finished
-- MPI and MPI3
+- Half precision comms
-- HiRep, Smearing, Generic gauge group

benchmarks/Benchmark_su3.cc

@@ -35,8 +35,9 @@ using namespace Grid::QCD;
 int main (int argc, char ** argv)
 {
   Grid_init(&argc,&argv);
+#define LMAX (32)
 
-  int Nloop=1000;
+  int Nloop=200;
 
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
@@ -50,7 +51,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
@@ -82,7 +83,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
@@ -113,7 +114,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
@@ -144,7 +145,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s\t\t GFlop/s"<<std::endl;
   std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
 
-  for(int lat=2;lat<=32;lat+=2){
+  for(int lat=2;lat<=LMAX;lat+=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
       int vol = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];

benchmarks/Makefile.am

@@ -1,11 +1,7 @@
 include Make.inc
 
-simple: simple_su3_test.o simple_su3_expr.o simple_simd_test.o
+bench-local: all
-
+	./Benchmark_su3
-EXTRA_LIBRARIES = libsimple_su3_test.a libsimple_su3_expr.a libsimple_simd_test.a
+	./Benchmark_memory_bandwidth
-
+	./Benchmark_wilson
-libsimple_su3_test_a_SOURCES = simple_su3_test.cc
+	./Benchmark_dwf --dslash-unroll
-
-libsimple_su3_expr_a_SOURCES = simple_su3_expr.cc
-
-libsimple_simd_test_a_SOURCES = simple_simd_test.cc

bootstrap.sh

@@ -1,6 +1,6 @@
 ]#!/usr/bin/env bash
 
-EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.2.9.tar.bz2'
+EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2'
 
 echo "-- deploying Eigen source..."
 wget ${EIGEN_URL} --no-check-certificate

configure.ac

@@ -1,9 +1,10 @@
 AC_PREREQ([2.63])
-AC_INIT([Grid], [0.6.0-dev], [https://github.com/paboyle/Grid], [Grid])
+AC_INIT([Grid], [0.7.0], [https://github.com/paboyle/Grid], [Grid])
 AC_CANONICAL_BUILD
 AC_CANONICAL_HOST
 AC_CANONICAL_TARGET
-AM_INIT_AUTOMAKE(subdir-objects)
+AM_INIT_AUTOMAKE([subdir-objects 1.13])
+AM_EXTRA_RECURSIVE_TARGETS([tests bench])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
 AC_CONFIG_HEADERS([lib/Config.h],[sed -i 's|PACKAGE_|GRID_|' lib/Config.h])
@@ -13,7 +14,6 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 #AC_REVISION([m4_esyscmd_s([./scripts/configure.commit])])
 
 ############### Checks for programs
-CXXFLAGS="-O3 $CXXFLAGS"
 AC_PROG_CXX
 AC_PROG_RANLIB
 
@@ -27,6 +27,9 @@ AX_GXX_VERSION
 AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"],
       [version of g++ that will compile the code])
 
+CXXFLAGS="-O3 $CXXFLAGS"
+
+
 ############### Checks for typedefs, structures, and compiler characteristics
 AC_TYPE_SIZE_T
 AC_TYPE_UINT32_T

gcc-bug-report/README

@@ -20,4 +20,17 @@ The simple testcase in this directory is the submitted bug report that encapsula
 problem. The test case works with icpc and with clang++, but fails consistently on g++
 current variants.
 
-Peter
+Peter
+
+
+************
+
+Second GCC bug reported, see Issue 100.
+
+https://wandbox.org/permlink/tzssJza6R9XnqANw
+https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80652
+
+Getting Travis fails under gcc-5 for Test_simd, now that I added more comprehensive testing to the
+CI test suite. The limitations of Travis runtime limits & weak cores are being shown.
+
+Travis uses 5.4.1 for g++-5.

lib/Grid_Eigen_Dense.h

@@ -0,0 +1,9 @@
+#pragma once
+#if defined __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+#include <Grid/Eigen/Dense>
+#if defined __GNUC__
+#pragma GCC diagnostic pop
+#endif

lib/algorithms/iterative/ConjugateGradient.h

@@ -135,8 +135,6 @@ class ConjugateGradient : public OperatorFunction<Field> {
         Linop.HermOpAndNorm(psi, mmp, d, qq);
         p = mmp - src;
 
-        RealD mmpnorm = sqrt(norm2(mmp));
-        RealD psinorm = sqrt(norm2(psi));
         RealD srcnorm = sqrt(norm2(src));
         RealD resnorm = sqrt(norm2(p));
         RealD true_residual = resnorm / srcnorm;

lib/json/json.hpp

@@ -28,7 +28,7 @@ SOFTWARE.
 
 #ifndef NLOHMANN_JSON_HPP
 #define NLOHMANN_JSON_HPP
-
+#include <Grid/DisableWarnings.h>
 #include <algorithm> // all_of, for_each, transform
 #include <array> // array
 #include <cassert> // assert

lib/lattice/Lattice_reduction.h

@@ -22,7 +22,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_LATTICE_REDUCTION_H
 #define GRID_LATTICE_REDUCTION_H
 
-#include <Grid/Eigen/Dense>
+#include <Grid/Grid_Eigen_Dense.h>
 
 namespace Grid {
 #ifdef GRID_WARN_SUBOPTIMAL

lib/lattice/Lattice_rng.h

@@ -158,7 +158,7 @@ namespace Grid {
 #ifdef RNG_SITMO
     typedef sitmo::prng_engine 	RngEngine;
     typedef uint64_t    	RngStateType;
-    static const int    	RngStateCount = 4;
+    static const int    	RngStateCount = 13;
 #endif
 
     std::vector<RngEngine>                             _generators;

lib/log/Log.cc

@@ -30,6 +30,7 @@ directory
 *************************************************************************************/
 /*  END LEGAL */
 #include <Grid/GridCore.h>
+#include <Grid/util/CompilerCompatible.h>
 
 #include <cxxabi.h>
 #include <memory>

lib/perfmon/PerfCount.h

@@ -205,13 +205,14 @@ public:
   void Stop(void) {
     count=0;
     cycles=0;
-    size_t ign;
 #ifdef __linux__
+    ssize_t ign;
     if ( fd!= -1) {
       ::ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
       ::ioctl(cyclefd, PERF_EVENT_IOC_DISABLE, 0);
       ign=::read(fd, &count, sizeof(long long));
-      ign=::read(cyclefd, &cycles, sizeof(long long));
+      ign+=::read(cyclefd, &cycles, sizeof(long long));
+      assert(ign=2*sizeof(long long));
     }
     elapsed = cyclecount() - begin;
 #else

lib/qcd/action/fermion/CayleyFermion5D.cc

@@ -29,7 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Eigen/Dense>
+#include <Grid/Grid_Eigen_Dense.h>
 #include <Grid/qcd/action/fermion/FermionCore.h>
 #include <Grid/qcd/action/fermion/CayleyFermion5D.h>
 

lib/qcd/action/fermion/CayleyFermion5Ddense.cc

@@ -29,7 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Eigen/Dense>
+#include <Grid/Grid_Eigen_Dense.h>
 #include <Grid/qcd/action/fermion/FermionCore.h>
 #include <Grid/qcd/action/fermion/CayleyFermion5D.h>
 

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -220,20 +220,32 @@ namespace QCD {
       
     inline void DoubleStore(GridBase *GaugeGrid,
                             DoubledGaugeField &Uds,
-                            const GaugeField &Umu) {
+                            const GaugeField &Umu) 
+    {
+      typedef typename Simd::scalar_type scalar_type;
+
       conformable(Uds._grid, GaugeGrid);
       conformable(Umu._grid, GaugeGrid);
+
       GaugeLinkField U(GaugeGrid);
       GaugeLinkField tmp(GaugeGrid);
+
       Lattice<iScalar<vInteger> > coor(GaugeGrid);
       for (int mu = 0; mu < Nd; mu++) {
-        LatticeCoordinate(coor, mu);
+
+	      auto pha = Params.boundary_phases[mu];
+	      scalar_type phase( real(pha),imag(pha) );
+
         int Lmu = GaugeGrid->GlobalDimensions()[mu] - 1;
+
+        LatticeCoordinate(coor, mu);
+
         U = PeekIndex<LorentzIndex>(Umu, mu);
-        tmp = where(coor == Lmu, Params.boundary_phases[mu] * U, U);
+        tmp = where(coor == Lmu, phase * U, U);
         PokeIndex<LorentzIndex>(Uds, tmp, mu);
+
         U = adj(Cshift(U, mu, -1));
-        U = where(coor == 0, Params.boundary_phases[mu] * U, U);
+        U = where(coor == 0, conjugate(phase) * U, U); 
         PokeIndex<LorentzIndex>(Uds, U, mu + 4);
       }
     }
@@ -251,11 +263,11 @@ namespace QCD {
       tmp = zero;
       
       parallel_for(int sss=0;sss<tmp._grid->oSites();sss++){
-	int sU=sss;
+	    int sU=sss;
-	for(int s=0;s<Ls;s++){
+	    for(int s=0;s<Ls;s++){
-	  int sF = s+Ls*sU;
+	        int sF = s+Ls*sU;
-	  tmp[sU] = tmp[sU]+ traceIndex<SpinIndex>(outerProduct(Btilde[sF],Atilde[sF])); // ordering here
+	        tmp[sU] = tmp[sU]+ traceIndex<SpinIndex>(outerProduct(Btilde[sF],Atilde[sF])); // ordering here
-	}
+	    }
       }
       PokeIndex<LorentzIndex>(mat,tmp,mu);
       
@@ -587,7 +599,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    // use lorentz for flavour as hack.
    auto tmp = TraceIndex<SpinIndex>(outerProduct(Btilde, A));
    parallel_for(auto ss = tmp.begin(); ss < tmp.end(); ss++) {
-     link[ss]() = tmp[ss](0, 0) - conjugate(tmp[ss](1, 1));
+     link[ss]() = tmp[ss](0, 0) + conjugate(tmp[ss](1, 1));
    }
    PokeIndex<LorentzIndex>(mat, link, mu);
    return;

lib/serialisation/JSON_IO.cc

@@ -40,7 +40,6 @@ JSONWriter::~JSONWriter(void)
   delete_comma();
   ss_ << "}";  
 
-  cout  << ss_.str() << endl;
   // write prettified JSON to file
   std::ofstream os(fileName_);
   os << std::setw(2) << json::parse(ss_.str()) << std::endl;
@@ -95,7 +94,7 @@ JSONReader::JSONReader(const string &fileName)
 
   // test
   // serialize to standard output
-  std::cout << "JSONReader::JSONReader : " << jobject_ << endl; 
+  //std::cout << "JSONReader::JSONReader : " << jobject_ << endl; 
   jcur_ = jobject_;
 }
 
@@ -113,7 +112,7 @@ bool JSONReader::push(const string &s)
       std::cout << "out of range: " << e.what() << '\n';
       return false;
     }
-    cout << "JSONReader::push : " << s << " : "<< jcur_ << endl;
+    //cout << "JSONReader::push : " << s << " : "<< jcur_ << endl;
   }
   else
   {
@@ -134,7 +133,7 @@ void JSONReader::pop(void)
   else
     do_pop.pop_back();
 
-  cout << "JSONReader::pop : " << jcur_ << endl;
+  //cout << "JSONReader::pop : " << jcur_ << endl;
 }
 
 bool JSONReader::nextElement(const std::string &s)
@@ -157,14 +156,14 @@ bool JSONReader::nextElement(const std::string &s)
 template <>
 void JSONReader::readDefault(const string &s, string &output)
 {
-  cout << "JSONReader::readDefault(string) : " << s<< " " << jcur_ << endl;
+  //cout << "JSONReader::readDefault(string) : " << s<< " " << jcur_ << endl;
   if (s.size()){
-    std::cout << "String: "<< jcur_[s] << std::endl;
+    //std::cout << "String: "<< jcur_[s] << std::endl;
     output = jcur_[s];
   }
   else
   {
-    std::cout << "String: "<< jcur_ << std::endl;
+    //std::cout << "String: "<< jcur_ << std::endl;
     output = jcur_;    
   }
 }

lib/serialisation/JSON_IO.h

@@ -1,6 +1,6 @@
     /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    Grid physics library, www.github.com/paboyle/Grid
 
     Source file: ./lib/serialisation/JSON_IO.h
 
@@ -43,10 +43,10 @@ using json = nlohmann::json;
 
 namespace Grid
 {
-  
+
   class JSONWriter: public Writer<JSONWriter>
   {
-    
+
   public:
     JSONWriter(const std::string &fileName);
     virtual ~JSONWriter(void);
@@ -55,6 +55,8 @@ namespace Grid
     template <typename U>
     void writeDefault(const std::string &s, const U &x);
     template <typename U>
+    void writeDefault(const std::string &s, const std::complex<U> &x);
+    template <typename U>
     void writeDefault(const std::string &s, const std::vector<U> &x);
 
     template<std::size_t N>
@@ -65,7 +67,7 @@ namespace Grid
     std::string         fileName_;
     std::ostringstream  ss_;
   };
-  
+
   class JSONReader: public Reader<JSONReader>
   {
   public:
@@ -77,6 +79,8 @@ namespace Grid
     template <typename U>
     void readDefault(const std::string &s, U &output);
     template <typename U>
+    void readDefault(const std::string &s, std::complex<U> &output);
+    template <typename U>
     void readDefault(const std::string &s, std::vector<U> &output);
   private:
     json                jobject_; // main object
@@ -97,11 +101,12 @@ namespace Grid
   struct isWriter< JSONWriter > {
     static const bool value = true;
   };
-  
+
   // Writer template implementation ////////////////////////////////////////////
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const U &x)
   {
+    //std::cout << "JSONReader::writeDefault(U) : " << s <<  std::endl;
     std::ostringstream os;
     os << std::boolalpha << x;
     if (s.size())
@@ -110,9 +115,23 @@ namespace Grid
      ss_ << os.str() << " ," ;
   }
 
+  template <typename U>
+  void JSONWriter::writeDefault(const std::string &s, const std::complex<U> &x)
+  {
+    //std::cout << "JSONReader::writeDefault(complex) : " << s <<  std::endl;
+    std::ostringstream os;
+    os << "["<< std::boolalpha << x.real() << ", " << x.imag() << "]";
+    if (s.size())
+      ss_ << "\""<< s << "\" : " << os.str() << " ," ;
+    else
+     ss_ << os.str() << " ," ;
+  }
+
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const std::vector<U> &x)
   {
+    //std::cout << "JSONReader::writeDefault(vec U) : " << s <<  std::endl;
+
     if (s.size())
       ss_ << " \""<<s<<"\" : [";
     else
@@ -124,59 +143,77 @@ namespace Grid
     delete_comma();
     ss_<< "],";
   }
-  
+
   template<std::size_t N>
   void JSONWriter::writeDefault(const std::string &s, const char(&x)[N]){
+    //std::cout << "JSONReader::writeDefault(char U) : " << s <<  std::endl;
+
     if (s.size())
     ss_ << "\""<< s << "\" : \"" << x << "\" ," ;
     else
-    ss_ << "\"" << x << "\" ," ; 
+    ss_ << "\"" << x << "\" ," ;
   }
 
   // Reader template implementation ////////////////////////////////////////////
   template <typename U>
   void JSONReader::readDefault(const std::string &s, U &output)
   {
-    std::cout << "JSONReader::readDefault(U) : " << s << "  :  "<< jcur_ << std::endl;
+    //std::cout << "JSONReader::readDefault(U) : " << s << "  :  "<< jcur_ << std::endl;
-  
+
     if (s.size()){
-      std::cout << "String: "<< jcur_[s] << std::endl;
+      //std::cout << "String: "<< jcur_[s] << std::endl;
       output = jcur_[s];
     }
     else
     {
-      std::cout << "String: "<< jcur_ << std::endl;
+      //std::cout << "String: "<< jcur_ << std::endl;
-      output = jcur_;    
+      output = jcur_;
     }
 
 
   }
-  
+
+  template <typename U>
+  void JSONReader::readDefault(const std::string &s, std::complex<U> &output)
+  {
+    U tmp1, tmp2;
+    //std::cout << "JSONReader::readDefault( complex U) : " << s << "  :  "<< jcur_ << std::endl;
+    json j = jcur_;
+    json::iterator it = j.begin();
+    jcur_ = *it;
+    read("", tmp1);
+    it++;
+    jcur_ = *it;
+    read("", tmp2);
+    output = std::complex<U>(tmp1,tmp2);
+  }
+
+
   template <>
   void JSONReader::readDefault(const std::string &s, std::string &output);
-  
+
   template <typename U>
   void JSONReader::readDefault(const std::string &s, std::vector<U> &output)
   {
     std::string    buf;
     unsigned int   i = 0;
-    std::cout << "JSONReader::readDefault(vec) : " << jcur_ << std::endl;
+    //std::cout << "JSONReader::readDefault(vec) : " << jcur_ << std::endl;
     if (s.size())
       push(s);
-    
+
     json j = jcur_;
     for (json::iterator it = j.begin(); it != j.end(); ++it) {
       jcur_ = *it;
-      std::cout << "Value: " << it.value() << "\n";
+      //std::cout << "Value: " << it.value() << "\n";
       output.resize(i + 1);
       read("", output[i++]);
     }
-    
+
 
     jcur_ = j;
     if (s.size())
       pop();
   }
-  
+
 }
 #endif

lib/simd/Grid_vector_types.h

@@ -327,6 +327,10 @@ class Grid_simd {
   // provides support
   ///////////////////////////////////////
 
+#if (__GNUC__ == 5 ) || ( ( __GNUC__ == 6 ) && __GNUC_MINOR__ < 3 )
+#pragma GCC push_options 
+#pragma GCC optimize ("O0") 
+#endif
   template <class functor>
   friend inline Grid_simd SimdApply(const functor &func, const Grid_simd &v) {
     Grid_simd ret;
@@ -355,7 +359,9 @@ class Grid_simd {
     ret.v = cx.v;
     return ret;
   }
-
+#if (__GNUC__ == 5 ) || ( ( __GNUC__ == 6 ) && __GNUC_MINOR__ < 3 )
+#pragma GCC pop_options
+#endif
   ///////////////////////
   // Exchange 
   // Al Ah , Bl Bh -> Al Bl Ah,Bh

lib/sitmo_rng/README

@@ -0,0 +1,2 @@
+downloaded file from : http://www.sitmo.com/wp-content/uploads/2016/03/prng_engine.hpp
+Unmodified beyond filename

lib/stencil/Stencil.h

@@ -106,7 +106,6 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
  public:
 
   typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
-  typedef uint32_t StencilInteger;
   typedef typename cobj::vector_type vector_type;
   typedef typename cobj::scalar_type scalar_type;
   typedef typename cobj::scalar_object scalar_object;

lib/util/CompilerCompatible.h

@@ -0,0 +1,56 @@
+#pragma once 
+
+#if defined(__clang__)
+
+  #if __clang_major__ < 3
+    #error "This clang++ version is known to not work with Grid due to compiler bugs"
+  #endif
+
+  #if __clang_major__ == 3
+    #if __clang_minor__ < 5
+    #error "This clang++ version is known to not work with Grid due to compiler bugs"
+    #endif
+  #endif 
+
+// Intel compiler *ALSO* has __GNUC__ defined so must if/else GCC checks
+#elif defined(__INTEL_COMPILER)
+
+  #if __INTEL_COMPILER < 1603
+    #error "This icpc version is known to not work with Grid due to compiler bugs"
+  #endif
+
+#else 
+
+// This macro is annoying many other compilers just define __GNUC__ and claim GCC compat
+// but this defeats the use of __GNUC__ to really detect G++
+  #if defined(__GNUC__)
+
+    #if __GNUC__ < 4 
+      #error "g++ prior to version 4 is known to not work with Grid due to compiler bugs"
+    #endif
+
+    #if __GNUC__ == 4 
+      #if __GNUC_MINOR__ != 9 
+      #error "g++ 4.9 is the only gcc-4.x version known to work with Grid due to compiler bugs"
+      #endif
+    #endif
+
+    #if __GNUC__ == 5
+      #warning "g++ version 5 is known to not work with Grid due to compiler bugs under -O3 : ensure you run make check"
+    #endif
+
+    #if __GNUC__ == 6
+      #if __GNUC_MINOR__ < 3
+        #warning "This g++6.3 is the first recent g++ version known to work with Grid: ensure you run make check"
+      #endif
+    #endif
+
+  #else
+
+    #warning "Unknown compiler detected: cannot guarantee compatability since Grid tends to break compilers"
+    #warning "Ensure to run :  make check"
+
+  #endif
+
+#endif
+

lib/util/Init.cc

@@ -48,6 +48,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <Grid/Grid.h>
 
+#include <Grid/util/CompilerCompatible.h>
+
 
 #include <fenv.h>
 #ifdef __APPLE__
@@ -229,10 +231,12 @@ void Grid_init(int *argc,char ***argv)
   if( !GridCmdOptionExists(*argv,*argv+*argc,"--debug-stdout") ){
     Grid_quiesce_nodes();
   } else { 
+    FILE *fp;
     std::ostringstream fname;
     fname<<"Grid.stdout.";
     fname<<CartesianCommunicator::RankWorld();
-    freopen(fname.str().c_str(),"w",stdout);
+    fp=freopen(fname.str().c_str(),"w",stdout);
+    assert(fp!=(FILE *)NULL);
   }
 
   ////////////////////////////////////

scripts/filelist

@@ -23,8 +23,7 @@ for subdir in $dirs; do
     TESTS=`ls T*.cc`
     TESTLIST=`echo ${TESTS} | sed s/.cc//g `
     PREF=`[ $subdir = '.' ] && echo noinst || echo EXTRA`
-    SUB=`[ $subdir = '.' ] && echo subtests`
+    echo "tests-local: ${TESTLIST} " > Make.inc
-    echo "tests: ${TESTLIST} ${SUB}" > Make.inc
     echo ${PREF}_PROGRAMS = ${TESTLIST} >> Make.inc
     echo >> Make.inc
     HADLINK=`[ $subdir = './hadrons' ] && echo '-lHadrons '`

tests/IO/Test_nersc_io.cc

@@ -41,7 +41,7 @@ int main (int argc, char ** argv)
 
   std::vector<int> simd_layout = GridDefaultSimd(4,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
-  std::vector<int> latt_size  ({32,32,32,32});
+  std::vector<int> latt_size  ({16,16,16,16});
   std::vector<int> clatt_size  ({4,4,4,8});
   int orthodir=3;
   int orthosz =latt_size[orthodir];
@@ -54,8 +54,8 @@ int main (int argc, char ** argv)
   GridSerialRNG     sRNGa;
   GridSerialRNG     sRNGb;
 
-  pRNGa.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+  pRNGa.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
-  sRNGa.SeedFixedIntegers(std::vector<int>({45,12,81,9});
+  sRNGa.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
   
   std::string rfile("./ckpoint_rng.4000");
   NerscIO::writeRNGState(sRNGa,pRNGa,rfile);
@@ -80,8 +80,14 @@ int main (int argc, char ** argv)
 
   std::vector<LatticeColourMatrix> U(4,&Fine);
   
+  SU3::ColdConfiguration(pRNGa,Umu);
+
   NerscField header;
   std::string file("./ckpoint_lat.4000");
+
+  int precision32 = 0;
+  int tworow      = 0;
+  NerscIO::writeConfiguration(Umu,file,tworow,precision32);
   NerscIO::readConfiguration(Umu,header,file);
 
   for(int mu=0;mu<Nd;mu++){
@@ -147,8 +153,6 @@ int main (int argc, char ** argv)
   std::string clone2x3("./ckpoint_clone2x3.4000");
   std::string clone3x3("./ckpoint_clone3x3.4000");
 
-  int precision32 = 1;
-  int tworow      = 1;
   NerscIO::writeConfiguration(Umu,clone3x3,0,precision32);
   NerscIO::writeConfiguration(Umu,clone2x3,1,precision32);
   

tests/IO/Test_serialisation.cc

@@ -214,6 +214,7 @@ int main(int argc,char **argv)
     
     // test serializable class writing
     myclass obj(1234); // non-trivial constructor
+    std::cout << "-- serialisable class writing to 'bother.json'..." << std::endl;
     write(JW,"obj",obj);
     JW.write("obj2", obj);
     
@@ -236,7 +237,9 @@ int main(int argc,char **argv)
     std::cout << "Loaded (JSON) -----------------" << std::endl;
     std::cout << jcopy1 << std::endl << jveccopy1 << std::endl;
   }
-  
+
+/* 
+  // This is still work in progress
   {
     // Testing the next element function
     JSONReader RD("test.json");
@@ -245,7 +248,7 @@ int main(int argc,char **argv)
     std::string name;
     read(RD,"name", name);
   }
-  
+*/
 
 
 }

tests/Makefile.am

@@ -1,12 +1,16 @@
-SUBDIRS = core forces hmc solver debug smearing
+SUBDIRS = . core forces hmc solver debug smearing IO
 
 if BUILD_CHROMA_REGRESSION
   SUBDIRS+= qdpxx
 endif
 
-.PHONY: subtests
-
 include Make.inc
 
-subtests:
+check-local: all
-	for d in $(SUBDIRS); do $(MAKE) -C $${d} tests; done
+	./Test_simd
+	./Test_cshift
+	./Test_stencil
+	./Test_dwf_mixedcg_prec
+
+
+

tests/forces/Makefile.am

@@ -1 +1,14 @@
+.PHONY: check
+
 include Make.inc
+
+check: tests
+	./Test_rect_force
+	./Test_gp_rect_force
+	./Test_laplacian_force
+	./Test_wilson_force
+	./Test_gpwilson_force
+	./Test_dwf_force
+	./Test_dwf_gpforce
+	./Test_mobius_force
+	./Test_zmobius_force

tests/forces/Test_contfrac_force.cc

@@ -145,6 +145,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << " Sprime "<<Sprime<<std::endl;
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0e-2 ) ;
 
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();

tests/forces/Test_dwf_force.cc

@@ -157,6 +157,8 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
+
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_dwf_gpforce.cc

@@ -212,6 +212,7 @@ int main (int argc, char ** argv)
 
   std::cout << GridLogMessage << "Total dS    "<< Hmomprime - Hmom + Sprime - S <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 5.0 ) ;
 
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();

tests/forces/Test_gp_rect_force.cc

@@ -119,7 +119,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << " Sprime "<<Sprime<<std::endl;
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "pred dS "<< dSpred <<std::endl;
-
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0e-2 ) ;
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_gpdwf_force.cc

@@ -91,7 +91,7 @@ int main (int argc, char ** argv)
   ////////////////////////////////////
   // Modify the gauge field a little 
   ////////////////////////////////////
-  RealD dt = 0.001;
+  RealD dt = 0.0001;
 
   LatticeColourMatrix mommu(UGrid); 
   LatticeColourMatrix forcemu(UGrid); 
@@ -128,7 +128,6 @@ int main (int argc, char ** argv)
   // Use derivative to estimate dS
   //////////////////////////////////////////////
 
-
   LatticeComplex dS(UGrid); dS = zero;
   for(int mu=0;mu<Nd;mu++){
     mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
@@ -142,7 +141,6 @@ int main (int argc, char ** argv)
 
     // Update PF action density
     dS = dS+trace(mommu*forcemu)*dt;
-
   }
 
   Complex dSpred    = sum(dS);
@@ -200,7 +198,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << " Sprime "<<Sprime<<std::endl;
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
-
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_gpwilson_force.cc

@@ -91,7 +91,7 @@ int main (int argc, char ** argv)
   ////////////////////////////////////
   // Modify the gauge field a little 
   ////////////////////////////////////
-  RealD dt = 0.00001;
+  RealD dt = 0.0001;
 
   LatticeColourMatrix mommu(UGrid); 
   LatticeColourMatrix forcemu(UGrid); 
@@ -100,7 +100,8 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); 
 
     PokeIndex<LorentzIndex>(mom,mommu,mu);
 
@@ -116,7 +117,6 @@ int main (int argc, char ** argv)
 	+ mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0)
 	;
     }
-
   }
   
   Wil.ImportGauge(Uprime);
@@ -128,42 +128,30 @@ int main (int argc, char ** argv)
   // Use derivative to estimate dS
   //////////////////////////////////////////////
 
-
-  LatticeComplex dS(UGrid); dS = zero;
   for(int mu=0;mu<Nd;mu++){
     mommu   = PeekIndex<LorentzIndex>(UdSdU,mu);
     mommu=Ta(mommu)*2.0;
     PokeIndex<LorentzIndex>(UdSdU,mommu,mu);
   }
 
+  LatticeComplex dS(UGrid); dS = zero;
   for(int mu=0;mu<Nd;mu++){
     forcemu = PeekIndex<LorentzIndex>(UdSdU,mu);
     mommu   = PeekIndex<LorentzIndex>(mom,mu);
 
     // Update PF action density
     dS = dS+trace(mommu*forcemu)*dt;
-
   }
 
   Complex dSpred    = sum(dS);
 
-  // From TwoFlavourPseudoFermion:
-  //////////////////////////////////////////////////////
-  // dS/du = - phi^dag  (Mdag M)^-1 [ Mdag dM + dMdag M ]  (Mdag M)^-1 phi
-  //       = - phi^dag M^-1 dM (MdagM)^-1 phi -  phi^dag (MdagM)^-1 dMdag dM (Mdag)^-1 phi 
-  //
-  //       = - Ydag dM X  - Xdag dMdag Y
-  //
-  //////////////////////////////////////////////////////
-  // So must take dSdU - adj(dSdU) and left multiply by mom to get dS/dt.
-  //
-  // 
-
   std::cout << GridLogMessage << " S      "<<S<<std::endl;
   std::cout << GridLogMessage << " Sprime "<<Sprime<<std::endl;
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 2.0 ) ;
+
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_laplacian_force.cc

@@ -168,12 +168,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "pred dS "<< dSpred <<std::endl; 
 
-
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
-
-
-
-
-
 
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();

tests/forces/Test_mobius_force.cc

@@ -148,6 +148,8 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "dS            "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
+
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_partfrac_force.cc

@@ -148,6 +148,8 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0e-2 ) ;
+
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_rect_force.cc

@@ -119,6 +119,8 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "pred dS "<< dSpred <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0e-2 ) ;
+
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/forces/Test_wilson_force.cc

@@ -76,6 +76,7 @@ int main (int argc, char ** argv)
 
   Dw.MDeriv(tmp , Mphi,  phi,DaggerNo );  UdSdU=tmp;
   Dw.MDeriv(tmp , phi,  Mphi,DaggerYes ); UdSdU=(UdSdU+tmp);
+
   // Take the trace
   UdSdU = Ta(UdSdU);
   
@@ -95,7 +96,8 @@ int main (int argc, char ** argv)
 
   for(int mu=0;mu<Nd;mu++){
 
-    SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+    // Traceless antihermitian momentum; gaussian in lie alg
+    SU3::GaussianFundamentalLieAlgebraMatrix(pRNG, mommu); 
 
     Hmom -= real(sum(trace(mommu*mommu)));
 
@@ -194,6 +196,7 @@ int main (int argc, char ** argv)
 
   std::cout << GridLogMessage << "Total dS    "<< Hmomprime - Hmom + Sprime - S <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
 
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();

tests/forces/Test_zmobius_force.cc

@@ -162,6 +162,8 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "dS      "<<Sprime-S<<std::endl;
   std::cout << GridLogMessage << "predict dS    "<< dSpred <<std::endl;
 
+  assert( fabs(real(Sprime-S-dSpred)) < 3.0 ) ;
+
   std::cout<< GridLogMessage << "Done" <<std::endl;
   Grid_finalize();
 }

tests/hmc/Test_hmc_EODWFRatio.cc

@@ -63,7 +63,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_EODWFRatioLsVectorised.cc

@@ -65,7 +65,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_EODWFRatio_Gparity.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_EOMobiusRatio.cc

@@ -130,7 +130,7 @@ int main(int argc, char **argv) {
   CPparams.format = "IEEE64BIG";
   */
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   // RNG definition (Name: RandomNumberGenerator)
   RNGModuleParameters RNGpar(Reader);

tests/hmc/Test_hmc_EOMobiusRatioManyFlavour.cc

@@ -0,0 +1,302 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/Test_hmc_EODWFRatio.cc
+
+Copyright (C) 2015-2016
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+namespace Grid{
+  struct FermionParameters: Serializable {
+    GRID_SERIALIZABLE_CLASS_MEMBERS(FermionParameters,
+				    int, Ls,
+				    double, mass,
+				    double, M5,
+				    double, b,
+				    double, c,
+				    double, StoppingCondition,
+				    int, MaxCGIterations,
+				    bool, ApplySmearing);
+
+    //template <class ReaderClass >
+    //FermionParameters(Reader<ReaderClass>& Reader){
+    //  read(Reader, "Mobius", *this);
+    //}
+
+  };
+
+  
+  struct MobiusHMCParameters: Serializable {
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MobiusHMCParameters,
+				  double, gauge_beta,
+				  FermionParameters, Mobius)
+
+  template <class ReaderClass >
+  MobiusHMCParameters(Reader<ReaderClass>& Reader){
+    read(Reader, "Action", *this);
+  }
+
+};
+
+  struct SmearingParameters: Serializable {
+    GRID_SERIALIZABLE_CLASS_MEMBERS(SmearingParameters,
+				    double, rho,
+				    Integer, Nsmear)
+
+    template <class ReaderClass >
+    SmearingParameters(Reader<ReaderClass>& Reader){
+      read(Reader, "StoutSmearing", *this);
+    }
+
+  };
+  
+  
+}
+
+
+int main(int argc, char **argv) {
+  using namespace Grid;
+  using namespace Grid::QCD;
+
+  Grid_init(&argc, &argv);
+  int threads = GridThread::GetThreads();
+  // here make a routine to print all the relevant information on the run
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+   // Typedefs to simplify notation
+  typedef GenericHMCRunner<MinimumNorm2> HMCWrapper;  // Uses the default minimum norm
+  typedef WilsonImplR FermionImplPolicy;
+  typedef MobiusFermionR FermionAction;
+  typedef typename FermionAction::FermionField FermionField;
+  // Serialiser
+  //typedef Grid::XmlReader       Serialiser;
+  typedef Grid::JSONReader       Serialiser;
+  
+  //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+  HMCWrapper TheHMC;
+  TheHMC.ReadCommandLine(argc, argv); // these can be parameters from file
+ 
+  // Reader, file should come from command line
+  if (TheHMC.ParameterFile.empty()){
+    std::cout << "Input file not specified."
+              << "Use --ParameterFile option in the command line.\nAborting" 
+              << std::endl;
+    exit(1);
+  }
+  Serialiser Reader(TheHMC.ParameterFile);
+
+  MobiusHMCParameters MyParams(Reader);  
+  // Apply smearing to the fermionic action
+  bool ApplySmearing = MyParams.Mobius.ApplySmearing;
+  
+  
+
+  // Grid from the command line
+  TheHMC.Resources.AddFourDimGrid("gauge");
+  // Possibile to create the module by hand 
+  // hardcoding parameters or using a Reader
+
+
+  // Checkpointer definition (Name: Checkpointer)
+  CheckpointerParameters CPparams(Reader);
+  // Commenting out since we are using the reader 
+  /*
+  CPparams.config_prefix = "ckpoint_EODWF_lat";
+  CPparams.rng_prefix = "ckpoint_EODWF_rng";
+  CPparams.saveInterval = 5;
+  CPparams.format = "IEEE64BIG";
+  */
+  
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+  //  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+
+  // RNG definition (Name: RandomNumberGenerator)
+  RNGModuleParameters RNGpar(Reader);
+  // Commenting out since we are using the reader
+  /*
+  RNGpar.serial_seeds = "1 2 3 4 5";
+  RNGpar.parallel_seeds = "6 7 8 9 10";
+  */
+  TheHMC.Resources.SetRNGSeeds(RNGpar);
+
+  // Construct observables
+  // here there is too much indirection 
+  typedef PlaquetteMod<HMCWrapper::ImplPolicy> PlaqObs;
+  TheHMC.Resources.AddObservable<PlaqObs>();
+  //////////////////////////////////////////////
+
+  /////////////////////////////////////////////////////////////
+  // Collect actions, here use more encapsulation
+  // need wrappers of the fermionic classes 
+  // that have a complex construction
+  // standard
+
+  //RealD beta = 5.6 ;
+  //WilsonGaugeActionR Waction(MyParams.gauge_beta);
+  SymanzikGaugeActionR Syzaction(MyParams.gauge_beta);
+    
+
+  //const int Ls = 8;
+  const int Ls = MyParams.Mobius.Ls;
+  auto GridPtr   = TheHMC.Resources.GetCartesian();
+  auto GridRBPtr = TheHMC.Resources.GetRBCartesian();
+  auto FGrid     = SpaceTimeGrid::makeFiveDimGrid(Ls,GridPtr);
+  auto FrbGrid   = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,GridPtr);
+
+
+  // temporarily need a gauge field
+  LatticeGaugeField U(GridPtr);
+
+  Real mass = MyParams.Mobius.mass; //0.04;
+  Real pv   = 1.0;
+  RealD M5  = MyParams.Mobius.M5; //1.5;
+  // Note: IroIro and Grid notation for b and c differ
+  RealD b   = MyParams.Mobius.b; //  3./2.;
+  RealD c   = MyParams.Mobius.c; //  1./2.;
+
+  // These lines are unecessary if BC are all periodic
+  std::vector<Complex> boundary = {1,1,1,-1};
+  FermionAction::ImplParams Params(boundary);
+  
+  FermionAction DenOp(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,mass,M5,b,c, Params);
+  FermionAction NumOp(U,*FGrid,*FrbGrid,*GridPtr,*GridRBPtr,pv,  M5,b,c, Params);
+
+  //double StoppingCondition = 1e-8;
+  //double MaxCGIterations = 10000;
+  ConjugateGradient<FermionField>  CG(MyParams.Mobius.StoppingCondition,MyParams.Mobius.MaxCGIterations);
+  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2a(NumOp, DenOp,CG,CG);
+  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2b(NumOp, DenOp,CG,CG);
+  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2c(NumOp, DenOp,CG,CG);
+  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2d(NumOp, DenOp,CG,CG);
+  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2e(NumOp, DenOp,CG,CG);
+  //  TwoFlavourEvenOddRatioPseudoFermionAction<FermionImplPolicy> Nf2f(NumOp, DenOp,CG,CG);
+
+  // Set smearing (true/false), default: false
+  Nf2a.is_smeared = ApplySmearing;
+  Nf2b.is_smeared = ApplySmearing;
+  Nf2c.is_smeared = ApplySmearing;
+  Nf2d.is_smeared = ApplySmearing;
+  Nf2e.is_smeared = ApplySmearing;
+  //Nf2f.is_smeared = ApplySmearing;
+  
+  // Collect actions
+  ActionLevel<HMCWrapper::Field> Level1(1);
+  Level1.push_back(&Nf2a);
+  Level1.push_back(&Nf2b);
+  Level1.push_back(&Nf2c);
+  Level1.push_back(&Nf2d);
+  Level1.push_back(&Nf2e);
+  //Level1.push_back(&Nf2f);
+
+  ActionLevel<HMCWrapper::Field> Level2(4);
+  Level2.push_back(&Syzaction);
+
+  TheHMC.TheAction.push_back(Level1);
+  TheHMC.TheAction.push_back(Level2);
+
+  /////////////////////////////////////////////////////////////
+  // HMC parameters are serialisable
+  TheHMC.Parameters.initialize(Reader);
+  /*
+  TheHMC.Parameters.MD.MDsteps = 20;
+  TheHMC.Parameters.MD.trajL   = 1.0;
+  */
+
+  // Reset performance counters 
+  NumOp.ZeroCounters();
+  DenOp.ZeroCounters();
+
+  if (ApplySmearing){
+    SmearingParameters SmPar(Reader);
+    //double rho = 0.1;  // smearing parameter
+    //int Nsmear = 3;    // number of smearing levels
+    Smear_Stout<HMCWrapper::ImplPolicy> Stout(SmPar.rho);
+    SmearedConfiguration<HMCWrapper::ImplPolicy> SmearingPolicy(GridPtr, SmPar.Nsmear, Stout);
+    TheHMC.Run(SmearingPolicy); // for smearing
+  } else {
+    TheHMC.Run();  // no smearing
+  }
+
+  std::cout << GridLogMessage << "Numerator report, Pauli-Villars term         : " << std::endl;
+  NumOp.Report();
+  std::cout << GridLogMessage << "Denominator report, Dw(m) term (includes CG) : " << std::endl;
+  DenOp.Report();
+
+  Grid_finalize();
+} // main
+
+
+
+/* Examples for input files
+
+JSON
+
+{
+    "Checkpointer": {
+    "config_prefix": "ckpoint_json_lat",
+    "rng_prefix": "ckpoint_json_rng",
+    "saveInterval": 1,
+    "format": "IEEE64BIG"
+    },
+    "RandomNumberGenerator": {
+    "serial_seeds": "1 2 3 4 6",
+    "parallel_seeds": "6 7 8 9 11"
+    },
+    "Action":{
+    "gauge_beta": 5.6,
+    "Mobius": {
+        "Ls"  : 10,
+	"mass": 0.01,
+	"M5"  : 1.0,
+	"b"   : 1.5,
+	"c"   : 0.5,
+	"StoppingCondition": 1e-8,
+	"MaxCGIterations": 10000,
+	"ApplySmearing": true
+       }
+    },
+    "HMC":{
+    "StartTrajectory": 0,
+    "Trajectories": 100,
+    "MetropolisTest": true,
+    "NoMetropolisUntil": 10,
+    "StartingType": "HotStart",
+    "MD":{
+        "name": "MinimumNorm2",
+	"MDsteps": 15,
+	"trajL": 2.0
+	}
+    },
+    "StoutSmearing":{
+    "rho": 0.1,
+    "Nsmear": 3
+    }
+}
+
+
+XML example not provided yet
+
+*/

tests/hmc/Test_hmc_EOWilsonFermionGauge.cc

@@ -60,7 +60,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_EOWilsonRatio.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_GparityIwasakiGauge.cc

@@ -61,7 +61,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_GparityWilsonGauge.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_IwasakiGauge.cc

@@ -54,7 +54,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 20;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_RectGauge.cc

@@ -55,7 +55,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 20;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonAdjointFermionGauge.cc

@@ -65,7 +65,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonFermionGauge.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonGauge.cc

@@ -51,10 +51,10 @@ int main(int argc, char **argv) {
   CheckpointerParameters CPparams;  
   CPparams.config_prefix = "ckpoint_lat";
   CPparams.rng_prefix = "ckpoint_rng";
-  CPparams.saveInterval = 20;
+  CPparams.saveInterval = 1;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonMixedRepresentationsFermionGauge.cc

@@ -74,7 +74,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonRatio.cc

@@ -61,7 +61,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonTMFermionGauge.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_hmc_WilsonTwoIndexSymmetricFermionGauge.cc

@@ -64,7 +64,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_rhmc_EOWilson1p1.cc

@@ -61,7 +61,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_rhmc_EOWilsonRatio.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_rhmc_Wilson1p1.cc

@@ -62,7 +62,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";

tests/hmc/Test_rhmc_WilsonRatio.cc

@@ -61,7 +61,7 @@ int main(int argc, char **argv) {
   CPparams.saveInterval = 5;
   CPparams.format = "IEEE64BIG";
   
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
 
   RNGModuleParameters RNGpar;
   RNGpar.serial_seeds = "1 2 3 4 5";