Just commenting

.gitignore

@@ -9,7 +9,6 @@
 ################
 *~
 *#
-*.sublime-*
 
 # Precompiled Headers #
 #######################
@@ -105,15 +104,3 @@ lib/fftw/*
 ##################
 m4/lt*
 m4/libtool.m4
-
-# Buck files #
-##############
-.buck*
-buck-out
-BUCK
-make-bin-BUCK.sh
-
-# generated sources #
-#####################
-lib/qcd/spin/gamma-gen/*.h
-lib/qcd/spin/gamma-gen/*.cc

Makefile.am

@@ -1,5 +1,5 @@
 # additional include paths necessary to compile the C++ library
-SUBDIRS = lib benchmarks tests extras
+SUBDIRS = lib benchmarks tests
 
 include $(top_srcdir)/doxygen.inc
 

benchmarks/Benchmark_dwf.cc

@@ -37,11 +37,11 @@ struct scal {
   d internal;
 };
 
-  Gamma::Algebra Gmu [] = {
+  Gamma::GammaMatrix Gmu [] = {
-    Gamma::Algebra::GammaX,
+    Gamma::GammaX,
-    Gamma::Algebra::GammaY,
+    Gamma::GammaY,
-    Gamma::Algebra::GammaZ,
+    Gamma::GammaZ,
-    Gamma::Algebra::GammaT
+    Gamma::GammaT
   };
 
 typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
@@ -321,7 +321,7 @@ int main (int argc, char ** argv)
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
 
-      //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
+      //    ref =  src - Gamma(Gamma::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu+1,1);
       for(int i=0;i<ref._odata.size();i++){
   ref._odata[i]+= tmp._odata[i] + Gamma(Gmu[mu])*tmp._odata[i]; ;

benchmarks/Benchmark_dwf_sweep.cc

@@ -37,11 +37,11 @@ struct scal {
   d internal;
 };
 
-  Gamma::Algebra Gmu [] = {
+  Gamma::GammaMatrix Gmu [] = {
-    Gamma::Algebra::GammaX,
+    Gamma::GammaX,
-    Gamma::Algebra::GammaY,
+    Gamma::GammaY,
-    Gamma::Algebra::GammaZ,
+    Gamma::GammaZ,
-    Gamma::Algebra::GammaT
+    Gamma::GammaT
   };
 
 void benchDw(std::vector<int> & L, int Ls, int threads, int report =0 );

benchmarks/Benchmark_mooee.cc

@@ -113,36 +113,6 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << "Called " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
     std::cout<<GridLogMessage << "******************"<<std::endl;
 
-#define BENCH_ZDW(A,in,out)			\
-    zDw.CayleyZeroCounters();			\
-    zDw. A (in,out);				\
-    FGrid->Barrier();				\
-    t0=usecond();				\
-    for(int i=0;i<ncall;i++){			\
-      zDw. A (in,out);				\
-    }						\
-    t1=usecond();				\
-    FGrid->Barrier();				\
-    zDw.CayleyReport();							\
-    std::cout<<GridLogMessage << "Called ZDw " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
-    std::cout<<GridLogMessage << "******************"<<std::endl;
-
-#define BENCH_DW_SSC(A,in,out)			\
-    Dw.CayleyZeroCounters();			\
-    Dw. A (in,out);				\
-    FGrid->Barrier();				\
-    t0=usecond();				\
-    for(int i=0;i<ncall;i++){			\
-      __SSC_START ;				\
-      Dw. A (in,out);				\
-      __SSC_STOP ;				\
-    }						\
-    t1=usecond();				\
-    FGrid->Barrier();				\
-    Dw.CayleyReport();					\
-    std::cout<<GridLogMessage << "Called " #A " "<< (t1-t0)/ncall<<" us"<<std::endl;\
-    std::cout<<GridLogMessage << "******************"<<std::endl;
-
 #define BENCH_DW_MEO(A,in,out)			\
     Dw.CayleyZeroCounters();			\
     Dw. A (in,out,0);				\
@@ -178,15 +148,9 @@ int main (int argc, char ** argv)
     LatticeFermion sref(sFGrid);
     LatticeFermion result(sFGrid);
 
-
     std::cout<<GridLogMessage << "Constructing Vec5D Dw "<<std::endl;
     DomainWallFermionVec5dR Dw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,mass,M5);
 
-    RealD b=1.5;// Scale factor b+c=2, b-c=1
-    RealD c=0.5;
-    std::vector<ComplexD> gamma(Ls,std::complex<double>(1.0,0.0));
-    ZMobiusFermionVec5dR zDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,mass,M5,gamma,b,c);
-
     std::cout<<GridLogMessage << "Calling Dhop "<<std::endl;
     FGrid->Barrier();
 
@@ -209,13 +173,10 @@ int main (int argc, char ** argv)
 
     BENCH_DW_MEO(Dhop    ,src,result);
     BENCH_DW_MEO(DhopEO  ,src_o,r_e);
-    BENCH_DW_SSC(Meooe   ,src_o,r_e);
+    BENCH_DW(Meooe   ,src_o,r_e);
     BENCH_DW(Mooee   ,src_o,r_o);
     BENCH_DW(MooeeInv,src_o,r_o);
 
-    BENCH_ZDW(Mooee   ,src_o,r_o);
-    BENCH_ZDW(MooeeInv,src_o,r_o);
-
   }
 
   Grid_finalize();

benchmarks/Benchmark_wilson.cc

@@ -37,11 +37,11 @@ struct scal {
   d internal;
 };
 
-  Gamma::Algebra Gmu [] = {
+  Gamma::GammaMatrix Gmu [] = {
-    Gamma::Algebra::GammaX,
+    Gamma::GammaX,
-    Gamma::Algebra::GammaY,
+    Gamma::GammaY,
-    Gamma::Algebra::GammaZ,
+    Gamma::GammaZ,
-    Gamma::Algebra::GammaT
+    Gamma::GammaT
   };
 
 bool overlapComms = false;
@@ -106,7 +106,7 @@ int main (int argc, char ** argv)
   { // Naive wilson implementation
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
-      //    ref =  src + Gamma(Gamma::Algebra::GammaX)* src ; // 1-gamma_x
+      //    ref =  src + Gamma(Gamma::GammaX)* src ; // 1-gamma_x
       tmp = U[mu]*Cshift(src,mu,1);
       for(int i=0;i<ref._odata.size();i++){
 	ref._odata[i]+= tmp._odata[i] - Gamma(Gmu[mu])*tmp._odata[i]; ;
@@ -159,7 +159,7 @@ int main (int argc, char ** argv)
     ref = zero;
     for(int mu=0;mu<Nd;mu++){
 
-      //    ref =  src - Gamma(Gamma::Algebra::GammaX)* src ; // 1+gamma_x
+      //    ref =  src - Gamma(Gamma::GammaX)* src ; // 1+gamma_x
       tmp = U[mu]*Cshift(src,mu,1);
       for(int i=0;i<ref._odata.size();i++){
 	ref._odata[i]+= tmp._odata[i] + Gamma(Gmu[mu])*tmp._odata[i]; ;

benchmarks/Benchmark_wilson_sweep.cc

@@ -30,11 +30,11 @@ struct scal {
   d internal;
 };
 
-Gamma::Algebra Gmu [] = {
+Gamma::GammaMatrix Gmu [] = {
-    Gamma::Algebra::GammaX,
+  Gamma::GammaX,
-    Gamma::Algebra::GammaY,
+  Gamma::GammaY,
-    Gamma::Algebra::GammaZ,
+  Gamma::GammaZ,
-    Gamma::Algebra::GammaT
+  Gamma::GammaT
 };
 
 bool overlapComms = false;

configure.ac

@@ -6,7 +6,7 @@ AC_CANONICAL_TARGET
 AM_INIT_AUTOMAKE(subdir-objects)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_SRCDIR([lib/Grid.h])
-AC_CONFIG_HEADERS([lib/Config.h],[sed -i 's|PACKAGE_|GRID_|' lib/Config.h])
+AC_CONFIG_HEADERS([lib/Config.h])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 ############### Checks for programs
@@ -99,13 +99,6 @@ case ${ac_MKL} in
         AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
 esac
 
-############### HDF5
-AC_ARG_WITH([hdf5],
-    [AS_HELP_STRING([--with-hdf5=prefix],
-    [try this for a non-standard install prefix of the HDF5 library])],
-    [AM_CXXFLAGS="-I$with_hdf5/include $AM_CXXFLAGS"]
-    [AM_LDFLAGS="-L$with_hdf5/lib $AM_LDFLAGS"])
-
 ############### first-touch
 AC_ARG_ENABLE([numa],
     [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
@@ -152,12 +145,6 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
                [have_fftw=true])
 
-AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
-               [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
-               [have_hdf5=true]
-               [LIBS="${LIBS} -lhdf5"], [], [-lhdf5])
-AM_CONDITIONAL(BUILD_HDF5, [ test "${have_hdf5}X" == "trueX" ])
-
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
@@ -319,7 +306,7 @@ AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] )
 AM_CONDITIONAL(BUILD_COMMS_NONE,  [ test "${comms_type}X" == "noneX" ])
 
 ############### RNG selection
-AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937|sitmo],\
+AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\
 	            [Select Random Number Generator to be used])],\
 	            [ac_RNG=${enable_rng}],[ac_RNG=ranlux48])
 
@@ -330,9 +317,6 @@ case ${ac_RNG} in
      mt19937)
       AC_DEFINE([RNG_MT19937],[1],[RNG_MT19937] )
      ;;
-     sitmo)
-      AC_DEFINE([RNG_SITMO],[1],[RNG_SITMO] )
-     ;;
      *)
       AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]); 
      ;;
@@ -397,13 +381,10 @@ AC_CONFIG_FILES(tests/IO/Makefile)
 AC_CONFIG_FILES(tests/core/Makefile)
 AC_CONFIG_FILES(tests/debug/Makefile)
 AC_CONFIG_FILES(tests/forces/Makefile)
-AC_CONFIG_FILES(tests/hadrons/Makefile)
 AC_CONFIG_FILES(tests/hmc/Makefile)
 AC_CONFIG_FILES(tests/solver/Makefile)
 AC_CONFIG_FILES(tests/qdpxx/Makefile)
 AC_CONFIG_FILES(benchmarks/Makefile)
-AC_CONFIG_FILES(extras/Makefile)
-AC_CONFIG_FILES(extras/Hadrons/Makefile)
 AC_OUTPUT
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -426,7 +407,6 @@ RNG choice                  : ${ac_RNG}
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 LAPACK                      : ${ac_LAPACK}
 FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
-HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
 build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
 CXXFLAGS:

extras/Hadrons/Application.cc

@@ -1,317 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Application.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Application.hpp>
-#include <Grid/Hadrons/GeneticScheduler.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-#define BIG_SEP "==============="
-#define SEP     "---------------"
-
-/******************************************************************************
- *                       Application implementation                           *
- ******************************************************************************/
-// constructors ////////////////////////////////////////////////////////////////
-Application::Application(void)
-{
-    LOG(Message) << "Modules available:" << std::endl;
-    auto list = ModuleFactory::getInstance().getBuilderList();
-    for (auto &m: list)
-    {
-        LOG(Message) << "  " << m << std::endl;
-    }
-    auto dim = GridDefaultLatt(), mpi = GridDefaultMpi(), loc(dim);
-    locVol_ = 1;
-    for (unsigned int d = 0; d < dim.size(); ++d)
-    {
-        loc[d]  /= mpi[d];
-        locVol_ *= loc[d];
-    }
-    LOG(Message) << "Global lattice: " << dim << std::endl;
-    LOG(Message) << "MPI partition : " << mpi << std::endl;
-    LOG(Message) << "Local lattice : " << loc << std::endl;
-}
-
-Application::Application(const Application::GlobalPar &par)
-: Application()
-{
-    setPar(par);
-}
-
-Application::Application(const std::string parameterFileName)
-: Application()
-{
-    parameterFileName_ = parameterFileName;
-}
-
-// environment shortcut ////////////////////////////////////////////////////////
-Environment & Application::env(void) const
-{
-    return Environment::getInstance();
-}
-
-// access //////////////////////////////////////////////////////////////////////
-void Application::setPar(const Application::GlobalPar &par)
-{
-    par_ = par;
-    env().setSeed(strToVec<int>(par_.seed));
-}
-
-const Application::GlobalPar & Application::getPar(void)
-{
-    return par_;
-}
-
-// execute /////////////////////////////////////////////////////////////////////
-void Application::run(void)
-{
-    if (!parameterFileName_.empty() and (env().getNModule() == 0))
-    {
-        parseParameterFile(parameterFileName_);
-    }
-    if (!scheduled_)
-    {
-        schedule();
-    }
-    printSchedule();
-    configLoop();
-}
-
-// parse parameter file ////////////////////////////////////////////////////////
-class ObjectId: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(ObjectId,
-                                    std::string, name,
-                                    std::string, type);
-};
-
-void Application::parseParameterFile(const std::string parameterFileName)
-{
-    XmlReader reader(parameterFileName);
-    GlobalPar par;
-    ObjectId  id;
-    
-    LOG(Message) << "Building application from '" << parameterFileName << "'..." << std::endl;
-    read(reader, "parameters", par);
-    setPar(par);
-    push(reader, "modules");
-    push(reader, "module");
-    do
-    {
-        read(reader, "id", id);
-        env().createModule(id.name, id.type, reader);
-    } while (reader.nextElement("module"));
-    pop(reader);
-    pop(reader);
-}
-
-void Application::saveParameterFile(const std::string parameterFileName)
-{
-    XmlWriter          writer(parameterFileName);
-    ObjectId           id;
-    const unsigned int nMod = env().getNModule();
-    
-    LOG(Message) << "Saving application to '" << parameterFileName << "'..." << std::endl;
-    write(writer, "parameters", getPar());
-    push(writer, "modules");
-    for (unsigned int i = 0; i < nMod; ++i)
-    {
-        push(writer, "module");
-        id.name = env().getModuleName(i);
-        id.type = env().getModule(i)->getRegisteredName();
-        write(writer, "id", id);
-        env().getModule(i)->saveParameters(writer, "options");
-        pop(writer);
-    }
-    pop(writer);
-    pop(writer);
-}
-
-// schedule computation ////////////////////////////////////////////////////////
-#define MEM_MSG(size)\
-sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
-
-#define DEFINE_MEMPEAK \
-auto memPeak = [this](const std::vector<unsigned int> &program)\
-{\
-    unsigned int memPeak;\
-    bool         msg;\
-    \
-    msg = HadronsLogMessage.isActive();\
-    HadronsLogMessage.Active(false);\
-    env().dryRun(true);\
-    memPeak = env().executeProgram(program);\
-    env().dryRun(false);\
-    env().freeAll();\
-    HadronsLogMessage.Active(true);\
-    \
-    return memPeak;\
-}
-
-void Application::schedule(void)
-{
-    DEFINE_MEMPEAK;
-    
-    // build module dependency graph
-    LOG(Message) << "Building module graph..." << std::endl;
-    auto graph = env().makeModuleGraph();
-    auto con = graph.getConnectedComponents();
-    
-    // constrained topological sort using a genetic algorithm
-    LOG(Message) << "Scheduling computation..." << std::endl;
-    LOG(Message) << "               #module= " << graph.size() << std::endl;
-    LOG(Message) << "       population size= " << par_.genetic.popSize << std::endl;
-    LOG(Message) << "       max. generation= " << par_.genetic.maxGen << std::endl;
-    LOG(Message) << "  max. cst. generation= " << par_.genetic.maxCstGen << std::endl;
-    LOG(Message) << "         mutation rate= " << par_.genetic.mutationRate << std::endl;
-    
-    unsigned int                               k = 0, gen, prevPeak, nCstPeak = 0;
-    std::random_device                         rd;
-    GeneticScheduler<unsigned int>::Parameters par;
-    
-    par.popSize      = par_.genetic.popSize;
-    par.mutationRate = par_.genetic.mutationRate;
-    par.seed         = rd();
-    memPeak_         = 0;
-    CartesianCommunicator::BroadcastWorld(0, &(par.seed), sizeof(par.seed));
-    for (unsigned int i = 0; i < con.size(); ++i)
-    {
-        GeneticScheduler<unsigned int> scheduler(con[i], memPeak, par);
-        
-        gen = 0;
-        do
-        {
-            LOG(Debug) << "Generation " << gen << ":" << std::endl;
-            scheduler.nextGeneration();
-            if (gen != 0)
-            {
-                if (prevPeak == scheduler.getMinValue())
-                {
-                    nCstPeak++;
-                }
-                else
-                {
-                    nCstPeak = 0;
-                }
-            }
-            
-            prevPeak = scheduler.getMinValue();
-            if (gen % 10 == 0)
-            {
-                LOG(Iterative) << "Generation " << gen << ": "
-                               << MEM_MSG(scheduler.getMinValue()) << std::endl;
-            }
-            
-            gen++;
-        } while ((gen < par_.genetic.maxGen)
-                 and (nCstPeak < par_.genetic.maxCstGen));
-        auto &t = scheduler.getMinSchedule();
-        if (scheduler.getMinValue() > memPeak_)
-        {
-            memPeak_ = scheduler.getMinValue();
-        }
-        for (unsigned int j = 0; j < t.size(); ++j)
-        {
-            program_.push_back(t[j]);
-        }
-    }
-    scheduled_ = true;
-}
-
-void Application::saveSchedule(const std::string filename)
-{
-    TextWriter               writer(filename);
-    std::vector<std::string> program;
-    
-    if (!scheduled_)
-    {
-        HADRON_ERROR("Computation not scheduled");
-    }
-    LOG(Message) << "Saving current schedule to '" << filename << "'..."
-                 << std::endl;
-    for (auto address: program_)
-    {
-        program.push_back(env().getModuleName(address));
-    }
-    write(writer, "schedule", program);
-}
-
-void Application::loadSchedule(const std::string filename)
-{
-    DEFINE_MEMPEAK;
-    
-    TextReader               reader(filename);
-    std::vector<std::string> program;
-    
-    LOG(Message) << "Loading schedule from '" << filename << "'..."
-                 << std::endl;
-    read(reader, "schedule", program);
-    program_.clear();
-    for (auto &name: program)
-    {
-        program_.push_back(env().getModuleAddress(name));
-    }
-    scheduled_ = true;
-    memPeak_   = memPeak(program_);
-}
-
-void Application::printSchedule(void)
-{
-    if (!scheduled_)
-    {
-        HADRON_ERROR("Computation not scheduled");
-    }
-    LOG(Message) << "Schedule (memory peak: " << MEM_MSG(memPeak_) << "):"
-                 << std::endl;
-    for (unsigned int i = 0; i < program_.size(); ++i)
-    {
-        LOG(Message) << std::setw(4) << i + 1 << ": "
-                     << env().getModuleName(program_[i]) << std::endl;
-    }
-}
-
-// loop on configurations //////////////////////////////////////////////////////
-void Application::configLoop(void)
-{
-    auto range = par_.trajCounter;
-    
-    for (unsigned int t = range.start; t < range.end; t += range.step)
-    {
-        LOG(Message) << BIG_SEP << " Starting measurement for trajectory " << t
-                     << " " << BIG_SEP << std::endl;
-        env().setTrajectory(t);
-        env().executeProgram(program_);
-    }
-    LOG(Message) << BIG_SEP << " End of measurement " << BIG_SEP << std::endl;
-    env().freeAll();
-}

extras/Hadrons/Application.hpp

@@ -1,132 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Application.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Application_hpp_
-#define Hadrons_Application_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Environment.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-#include <Grid/Hadrons/Modules.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         Main program manager                               *
- ******************************************************************************/
-class Application
-{
-public:
-    class TrajRange: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(TrajRange,
-                                        unsigned int, start,
-                                        unsigned int, end,
-                                        unsigned int, step);
-    };
-    class GeneticPar: Serializable
-    {
-    public:
-        GeneticPar(void):
-            popSize{20}, maxGen{1000}, maxCstGen{100}, mutationRate{.1} {};
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(GeneticPar,
-                                        unsigned int, popSize,
-                                        unsigned int, maxGen,
-                                        unsigned int, maxCstGen,
-                                        double      , mutationRate);
-    };
-    class GlobalPar: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(GlobalPar,
-                                        TrajRange,   trajCounter,
-                                        GeneticPar,  genetic,
-                                        std::string, seed);
-    };
-public:
-    // constructors
-    Application(void);
-    Application(const GlobalPar &par);
-    Application(const std::string parameterFileName);
-    // destructor
-    virtual ~Application(void) = default;
-    // access
-    void              setPar(const GlobalPar &par);
-    const GlobalPar & getPar(void);
-    // module creation
-    template <typename M>
-    void createModule(const std::string name);
-    template <typename M>
-    void createModule(const std::string name, const typename M::Par &par);
-    // execute
-    void run(void);
-    // XML parameter file I/O
-    void parseParameterFile(const std::string parameterFileName);
-    void saveParameterFile(const std::string parameterFileName);
-    // schedule computation
-    void schedule(void);
-    void saveSchedule(const std::string filename);
-    void loadSchedule(const std::string filename);
-    void printSchedule(void);
-    // loop on configurations
-    void configLoop(void);
-private:
-    // environment shortcut
-    Environment & env(void) const;
-private:
-    long unsigned int         locVol_;
-    std::string               parameterFileName_{""};
-    GlobalPar                 par_;
-    std::vector<unsigned int> program_;
-    Environment::Size         memPeak_;
-    bool                      scheduled_{false};
-};
-
-/******************************************************************************
- *                     Application template implementation                    *
- ******************************************************************************/
-// module creation /////////////////////////////////////////////////////////////
-template <typename M>
-void Application::createModule(const std::string name)
-{
-    env().createModule<M>(name);
-}
-
-template <typename M>
-void Application::createModule(const std::string name,
-                               const typename M::Par &par)
-{
-    env().createModule<M>(name, par);
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Application_hpp_

extras/Hadrons/Environment.cc

@@ -1,743 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Environment.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Environment.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-/******************************************************************************
- *                       Environment implementation                           *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-Environment::Environment(void)
-{
-    nd_ = GridDefaultLatt().size();
-    grid4d_.reset(SpaceTimeGrid::makeFourDimGrid(
-        GridDefaultLatt(), GridDefaultSimd(nd_, vComplex::Nsimd()),
-        GridDefaultMpi()));
-    gridRb4d_.reset(SpaceTimeGrid::makeFourDimRedBlackGrid(grid4d_.get()));
-    auto loc = getGrid()->LocalDimensions();
-    locVol_ = 1;
-    for (unsigned int d = 0; d < loc.size(); ++d)
-    {
-        locVol_ *= loc[d];
-    }
-    rng4d_.reset(new GridParallelRNG(grid4d_.get()));
-}
-
-// dry run /////////////////////////////////////////////////////////////////////
-void Environment::dryRun(const bool isDry)
-{
-    dryRun_ = isDry;
-}
-
-bool Environment::isDryRun(void) const
-{
-    return dryRun_;
-}
-
-// trajectory number ///////////////////////////////////////////////////////////
-void Environment::setTrajectory(const unsigned int traj)
-{
-    traj_ = traj;
-}
-
-unsigned int Environment::getTrajectory(void) const
-{
-    return traj_;
-}
-
-// grids ///////////////////////////////////////////////////////////////////////
-void Environment::createGrid(const unsigned int Ls)
-{
-    if (grid5d_.find(Ls) == grid5d_.end())
-    {
-        auto g = getGrid();
-        
-        grid5d_[Ls].reset(SpaceTimeGrid::makeFiveDimGrid(Ls, g));
-        gridRb5d_[Ls].reset(SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, g));
-    }
-}
-
-GridCartesian * Environment::getGrid(const unsigned int Ls) const
-{
-    try
-    {
-        if (Ls == 1)
-        {
-            return grid4d_.get();
-        }
-        else
-        {
-            return grid5d_.at(Ls).get();
-        }
-    }
-    catch(std::out_of_range &)
-    {
-        HADRON_ERROR("no grid with Ls= " << Ls);
-    }
-}
-
-GridRedBlackCartesian * Environment::getRbGrid(const unsigned int Ls) const
-{
-    try
-    {
-        if (Ls == 1)
-        {
-            return gridRb4d_.get();
-        }
-        else
-        {
-            return gridRb5d_.at(Ls).get();
-        }
-    }
-    catch(std::out_of_range &)
-    {
-        HADRON_ERROR("no red-black 5D grid with Ls= " << Ls);
-    }
-}
-
-unsigned int Environment::getNd(void) const
-{
-    return nd_;
-}
-
-// random number generator /////////////////////////////////////////////////////
-void Environment::setSeed(const std::vector<int> &seed)
-{
-    rng4d_->SeedFixedIntegers(seed);
-}
-
-GridParallelRNG * Environment::get4dRng(void) const
-{
-    return rng4d_.get();
-}
-
-// module management ///////////////////////////////////////////////////////////
-void Environment::pushModule(Environment::ModPt &pt)
-{
-    std::string name = pt->getName();
-    
-    if (!hasModule(name))
-    {
-        std::vector<unsigned int> inputAddress;
-        unsigned int              address;
-        ModuleInfo                m;
-        
-        m.data = std::move(pt);
-        m.type = typeIdPt(*m.data.get());
-        m.name = name;
-        auto input  = m.data->getInput();
-        for (auto &in: input)
-        {
-            if (!hasObject(in))
-            {
-                addObject(in , -1);
-            }
-            m.input.push_back(objectAddress_[in]);
-        }
-        auto output = m.data->getOutput();
-        module_.push_back(std::move(m));
-        address              = static_cast<unsigned int>(module_.size() - 1);
-        moduleAddress_[name] = address;
-        for (auto &out: output)
-        {
-            if (!hasObject(out))
-            {
-                addObject(out, address);
-            }
-            else
-            {
-                if (object_[objectAddress_[out]].module < 0)
-                {
-                    object_[objectAddress_[out]].module = address;
-                }
-                else
-                {
-                    HADRON_ERROR("object '" + out
-                                 + "' is already produced by module '"
-                                 + module_[object_[getObjectAddress(out)].module].name
-                                 + "' (while pushing module '" + name + "')");
-                }
-            }
-        }
-    }
-    else
-    {
-        HADRON_ERROR("module '" + name + "' already exists");
-    }
-}
-
-unsigned int Environment::getNModule(void) const
-{
-    return module_.size();
-}
-
-void Environment::createModule(const std::string name, const std::string type,
-                               XmlReader &reader)
-{
-    auto &factory = ModuleFactory::getInstance();
-    auto pt       = factory.create(type, name);
-    
-    pt->parseParameters(reader, "options");
-    pushModule(pt);
-}
-
-ModuleBase * Environment::getModule(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return module_[address].data.get();
-    }
-    else
-    {
-        HADRON_ERROR("no module with address " + std::to_string(address));
-    }
-}
-
-ModuleBase * Environment::getModule(const std::string name) const
-{
-    return getModule(getModuleAddress(name));
-}
-
-unsigned int Environment::getModuleAddress(const std::string name) const
-{
-    if (hasModule(name))
-    {
-        return moduleAddress_.at(name);
-    }
-    else
-    {
-        HADRON_ERROR("no module with name '" + name + "'");
-    }
-}
-
-std::string Environment::getModuleName(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return module_[address].name;
-    }
-    else
-    {
-        HADRON_ERROR("no module with address " + std::to_string(address));
-    }
-}
-
-std::string Environment::getModuleType(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return typeName(module_[address].type);
-    }
-    else
-    {
-        HADRON_ERROR("no module with address " + std::to_string(address));
-    }
-}
-
-std::string Environment::getModuleType(const std::string name) const
-{
-    return getModuleType(getModuleAddress(name));
-}
-
-bool Environment::hasModule(const unsigned int address) const
-{
-    return (address < module_.size());
-}
-
-bool Environment::hasModule(const std::string name) const
-{
-    return (moduleAddress_.find(name) != moduleAddress_.end());
-}
-
-Graph<unsigned int> Environment::makeModuleGraph(void) const
-{
-    Graph<unsigned int> moduleGraph;
-    
-    for (unsigned int i = 0; i < module_.size(); ++i)
-    {
-        moduleGraph.addVertex(i);
-        for (auto &j: module_[i].input)
-        {
-            moduleGraph.addEdge(object_[j].module, i);
-        }
-    }
-    
-    return moduleGraph;
-}
-
-#define BIG_SEP "==============="
-#define SEP     "---------------"
-#define MEM_MSG(size)\
-sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
-
-Environment::Size
-Environment::executeProgram(const std::vector<unsigned int> &p)
-{
-    Size                                memPeak = 0, sizeBefore, sizeAfter;
-    std::vector<std::set<unsigned int>> freeProg;
-    bool                                continueCollect, nothingFreed;
-    
-    // build garbage collection schedule
-    freeProg.resize(p.size());
-    for (unsigned int i = 0; i < object_.size(); ++i)
-    {
-        auto pred = [i, this](const unsigned int j)
-        {
-            auto &in = module_[j].input;
-            auto it  = std::find(in.begin(), in.end(), i);
-            
-            return (it != in.end()) or (j == object_[i].module);
-        };
-        auto it = std::find_if(p.rbegin(), p.rend(), pred);
-        if (it != p.rend())
-        {
-            freeProg[p.rend() - it - 1].insert(i);
-        }
-    }
-    
-    // program execution
-    for (unsigned int i = 0; i < p.size(); ++i)
-    {
-        // execute module
-        if (!isDryRun())
-        {
-            LOG(Message) << SEP << " Measurement step " << i+1 << "/"
-                         << p.size() << " (module '" << module_[p[i]].name
-                         << "') " << SEP << std::endl;
-        }
-        (*module_[p[i]].data)();
-        sizeBefore = getTotalSize();
-        // print used memory after execution
-        if (!isDryRun())
-        {
-            LOG(Message) << "Allocated objects: " << MEM_MSG(sizeBefore)
-                         << std::endl;
-        }
-        if (sizeBefore > memPeak)
-        {
-            memPeak = sizeBefore;
-        }
-        // garbage collection for step i
-        if (!isDryRun())
-        {
-            LOG(Message) << "Garbage collection..." << std::endl;
-        }
-        nothingFreed = true;
-        do
-        {
-            continueCollect = false;
-            auto toFree = freeProg[i];
-            for (auto &j: toFree)
-            {
-                // continue garbage collection while there are still
-                // objects without owners
-                continueCollect = continueCollect or !hasOwners(j);
-                if(freeObject(j))
-                {
-                    // if an object has been freed, remove it from
-                    // the garbage collection schedule
-                    freeProg[i].erase(j);
-                    nothingFreed = false;
-                }
-            }
-        } while (continueCollect);
-        // any remaining objects in step i garbage collection schedule
-        // is scheduled for step i + 1
-        if (i + 1 < p.size())
-        {
-            for (auto &j: freeProg[i])
-            {
-                freeProg[i + 1].insert(j);
-            }
-        }
-        // print used memory after garbage collection if necessary
-        if (!isDryRun())
-        {
-            sizeAfter = getTotalSize();
-            if (sizeBefore != sizeAfter)
-            {
-                LOG(Message) << "Allocated objects: " << MEM_MSG(sizeAfter)
-                             << std::endl;
-            }
-            else
-            {
-                LOG(Message) << "Nothing to free" << std::endl;
-            }
-        }
-    }
-    
-    return memPeak;
-}
-
-Environment::Size Environment::executeProgram(const std::vector<std::string> &p)
-{
-    std::vector<unsigned int> pAddress;
-    
-    for (auto &n: p)
-    {
-        pAddress.push_back(getModuleAddress(n));
-    }
-    
-    return executeProgram(pAddress);
-}
-
-// general memory management ///////////////////////////////////////////////////
-void Environment::addObject(const std::string name, const int moduleAddress)
-{
-    if (!hasObject(name))
-    {
-        ObjInfo info;
-        
-        info.name   = name;
-        info.module = moduleAddress;
-        object_.push_back(std::move(info));
-        objectAddress_[name] = static_cast<unsigned int>(object_.size() - 1);
-    }
-    else
-    {
-        HADRON_ERROR("object '" + name + "' already exists");
-    }
-}
-
-void Environment::registerObject(const unsigned int address,
-                                 const unsigned int size, const unsigned int Ls)
-{
-    if (!hasRegisteredObject(address))
-    {
-        if (hasObject(address))
-        {
-            object_[address].size         = size;
-            object_[address].Ls           = Ls;
-            object_[address].isRegistered = true;
-        }
-        else
-        {
-            HADRON_ERROR("no object with address " + std::to_string(address));
-        }
-    }
-    else
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " already registered");
-    }
-}
-
-void Environment::registerObject(const std::string name,
-                                 const unsigned int size, const unsigned int Ls)
-{
-    if (!hasObject(name))
-    {
-        addObject(name);
-    }
-    registerObject(getObjectAddress(name), size, Ls);
-}
-
-unsigned int Environment::getObjectAddress(const std::string name) const
-{
-    if (hasObject(name))
-    {
-        return objectAddress_.at(name);
-    }
-    else
-    {
-        HADRON_ERROR("no object with name '" + name + "'");
-    }
-}
-
-std::string Environment::getObjectName(const unsigned int address) const
-{
-    if (hasObject(address))
-    {
-        return object_[address].name;
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-std::string Environment::getObjectType(const unsigned int address) const
-{
-    if (hasRegisteredObject(address))
-    {
-        return typeName(object_[address].type);
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-std::string Environment::getObjectType(const std::string name) const
-{
-    return getObjectType(getObjectAddress(name));
-}
-
-Environment::Size Environment::getObjectSize(const unsigned int address) const
-{
-    if (hasRegisteredObject(address))
-    {
-        return object_[address].size;
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-Environment::Size Environment::getObjectSize(const std::string name) const
-{
-    return getObjectSize(getObjectAddress(name));
-}
-
-unsigned int Environment::getObjectLs(const unsigned int address) const
-{
-    if (hasRegisteredObject(address))
-    {
-        return object_[address].Ls;
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address)
-                     + " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-unsigned int Environment::getObjectLs(const std::string name) const
-{
-    return getObjectLs(getObjectAddress(name));
-}
-
-bool Environment::hasObject(const unsigned int address) const
-{
-    return (address < object_.size());
-}
-
-bool Environment::hasObject(const std::string name) const
-{
-    auto it = objectAddress_.find(name);
-    
-    return ((it != objectAddress_.end()) and hasObject(it->second));
-}
-
-bool Environment::hasRegisteredObject(const unsigned int address) const
-{
-    if (hasObject(address))
-    {
-        return object_[address].isRegistered;
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::hasRegisteredObject(const std::string name) const
-{
-    if (hasObject(name))
-    {
-        return hasRegisteredObject(getObjectAddress(name));
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::hasCreatedObject(const unsigned int address) const
-{
-    if (hasObject(address))
-    {
-        return (object_[address].data != nullptr);
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::hasCreatedObject(const std::string name) const
-{
-    if (hasObject(name))
-    {
-        return hasCreatedObject(getObjectAddress(name));
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::isObject5d(const unsigned int address) const
-{
-    return (getObjectLs(address) > 1);
-}
-
-bool Environment::isObject5d(const std::string name) const
-{
-    return (getObjectLs(name) > 1);
-}
-
-Environment::Size Environment::getTotalSize(void) const
-{
-    Environment::Size size = 0;
-    
-    for (auto &o: object_)
-    {
-        if (o.isRegistered)
-        {
-            size += o.size;
-        }
-    }
-    
-    return size;
-}
-
-void Environment::addOwnership(const unsigned int owner,
-                               const unsigned int property)
-{
-    if (hasObject(property))
-    {
-        object_[property].owners.insert(owner);
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(property));
-    }
-    if (hasObject(owner))
-    {
-        object_[owner].properties.insert(property);
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(owner));
-    }
-}
-
-void Environment::addOwnership(const std::string owner,
-                               const std::string property)
-{
-    addOwnership(getObjectAddress(owner), getObjectAddress(property));
-}
-
-bool Environment::hasOwners(const unsigned int address) const
-{
-    
-    if (hasObject(address))
-    {
-        return (!object_[address].owners.empty());
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-bool Environment::hasOwners(const std::string name) const
-{
-    return hasOwners(getObjectAddress(name));
-}
-
-bool Environment::freeObject(const unsigned int address)
-{
-    if (!hasOwners(address))
-    {
-        if (!isDryRun() and object_[address].isRegistered)
-        {
-            LOG(Message) << "Destroying object '" << object_[address].name
-                         << "'" << std::endl;
-        }
-        for (auto &p: object_[address].properties)
-        {
-            object_[p].owners.erase(address);
-        }
-        object_[address].size         = 0;
-        object_[address].Ls           = 0;
-        object_[address].isRegistered = false;
-        object_[address].type         = nullptr;
-        object_[address].owners.clear();
-        object_[address].properties.clear();
-        object_[address].data.reset(nullptr);
-        
-        return true;
-    }
-    else
-    {
-        return false;
-    }
-}
-
-bool Environment::freeObject(const std::string name)
-{
-    return freeObject(getObjectAddress(name));
-}
-
-void Environment::freeAll(void)
-{
-    for (unsigned int i = 0; i < object_.size(); ++i)
-    {
-        freeObject(i);
-    }
-}
-
-void Environment::printContent(void)
-{
-    LOG(Message) << "Modules: " << std::endl;
-    for (unsigned int i = 0; i < module_.size(); ++i)
-    {
-        LOG(Message) << std::setw(4) << i << ": "
-                     << getModuleName(i) << std::endl;
-    }
-    LOG(Message) << "Objects: " << std::endl;
-    for (unsigned int i = 0; i < object_.size(); ++i)
-    {
-        LOG(Message) << std::setw(4) << i << ": "
-                     << getObjectName(i) << std::endl;
-    }
-}

extras/Hadrons/Environment.hpp

@@ -1,385 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Environment.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Environment_hpp_
-#define Hadrons_Environment_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Graph.hpp>
-
-#ifndef SITE_SIZE_TYPE
-#define SITE_SIZE_TYPE unsigned int
-#endif
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         Global environment                                 *
- ******************************************************************************/
-// forward declaration of Module
-class ModuleBase;
-
-class Object
-{
-public:
-    Object(void) = default;
-    virtual ~Object(void) = default;
-};
-
-template <typename T>
-class Holder: public Object
-{
-public:
-    Holder(void) = default;
-    Holder(T *pt);
-    virtual ~Holder(void) = default;
-    T &       get(void) const;
-    T *       getPt(void) const;
-    void      reset(T *pt);
-private:
-    std::unique_ptr<T> objPt_{nullptr};
-};
-
-class Environment
-{
-    SINGLETON(Environment);
-public:
-    typedef SITE_SIZE_TYPE                         Size;
-    typedef std::unique_ptr<ModuleBase>            ModPt;
-    typedef std::unique_ptr<GridCartesian>         GridPt;
-    typedef std::unique_ptr<GridRedBlackCartesian> GridRbPt;
-    typedef std::unique_ptr<GridParallelRNG>       RngPt;
-    typedef std::unique_ptr<LatticeBase>           LatticePt;
-private:
-    struct ModuleInfo
-    {
-        const std::type_info      *type{nullptr};
-        std::string               name;
-        ModPt                     data{nullptr};
-        std::vector<unsigned int> input;
-    };
-    struct ObjInfo
-    {
-        Size                    size{0};
-        unsigned int            Ls{0};
-        bool                    isRegistered{false};
-        const std::type_info    *type{nullptr};
-        std::string             name;
-        int                     module{-1};
-        std::set<unsigned int>  owners, properties;
-        std::unique_ptr<Object> data{nullptr};
-    };
-public:
-    // dry run
-    void                    dryRun(const bool isDry);
-    bool                    isDryRun(void) const;
-    // trajectory number
-    void                    setTrajectory(const unsigned int traj);
-    unsigned int            getTrajectory(void) const;
-    // grids
-    void                    createGrid(const unsigned int Ls);
-    GridCartesian *         getGrid(const unsigned int Ls = 1) const;
-    GridRedBlackCartesian * getRbGrid(const unsigned int Ls = 1) const;
-    unsigned int            getNd(void) const;
-    // random number generator
-    void                    setSeed(const std::vector<int> &seed);
-    GridParallelRNG *       get4dRng(void) const;
-    // module management
-    void                    pushModule(ModPt &pt);
-    template <typename M>
-    void                    createModule(const std::string name);
-    template <typename M>
-    void                    createModule(const std::string name,
-                                         const typename M::Par &par);
-    void                    createModule(const std::string name,
-                                         const std::string type,
-                                         XmlReader &reader);
-    unsigned int            getNModule(void) const;
-    ModuleBase *            getModule(const unsigned int address) const;
-    ModuleBase *            getModule(const std::string name) const;
-    template <typename M>
-    M *                     getModule(const unsigned int address) const;
-    template <typename M>
-    M *                     getModule(const std::string name) const;
-    unsigned int            getModuleAddress(const std::string name) const;
-    std::string             getModuleName(const unsigned int address) const;
-    std::string             getModuleType(const unsigned int address) const;
-    std::string             getModuleType(const std::string name) const;
-    bool                    hasModule(const unsigned int address) const;
-    bool                    hasModule(const std::string name) const;
-    Graph<unsigned int>     makeModuleGraph(void) const;
-    Size                    executeProgram(const std::vector<unsigned int> &p);
-    Size                    executeProgram(const std::vector<std::string> &p);
-    // general memory management
-    void                    addObject(const std::string name,
-                                      const int moduleAddress = -1);
-    void                    registerObject(const unsigned int address,
-                                           const unsigned int size,
-                                           const unsigned int Ls = 1);
-    void                    registerObject(const std::string name,
-                                           const unsigned int size,
-                                           const unsigned int Ls = 1);
-    template <typename T>
-    unsigned int            lattice4dSize(void) const;
-    template <typename T>
-    void                    registerLattice(const unsigned int address,
-                                            const unsigned int Ls = 1);
-    template <typename T>
-    void                    registerLattice(const std::string name,
-                                            const unsigned int Ls = 1);
-    template <typename T>
-    void                    setObject(const unsigned int address, T *object);
-    template <typename T>
-    void                    setObject(const std::string name, T *object);
-    template <typename T>
-    T *                     getObject(const unsigned int address) const;
-    template <typename T>
-    T *                     getObject(const std::string name) const;
-    template <typename T>
-    T *                     createLattice(const unsigned int address);
-    template <typename T>
-    T *                     createLattice(const std::string name);
-    unsigned int            getObjectAddress(const std::string name) const;
-    std::string             getObjectName(const unsigned int address) const;
-    std::string             getObjectType(const unsigned int address) const;
-    std::string             getObjectType(const std::string name) const;
-    Size                    getObjectSize(const unsigned int address) const;
-    Size                    getObjectSize(const std::string name) const;
-    unsigned int            getObjectLs(const unsigned int address) const;
-    unsigned int            getObjectLs(const std::string name) const;
-    bool                    hasObject(const unsigned int address) const;
-    bool                    hasObject(const std::string name) const;
-    bool                    hasRegisteredObject(const unsigned int address) const;
-    bool                    hasRegisteredObject(const std::string name) const;
-    bool                    hasCreatedObject(const unsigned int address) const;
-    bool                    hasCreatedObject(const std::string name) const;
-    bool                    isObject5d(const unsigned int address) const;
-    bool                    isObject5d(const std::string name) const;
-    Environment::Size       getTotalSize(void) const;
-    void                    addOwnership(const unsigned int owner,
-                                         const unsigned int property);
-    void                    addOwnership(const std::string owner,
-                                         const std::string property);
-    bool                    hasOwners(const unsigned int address) const;
-    bool                    hasOwners(const std::string name) const;
-    bool                    freeObject(const unsigned int address);
-    bool                    freeObject(const std::string name);
-    void                    freeAll(void);
-    void                    printContent(void);
-private:
-    // general
-    bool                                   dryRun_{false};
-    unsigned int                           traj_, locVol_;
-    // grids
-    GridPt                                 grid4d_;
-    std::map<unsigned int, GridPt>         grid5d_;
-    GridRbPt                               gridRb4d_;
-    std::map<unsigned int, GridRbPt>       gridRb5d_;
-    unsigned int                           nd_;
-    // random number generator
-    RngPt                                  rng4d_;
-    // module and related maps
-    std::vector<ModuleInfo>                module_;
-    std::map<std::string, unsigned int>    moduleAddress_;
-    // lattice store
-    std::map<unsigned int, LatticePt>      lattice_;
-    // object store
-    std::vector<ObjInfo>                   object_;
-    std::map<std::string, unsigned int>    objectAddress_;
-};
-
-/******************************************************************************
- *                       Holder template implementation                       *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename T>
-Holder<T>::Holder(T *pt)
-: objPt_(pt)
-{}
-
-// access //////////////////////////////////////////////////////////////////////
-template <typename T>
-T & Holder<T>::get(void) const
-{
-    return &objPt_.get();
-}
-
-template <typename T>
-T * Holder<T>::getPt(void) const
-{
-    return objPt_.get();
-}
-
-template <typename T>
-void Holder<T>::reset(T *pt)
-{
-    objPt_.reset(pt);
-}
-
-/******************************************************************************
- *                     Environment template implementation                    *
- ******************************************************************************/
-// module management ///////////////////////////////////////////////////////////
-template <typename M>
-void Environment::createModule(const std::string name)
-{
-    ModPt pt(new M(name));
-    
-    pushModule(pt);
-}
-
-template <typename M>
-void Environment::createModule(const std::string name,
-                               const typename M::Par &par)
-{
-    ModPt pt(new M(name));
-    
-    static_cast<M *>(pt.get())->setPar(par);
-    pushModule(pt);
-}
-
-template <typename M>
-M * Environment::getModule(const unsigned int address) const
-{
-    if (auto *pt = dynamic_cast<M *>(getModule(address)))
-    {
-        return pt;
-    }
-    else
-    {
-        HADRON_ERROR("module '" + module_[address].name
-                     + "' does not have type " + typeid(M).name()
-                     + "(object type: " + getModuleType(address) + ")");
-    }
-}
-
-template <typename M>
-M * Environment::getModule(const std::string name) const
-{
-    return getModule<M>(getModuleAddress(name));
-}
-
-template <typename T>
-unsigned int Environment::lattice4dSize(void) const
-{
-    return sizeof(typename T::vector_object)/getGrid()->Nsimd();
-}
-
-template <typename T>
-void Environment::registerLattice(const unsigned int address,
-                                  const unsigned int Ls)
-{
-    createGrid(Ls);
-    registerObject(address, Ls*lattice4dSize<T>(), Ls);
-}
-
-template <typename T>
-void Environment::registerLattice(const std::string name, const unsigned int Ls)
-{
-    createGrid(Ls);
-    registerObject(name, Ls*lattice4dSize<T>(), Ls);
-}
-
-template <typename T>
-void Environment::setObject(const unsigned int address, T *object)
-{
-    if (hasRegisteredObject(address))
-    {
-        object_[address].data.reset(new Holder<T>(object));
-        object_[address].type = &typeid(T);
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address) +
-                     " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-template <typename T>
-void Environment::setObject(const std::string name, T *object)
-{
-    setObject(getObjectAddress(name), object);
-}
-
-template <typename T>
-T * Environment::getObject(const unsigned int address) const
-{
-    if (hasRegisteredObject(address))
-    {
-        if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
-        {
-            return h->getPt();
-        }
-        else
-        {
-            HADRON_ERROR("object with address " + std::to_string(address) +
-                         " does not have type '" + typeid(T).name() +
-                         "' (has type '" + getObjectType(address) + "')");
-        }
-    }
-    else if (hasObject(address))
-    {
-        HADRON_ERROR("object with address " + std::to_string(address) +
-                     " exists but is not registered");
-    }
-    else
-    {
-        HADRON_ERROR("no object with address " + std::to_string(address));
-    }
-}
-
-template <typename T>
-T * Environment::getObject(const std::string name) const
-{
-    return getObject<T>(getObjectAddress(name));
-}
-
-template <typename T>
-T * Environment::createLattice(const unsigned int address)
-{
-    GridCartesian *g = getGrid(getObjectLs(address));
-    
-    setObject(address, new T(g));
-    
-    return getObject<T>(address);
-}
-
-template <typename T>
-T * Environment::createLattice(const std::string name)
-{
-    return createLattice<T>(getObjectAddress(name));
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Environment_hpp_

extras/Hadrons/Factory.hpp

@@ -1,106 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Factory.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Factory_hpp_
-#define Hadrons_Factory_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                        abstract factory class                              *
- ******************************************************************************/
-template <typename T>
-class Factory
-{
-public:
-    typedef std::function<std::unique_ptr<T>(const std::string)> Func;
-public:
-    // constructor
-    Factory(void) = default;
-    // destructor
-    virtual ~Factory(void) = default;
-    // registration
-    void registerBuilder(const std::string type, const Func &f);
-    // get builder list
-    std::vector<std::string> getBuilderList(void) const;
-    // factory
-    std::unique_ptr<T> create(const std::string type,
-                              const std::string name) const;
-private:
-    std::map<std::string, Func> builder_;
-};
-
-/******************************************************************************
- *                         template implementation                            *
- ******************************************************************************/
-// registration ////////////////////////////////////////////////////////////////
-template <typename T>
-void Factory<T>::registerBuilder(const std::string type, const Func &f)
-{
-    builder_[type] = f;
-}
-
-// get module list /////////////////////////////////////////////////////////////
-template <typename T>
-std::vector<std::string> Factory<T>::getBuilderList(void) const
-{
-    std::vector<std::string> list;
-    
-    for (auto &b: builder_)
-    {
-        list.push_back(b.first);
-    }
-    
-    return list;
-}
-
-// factory /////////////////////////////////////////////////////////////////////
-template <typename T>
-std::unique_ptr<T> Factory<T>::create(const std::string type,
-                                      const std::string name) const
-{
-    Func func;
-    
-    try
-    {
-        func = builder_.at(type);
-    }
-    catch (std::out_of_range &)
-    {
-        HADRON_ERROR("object of type '" + type + "' unknown");
-    }
-    
-    return func(name);
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Factory_hpp_

extras/Hadrons/GeneticScheduler.hpp

@@ -1,329 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/GeneticScheduler.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_GeneticScheduler_hpp_
-#define Hadrons_GeneticScheduler_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Graph.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                   Scheduler based on a genetic algorithm                   *
- ******************************************************************************/
-template <typename T>
-class GeneticScheduler
-{
-public:
-    typedef std::vector<T>                   Gene;
-    typedef std::pair<Gene *, Gene *>        GenePair;
-    typedef std::function<int(const Gene &)> ObjFunc;
-    struct Parameters
-    {
-        double       mutationRate;
-        unsigned int popSize, seed;
-    };
-public:
-    // constructor
-    GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
-                     const Parameters &par);
-    // destructor
-    virtual ~GeneticScheduler(void) = default;
-    // access
-    const Gene & getMinSchedule(void);
-    int          getMinValue(void);
-    // breed a new generation
-    void nextGeneration(void);
-    // heuristic benchmarks
-    void benchmarkCrossover(const unsigned int nIt);
-    // print population
-    friend std::ostream & operator<<(std::ostream &out,
-                                     const GeneticScheduler<T> &s)
-    {
-        out << "[";
-        for (auto &p: s.population_)
-        {
-            out << p.first << ", ";
-        }
-        out << "\b\b]";
-        
-        return out;
-    }
-private:
-    // evolution steps
-    void initPopulation(void);
-    void doCrossover(void);
-    void doMutation(void);
-    // genetic operators
-    GenePair selectPair(void);
-    void     crossover(Gene &c1, Gene &c2, const Gene &p1, const Gene &p2);
-    void     mutation(Gene &m, const Gene &c);
-    
-private:
-    Graph<T>                 &graph_;
-    const ObjFunc            &func_;
-    const Parameters         par_;
-    std::multimap<int, Gene> population_;
-    std::mt19937             gen_;
-};
-
-/******************************************************************************
- *                       template implementation                              *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename T>
-GeneticScheduler<T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
-                                      const Parameters &par)
-: graph_(graph)
-, func_(func)
-, par_(par)
-{
-    gen_.seed(par_.seed);
-}
-
-// access //////////////////////////////////////////////////////////////////////
-template <typename T>
-const typename GeneticScheduler<T>::Gene &
-GeneticScheduler<T>::getMinSchedule(void)
-{
-    return population_.begin()->second;
-}
-
-template <typename T>
-int GeneticScheduler<T>::getMinValue(void)
-{
-    return population_.begin()->first;
-}
-
-// breed a new generation //////////////////////////////////////////////////////
-template <typename T>
-void GeneticScheduler<T>::nextGeneration(void)
-{
-    // random initialization of the population if necessary
-    if (population_.size() != par_.popSize)
-    {
-        initPopulation();
-    }
-    LOG(Debug) << "Starting population:\n" << *this << std::endl;
-    
-    // random mutations
-    //PARALLEL_FOR_LOOP
-    for (unsigned int i = 0; i < par_.popSize; ++i)
-    {
-        doMutation();
-    }
-    LOG(Debug) << "After mutations:\n" << *this << std::endl;
-    
-    // mating
-    //PARALLEL_FOR_LOOP
-    for (unsigned int i = 0; i < par_.popSize/2; ++i)
-    {
-        doCrossover();
-    }
-    LOG(Debug) << "After mating:\n" << *this << std::endl;
-    
-    // grim reaper
-    auto it = population_.begin();
-    
-    std::advance(it, par_.popSize);
-    population_.erase(it, population_.end());
-    LOG(Debug) << "After grim reaper:\n" << *this << std::endl;
-}
-
-// evolution steps /////////////////////////////////////////////////////////////
-template <typename T>
-void GeneticScheduler<T>::initPopulation(void)
-{
-    population_.clear();
-    for (unsigned int i = 0; i < par_.popSize; ++i)
-    {
-        auto p = graph_.topoSort(gen_);
-        
-        population_.insert(std::make_pair(func_(p), p));
-    }
-}
-
-template <typename T>
-void GeneticScheduler<T>::doCrossover(void)
-{
-    auto p = selectPair();
-    Gene &p1 = *(p.first), &p2 = *(p.second);
-    Gene c1, c2;
-    
-    crossover(c1, c2, p1, p2);
-    PARALLEL_CRITICAL
-    {
-        population_.insert(std::make_pair(func_(c1), c1));
-        population_.insert(std::make_pair(func_(c2), c2));
-    }
-}
-
-template <typename T>
-void GeneticScheduler<T>::doMutation(void)
-{
-    std::uniform_real_distribution<double>      mdis(0., 1.);
-    std::uniform_int_distribution<unsigned int> pdis(0, population_.size() - 1);
-    
-    if (mdis(gen_) < par_.mutationRate)
-    {
-        Gene m;
-        auto it = population_.begin();
-        
-        std::advance(it, pdis(gen_));
-        mutation(m, it->second);
-        PARALLEL_CRITICAL
-        {
-            population_.insert(std::make_pair(func_(m), m));
-        }
-    }
-}
-
-// genetic operators ///////////////////////////////////////////////////////////
-template <typename T>
-typename GeneticScheduler<T>::GenePair GeneticScheduler<T>::selectPair(void)
-{
-    std::vector<double> prob;
-    unsigned int        ind;
-    Gene                *p1, *p2;
-    
-    for (auto &c: population_)
-    {
-        prob.push_back(1./c.first);
-    }
-    do
-    {
-        double probCpy;
-        
-        std::discrete_distribution<unsigned int> dis1(prob.begin(), prob.end());
-        auto rIt = population_.begin();
-        ind = dis1(gen_);
-        std::advance(rIt, ind);
-        p1 = &(rIt->second);
-        probCpy   = prob[ind];
-        prob[ind] = 0.;
-        std::discrete_distribution<unsigned int> dis2(prob.begin(), prob.end());
-        rIt = population_.begin();
-        std::advance(rIt, dis2(gen_));
-        p2 = &(rIt->second);
-        prob[ind] = probCpy;
-    } while (p1 == p2);
-    
-    return std::make_pair(p1, p2);
-}
-
-template <typename T>
-void GeneticScheduler<T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
-                                    const Gene &p2)
-{
-    Gene                                        buf;
-    std::uniform_int_distribution<unsigned int> dis(0, p1.size() - 1);
-    unsigned int                                cut = dis(gen_);
-    
-    c1.clear();
-    buf = p2;
-    for (unsigned int i = 0; i < cut; ++i)
-    {
-        c1.push_back(p1[i]);
-        buf.erase(std::find(buf.begin(), buf.end(), p1[i]));
-    }
-    for (unsigned int i = 0; i < buf.size(); ++i)
-    {
-        c1.push_back(buf[i]);
-    }
-    c2.clear();
-    buf = p2;
-    for (unsigned int i = cut; i < p1.size(); ++i)
-    {
-        buf.erase(std::find(buf.begin(), buf.end(), p1[i]));
-    }
-    for (unsigned int i = 0; i < buf.size(); ++i)
-    {
-        c2.push_back(buf[i]);
-    }
-    for (unsigned int i = cut; i < p1.size(); ++i)
-    {
-        c2.push_back(p1[i]);
-    }
-}
-
-template <typename T>
-void GeneticScheduler<T>::mutation(Gene &m, const Gene &c)
-{
-    Gene                                        buf;
-    std::uniform_int_distribution<unsigned int> dis(0, c.size() - 1);
-    unsigned int                                cut = dis(gen_);
-    Graph<T>                                    g1 = graph_, g2 = graph_;
-    
-    for (unsigned int i = 0; i < cut; ++i)
-    {
-        g1.removeVertex(c[i]);
-    }
-    for (unsigned int i = cut; i < c.size(); ++i)
-    {
-        g2.removeVertex(c[i]);
-    }
-    if (g1.size() > 0)
-    {
-        buf = g1.topoSort(gen_);
-    }
-    if (g2.size() > 0)
-    {
-        m = g2.topoSort(gen_);
-    }
-    for (unsigned int i = cut; i < c.size(); ++i)
-    {
-        m.push_back(buf[i - cut]);
-    }
-}
-
-template <typename T>
-void GeneticScheduler<T>::benchmarkCrossover(const unsigned int nIt)
-{
-    Gene   p1, p2, c1, c2;
-    double neg = 0., eq = 0., pos = 0., total;
-    int    improvement;
-    
-    LOG(Message) << "Benchmarking crossover..." << std::endl;
-    for (unsigned int i = 0; i < nIt; ++i)
-    {
-        p1 = graph_.topoSort(gen_);
-        p2 = graph_.topoSort(gen_);
-        crossover(c1, c2, p1, p2);
-        improvement = (func_(c1) + func_(c2) - func_(p1) - func_(p2))/2;
-        if (improvement < 0) neg++; else if (improvement == 0) eq++; else pos++;
-    }
-    total = neg + eq + pos;
-    LOG(Message) << "  -: " << neg/total << "  =: " << eq/total
-                 << "  +: " << pos/total << std::endl;
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_GeneticScheduler_hpp_

extras/Hadrons/Global.cc

@@ -1,82 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Global.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Global.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-HadronsLogger Hadrons::HadronsLogError(1,"Error");
-HadronsLogger Hadrons::HadronsLogWarning(1,"Warning");
-HadronsLogger Hadrons::HadronsLogMessage(1,"Message");
-HadronsLogger Hadrons::HadronsLogIterative(1,"Iterative");
-HadronsLogger Hadrons::HadronsLogDebug(1,"Debug");
-
-// pretty size formatting //////////////////////////////////////////////////////
-std::string Hadrons::sizeString(long unsigned int bytes)
-
-{
-    constexpr unsigned int bufSize = 256;
-    const char             *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
-    char                   buf[256];
-    long unsigned int      s     = 0;
-    double                 count = bytes;
-    
-    while (count >= 1024 && s < 7)
-    {
-        s++;
-        count /= 1024;
-    }
-    if (count - floor(count) == 0.0)
-    {
-        snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
-    }
-    else
-    {
-        snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
-    }
-    
-    return std::string(buf);
-}
-
-// type utilities //////////////////////////////////////////////////////////////
-constexpr unsigned int maxNameSize = 1024u;
-
-std::string Hadrons::typeName(const std::type_info *info)
-{
-    char        *buf;
-    std::string name;
-    
-    buf  = abi::__cxa_demangle(info->name(), nullptr, nullptr, nullptr);
-    name = buf;
-    free(buf);
-    
-    return name;
-}

extras/Hadrons/Global.hpp

@@ -1,150 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Global.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Global_hpp_
-#define Hadrons_Global_hpp_
-
-#include <set>
-#include <stack>
-#include <Grid/Grid.h>
-#include <cxxabi.h>
-
-#define BEGIN_HADRONS_NAMESPACE \
-namespace Grid {\
-using namespace QCD;\
-namespace Hadrons {\
-using Grid::operator<<;
-#define END_HADRONS_NAMESPACE }}
-
-#define BEGIN_MODULE_NAMESPACE(name)\
-namespace name {\
-using Grid::operator<<;
-#define END_MODULE_NAMESPACE }
-
-/* the 'using Grid::operator<<;' statement prevents a very nasty compilation
- * error with GCC 5 (clang & GCC 6 compile fine without it).
- */
-
-// FIXME: find a way to do that in a more general fashion
-#ifndef FIMPL
-#define FIMPL WilsonImplR
-#endif
-
-BEGIN_HADRONS_NAMESPACE
-
-// type aliases
-#define TYPE_ALIASES(FImpl, suffix)\
-typedef FermionOperator<FImpl>                       FMat##suffix;             \
-typedef typename FImpl::FermionField                 FermionField##suffix;     \
-typedef typename FImpl::PropagatorField              PropagatorField##suffix;  \
-typedef typename FImpl::SitePropagator               SitePropagator##suffix;   \
-typedef typename FImpl::DoubledGaugeField            DoubledGaugeField##suffix;\
-typedef std::function<void(FermionField##suffix &,                             \
-                      const FermionField##suffix &)> SolverFn##suffix;
-
-// logger
-class HadronsLogger: public Logger
-{
-public:
-    HadronsLogger(int on, std::string nm): Logger("Hadrons", on, nm,
-                                                  GridLogColours, "BLACK"){};
-};
-
-#define LOG(channel) std::cout << HadronsLog##channel
-#define HADRON_ERROR(msg)\
-LOG(Error) << msg << " (" << __FUNCTION__ << " at " << __FILE__ << ":"\
-           << __LINE__ << ")" << std::endl;\
-abort();
-
-#define DEBUG_VAR(var) LOG(Debug) << #var << "= " << (var) << std::endl;
-
-extern HadronsLogger HadronsLogError;
-extern HadronsLogger HadronsLogWarning;
-extern HadronsLogger HadronsLogMessage;
-extern HadronsLogger HadronsLogIterative;
-extern HadronsLogger HadronsLogDebug;
-
-// singleton pattern
-#define SINGLETON(name)\
-public:\
-    name(const name &e) = delete;\
-    void operator=(const name &e) = delete;\
-    static name & getInstance(void)\
-    {\
-        static name e;\
-        return e;\
-    }\
-private:\
-    name(void);
-
-#define SINGLETON_DEFCTOR(name)\
-public:\
-    name(const name &e) = delete;\
-    void operator=(const name &e) = delete;\
-    static name & getInstance(void)\
-    {\
-        static name e;\
-        return e;\
-    }\
-private:\
-    name(void) = default;
-
-// pretty size formating
-std::string sizeString(long unsigned int bytes);
-
-// type utilities
-template <typename T>
-const std::type_info * typeIdPt(const T &x)
-{
-    return &typeid(x);
-}
-
-std::string typeName(const std::type_info *info);
-
-template <typename T>
-const std::type_info * typeIdPt(void)
-{
-    return &typeid(T);
-}
-
-template <typename T>
-std::string typeName(const T &x)
-{
-    return typeName(typeIdPt(x));
-}
-
-template <typename T>
-std::string typeName(void)
-{
-    return typeName(typeIdPt<T>());
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Global_hpp_

extras/Hadrons/Graph.hpp

@@ -1,760 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Graph.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Graph_hpp_
-#define Hadrons_Graph_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                          Oriented graph class                              *
- ******************************************************************************/
-// I/O for edges
-template <typename T>
-std::ostream & operator<<(std::ostream &out, const std::pair<T, T> &e)
-{
-    out << "\""  << e.first << "\" -> \"" << e.second << "\"";
-    
-    return out;
-}
-
-// main class
-template <typename T>
-class Graph
-{
-public:
-    typedef std::pair<T, T> Edge;
-public:
-    // constructor
-    Graph(void);
-    // destructor
-    virtual ~Graph(void) = default;
-    // access
-    void           addVertex(const T &value);
-    void           addEdge(const Edge &e);
-    void           addEdge(const T &start, const T &end);
-    std::vector<T> getVertices(void) const;
-    void           removeVertex(const T &value);
-    void           removeEdge(const Edge &e);
-    void           removeEdge(const T &start, const T &end);
-    unsigned int   size(void) const;
-    // tests
-    bool gotValue(const T &value) const;
-    // graph topological manipulations
-    std::vector<T>              getAdjacentVertices(const T &value) const;
-    std::vector<T>              getChildren(const T &value) const;
-    std::vector<T>              getParents(const T &value) const;
-    std::vector<T>              getRoots(void) const;
-    std::vector<Graph<T>>       getConnectedComponents(void) const;
-    std::vector<T>              topoSort(void);
-    template <typename Gen>
-    std::vector<T>              topoSort(Gen &gen);
-    std::vector<std::vector<T>> allTopoSort(void);
-    // I/O
-    friend std::ostream & operator<<(std::ostream &out, const Graph<T> &g)
-    {
-        out << "{";
-        for (auto &e: g.edgeSet_)
-        {
-            out << e << ", ";
-        }
-        if (g.edgeSet_.size() != 0)
-        {
-            out << "\b\b";
-        }
-        out << "}";
-        
-        return out;
-    }
-private:
-    // vertex marking
-    void      mark(const T &value, const bool doMark = true);
-    void      markAll(const bool doMark = true);
-    void      unmark(const T &value);
-    void      unmarkAll(void);
-    bool      isMarked(const T &value) const;
-    const T * getFirstMarked(const bool isMarked = true) const;
-    template <typename Gen>
-    const T * getRandomMarked(const bool isMarked, Gen &gen);
-    const T * getFirstUnmarked(void) const;
-    template <typename Gen>
-    const T * getRandomUnmarked(Gen &gen);
-    // prune marked/unmarked vertices
-    void removeMarked(const bool isMarked = true);
-    void removeUnmarked(void);
-    // depth-first search marking
-    void depthFirstSearch(void);
-    void depthFirstSearch(const T &root);
-private:
-    std::map<T, bool>  isMarked_;
-    std::set<Edge>     edgeSet_;
-};
-
-// build depedency matrix from topological sorts
-template <typename T>
-std::map<T, std::map<T, bool>>
-makeDependencyMatrix(const std::vector<std::vector<T>> &topSort);
-
-/******************************************************************************
- *                       template implementation                              *
- ******************************************************************************
- * in all the following V is the number of vertex and E is the number of edge
- * in the worst case E = V^2
- */
-
-// constructor /////////////////////////////////////////////////////////////////
-template <typename T>
-Graph<T>::Graph(void)
-{}
-
-// access //////////////////////////////////////////////////////////////////////
-// complexity: log(V)
-template <typename T>
-void Graph<T>::addVertex(const T &value)
-{
-    isMarked_[value] = false;
-}
-
-// complexity: O(log(V))
-template <typename T>
-void Graph<T>::addEdge(const Edge &e)
-{
-    addVertex(e.first);
-    addVertex(e.second);
-    edgeSet_.insert(e);
-}
-
-// complexity: O(log(V))
-template <typename T>
-void Graph<T>::addEdge(const T &start, const T &end)
-{
-    addEdge(Edge(start, end));
-}
-
-template <typename T>
-std::vector<T> Graph<T>::getVertices(void) const
-{
-    std::vector<T> vertex;
-    
-    for (auto &v: isMarked_)
-    {
-        vertex.push_back(v.first);
-    }
-    
-    return vertex;
-}
-
-// complexity: O(V*log(V))
-template <typename T>
-void Graph<T>::removeVertex(const T &value)
-{
-    // remove vertex from the mark table
-    auto vIt = isMarked_.find(value);
-    
-    if (vIt != isMarked_.end())
-    {
-        isMarked_.erase(vIt);
-    }
-    else
-    {
-        HADRON_ERROR("vertex " << value << " does not exists");
-    }
-
-    // remove all edges containing the vertex
-    auto pred = [&value](const Edge &e)
-    {
-        return ((e.first == value) or (e.second == value));
-    };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
-    
-    while (eIt != edgeSet_.end())
-    {
-        edgeSet_.erase(eIt);
-        eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
-    }
-}
-
-// complexity: O(log(V))
-template <typename T>
-void Graph<T>::removeEdge(const Edge &e)
-{
-    auto eIt = edgeSet_.find(e);
-    
-    if (eIt != edgeSet_.end())
-    {
-        edgeSet_.erase(eIt);
-    }
-    else
-    {
-        HADRON_ERROR("edge "  << e << " does not exists");
-    }
-}
-
-// complexity: O(log(V))
-template <typename T>
-void Graph<T>::removeEdge(const T &start, const T &end)
-{
-    removeEdge(Edge(start, end));
-}
-
-// complexity: O(1)
-template <typename T>
-unsigned int Graph<T>::size(void) const
-{
-    return isMarked_.size();
-}
-
-// tests ///////////////////////////////////////////////////////////////////////
-// complexity: O(log(V))
-template <typename T>
-bool Graph<T>::gotValue(const T &value) const
-{
-    auto it = isMarked_.find(value);
-    
-    if (it == isMarked_.end())
-    {
-        return false;
-    }
-    else
-    {
-        return true;
-    }
-}
-
-// vertex marking //////////////////////////////////////////////////////////////
-// complexity: O(log(V))
-template <typename T>
-void Graph<T>::mark(const T &value, const bool doMark)
-{
-    if (gotValue(value))
-    {
-        isMarked_[value] = doMark;
-    }
-    else
-    {
-        HADRON_ERROR("vertex " << value << " does not exists");
-    }
-}
-
-// complexity: O(V*log(V))
-template <typename T>
-void Graph<T>::markAll(const bool doMark)
-{
-    for (auto &v: isMarked_)
-    {
-        mark(v.first, doMark);
-    }
-}
-
-// complexity: O(log(V))
-template <typename T>
-void Graph<T>::unmark(const T &value)
-{
-    mark(value, false);
-}
-
-// complexity: O(V*log(V))
-template <typename T>
-void Graph<T>::unmarkAll(void)
-{
-    markAll(false);
-}
-
-// complexity: O(log(V))
-template <typename T>
-bool Graph<T>::isMarked(const T &value) const
-{
-    if (gotValue(value))
-    {
-        return isMarked_.at(value);
-    }
-    else
-    {
-        HADRON_ERROR("vertex " << value << " does not exists");
-        
-        return false;
-    }
-}
-
-// complexity: O(log(V))
-template <typename T>
-const T * Graph<T>::getFirstMarked(const bool isMarked) const
-{
-    auto pred = [&isMarked](const std::pair<T, bool> &v)
-    {
-        return (v.second == isMarked);
-    };
-    auto vIt = std::find_if(isMarked_.begin(), isMarked_.end(), pred);
-    
-    if (vIt != isMarked_.end())
-    {
-        return &(vIt->first);
-    }
-    else
-    {
-        return nullptr;
-    }
-}
-
-// complexity: O(log(V))
-template <typename T>
-template <typename Gen>
-const T * Graph<T>::getRandomMarked(const bool isMarked, Gen &gen)
-{
-    auto pred = [&isMarked](const std::pair<T, bool> &v)
-    {
-        return (v.second == isMarked);
-    };
-    std::uniform_int_distribution<unsigned int> dis(0, size() - 1);
-    auto                                        rIt = isMarked_.begin();
-    
-    std::advance(rIt, dis(gen));
-    auto vIt = std::find_if(rIt, isMarked_.end(), pred);
-    if (vIt != isMarked_.end())
-    {
-        return &(vIt->first);
-    }
-    else
-    {
-        vIt = std::find_if(isMarked_.begin(), rIt, pred);
-        if (vIt != rIt)
-        {
-            return &(vIt->first);
-        }
-        else
-        {
-            return nullptr;
-        }
-    }
-}
-
-// complexity: O(log(V))
-template <typename T>
-const T * Graph<T>::getFirstUnmarked(void) const
-{
-    return getFirstMarked(false);
-}
-
-// complexity: O(log(V))
-template <typename T>
-template <typename Gen>
-const T * Graph<T>::getRandomUnmarked(Gen &gen)
-{
-    return getRandomMarked(false, gen);
-}
-
-// prune marked/unmarked vertices //////////////////////////////////////////////
-// complexity: O(V^2*log(V))
-template <typename T>
-void Graph<T>::removeMarked(const bool isMarked)
-{
-    auto isMarkedCopy = isMarked_;
-    
-    for (auto &v: isMarkedCopy)
-    {
-        if (v.second == isMarked)
-        {
-            removeVertex(v.first);
-        }
-    }
-}
-
-// complexity: O(V^2*log(V))
-template <typename T>
-void Graph<T>::removeUnmarked(void)
-{
-    removeMarked(false);
-}
-
-// depth-first search marking //////////////////////////////////////////////////
-// complexity: O(V*log(V))
-template <typename T>
-void Graph<T>::depthFirstSearch(void)
-{
-    depthFirstSearch(isMarked_.begin()->first);
-}
-
-// complexity: O(V*log(V))
-template <typename T>
-void Graph<T>::depthFirstSearch(const T &root)
-{
-    std::vector<T> adjacentVertex;
-    
-    mark(root);
-    adjacentVertex = getAdjacentVertices(root);
-    for (auto &v: adjacentVertex)
-    {
-        if (!isMarked(v))
-        {
-            depthFirstSearch(v);
-        }
-    }
-}
-
-// graph topological manipulations /////////////////////////////////////////////
-// complexity: O(V*log(V))
-template <typename T>
-std::vector<T> Graph<T>::getAdjacentVertices(const T &value) const
-{
-    std::vector<T> adjacentVertex;
-    
-    auto pred = [&value](const Edge &e)
-    {
-        return ((e.first == value) or (e.second == value));
-    };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
-    
-    while (eIt != edgeSet_.end())
-    {
-        if (eIt->first == value)
-        {
-            adjacentVertex.push_back((*eIt).second);
-        }
-        else if (eIt->second == value)
-        {
-            adjacentVertex.push_back((*eIt).first);
-        }
-        eIt = find_if(++eIt, edgeSet_.end(), pred);
-    }
-    
-    return adjacentVertex;
-}
-
-// complexity: O(V*log(V))
-template <typename T>
-std::vector<T> Graph<T>::getChildren(const T &value) const
-{
-    std::vector<T> child;
-    
-    auto pred = [&value](const Edge &e)
-    {
-        return (e.first == value);
-    };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
-    
-    while (eIt != edgeSet_.end())
-    {
-        child.push_back((*eIt).second);
-        eIt = find_if(++eIt, edgeSet_.end(), pred);
-    }
-    
-    return child;
-}
-
-// complexity: O(V*log(V))
-template <typename T>
-std::vector<T> Graph<T>::getParents(const T &value) const
-{
-    std::vector<T> parent;
-    
-    auto pred = [&value](const Edge &e)
-    {
-        return (e.second == value);
-    };
-    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
-    
-    while (eIt != edgeSet_.end())
-    {
-        parent.push_back((*eIt).first);
-        eIt = find_if(++eIt, edgeSet_.end(), pred);
-    }
-    
-    return parent;
-}
-
-// complexity: O(V^2*log(V))
-template <typename T>
-std::vector<T> Graph<T>::getRoots(void) const
-{
-    std::vector<T> root;
-    
-    for (auto &v: isMarked_)
-    {
-        auto parent = getParents(v.first);
-        
-        if (parent.size() == 0)
-        {
-            root.push_back(v.first);
-        }
-    }
-    
-    return root;
-}
-
-// complexity: O(V^2*log(V))
-template <typename T>
-std::vector<Graph<T>> Graph<T>::getConnectedComponents(void) const
-{
-    std::vector<Graph<T>> res;
-    Graph<T>              copy(*this);
-    
-    while (copy.size() > 0)
-    {
-        copy.depthFirstSearch();
-        res.push_back(copy);
-        res.back().removeUnmarked();
-        res.back().unmarkAll();
-        copy.removeMarked();
-        copy.unmarkAll();
-    }
-    
-    return res;
-}
-
-// topological sort using a directed DFS algorithm
-// complexity: O(V*log(V))
-template <typename T>
-std::vector<T> Graph<T>::topoSort(void)
-{
-    std::stack<T>     buf;
-    std::vector<T>    res;
-    const T           *vPt;
-    std::map<T, bool> tmpMarked(isMarked_);
-
-    // visit function
-    std::function<void(const T &)> visit = [&](const T &v)
-    {
-        if (tmpMarked.at(v))
-        {
-            HADRON_ERROR("cannot topologically sort a cyclic graph");
-        }
-        if (!isMarked(v))
-        {
-            std::vector<T> child = getChildren(v);
-
-            tmpMarked[v] = true;
-            for (auto &c: child)
-            {
-                visit(c);
-            }
-            mark(v);
-            tmpMarked[v] = false;
-            buf.push(v);
-        }
-    };
-    
-    // reset temporary marks
-    for (auto &v: tmpMarked)
-    {
-        tmpMarked.at(v.first) = false;
-    }
-    
-    // loop on unmarked vertices
-    unmarkAll();
-    vPt = getFirstUnmarked();
-    while (vPt)
-    {
-        visit(*vPt);
-        vPt = getFirstUnmarked();
-    }
-    unmarkAll();
-    
-    // create result vector
-    while (!buf.empty())
-    {
-        res.push_back(buf.top());
-        buf.pop();
-    }
-    
-    return res;
-}
-
-// random version of the topological sort
-// complexity: O(V*log(V))
-template <typename T>
-template <typename Gen>
-std::vector<T> Graph<T>::topoSort(Gen &gen)
-{
-    std::stack<T>     buf;
-    std::vector<T>    res;
-    const T           *vPt;
-    std::map<T, bool> tmpMarked(isMarked_);
-    
-    // visit function
-    std::function<void(const T &)> visit = [&](const T &v)
-    {
-        if (tmpMarked.at(v))
-        {
-            HADRON_ERROR("cannot topologically sort a cyclic graph");
-        }
-        if (!isMarked(v))
-        {
-            std::vector<T> child = getChildren(v);
-            
-            tmpMarked[v] = true;
-            std::shuffle(child.begin(), child.end(), gen);
-            for (auto &c: child)
-            {
-                visit(c);
-            }
-            mark(v);
-            tmpMarked[v] = false;
-            buf.push(v);
-        }
-    };
-    
-    // reset temporary marks
-    for (auto &v: tmpMarked)
-    {
-        tmpMarked.at(v.first) = false;
-    }
-    
-    // loop on unmarked vertices
-    unmarkAll();
-    vPt = getRandomUnmarked(gen);
-    while (vPt)
-    {
-        visit(*vPt);
-        vPt = getRandomUnmarked(gen);
-    }
-    unmarkAll();
-    
-    // create result vector
-    while (!buf.empty())
-    {
-        res.push_back(buf.top());
-        buf.pop();
-    }
-    
-    return res;
-}
-
-// generate all possible topological sorts
-// Y. L. Varol & D. Rotem, Comput. J. 24(1), pp. 83–84, 1981
-// http://comjnl.oupjournals.org/cgi/doi/10.1093/comjnl/24.1.83
-// complexity: O(V*log(V)) (from the paper, but really ?)
-template <typename T>
-std::vector<std::vector<T>> Graph<T>::allTopoSort(void)
-{
-    std::vector<std::vector<T>>    res;
-    std::map<T, std::map<T, bool>> iMat;
-    
-    // create incidence matrix
-    for (auto &v1: isMarked_)
-    for (auto &v2: isMarked_)
-    {
-        iMat[v1.first][v2.first] = false;
-    }
-    for (auto &v: isMarked_)
-    {
-        auto cVec = getChildren(v.first);
-        
-        for (auto &c: cVec)
-        {
-            iMat[v.first][c] = true;
-        }
-    }
-    
-    // generate initial topological sort
-    res.push_back(topoSort());
-    
-    // generate all other topological sorts by permutation
-    std::vector<T>            p = res[0];
-    const unsigned int        n = size();
-    std::vector<unsigned int> loc(n);
-    unsigned int              i, k, k1;
-    T                         obj_k, obj_k1;
-    bool                      isFinal;
-    
-    for (unsigned int j = 0; j < n; ++j)
-    {
-        loc[j] = j;
-    }
-    i = 0;
-    while (i < n-1)
-    {
-        k      = loc[i];
-        k1     = k + 1;
-        obj_k  = p[k];
-        if (k1 >= n)
-        {
-            isFinal = true;
-            obj_k1  = obj_k;
-        }
-        else
-        {
-            isFinal = false;
-            obj_k1  = p[k1];
-        }
-        if (iMat[res[0][i]][obj_k1] or isFinal)
-        {
-            for (unsigned int l = k; l >= i + 1; --l)
-            {
-                p[l]   = p[l-1];
-            }
-            p[i]   = obj_k;
-            loc[i] = i;
-            i++;
-        }
-        else
-        {
-            p[k]   = obj_k1;
-            p[k1]  = obj_k;
-            loc[i] = k1;
-            i      = 0;
-            res.push_back(p);
-        }
-    }
-    
-    return res;
-}
-
-// build depedency matrix from topological sorts ///////////////////////////////
-// complexity: something like O(V^2*log(V!))
-template <typename T>
-std::map<T, std::map<T, bool>>
-makeDependencyMatrix(const std::vector<std::vector<T>> &topSort)
-{
-    std::map<T, std::map<T, bool>> m;
-    const std::vector<T>           &vList = topSort[0];
-    
-    for (auto &v1: vList)
-    for (auto &v2: vList)
-    {
-        bool dep = true;
-        
-        for (auto &t: topSort)
-        {
-            auto i1 = std::find(t.begin(), t.end(), v1);
-            auto i2 = std::find(t.begin(), t.end(), v2);
-            
-            dep = dep and (i1 - i2 > 0);
-            if (!dep) break;
-        }
-        m[v1][v2] = dep;
-    }
-    
-    return m;
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Graph_hpp_

extras/Hadrons/HadronsXmlRun.cc

@@ -1,80 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/HadronsXmlRun.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Application.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-int main(int argc, char *argv[])
-{
-    // parse command line
-    std::string parameterFileName, scheduleFileName = "";
-    
-    if (argc < 2)
-    {
-        std::cerr << "usage: " << argv[0] << " <parameter file> [<precomputed schedule>] [Grid options]";
-        std::cerr << std::endl;
-        std::exit(EXIT_FAILURE);
-    }
-    parameterFileName = argv[1];
-    if (argc > 2)
-    {
-        if (argv[2][0] != '-')
-        {
-            scheduleFileName = argv[2];
-        }
-    }
-    
-    // initialization
-    Grid_init(&argc, &argv);
-    HadronsLogError.Active(GridLogError.isActive());
-    HadronsLogWarning.Active(GridLogWarning.isActive());
-    HadronsLogMessage.Active(GridLogMessage.isActive());
-    HadronsLogIterative.Active(GridLogIterative.isActive());
-    HadronsLogDebug.Active(GridLogDebug.isActive());
-    LOG(Message) << "Grid initialized" << std::endl;
-    
-    // execution
-    Application application(parameterFileName);
-    
-    application.parseParameterFile(parameterFileName);
-    if (!scheduleFileName.empty())
-    {
-        application.loadSchedule(scheduleFileName);
-    }
-    application.run();
-    
-    // epilogue
-    LOG(Message) << "Grid is finalizing now" << std::endl;
-    Grid_finalize();
-    
-    return EXIT_SUCCESS;
-}

extras/Hadrons/HadronsXmlSchedule.cc

@@ -1,72 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/HadronsXmlSchedule.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Application.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-int main(int argc, char *argv[])
-{
-    // parse command line
-    std::string parameterFileName, scheduleFileName;
-    
-    if (argc < 3)
-    {
-        std::cerr << "usage: " << argv[0] << " <parameter file> <schedule output> [Grid options]";
-        std::cerr << std::endl;
-        std::exit(EXIT_FAILURE);
-    }
-    parameterFileName = argv[1];
-    scheduleFileName  = argv[2];
-    
-    // initialization
-    Grid_init(&argc, &argv);
-    HadronsLogError.Active(GridLogError.isActive());
-    HadronsLogWarning.Active(GridLogWarning.isActive());
-    HadronsLogMessage.Active(GridLogMessage.isActive());
-    HadronsLogIterative.Active(GridLogIterative.isActive());
-    HadronsLogDebug.Active(GridLogDebug.isActive());
-    LOG(Message) << "Grid initialized" << std::endl;
-    
-    // execution
-    Application application;
-    
-    application.parseParameterFile(parameterFileName);
-    application.schedule();
-    application.printSchedule();
-    application.saveSchedule(scheduleFileName);
-    
-    // epilogue
-    LOG(Message) << "Grid is finalizing now" << std::endl;
-    Grid_finalize();
-    
-    return EXIT_SUCCESS;
-}

extras/Hadrons/Makefile.am

@@ -1,29 +0,0 @@
-lib_LIBRARIES = libHadrons.a
-bin_PROGRAMS  = HadronsXmlRun HadronsXmlSchedule
-
-include modules.inc
-
-libHadrons_a_SOURCES = \
-    $(modules_cc)      \
-    Application.cc     \
-    Environment.cc     \
-    Global.cc          \
-    Module.cc
-libHadrons_adir = $(pkgincludedir)/Hadrons
-nobase_libHadrons_a_HEADERS = \
-	$(modules_hpp)            \
-	Application.hpp           \
-	Environment.hpp           \
-	Factory.hpp               \
-	GeneticScheduler.hpp      \
-	Global.hpp                \
-	Graph.hpp                 \
-	Module.hpp                \
-	Modules.hpp               \
-	ModuleFactory.hpp
-
-HadronsXmlRun_SOURCES = HadronsXmlRun.cc
-HadronsXmlRun_LDADD   = libHadrons.a -lGrid
-
-HadronsXmlSchedule_SOURCES = HadronsXmlSchedule.cc
-HadronsXmlSchedule_LDADD   = libHadrons.a -lGrid

extras/Hadrons/Module.cc

@@ -1,71 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Module.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Module.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-/******************************************************************************
- *                       ModuleBase implementation                            *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-ModuleBase::ModuleBase(const std::string name)
-: name_(name)
-, env_(Environment::getInstance())
-{}
-
-// access //////////////////////////////////////////////////////////////////////
-std::string ModuleBase::getName(void) const
-{
-    return name_;
-}
-
-Environment & ModuleBase::env(void) const
-{
-    return env_;
-}
-
-// get factory registration name if available
-std::string ModuleBase::getRegisteredName(void)
-{
-    HADRON_ERROR("module '" + getName() + "' has a type not registered"
-                 + " in the factory");
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-void ModuleBase::operator()(void)
-{
-    setup();
-    if (!env().isDryRun())
-    {
-        execute();
-    }
-}

extras/Hadrons/Module.hpp

@@ -1,198 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Module.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Module_hpp_
-#define Hadrons_Module_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Environment.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-// module registration macros
-#define MODULE_REGISTER(mod, base)\
-class mod: public base\
-{\
-public:\
-    typedef base Base;\
-    using Base::Base;\
-    virtual std::string getRegisteredName(void)\
-    {\
-        return std::string(#mod);\
-    }\
-};\
-class mod##ModuleRegistrar\
-{\
-public:\
-    mod##ModuleRegistrar(void)\
-    {\
-        ModuleFactory &modFac = ModuleFactory::getInstance();\
-        modFac.registerBuilder(#mod, [&](const std::string name)\
-                              {\
-                                  return std::unique_ptr<mod>(new mod(name));\
-                              });\
-    }\
-};\
-static mod##ModuleRegistrar mod##ModuleRegistrarInstance;
-
-#define MODULE_REGISTER_NS(mod, base, ns)\
-class mod: public base\
-{\
-public:\
-    typedef base Base;\
-    using Base::Base;\
-    virtual std::string getRegisteredName(void)\
-    {\
-        return std::string(#ns "::" #mod);\
-    }\
-};\
-class ns##mod##ModuleRegistrar\
-{\
-public:\
-    ns##mod##ModuleRegistrar(void)\
-    {\
-        ModuleFactory &modFac = ModuleFactory::getInstance();\
-        modFac.registerBuilder(#ns "::" #mod, [&](const std::string name)\
-                              {\
-                                  return std::unique_ptr<ns::mod>(new ns::mod(name));\
-                              });\
-    }\
-};\
-static ns##mod##ModuleRegistrar ns##mod##ModuleRegistrarInstance;
-
-#define ARG(...) __VA_ARGS__
-
-/******************************************************************************
- *                            Module class                                    *
- ******************************************************************************/
-// base class
-class ModuleBase
-{
-public:
-    // constructor
-    ModuleBase(const std::string name);
-    // destructor
-    virtual ~ModuleBase(void) = default;
-    // access
-    std::string getName(void) const;
-    Environment &env(void) const;
-    // get factory registration name if available
-    virtual std::string getRegisteredName(void);
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void) = 0;
-    virtual std::vector<std::string> getOutput(void) = 0;
-    // parse parameters
-    virtual void parseParameters(XmlReader &reader, const std::string name) = 0;
-    virtual void saveParameters(XmlWriter &writer, const std::string name) = 0;
-    // setup
-    virtual void setup(void) {};
-    // execution
-    void operator()(void);
-    virtual void execute(void) = 0;
-private:
-    std::string name_;
-    Environment &env_;
-};
-
-// derived class, templating the parameter class
-template <typename P>
-class Module: public ModuleBase
-{
-public:
-    typedef P Par;
-public:
-    // constructor
-    Module(const std::string name);
-    // destructor
-    virtual ~Module(void) = default;
-    // parse parameters
-    virtual void parseParameters(XmlReader &reader, const std::string name);
-    virtual void saveParameters(XmlWriter &writer, const std::string name);
-    // parameter access
-    const P & par(void) const;
-    void      setPar(const P &par);
-private:
-    P par_;
-};
-
-// no parameter type
-class NoPar {};
-
-template <>
-class Module<NoPar>: public ModuleBase
-{
-public:
-    // constructor
-    Module(const std::string name): ModuleBase(name) {};
-    // destructor
-    virtual ~Module(void) = default;
-    // parse parameters (do nothing)
-    virtual void parseParameters(XmlReader &reader, const std::string name) {};
-    virtual void saveParameters(XmlWriter &writer, const std::string name)
-    {
-        push(writer, "options");
-        pop(writer);
-    };
-};
-
-/******************************************************************************
- *                           Template implementation                          *
- ******************************************************************************/
-template <typename P>
-Module<P>::Module(const std::string name)
-: ModuleBase(name)
-{}
-
-template <typename P>
-void Module<P>::parseParameters(XmlReader &reader, const std::string name)
-{
-    read(reader, name, par_);
-}
-
-template <typename P>
-void Module<P>::saveParameters(XmlWriter &writer, const std::string name)
-{
-    write(writer, name, par_);
-}
-
-template <typename P>
-const P & Module<P>::par(void) const
-{
-    return par_;
-}
-
-template <typename P>
-void Module<P>::setPar(const P &par)
-{
-    par_ = par;
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Module_hpp_

extras/Hadrons/Modules.hpp

@@ -1,40 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/Hadrons/Modules/MAction/DWF.hpp>
-#include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
-#include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
-#include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
-#include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
-#include <Grid/Hadrons/Modules/MSource/Point.hpp>
-#include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
-#include <Grid/Hadrons/Modules/MSource/Z2.hpp>
-#include <Grid/Hadrons/Modules/Quark.hpp>

extras/Hadrons/Modules/MAction/DWF.hpp

@@ -1,134 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MAction/DWF.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_DWF_hpp_
-#define Hadrons_DWF_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                     Domain wall quark action                               *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MAction)
-
-class DWFPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(DWFPar,
-                                    std::string, gauge,
-                                    unsigned int, Ls,
-                                    double      , mass,
-                                    double      , M5);
-};
-
-template <typename FImpl>
-class TDWF: public Module<DWFPar>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TDWF(const std::string name);
-    // destructor
-    virtual ~TDWF(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(DWF, TDWF<FIMPL>, MAction);
-
-/******************************************************************************
- *                        DWF template implementation                         *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TDWF<FImpl>::TDWF(const std::string name)
-: Module<DWFPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TDWF<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().gauge};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TDWF<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TDWF<FImpl>::setup(void)
-{
-    unsigned int size;
-    
-    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
-    env().registerObject(getName(), size, par().Ls);
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TDWF<FImpl>::execute(void)
-{
-    LOG(Message) << "Setting up domain wall fermion matrix with m= "
-                 << par().mass << ", M5= " << par().M5 << " and Ls= "
-                 << par().Ls << " using gauge field '" << par().gauge << "'"
-                 << std::endl;
-    env().createGrid(par().Ls);
-    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
-    auto &g4     = *env().getGrid();
-    auto &grb4   = *env().getRbGrid();
-    auto &g5     = *env().getGrid(par().Ls);
-    auto &grb5   = *env().getRbGrid(par().Ls);
-    FMat *fMatPt = new DomainWallFermion<FImpl>(U, g5, grb5, g4, grb4,
-                                                par().mass, par().M5);
-    env().setObject(getName(), fMatPt);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_DWF_hpp_

extras/Hadrons/Modules/MAction/Wilson.hpp

@@ -1,126 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MAction/Wilson.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Wilson_hpp_
-#define Hadrons_Wilson_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                            TWilson quark action                            *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MAction)
-
-class WilsonPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonPar,
-                                    std::string, gauge,
-                                    double     , mass);
-};
-
-template <typename FImpl>
-class TWilson: public Module<WilsonPar>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TWilson(const std::string name);
-    // destructor
-    virtual ~TWilson(void) = default;
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Wilson, TWilson<FIMPL>, MAction);
-
-/******************************************************************************
- *                     TWilson template implementation                        *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TWilson<FImpl>::TWilson(const std::string name)
-: Module<WilsonPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TWilson<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().gauge};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TWilson<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TWilson<FImpl>::setup(void)
-{
-    unsigned int size;
-    
-    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
-    env().registerObject(getName(), size);
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TWilson<FImpl>::execute()
-{
-    LOG(Message) << "Setting up TWilson fermion matrix with m= " << par().mass
-                 << " using gauge field '" << par().gauge << "'" << std::endl;
-    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
-    auto &grid   = *env().getGrid();
-    auto &gridRb = *env().getRbGrid();
-    FMat *fMatPt = new WilsonFermion<FImpl>(U, grid, gridRb, par().mass);
-    env().setObject(getName(), fMatPt);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Wilson_hpp_

extras/Hadrons/Modules/MContraction/Baryon.hpp

@@ -1,131 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MContraction/Baryon.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Baryon_hpp_
-#define Hadrons_Baryon_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                               Baryon                                       *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MContraction)
-
-class BaryonPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(BaryonPar,
-                                    std::string, q1,
-                                    std::string, q2,
-                                    std::string, q3,
-                                    std::string, output);
-};
-
-template <typename FImpl1, typename FImpl2, typename FImpl3>
-class TBaryon: public Module<BaryonPar>
-{
-public:
-    TYPE_ALIASES(FImpl1, 1);
-    TYPE_ALIASES(FImpl2, 2);
-    TYPE_ALIASES(FImpl3, 3);
-    class Result: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
-                                        std::vector<std::vector<std::vector<Complex>>>, corr);
-    };
-public:
-    // constructor
-    TBaryon(const std::string name);
-    // destructor
-    virtual ~TBaryon(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Baryon, ARG(TBaryon<FIMPL, FIMPL, FIMPL>), MContraction);
-
-/******************************************************************************
- *                         TBaryon implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2, typename FImpl3>
-TBaryon<FImpl1, FImpl2, FImpl3>::TBaryon(const std::string name)
-: Module<BaryonPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2, typename FImpl3>
-std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getInput(void)
-{
-    std::vector<std::string> input = {par().q1, par().q2, par().q3};
-    
-    return input;
-}
-
-template <typename FImpl1, typename FImpl2, typename FImpl3>
-std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2, typename FImpl3>
-void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
-{
-    LOG(Message) << "Computing baryon contractions '" << getName() << "' using"
-                 << " quarks '" << par().q1 << "', '" << par().q2 << "', and '"
-                 << par().q3 << "'" << std::endl;
-    
-    XmlWriter             writer(par().output);
-    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q2);
-    LatticeComplex        c(env().getGrid());
-    Result                result;
-    
-    // FIXME: do contractions
-    
-    write(writer, "meson", result);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Baryon_hpp_

extras/Hadrons/Modules/MContraction/Meson.hpp

@@ -1,137 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MContraction/Meson.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Meson_hpp_
-#define Hadrons_Meson_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                                TMeson                                       *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MContraction)
-
-class MesonPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(MesonPar,
-                                    std::string,    q1,
-                                    std::string,    q2,
-                                    std::string,    output,
-                                    Gamma::Algebra, gammaSource,
-                                    Gamma::Algebra, gammaSink);
-};
-
-template <typename FImpl1, typename FImpl2>
-class TMeson: public Module<MesonPar>
-{
-public:
-    TYPE_ALIASES(FImpl1, 1);
-    TYPE_ALIASES(FImpl2, 2);
-    class Result: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result, std::vector<Complex>, corr);
-    };
-public:
-    // constructor
-    TMeson(const std::string name);
-    // destructor
-    virtual ~TMeson(void) = default;
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Meson, ARG(TMeson<FIMPL, FIMPL>), MContraction);
-
-/******************************************************************************
- *                           TMeson implementation                            *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2>
-TMeson<FImpl1, FImpl2>::TMeson(const std::string name)
-: Module<MesonPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2>
-std::vector<std::string> TMeson<FImpl1, FImpl2>::getInput(void)
-{
-    std::vector<std::string> input = {par().q1, par().q2};
-    
-    return input;
-}
-
-template <typename FImpl1, typename FImpl2>
-std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
-{
-    std::vector<std::string> output = {getName()};
-    
-    return output;
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2>
-void TMeson<FImpl1, FImpl2>::execute(void)
-{
-    LOG(Message) << "Computing meson contractions '" << getName() << "' using"
-                 << " quarks '" << par().q1 << "' and '" << par().q2 << "'"
-                 << std::endl;
-    
-    XmlWriter             writer(par().output);
-    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
-    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
-    LatticeComplex        c(env().getGrid());
-    Gamma                 gSrc(par().gammaSource), gSnk(par().gammaSink);
-    Gamma                 g5(Gamma::Algebra::Gamma5);
-    std::vector<TComplex> buf;
-    Result                result;
-    
-    c = trace(gSnk*q1*adj(gSrc)*g5*adj(q2)*g5);
-    sliceSum(c, buf, Tp);
-    result.corr.resize(buf.size());
-    for (unsigned int t = 0; t < buf.size(); ++t)
-    {
-        result.corr[t] = TensorRemove(buf[t]);
-    }
-    write(writer, "meson", result);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Meson_hpp_

extras/Hadrons/Modules/MGauge/Load.cc

@@ -1,78 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/Load.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
-
-using namespace Grid;
-using namespace Hadrons;
-using namespace MGauge;
-
-/******************************************************************************
-*                           TLoad implementation                               *
-******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-TLoad::TLoad(const std::string name)
-: Module<LoadPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-std::vector<std::string> TLoad::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-std::vector<std::string> TLoad::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-void TLoad::setup(void)
-{
-    env().registerLattice<LatticeGaugeField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-void TLoad::execute(void)
-{
-    NerscField  header;
-    std::string fileName = par().file + "."
-                           + std::to_string(env().getTrajectory());
-    
-    LOG(Message) << "Loading NERSC configuration from file '" << fileName
-                 << "'" << std::endl;
-    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
-    NerscIO::readConfiguration(U, header, fileName);
-    LOG(Message) << "NERSC header:" << std::endl;
-    dump_nersc_header(header, LOG(Message));
-}

extras/Hadrons/Modules/MGauge/Load.hpp

@@ -1,73 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/Load.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Load_hpp_
-#define Hadrons_Load_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         Load a NERSC configuration                         *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MGauge)
-
-class LoadPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadPar,
-                                    std::string, file);
-};
-
-class TLoad: public Module<LoadPar>
-{
-public:
-    // constructor
-    TLoad(const std::string name);
-    // destructor
-    virtual ~TLoad(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Load, TLoad, MGauge);
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Load_hpp_

extras/Hadrons/Modules/MGauge/Random.cc

@@ -1,69 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/Random.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
-
-using namespace Grid;
-using namespace Hadrons;
-using namespace MGauge;
-
-/******************************************************************************
-*                           TRandom implementation                            *
-******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-TRandom::TRandom(const std::string name)
-: Module<NoPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-std::vector<std::string> TRandom::getInput(void)
-{
-    return std::vector<std::string>();
-}
-
-std::vector<std::string> TRandom::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-void TRandom::setup(void)
-{
-    env().registerLattice<LatticeGaugeField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-void TRandom::execute(void)
-{
-    LOG(Message) << "Generating random gauge configuration" << std::endl;
-    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
-    SU3::HotConfiguration(*env().get4dRng(), U);
-}

extras/Hadrons/Modules/MGauge/Random.hpp

@@ -1,66 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/Random.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Random_hpp_
-#define Hadrons_Random_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                             Random gauge                                   *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MGauge)
-
-class TRandom: public Module<NoPar>
-{
-public:
-    // constructor
-    TRandom(const std::string name);
-    // destructor
-    virtual ~TRandom(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Random, TRandom, MGauge);
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Random_hpp_

extras/Hadrons/Modules/MGauge/Unit.cc

@@ -1,69 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/Unit.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
-
-using namespace Grid;
-using namespace Hadrons;
-using namespace MGauge;
-
-/******************************************************************************
-*                            TUnit implementation                             *
-******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-TUnit::TUnit(const std::string name)
-: Module<NoPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-std::vector<std::string> TUnit::getInput(void)
-{
-    return std::vector<std::string>();
-}
-
-std::vector<std::string> TUnit::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-void TUnit::setup(void)
-{
-    env().registerLattice<LatticeGaugeField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-void TUnit::execute(void)
-{
-    LOG(Message) << "Creating unit gauge configuration" << std::endl;
-    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
-    SU3::ColdConfiguration(*env().get4dRng(), U);
-}

extras/Hadrons/Modules/MGauge/Unit.hpp

@@ -1,66 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/Unit.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Unit_hpp_
-#define Hadrons_Unit_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                              Unit gauge                                    *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MGauge)
-
-class TUnit: public Module<NoPar>
-{
-public:
-    // constructor
-    TUnit(const std::string name);
-    // destructor
-    virtual ~TUnit(void) = default;
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Unit, TUnit, MGauge);
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Unit_hpp_

extras/Hadrons/Modules/MSolver/RBPrecCG.hpp

@@ -1,132 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSolver/RBPrecCG.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_RBPrecCG_hpp_
-#define Hadrons_RBPrecCG_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                     Schur red-black preconditioned CG                      *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MSolver)
-
-class RBPrecCGPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(RBPrecCGPar,
-                                    std::string, action,
-                                    double     , residual);
-};
-
-template <typename FImpl>
-class TRBPrecCG: public Module<RBPrecCGPar>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TRBPrecCG(const std::string name);
-    // destructor
-    virtual ~TRBPrecCG(void) = default;
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(RBPrecCG, TRBPrecCG<FIMPL>, MSolver);
-
-/******************************************************************************
- *                      TRBPrecCG template implementation                     *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TRBPrecCG<FImpl>::TRBPrecCG(const std::string name)
-: Module(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TRBPrecCG<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().action};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TRBPrecCG<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TRBPrecCG<FImpl>::setup(void)
-{
-    auto Ls = env().getObjectLs(par().action);
-    
-    env().registerObject(getName(), 0, Ls);
-    env().addOwnership(getName(), par().action);
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TRBPrecCG<FImpl>::execute(void)
-{
-    auto &mat   = *(env().template getObject<FMat>(par().action));
-    auto solver = [&mat, this](FermionField &sol, const FermionField &source)
-    {
-        ConjugateGradient<FermionField>           cg(par().residual, 10000);
-        SchurRedBlackDiagMooeeSolve<FermionField> schurSolver(cg);
-        
-        schurSolver(mat, source, sol);
-    };
-    
-    LOG(Message) << "setting up Schur red-black preconditioned CG for"
-                 << " action '" << par().action << "' with residual "
-                 << par().residual << std::endl;
-    env().setObject(getName(), new SolverFn(solver));
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_RBPrecCG_hpp_

extras/Hadrons/Modules/MSource/Point.hpp

@@ -1,135 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSource/Point.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Point_hpp_
-#define Hadrons_Point_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/*
- 
- Point source
- ------------
- * src_x = delta_x,position
- 
- * options:
- - position: space-separated integer sequence (e.g. "0 1 1 0")
- 
- */
-
-/******************************************************************************
- *                                  TPoint                                     *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MSource)
-
-class PointPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(PointPar,
-                                    std::string, position);
-};
-
-template <typename FImpl>
-class TPoint: public Module<PointPar>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TPoint(const std::string name);
-    // destructor
-    virtual ~TPoint(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Point, TPoint<FIMPL>, MSource);
-
-/******************************************************************************
- *                       TPoint template implementation                       *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TPoint<FImpl>::TPoint(const std::string name)
-: Module<PointPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TPoint<FImpl>::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TPoint<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TPoint<FImpl>::setup(void)
-{
-    env().template registerLattice<PropagatorField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TPoint<FImpl>::execute(void)
-{
-    std::vector<int> position = strToVec<int>(par().position);
-    typename SitePropagator::scalar_object id;
-    
-    LOG(Message) << "Creating point source at position [" << par().position
-                 << "]" << std::endl;
-    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
-    id  = 1.;
-    src = zero;
-    pokeSite(id, src, position);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Point_hpp_

extras/Hadrons/Modules/MSource/SeqGamma.hpp

@@ -1,163 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSource/SeqGamma.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_SeqGamma_hpp_
-#define Hadrons_SeqGamma_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/*
- 
- Sequential source
- -----------------------------
- * src_x = q_x * theta(x_3 - tA) * theta(tB - x_3) * gamma * exp(i x.mom)
- 
- * options:
- - q: input propagator (string)
- - tA: begin timeslice (integer)
- - tB: end timesilce (integer)
- - gamma: gamma product to insert (integer)
- - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0.")
- 
- */
-
-/******************************************************************************
- *                         SeqGamma                                 *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MSource)
-
-class SeqGammaPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(SeqGammaPar,
-                                    std::string,    q,
-                                    unsigned int,   tA,
-                                    unsigned int,   tB,
-                                    Gamma::Algebra, gamma,
-                                    std::string,    mom);
-};
-
-template <typename FImpl>
-class TSeqGamma: public Module<SeqGammaPar>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TSeqGamma(const std::string name);
-    // destructor
-    virtual ~TSeqGamma(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(SeqGamma, TSeqGamma<FIMPL>, MSource);
-
-/******************************************************************************
- *                         TSeqGamma implementation                           *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TSeqGamma<FImpl>::TSeqGamma(const std::string name)
-: Module<SeqGammaPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TSeqGamma<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().q};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TSeqGamma<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TSeqGamma<FImpl>::setup(void)
-{
-    env().template registerLattice<PropagatorField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TSeqGamma<FImpl>::execute(void)
-{
-    if (par().tA == par().tB)
-    {
-        LOG(Message) << "Generating gamma_" << par().gamma
-                     << " sequential source at t= " << par().tA << std::endl;
-    }
-    else
-    {
-        LOG(Message) << "Generating gamma_" << par().gamma
-                     << " sequential source for "
-                     << par().tA << " <= t <= " << par().tB << std::endl;
-    }
-    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
-    PropagatorField &q   = *env().template getObject<PropagatorField>(par().q);
-    Lattice<iScalar<vInteger>> t(env().getGrid());
-    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
-    Gamma                      g(par().gamma);
-    std::vector<Real>          p;
-    Complex                    i(0.0,1.0);
-    
-    p  = strToVec<Real>(par().mom);
-    ph = zero;
-    for(unsigned int mu = 0; mu < env().getNd(); mu++)
-    {
-        LatticeCoordinate(coor, mu);
-        ph = ph + p[mu]*coor;
-    }
-    ph = exp(i*ph);
-    LatticeCoordinate(t, Tp);
-    src = where((t >= par().tA) and (t <= par().tB), ph*(g*q), 0.*q);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_SeqGamma_hpp_

extras/Hadrons/Modules/MSource/Z2.hpp

@@ -1,151 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSource/Z2.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Z2_hpp_
-#define Hadrons_Z2_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/*
- 
- Z_2 stochastic source
- -----------------------------
- * src_x = eta_x * theta(x_3 - tA) * theta(tB - x_3)
- 
- the eta_x are independent uniform random numbers in {+/- 1 +/- i}
- 
- * options:
- - tA: begin timeslice (integer)
- - tB: end timesilce (integer)
- 
- */
- 
-/******************************************************************************
- *                          Z2 stochastic source                              *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MSource)
-
-class Z2Par: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(Z2Par,
-                                    unsigned int, tA,
-                                    unsigned int, tB);
-};
-
-template <typename FImpl>
-class TZ2: public Module<Z2Par>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TZ2(const std::string name);
-    // destructor
-    virtual ~TZ2(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Z2, TZ2<FIMPL>, MSource);
-
-/******************************************************************************
- *                       TZ2 template implementation                          *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TZ2<FImpl>::TZ2(const std::string name)
-: Module<Z2Par>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TZ2<FImpl>::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TZ2<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TZ2<FImpl>::setup(void)
-{
-    env().template registerLattice<PropagatorField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TZ2<FImpl>::execute(void)
-{
-    Lattice<iScalar<vInteger>> t(env().getGrid());
-    LatticeComplex             eta(env().getGrid());
-    Complex                    shift(1., 1.);
-    
-    if (par().tA == par().tB)
-    {
-        LOG(Message) << "Generating Z_2 wall source at t= " << par().tA
-        << std::endl;
-    }
-    else
-    {
-        LOG(Message) << "Generating Z_2 band for " << par().tA << " <= t <= "
-        << par().tB << std::endl;
-    }
-    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
-    LatticeCoordinate(t, Tp);
-    bernoulli(*env().get4dRng(), eta);
-    eta = (2.*eta - shift)*(1./::sqrt(2.));
-    eta = where((t >= par().tA) and (t <= par().tB), eta, 0.*eta);
-    src = 1.;
-    src = src*eta;
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Z2_hpp_

extras/Hadrons/Modules/Quark.hpp

@@ -1,185 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/Quark.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Quark_hpp_
-#define Hadrons_Quark_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                               TQuark                                       *
- ******************************************************************************/
-class QuarkPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(QuarkPar,
-                                    std::string, source,
-                                    std::string, solver);
-};
-
-template <typename FImpl>
-class TQuark: public Module<QuarkPar>
-{
-public:
-    TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TQuark(const std::string name);
-    // destructor
-    virtual ~TQuark(void) = default;
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-private:
-    unsigned int Ls_;
-    SolverFn     *solver_{nullptr};
-};
-
-MODULE_REGISTER(Quark, TQuark<FIMPL>);
-
-/******************************************************************************
- *                          TQuark implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TQuark<FImpl>::TQuark(const std::string name)
-: Module(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TQuark<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().source, par().solver};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TQuark<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName(), getName() + "_5d"};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TQuark<FImpl>::setup(void)
-{
-    Ls_ = env().getObjectLs(par().solver);
-    env().template registerLattice<PropagatorField>(getName());
-    if (Ls_ > 1)
-    {
-        env().template registerLattice<PropagatorField>(getName() + "_5d", Ls_);
-    }
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TQuark<FImpl>::execute(void)
-{
-    LOG(Message) << "Computing quark propagator '" << getName() << "'"
-                 << std::endl;
-    
-    FermionField    source(env().getGrid(Ls_)), sol(env().getGrid(Ls_)),
-                    tmp(env().getGrid());
-    std::string     propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
-    PropagatorField &prop    = *env().template createLattice<PropagatorField>(propName);
-    PropagatorField &fullSrc = *env().template getObject<PropagatorField>(par().source);
-    SolverFn        &solver  = *env().template getObject<SolverFn>(par().solver);
-    if (Ls_ > 1)
-    {
-        env().template createLattice<PropagatorField>(getName());
-    }
-    
-    LOG(Message) << "Inverting using solver '" << par().solver
-                 << "' on source '" << par().source << "'" << std::endl;
-    for (unsigned int s = 0; s < Ns; ++s)
-    for (unsigned int c = 0; c < Nc; ++c)
-    {
-        LOG(Message) << "Inversion for spin= " << s << ", color= " << c
-        << std::endl;
-        // source conversion for 4D sources
-        if (!env().isObject5d(par().source))
-        {
-            if (Ls_ == 1)
-            {
-                PropToFerm(source, fullSrc, s, c);
-            }
-            else
-            {
-                source = zero;
-                PropToFerm(tmp, fullSrc, s, c);
-                InsertSlice(tmp, source, 0, 0);
-                InsertSlice(tmp, source, Ls_-1, 0);
-                axpby_ssp_pplus(source, 0., source, 1., source, 0, 0);
-                axpby_ssp_pminus(source, 0., source, 1., source, Ls_-1, Ls_-1);
-            }
-        }
-        // source conversion for 5D sources
-        else
-        {
-            if (Ls_ != env().getObjectLs(par().source))
-            {
-                HADRON_ERROR("Ls mismatch between quark action and source");
-            }
-            else
-            {
-                PropToFerm(source, fullSrc, s, c);
-            }
-        }
-        sol = zero;
-        solver(sol, source);
-        FermToProp(prop, sol, s, c);
-        // create 4D propagators from 5D one if necessary
-        if (Ls_ > 1)
-        {
-            PropagatorField &p4d =
-                *env().template getObject<PropagatorField>(getName());
-            
-            axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0);
-            axpby_ssp_pplus(sol, 0., sol, 1., sol, 0, Ls_-1);
-            ExtractSlice(tmp, sol, 0, 0);
-            FermToProp(p4d, tmp, s, c);
-        }
-    }
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Quark_hpp_

extras/Hadrons/Modules/templates/Module.cc.template

@@ -1,39 +0,0 @@
-#include <Grid/Hadrons/Modules/___FILEBASENAME___.hpp>
-
-using namespace Grid;
-using namespace Hadrons;
-
-/******************************************************************************
-*                  T___FILEBASENAME___ implementation                             *
-******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-T___FILEBASENAME___::T___FILEBASENAME___(const std::string name)
-: Module<___FILEBASENAME___Par>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-std::vector<std::string> T___FILEBASENAME___::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-std::vector<std::string> T___FILEBASENAME___::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-void T___FILEBASENAME___::setup(void)
-{
-
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-void T___FILEBASENAME___::execute(void)
-{
-
-}

extras/Hadrons/Modules/templates/Module.hpp.template

@@ -1,40 +0,0 @@
-#ifndef Hadrons____FILEBASENAME____hpp_
-#define Hadrons____FILEBASENAME____hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         ___FILEBASENAME___                                 *
- ******************************************************************************/
-class ___FILEBASENAME___Par: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par, 
-                                    unsigned int, i);
-};
-
-class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
-{
-public:
-    // constructor
-    T___FILEBASENAME___(const std::string name);
-    // destructor
-    virtual ~T___FILEBASENAME___(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER(___FILEBASENAME___, T___FILEBASENAME___);
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons____FILEBASENAME____hpp_

extras/Hadrons/Modules/templates/Module_in_NS.cc.template

@@ -1,40 +0,0 @@
-#include <Grid/Hadrons/Modules/___NAMESPACE___/___FILEBASENAME___.hpp>
-
-using namespace Grid;
-using namespace Hadrons;
-using namespace ___NAMESPACE___;
-
-/******************************************************************************
-*                  T___FILEBASENAME___ implementation                             *
-******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-T___FILEBASENAME___::T___FILEBASENAME___(const std::string name)
-: Module<___FILEBASENAME___Par>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-std::vector<std::string> T___FILEBASENAME___::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-std::vector<std::string> T___FILEBASENAME___::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-void T___FILEBASENAME___::setup(void)
-{
-
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-void T___FILEBASENAME___::execute(void)
-{
-
-}

extras/Hadrons/Modules/templates/Module_in_NS.hpp.template

@@ -1,44 +0,0 @@
-#ifndef Hadrons____FILEBASENAME____hpp_
-#define Hadrons____FILEBASENAME____hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         ___FILEBASENAME___                                 *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(___NAMESPACE___)
-
-class ___FILEBASENAME___Par: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par,
-                                    unsigned int, i);
-};
-
-class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
-{
-public:
-    // constructor
-    T___FILEBASENAME___(const std::string name);
-    // destructor
-    virtual ~T___FILEBASENAME___(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(___FILEBASENAME___, T___FILEBASENAME___, ___NAMESPACE___);
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons____FILEBASENAME____hpp_

extras/Hadrons/Modules/templates/Module_tmp.hpp.template

@@ -1,81 +0,0 @@
-#ifndef Hadrons____FILEBASENAME____hpp_
-#define Hadrons____FILEBASENAME____hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         ___FILEBASENAME___                                 *
- ******************************************************************************/
-class ___FILEBASENAME___Par: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par,
-                                    unsigned int, i);
-};
-
-template <typename FImpl>
-class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
-{
-public:
-    // constructor
-    T___FILEBASENAME___(const std::string name);
-    // destructor
-    virtual ~T___FILEBASENAME___(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER(___FILEBASENAME___, T___FILEBASENAME___<FIMPL>);
-
-/******************************************************************************
- *                 T___FILEBASENAME___ implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-T___FILEBASENAME___<FImpl>::T___FILEBASENAME___(const std::string name)
-: Module<___FILEBASENAME___Par>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> T___FILEBASENAME___<FImpl>::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> T___FILEBASENAME___<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void T___FILEBASENAME___<FImpl>::setup(void)
-{
-    
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void T___FILEBASENAME___<FImpl>::execute(void)
-{
-    
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons____FILEBASENAME____hpp_

extras/Hadrons/Modules/templates/Module_tmp_in_NS.hpp.template

@@ -1,85 +0,0 @@
-#ifndef Hadrons____FILEBASENAME____hpp_
-#define Hadrons____FILEBASENAME____hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         ___FILEBASENAME___                                 *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(___NAMESPACE___)
-
-class ___FILEBASENAME___Par: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(___FILEBASENAME___Par,
-                                    unsigned int, i);
-};
-
-template <typename FImpl>
-class T___FILEBASENAME___: public Module<___FILEBASENAME___Par>
-{
-public:
-    // constructor
-    T___FILEBASENAME___(const std::string name);
-    // destructor
-    virtual ~T___FILEBASENAME___(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(___FILEBASENAME___, T___FILEBASENAME___<FIMPL>, ___NAMESPACE___);
-
-/******************************************************************************
- *                 T___FILEBASENAME___ implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-T___FILEBASENAME___<FImpl>::T___FILEBASENAME___(const std::string name)
-: Module<___FILEBASENAME___Par>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> T___FILEBASENAME___<FImpl>::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> T___FILEBASENAME___<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void T___FILEBASENAME___<FImpl>::setup(void)
-{
-    
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void T___FILEBASENAME___<FImpl>::execute(void)
-{
-    
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons____FILEBASENAME____hpp_

extras/Hadrons/add_module.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-if (( $# != 1 && $# != 2)); then
-    echo "usage: `basename $0` <module name> [<namespace>]" 1>&2
-    exit 1
-fi
-NAME=$1
-NS=$2
-
-if (( $# == 1 )); then
-	if [ -e "Modules/${NAME}.cc" ] || [ -e "Modules/${NAME}.hpp" ]; then
-	    echo "error: files Modules/${NAME}.* already exists" 1>&2
-	    exit 1
-	fi
-	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module.cc.template > Modules/${NAME}.cc
-	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module.hpp.template > Modules/${NAME}.hpp
-elif (( $# == 2 )); then
-	mkdir -p Modules/${NS}
-	if [ -e "Modules/${NS}/${NAME}.cc" ] || [ -e "Modules/${NS}/${NAME}.hpp" ]; then
-	    echo "error: files Modules/${NS}/${NAME}.* already exists" 1>&2
-	    exit 1
-	fi
-	TMPCC=".${NS}.${NAME}.tmp.cc"
-	TMPHPP=".${NS}.${NAME}.tmp.hpp"
-	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_in_NS.cc.template  > ${TMPCC}
-	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_in_NS.hpp.template > ${TMPHPP}
-	sed "s/___NAMESPACE___/${NS}/g" ${TMPCC}  > Modules/${NS}/${NAME}.cc
-	sed "s/___NAMESPACE___/${NS}/g" ${TMPHPP} > Modules/${NS}/${NAME}.hpp
-	rm -f ${TMPCC} ${TMPHPP}
-fi
-./make_module_list.sh

extras/Hadrons/add_module_template.sh

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-if (( $# != 1 && $# != 2)); then
-    echo "usage: `basename $0` <module name> [<namespace>]" 1>&2
-    exit 1
-fi
-NAME=$1
-NS=$2
-
-if (( $# == 1 )); then
-	if [ -e "Modules/${NAME}.cc" ] || [ -e "Modules/${NAME}.hpp" ]; then
-	    echo "error: files Modules/${NAME}.* already exists" 1>&2
-	    exit 1
-	fi
-	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_tmp.hpp.template > Modules/${NAME}.hpp
-elif (( $# == 2 )); then
-	mkdir -p Modules/${NS}
-	if [ -e "Modules/${NS}/${NAME}.cc" ] || [ -e "Modules/${NS}/${NAME}.hpp" ]; then
-	    echo "error: files Modules/${NS}/${NAME}.* already exists" 1>&2
-	    exit 1
-	fi
-	TMPCC=".${NS}.${NAME}.tmp.cc"
-	TMPHPP=".${NS}.${NAME}.tmp.hpp"
-	sed "s/___FILEBASENAME___/${NAME}/g" Modules/templates/Module_tmp_in_NS.hpp.template > ${TMPHPP}
-	sed "s/___NAMESPACE___/${NS}/g" ${TMPHPP} > Modules/${NS}/${NAME}.hpp
-	rm -f ${TMPCC} ${TMPHPP}
-fi
-./make_module_list.sh

extras/Hadrons/make_module_list.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-echo 'modules_cc =\' > modules.inc
-find Modules -name '*.cc' -type f -print | sed 's/^/  /;$q;s/$/ \\/' >> modules.inc
-echo '' >> modules.inc
-echo 'modules_hpp =\' >> modules.inc
-find Modules -name '*.hpp' -type f -print | sed 's/^/  /;$q;s/$/ \\/' >> modules.inc
-echo '' >> modules.inc
-rm -f Modules.hpp
-for f in `find Modules -name '*.hpp'`; do
-	echo "#include <Grid/Hadrons/${f}>" >> Modules.hpp
-done

extras/Hadrons/modules.inc

@@ -1,19 +0,0 @@
-modules_cc =\
-  Modules/MGauge/Load.cc \
-  Modules/MGauge/Random.cc \
-  Modules/MGauge/Unit.cc
-
-modules_hpp =\
-  Modules/MAction/DWF.hpp \
-  Modules/MAction/Wilson.hpp \
-  Modules/MContraction/Baryon.hpp \
-  Modules/MContraction/Meson.hpp \
-  Modules/MGauge/Load.hpp \
-  Modules/MGauge/Random.hpp \
-  Modules/MGauge/Unit.hpp \
-  Modules/MSolver/RBPrecCG.hpp \
-  Modules/MSource/Point.hpp \
-  Modules/MSource/SeqGamma.hpp \
-  Modules/MSource/Z2.hpp \
-  Modules/Quark.hpp
-

extras/Makefile.am

@@ -1 +0,0 @@
-SUBDIRS = Hadrons

lib/Algorithms.h

@@ -39,7 +39,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/approx/MultiShiftFunction.h>
 
 #include <Grid/algorithms/iterative/ConjugateGradient.h>
-#include <Grid/algorithms/iterative/ConjugateGradientShifted.h>
 #include <Grid/algorithms/iterative/ConjugateResidual.h>
 #include <Grid/algorithms/iterative/NormalEquations.h>
 #include <Grid/algorithms/iterative/SchurRedBlack.h>

lib/AlignedAllocator.cc

@@ -1,65 +0,0 @@
-
-
-
-#include <Grid/Grid.h>
-
-namespace Grid {
-
-int PointerCache::victim;
-
-  PointerCache::PointerCacheEntry PointerCache::Entries[PointerCache::Ncache];
-
-void *PointerCache::Insert(void *ptr,size_t bytes) {
-
-  if (bytes < 4096 ) return NULL;
-
-#ifdef _OPENMP
-  assert(omp_in_parallel()==0);
-#endif 
-  void * ret = NULL;
-  int v = -1;
-
-  for(int e=0;e<Ncache;e++) {
-    if ( Entries[e].valid==0 ) {
-      v=e; 
-      break;
-    }
-  }
-
-  if ( v==-1 ) {
-    v=victim;
-    victim = (victim+1)%Ncache;
-  }
-
-  if ( Entries[v].valid ) {
-    ret = Entries[v].address;
-    Entries[v].valid = 0;
-    Entries[v].address = NULL;
-    Entries[v].bytes = 0;
-  }
-
-  Entries[v].address=ptr;
-  Entries[v].bytes  =bytes;
-  Entries[v].valid  =1;
-
-  return ret;
-}
-
-void *PointerCache::Lookup(size_t bytes) {
-
- if (bytes < 4096 ) return NULL;
-
-#ifdef _OPENMP
-  assert(omp_in_parallel()==0);
-#endif 
-
-  for(int e=0;e<Ncache;e++){
-    if ( Entries[e].valid && ( Entries[e].bytes == bytes ) ) {
-      Entries[e].valid = 0;
-      return Entries[e].address;
-    }
-  }
-  return NULL;
-}
-
-}

lib/AlignedAllocator.h

@@ -1,4 +1,4 @@
-/*************************************************************************************
+    /*************************************************************************************
 
     Grid physics library, www.github.com/paboyle/Grid 
 
@@ -42,32 +42,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 
 namespace Grid {
 
-  class PointerCache {
-  private:
-
-    static const int Ncache=8;
-    static int victim;
-
-    typedef struct { 
-      void *address;
-      size_t bytes;
-      int valid;
-    } PointerCacheEntry;
-    
-    static PointerCacheEntry Entries[Ncache];
-
-  public:
-
-
-    static void *Insert(void *ptr,size_t bytes) ;
-    static void *Lookup(size_t bytes) ;
-
-  };
-
 ////////////////////////////////////////////////////////////////////
 // A lattice of something, but assume the something is SIMDized.
 ////////////////////////////////////////////////////////////////////
-
 template<typename _Tp>
 class alignedAllocator {
 public: 
@@ -89,27 +66,27 @@ public:
 
   pointer allocate(size_type __n, const void* _p= 0)
   { 
-    size_type bytes = __n*sizeof(_Tp);
-
-    _Tp *ptr = (_Tp *) PointerCache::Lookup(bytes);
-    
 #ifdef HAVE_MM_MALLOC_H
-    if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) _mm_malloc(bytes,128);
+    _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
 #else
-    if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) memalign(128,bytes);
+    _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp));
 #endif
 
+    _Tp tmp;
+#ifdef GRID_NUMA
+#pragma omp parallel for schedule(static)
+  for(int i=0;i<__n;i++){
+    ptr[i]=tmp;
+  }
+#endif 
     return ptr;
   }
 
-  void deallocate(pointer __p, size_type __n) { 
+  void deallocate(pointer __p, size_type) { 
-    size_type bytes = __n * sizeof(_Tp);
-    pointer __freeme = (pointer)PointerCache::Insert((void *)__p,bytes);
-
 #ifdef HAVE_MM_MALLOC_H
-    if ( __freeme ) _mm_free((void *)__freeme); 
+    _mm_free((void *)__p); 
 #else
-    if ( __freeme ) free((void *)__freeme);
+    free((void *)__p);
 #endif
   }
   void construct(pointer __p, const _Tp& __val) { };

lib/Bitwise.cc

@@ -2,12 +2,11 @@
 
 Grid physics library, www.github.com/paboyle/Grid
 
-Source file: extras/Hadrons/ModuleFactory.hpp
+Source file: ./lib/Bitwise.cc
 
-Copyright (C) 2015
 Copyright (C) 2016
 
-Author: Antonin Portelli <antonin.portelli@me.com>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -23,27 +22,28 @@ You should have received a copy of the GNU General Public License along
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-See the full license in the file "LICENSE" in the top level distribution directory
+See the full license in the file "LICENSE" in the top level distribution
+directory
 *************************************************************************************/
 /*  END LEGAL */
+#include <iostream>
+#include <Bitwise.h>
+#include <bitset>
+#include <climits>
 
-#ifndef Hadrons_ModuleFactory_hpp_
+namespace Grid {
-#define Hadrons_ModuleFactory_hpp_
 
-#include <Grid/Hadrons/Global.hpp>
+void show_binaryrep(const unsigned char* a, size_t size) {
-#include <Grid/Hadrons/Factory.hpp>
+  const unsigned char* beg = a;
-#include <Grid/Hadrons/Module.hpp>
+  const unsigned char* end = a + size;
+  unsigned int ctr = 0;
+  while (beg != end) {
+    std::cout << std::bitset<CHAR_BIT>(*beg++) << ' ';
+    ctr++;
+    if (ctr % GRID_REAL_BYTES == 0) std::cout << '\n';
+  }
+  std::cout << '\n';
+}
 
-BEGIN_HADRONS_NAMESPACE
+} // namespace 
 
-/******************************************************************************
- *                            ModuleFactory                                   *
- ******************************************************************************/
-class ModuleFactory: public Factory<ModuleBase>
-{
-    SINGLETON_DEFCTOR(ModuleFactory)
-};
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_ModuleFactory_hpp_

lib/Bitwise.h

@@ -0,0 +1,76 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/Bitwise.h
+
+Copyright (C) 2016
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef GRID_BITWISE_H
+#define GRID_BITWISE_H
+
+#include <cassert>
+#include <cfloat>
+#include <bitset>
+#include <climits>
+#include <Config.h>
+
+#ifdef GRID_DEFAULT_PRECISION_SINGLE
+#define GRID_REAL_BYTES 4
+#endif
+#ifdef GRID_DEFAULT_PRECISION_DOUBLE
+#define GRID_REAL_BYTES 8
+#endif
+
+
+namespace Grid {
+
+void show_binaryrep(const unsigned char* a, size_t size);
+
+template <typename T>
+void show_binaryrep(const T& a) {
+  const char* beg = reinterpret_cast<const char*>(&a);
+  const char* end = beg + sizeof(a);
+  unsigned int ctr = 0;
+  while (beg != end) {
+  	std::cout << std::bitset<CHAR_BIT>(*beg++) << ' ';
+  	ctr++;
+  	if (ctr % GRID_REAL_BYTES == 0) std::cout << '\n';
+  }
+  std::cout << '\n';
+}
+
+template <typename T>
+void bitwise_xor(T& l, T& r, unsigned char* xors) {
+  assert(sizeof(l) == sizeof(r));
+  unsigned char* org = reinterpret_cast<unsigned char*>(&l);
+  unsigned char* cur = reinterpret_cast<unsigned char*>(&r);
+  int words = sizeof(l) / sizeof(*org);
+  unsigned char result = 0;
+  for (int w = 0; w < words; w++) xors[w] = (org[w] ^ cur[w]);
+}
+
+}; // namespace 
+
+
+#endif

lib/Grid.h

@@ -59,13 +59,14 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 ///////////////////
 // Grid headers
 ///////////////////
+#include <Grid/serialisation/Serialisation.h>
 #include "Config.h"
 #include <Grid/Timer.h>
+#include <Grid/Bitwise.h>
 #include <Grid/PerfCount.h>
 #include <Grid/Log.h>
 #include <Grid/AlignedAllocator.h>
 #include <Grid/Simd.h>
-#include <Grid/serialisation/Serialisation.h>
 #include <Grid/Threads.h>
 #include <Grid/Lexicographic.h>
 #include <Grid/Init.h>

lib/Hadrons

@@ -1 +0,0 @@
-../extras/Hadrons

lib/Init.cc

@@ -41,7 +41,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #include <signal.h>
 #include <iostream>
 #include <iterator>
-#include <Grid/Grid.h>
+#include <Grid.h>
 #include <algorithm>
 #include <iterator>
 #include <cstdlib>

lib/Log.cc

@@ -29,7 +29,7 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 #include <cxxabi.h>
 
@@ -100,7 +100,7 @@ void Grid_quiesce_nodes(void) {
   me = shmem_my_pe();
 #endif
   if (me) {
-    std::cout.setstate(std::ios::badbit);
+    std::cout.setstate(std::ios::badbit);// mute all nodes except 0
   }
 }
 

lib/Log.h

@@ -110,8 +110,8 @@ public:
   friend std::ostream& operator<< (std::ostream& stream, Logger& log){
 
     if ( log.active ) {
-      stream << log.background()<< std::setw(10) << std::left << log.topName << log.background()<< " : ";
+      stream << log.background()<< log.topName << log.background()<< " : ";
-      stream << log.colour() << std::setw(14) << std::left << log.name << log.background() << " : ";
+      stream << log.colour() <<std::setw(14) << std::left << log.name << log.background() << " : ";
       if ( log.timestamp ) {
 	StopWatch.Stop();
 	GridTime now = StopWatch.Elapsed();

lib/Makefile.am

@@ -1,5 +1,4 @@
 extra_sources=
-extra_headers=
 if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
   extra_sources+=communicator/Communicator_base.cc
@@ -25,12 +24,6 @@ if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_base.cc
 endif
 
-if BUILD_HDF5
-  extra_sources+=serialisation/Hdf5IO.cc 
-  extra_headers+=serialisation/Hdf5IO.h
-  extra_headers+=serialisation/Hdf5Type.h
-endif
-
 #
 # Libraries
 #
@@ -39,9 +32,6 @@ include Eigen.inc
 
 lib_LIBRARIES = libGrid.a
 
-CCFILES += $(extra_sources)
+libGrid_a_SOURCES              = $(CCFILES) $(extra_sources)
-HFILES  += $(extra_headers)
-
-libGrid_a_SOURCES              = $(CCFILES)
 libGrid_adir                   = $(pkgincludedir)
 nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h

lib/PerfCount.cc

@@ -26,8 +26,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Grid.h>
+#include <Grid.h>
-#include <Grid/PerfCount.h>
+#include <PerfCount.h>
 
 namespace Grid {
 

lib/PerfCount.h

@@ -205,13 +205,12 @@ public:
   void Stop(void) {
     count=0;
     cycles=0;
-    size_t ign;
 #ifdef __linux__
     if ( fd!= -1) {
       ::ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
       ::ioctl(cyclefd, PERF_EVENT_IOC_DISABLE, 0);
-      ign=::read(fd, &count, sizeof(long long));
+      ::read(fd, &count, sizeof(long long));
-      ign=::read(cyclefd, &cycles, sizeof(long long));
+      ::read(cyclefd, &cycles, sizeof(long long));
     }
     elapsed = cyclecount() - begin;
 #else

lib/Stat.cc

@@ -1,6 +1,6 @@
-#include <Grid/Grid.h>
+#include <Grid.h>
-#include <Grid/PerfCount.h>
+#include <PerfCount.h>
-#include <Grid/Stat.h>
+#include <Stat.h>
 
 
 namespace Grid { 

lib/Stencil.h

@@ -113,7 +113,7 @@ Gather_plane_simple_table (std::vector<std::pair<int,int> >& table,const Lattice
 {
 PARALLEL_FOR_LOOP     
      for(int i=0;i<table.size();i++){
-       vstream(buffer[off+table[i].first],compress(rhs._odata[so+table[i].second]));
+       buffer[off+table[i].first]=compress(rhs._odata[so+table[i].second]);
      }
 }
 

lib/Threads.h

@@ -38,21 +38,21 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_OMP
 #include <omp.h>
 #ifdef GRID_NUMA
-#define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(static)")
+  #define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(static)")
-#define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(static)")
+  #define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(static)")
-#else
+  #else
-#define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(runtime)")
+  #define PARALLEL_FOR_LOOP        _Pragma("omp parallel for schedule(runtime)")
-#define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(runtime)")
+  #define PARALLEL_FOR_LOOP_INTERN _Pragma("omp for schedule(runtime)")
 #endif
-#define PARALLEL_NESTED_LOOP2 _Pragma("omp parallel for collapse(2)")
+#define PARALLEL_NESTED_LOOP2    _Pragma("omp parallel for collapse(2)")
-#define PARALLEL_REGION       _Pragma("omp parallel")
+#define PARALLEL_REGION          _Pragma("omp parallel")
-#define PARALLEL_CRITICAL     _Pragma("omp critical")
+#define PARALLEL_FOR_LOOP_STATIC _Pragma("omp parallel for schedule(static)")
 #else
 #define PARALLEL_FOR_LOOP
 #define PARALLEL_FOR_LOOP_INTERN
 #define PARALLEL_NESTED_LOOP2
 #define PARALLEL_REGION
-#define PARALLEL_CRITICAL
+#define PARALLEL_FOR_LOOP_STATIC
 #endif
 
 namespace Grid {

lib/algorithms/approx/MultiShiftFunction.cc

@@ -25,7 +25,7 @@ Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 namespace Grid {
 double MultiShiftFunction::approx(double x)

lib/algorithms/approx/Remez.cc

@@ -20,7 +20,7 @@
 #include<iomanip>
 #include<cassert>
 
-#include<Grid/algorithms/approx/Remez.h>
+#include<algorithms/approx/Remez.h>
 
 // Constructor
 AlgRemez::AlgRemez(double lower, double upper, long precision) 

lib/algorithms/iterative/ConjugateGradient.h

@@ -9,6 +9,7 @@ Copyright (C) 2015
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -33,6 +34,21 @@ directory
 
 namespace Grid {
 
+struct CG_state {
+  bool do_repro;
+  std::vector<RealD> residuals;
+
+  CG_state() {reset();}
+
+  void reset(){
+    do_repro = false;
+    residuals.clear();
+  }
+};
+
+
+enum CGexec_mode{ Default, ReproducibilityTest };
+
 /////////////////////////////////////////////////////////////
 // Base classes for iterative processes based on operators
 // single input vec, single output vec.
@@ -45,10 +61,30 @@ class ConjugateGradient : public OperatorFunction<Field> {
                            // Defaults true.
   RealD Tolerance;
   Integer MaxIterations;
-  ConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
+
-      : Tolerance(tol),
+  // Reproducibility controls
-        MaxIterations(maxit),
+  bool ReproTest;
-        ErrorOnNoConverge(err_on_no_conv){};
+  CG_state CGState; //to check reproducibility by repeating the CG
+  ReproducibilityState<typename Field::vector_object> ReprTest; // for the inner proucts
+
+  // Constructor
+  ConjugateGradient(RealD tol, Integer maxit, CGexec_mode Mode = Default)
+    : Tolerance(tol),MaxIterations(maxit){
+    switch(Mode)
+    {
+      case Default  : 
+      ErrorOnNoConverge = true;
+      ReproTest = false;
+      case ReproducibilityTest :
+      ErrorOnNoConverge = false;
+      ReproTest = true; 
+    }
+  };
+
+  void set_reproducibility_interval(unsigned int interval){
+    ReprTest.interval = interval;
+  }
+
 
   void operator()(LinearOperatorBase<Field> &Linop, const Field &src,
                   Field &psi) {
@@ -60,34 +96,37 @@ class ConjugateGradient : public OperatorFunction<Field> {
     Field p(src);
     Field mmp(src);
     Field r(src);
+    Field psi_start(psi);// save for the repro test
+
+    if (CGState.do_repro && ReproTest)
+        std::cout << GridLogMessage << "Starting reproducibility test, full check every "
+                  << ReprTest.interval << " calls" << std::endl;
+
+    if(!ReprTest.do_check)
+        ReprTest.reset();
+    ReprTest.enable_reprocheck=ReproTest;
+
+
 
     // Initial residual computation & set up
-    RealD guess = norm2(psi);
+    RealD guess = norm2(psi, ReprTest);
     assert(std::isnan(guess) == 0);
 
-    
+    Linop.HermOpAndNorm(psi, mmp, d, b);// eventually split this for the norm check
-    Linop.HermOpAndNorm(psi, mmp, d, b);
-    
     
     r = src - mmp;
     p = r;
 
-    a = norm2(p);
+    a = norm2(p, ReprTest);
     cp = a;
-    ssq = norm2(src);
+    ssq = norm2(src, ReprTest);
 
-    std::cout << GridLogIterative << std::setprecision(4)
+    std::cout << GridLogIterative << "ConjugateGradient: guess " << guess << std::endl;
-              << "ConjugateGradient: guess " << guess << std::endl;
+    std::cout << GridLogIterative << "ConjugateGradient:   src " << ssq << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
+    std::cout << GridLogIterative << "ConjugateGradient:    mp " << d << std::endl;
-              << "ConjugateGradient:   src " << ssq << std::endl;
+    std::cout << GridLogIterative << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
+    std::cout << GridLogIterative << "ConjugateGradient:  cp,r " << cp << std::endl;
-              << "ConjugateGradient:    mp " << d << std::endl;
+    std::cout << GridLogIterative << "ConjugateGradient:     p " << a << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:  cp,r " << cp << std::endl;
-    std::cout << GridLogIterative << std::setprecision(4)
-              << "ConjugateGradient:     p " << a << std::endl;
 
     RealD rsq = Tolerance * Tolerance * ssq;
 
@@ -107,10 +146,10 @@ class ConjugateGradient : public OperatorFunction<Field> {
     SolverTimer.Start();
     int k;
     for (k = 1; k <= MaxIterations; k++) {
-      c = cp;
+      c = cp;// old residual
 
       MatrixTimer.Start();
-      Linop.HermOpAndNorm(p, mmp, d, qq);
+      Linop.HermOpAndNorm(p, mmp, d, qq);// mmp = Ap, d=pAp
       MatrixTimer.Stop();
 
       LinalgTimer.Start();
@@ -118,14 +157,31 @@ class ConjugateGradient : public OperatorFunction<Field> {
       //  ComplexD dck  = innerProduct(p,mmp);
 
       a = c / d;
-      b_pred = a * (a * qq - d) / c;
+      b_pred = a * (a * qq - d) / c;// a check
 
-      cp = axpy_norm(r, -a, mmp, r);
+
+      axpy(r, -a, mmp, r);// new residual r = r_old - a * Ap
+      cp = norm2(r, ReprTest); // bookkeeping this norm
+      if (ReproTest && !CGState.do_repro) {
+        CGState.residuals.push_back(cp);  // save residuals state
+                std::cout << GridLogIterative << "ReproTest: Saving state" << std::endl;
+        }
+      if (ReproTest && CGState.do_repro){
+        // check that the residual agrees with the previous run
+        std::cout << GridLogIterative << "ReproTest: Checking state k=" << k << std::endl;
+        if (cp != CGState.residuals[k-1]){
+                std::cout << GridLogMessage << "Failing reproducibility test";
+                std::cout << GridLogMessage << " at k=" << k << std::endl;
+                std::cout << GridLogMessage << "saved residual = " << CGState.residuals[k-1] 
+                        << " cp = " << cp << std::endl;
+                exit(1);  // exit after the first failure
+        }
+      }
       b = cp / c;
 
       // Fuse these loops ; should be really easy
-      psi = a * p + psi;
+      psi = a * p + psi; // update solution
-      p = p * b + r;
+      p = p * b + r;  // update search direction
 
       LinalgTimer.Stop();
       std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
@@ -156,6 +212,22 @@ class ConjugateGradient : public OperatorFunction<Field> {
 
         if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
 
+        if (! (CGState.do_repro && ReproTest)){
+                CGState.do_repro = true;
+                ReprTest.do_check = true;
+                ReprTest.reset_counter();
+                this->operator()(Linop, src, psi_start);// run the repro test
+                if (ReprTest.success)
+                	std::cout << GridLogMessage << "Reproducibility test passed" << std::endl;
+                else{
+                	std::cout << GridLogMessage << "Reproducibility test failed" << std::endl;
+                	exit(1);
+                }
+        }
+
+        // Clear state
+        CGState.reset();
+        ReprTest.reset();
         return;
       }
     }

lib/algorithms/iterative/ConjugateGradientMixedPrec.h

@@ -55,9 +55,6 @@ namespace Grid {
     }
   
     void operator() (const FieldD &src_d_in, FieldD &sol_d){
-	(*this)(src_d_in,sol_d,NULL);
-    }
-    void operator() (const FieldD &src_d_in, FieldD &sol_d, RealD *shift){
       GridStopWatch TotalTimer;
       TotalTimer.Start();
     
@@ -85,7 +82,7 @@ namespace Grid {
       FieldF sol_f(SinglePrecGrid);
       sol_f.checkerboard = cb;
     
-      ConjugateGradientShifted<FieldF> CG_f(inner_tol, MaxInnerIterations);
+      ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
       CG_f.ErrorOnNoConverge = false;
 
       GridStopWatch InnerCGtimer;
@@ -95,7 +92,6 @@ namespace Grid {
       for(Integer outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
 	//Compute double precision rsd and also new RHS vector.
 	Linop_d.HermOp(sol_d, tmp_d);
-	if(shift) axpy(tmp_d,*shift,sol_d,tmp_d);
 	RealD norm = axpy_norm(src_d, -1., tmp_d, src_d_in); //src_d is residual vector
       
 	std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " <<outer_iter<<" residual "<< norm<< " target "<< stop<<std::endl;
@@ -119,7 +115,7 @@ namespace Grid {
 	//Inner CG
 	CG_f.Tolerance = inner_tol;
 	InnerCGtimer.Start();
-	CG_f(Linop_f, src_f, sol_f,shift);
+	CG_f(Linop_f, src_f, sol_f);
 	InnerCGtimer.Stop();
       
 	//Convert sol back to double and add to double prec solution
@@ -133,8 +129,8 @@ namespace Grid {
       //Final trial CG
       std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting final patch-up double-precision solve"<<std::endl;
     
-      ConjugateGradientShifted<FieldD> CG_d(Tolerance, MaxInnerIterations);
+      ConjugateGradient<FieldD> CG_d(Tolerance, MaxInnerIterations);
-      CG_d(Linop_d, src_d_in, sol_d,shift);
+      CG_d(Linop_d, src_d_in, sol_d);
 
       TotalTimer.Stop();
       std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Total " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;

lib/algorithms/iterative/ConjugateGradientMultiShift.h

@@ -45,7 +45,6 @@ public:
     Integer MaxIterations;
     int verbose;
     MultiShiftFunction shifts;
-    int iter;
 
     ConjugateGradientMultiShift(Integer maxit,MultiShiftFunction &_shifts) : 
 	MaxIterations(maxit),
@@ -61,7 +60,6 @@ void operator() (LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
   std::vector<Field> results(nshift,grid);
   (*this)(Linop,src,results,psi);
 }
-
 void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector<Field> &results, Field &psi)
 {
   int nshift = shifts.order;
@@ -107,12 +105,11 @@ void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector
   RealD a,b,c,d;
   RealD cp,bp,qq; //prev
   
-  int cb=src.checkerboard;
   // Matrix mult fields
   Field r(grid);
-  Field p(grid); p.checkerboard = src.checkerboard;
+  Field p(grid);
   Field tmp(grid);
-  Field mmp(grid);mmp.checkerboard = src.checkerboard;
+  Field mmp(grid);
   
   // Check lightest mass
   for(int s=0;s<nshift;s++){
@@ -135,9 +132,6 @@ void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector
   p=src;
   
   //MdagM+m[0]
-  std::cout << "p.checkerboard " << p.checkerboard
-  << "mmp.checkerboard " << mmp.checkerboard << std::endl;
-
   Linop.HermOpAndNorm(p,mmp,d,qq);
   axpy(mmp,mass[0],p,mmp);
   RealD rn = norm2(p);
@@ -275,7 +269,6 @@ void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector
 	RealD cn = norm2(src);
 	std::cout<<GridLogMessage<<"CGMultiShift: shift["<<s<<"] true residual "<<std::sqrt(rn/cn)<<std::endl;
       }
-      iter = k;
       return;
     }
   }

lib/algorithms/iterative/ConjugateGradientMultiShiftMixedPrec.h

@@ -1,404 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShiftMixedPrec.h
-
-    Copyright (C) 2015
-
-Author: Chulwoo Jung <chulwoo@quark.phy.bnl.gov>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END/ LEGAL */
-#ifndef GRID_CONJUGATE_GRADIENT_MULTI_MIXED_PREC_H
-#define GRID_CONJUGATE_GRADIENT_MULTI_MIXED_PREC_H
-
-namespace Grid {
-
-  //Mixed precision restarted defect correction CG
-  template<class FieldD,class FieldF
-//, typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0
-//, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0
-> 
-  class MixedPrecisionConjugateGradientMultiShift : public LinearFunction<FieldD> {
-  public:                                                
-//    RealD   Tolerance;
-    Integer MaxInnerIterations;
-    Integer MaxOuterIterations;
-    GridBase* SinglePrecGrid; //Grid for single-precision fields
-    RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
-    LinearOperatorBase<FieldF> &Linop_f;
-    LinearOperatorBase<FieldD> &Linop_d;
-    MultiShiftFunction shifts;
-    Integer iter;
-
-    //Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
-//    LinearFunction<FieldF> *guesser;
-    
-    MixedPrecisionConjugateGradientMultiShift(GridBase* _sp_grid, LinearOperatorBase<FieldF> &_Linop_f, LinearOperatorBase<FieldD> &_Linop_d, 
-Integer maxinnerit,	MultiShiftFunction &_shifts ) :
-      Linop_f(_Linop_f), Linop_d(_Linop_d),
-      MaxInnerIterations(maxinnerit), SinglePrecGrid(_sp_grid),
-      OuterLoopNormMult(100.), shifts(_shifts) {};
-
-  
-    void operator() (const FieldD &src_d_in, FieldD &sol_d){
-	assert(0); // not yet implemented
-    }
-    void operator() (const FieldD &src_d_in, std::vector<FieldD> &sol_d){
-      GridStopWatch TotalTimer;
-      TotalTimer.Start();
-    
-      int cb = src_d_in.checkerboard;
-
-      int nshift = shifts.order;
-      assert(nshift == sol_d.size());
-      for(int i=0;i<nshift;i++) sol_d[i].checkerboard = cb;
-    
-      RealD src_norm = norm2(src_d_in);
-//      RealD stop = src_norm * Tolerance*Tolerance;
-
-      GridBase* DoublePrecGrid = src_d_in._grid;
-      FieldD tmp_d(DoublePrecGrid); tmp_d.checkerboard = cb;
-    
-      FieldD tmp2_d(DoublePrecGrid); tmp2_d.checkerboard = cb;
-    
-      FieldD src_d(DoublePrecGrid);
-      src_d = src_d_in; //source for next inner iteration, computed from residual during operation
-    
-//      RealD inner_tol = Tolerance;
-  	FieldD psi_d(DoublePrecGrid);psi_d.checkerboard = cb;
-    
-      FieldF src_f(SinglePrecGrid);
-      src_f.checkerboard = cb;
-    
-      std::vector<FieldF> sol_f(nshift,SinglePrecGrid);
-      for(int i=0;i<nshift;i++) sol_f[i].checkerboard = cb;
-    
-//      ConjugateGradientShifted<FieldF> CG_f(inner_tol, MaxInnerIterations);
-      ConjugateGradientMultiShift<FieldF> MSCG(MaxInnerIterations,shifts);
-//      CG_f.ErrorOnNoConverge = false;
-
-      GridStopWatch InnerCGtimer;
-
-      GridStopWatch PrecChangeTimer;
-    
-{
-//	std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " <<outer_iter<<" residual "<< norm<< " target "<< stop<<std::endl;
-
-//	if(norm < OuterLoopNormMult * stop){
-//	  std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration converged on iteration " <<outer_iter <<std::endl;
-//	  break;
-//	}
-//	while(norm * inner_tol * inner_tol < stop) inner_tol *= 2;  // inner_tol = sqrt(stop/norm) ??
-
-	PrecChangeTimer.Start();
-	precisionChange(src_f, src_d);
-	PrecChangeTimer.Stop();
-      
-//	zeroit(sol_f);
-
-
-	//Inner CG
-	InnerCGtimer.Start();
-  int if_relup = 0;
-#if 0
-        MSCG(Linop_f,src_f,sol_f);
-#else
-{
-  
-  GridBase *grid = SinglePrecGrid;
-  
-  ////////////////////////////////////////////////////////////////////////
-  // Convenience references to the info stored in "MultiShiftFunction"
-  ////////////////////////////////////////////////////////////////////////
-  int nshift = shifts.order;
-
-
-  std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
-  std::vector<RealD> &mresidual(shifts.tolerances);
-  std::vector<RealD> alpha(nshift,1.);
-  std::vector<FieldF>   ps(nshift,grid);// Search directions
-
-  assert(sol_f.size()==nshift);
-  assert(mass.size()==nshift);
-  assert(mresidual.size()==nshift);
-  
-  // dynamic sized arrays on stack; 2d is a pain with vector
-  RealD  bs[nshift];
-  RealD  rsq[nshift];
-  RealD  z[nshift][2];
-  int     converged[nshift];
-  
-  const int       primary =0;
-  
-  //Primary shift fields CG iteration
-  RealD a,b,c,d;
-  RealD cp,bp,qq; //prev
-  
-  int cb=src_f.checkerboard;
-  // Matrix mult fields
-  FieldF r(grid); r.checkerboard = src_f.checkerboard;
-  FieldF p(grid); p.checkerboard = src_f.checkerboard;
-  FieldF tmp(grid); tmp.checkerboard = src_f.checkerboard;
-  FieldF mmp(grid);mmp.checkerboard = src_f.checkerboard;
-  FieldF psi(grid);psi.checkerboard = src_f.checkerboard;
-    std::cout.precision(12);
-    std::cout<<GridLogMessage<<"norm2(psi_d)= "<<norm2(psi_d)<<std::endl;
-    std::cout<<GridLogMessage<<"norm2(psi)= "<<norm2(psi)<<std::endl;
-  
-  
-  // Check lightest mass
-  for(int s=0;s<nshift;s++){
-    assert( mass[s]>= mass[primary] );
-    converged[s]=0;
-  }
-  
-  // Wire guess to zero
-  // Residuals "r" are src
-  // First search direction "p" is also src
-  cp = norm2(src_f);
-  Real c_relup = cp;
-  for(int s=0;s<nshift;s++){
-    rsq[s] = cp * mresidual[s] * mresidual[s];
-    std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientMultiShift: shift "<<s
-	     <<" target resid "<<rsq[s]<<std::endl;
-    ps[s] = src_f;
-  }
-  // r and p for primary
-  r=src_f;
-  p=src_f;
-  
-  //MdagM+m[0]
-  std::cout << "p.checkerboard " << p.checkerboard
-  << "mmp.checkerboard " << mmp.checkerboard << std::endl;
-
-  Linop_f.HermOpAndNorm(p,mmp,d,qq);
-  axpy(mmp,mass[0],p,mmp);
-  RealD rn = norm2(p);
-  d += rn*mass[0];
-  
-  // have verified that inner product of 
-  // p and mmp is equal to d after this since
-  // the d computation is tricky
-  //  qq = real(innerProduct(p,mmp));
-  //  std::cout<<GridLogMessage << "debug equal ?  qq "<<qq<<" d "<< d<<std::endl;
-  
-  b = -cp /d;
-  
-  // Set up the various shift variables
-  int       iz=0;
-  z[0][1-iz] = 1.0;
-  z[0][iz]   = 1.0;
-  bs[0]      = b;
-  for(int s=1;s<nshift;s++){
-    z[s][1-iz] = 1.0;
-    z[s][iz]   = 1.0/( 1.0 - b*(mass[s]-mass[0]));
-    bs[s]      = b*z[s][iz]; 
-  }
-  
-  // r += b[0] A.p[0]
-  // c= norm(r)
-  c=axpy_norm(r,b,mmp,r);
-  
- axpby(psi,0.,-bs[0],src_f,src_f);
-  for(int s=0;s<nshift;s++) {
-    axpby(sol_f[s],0.,-bs[s]*alpha[s],src_f,src_f);
-  }
-  
-  
-  // Iteration loop
-  int k;
- // inefficient zeroing, please replace!
-//  RealD sol_norm = axpy_norm(sol_d[0],-1.,sol_d[0],sol_d[0]);
-  zeroit(sol_d[0]);
-  std::cout<<GridLogMessage<<"norm(sol_d[0])= "<<norm2(sol_d[0])<<std::endl;
-  
-
-  int all_converged = 1;
-	RealD tmp1,tmp2;
-  for (k=1;k<=MaxOuterIterations;k++){
-    
-    a = c /cp;
-    axpy(p,a,p,r);
-    
-    // Note to self - direction ps is iterated seperately
-    // for each shift. Does not appear to have any scope
-    // for avoiding linear algebra in "single" case.
-    // 
-    // However SAME r is used. Could load "r" and update
-    // ALL ps[s]. 2/3 Bandwidth saving
-    // New Kernel: Load r, vector of coeffs, vector of pointers ps
-    for(int s=0;s<nshift;s++){
-      if ( ! converged[s] ) { 
-	if (s==0){
-	  axpy(ps[s],a,ps[s],r);
-	} else{
-	  RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
-	  axpby(ps[s],z[s][iz],as,r,ps[s]);
-	}
-      }
-    }
-    
-    cp=c;
-    
-    Linop_f.HermOpAndNorm(p,mmp,d,qq);
-    axpy(mmp,mass[0],p,mmp);
-    RealD rn = norm2(p);
-    d += rn*mass[0];
-    
-    bp=b;
-    b=-cp/d;
-    
-    c=axpy_norm(r,b,mmp,r);
-
-
-    // Toggle the recurrence history
-    bs[0] = b;
-    iz = 1-iz;
-    for(int s=1;s<nshift;s++){
-      if((!converged[s])){
-	RealD z0 = z[s][1-iz];
-	RealD z1 = z[s][iz];
-	z[s][iz] = z0*z1*bp
-	  / (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b)); 
-	bs[s] = b*z[s][iz]/z0; // NB sign  rel to Mike
-      }
-    }
-    
-    axpy(psi,-bs[0],ps[0],psi);
-    for(int s=0;s<nshift;s++){
-      int ss = s;
-      // Scope for optimisation here in case of "single".
-      // Could load sol_f[0] and pull all ps[s] in.
-      //      if ( single ) ss=primary;
-      // Bandwith saving in single case is Ls * 3 -> 2+Ls, so ~ 3x saving
-      // Pipelined CG gain:
-      //
-      // New Kernel: Load r, vector of coeffs, vector of pointers ps
-      // New Kernel: Load sol_f[0], vector of coeffs, vector of pointers ps
-      // If can predict the coefficient bs then we can fuse these and avoid write reread cyce
-      //  on ps[s].
-      // Before:  3 x npole  + 3 x npole
-      // After :  2 x npole (ps[s])        => 3x speed up of multishift CG.
-      
-      if( (!converged[s]) ) { 
-	axpy(sol_f[ss],-bs[s]*alpha[s],ps[s],sol_f[ss]);
-      }
-    }
-
-
-    if (k%MaxInnerIterations==0){
-//    if (c < 1e-4*c_relup){
-       RealD c_f=c;
-       precisionChange(tmp_d,psi);
-       RealD sol_norm =axpy_norm (psi_d,1.,tmp_d,psi_d);
-       tmp1 = norm2(psi);
-       zeroit(psi);
-       tmp2 = norm2(psi);
-       std::cout<<GridLogMessage<<"k= "<<k<<" norm2(sol)= "<<sol_norm<<" "<<tmp1<<" "<<tmp2<<std::endl;
-//       precisionChange(sol_d[0],sol_f[0]);
-       Linop_d.HermOpAndNorm(psi_d,tmp_d,tmp1,tmp2);
-       axpy(tmp2_d,mass[0],psi_d,tmp_d);
-       axpy(tmp_d,-1.,tmp2_d,src_d);
-       precisionChange(r,tmp_d);
-	c_relup = norm2(r);
-       std::cout<<GridLogMessage<<"k= "<<k<<" norm2(r)= "<<c<<" "<<c_relup<<" "<<c_f<<std::endl;
-	if_relup=1;
-    }
-    
-    // Convergence checks
-  all_converged=1;
-    for(int s=0;s<nshift;s++){
-      
-      if ( (!converged[s]) ){
-	
-	RealD css  = c * z[s][iz]* z[s][iz];
-	
-	if(css<rsq[s]){
-	  if ( ! converged[s] )
-	    std::cout<<GridLogMessage<<"ConjugateGradientMultiShift k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
-	      converged[s]=1;
-	} else {
-		if (k%MaxInnerIterations==0)
-	    std::cout<<GridLogMessage<<"ConjugateGradientMultiShift k="<<k<<" Shift "<<s<<" has not converged "<<css<<"<"<<rsq[s]<<std::endl;
-	  all_converged=0;
-	}
-
-      }
-    }
-    
-#if 0
-    if ( all_converged ){
-      std::cout<<GridLogMessage<< "CGMultiShift: All shifts have converged iteration "<<k<<std::endl;
-#else
-    if ( converged[0] ){
-      std::cout<<GridLogMessage<< "CGMultiShift: Shift 0 have converged iteration, terminating  "<<k<<std::endl;
-#endif
-      
-#if 1
-      for(int s=1; s < nshift; s++) { 
-	Linop_f.HermOpAndNorm(sol_f[s],mmp,d,qq);
-	axpy(tmp,mass[s],sol_f[s],mmp);
-	axpy(r,-alpha[s],src_f,tmp);
-	RealD rn = norm2(r);
-	RealD cn = norm2(src_f);
-	std::cout<<GridLogMessage<<"CGMultiShift: shift["<<s<<"] true residual "<<std::sqrt(rn/cn)<<std::endl;
-      }
-#endif
-     iter = k;
-      break;
-    }
-  }
-  // ugly hack
-  if ( !all_converged )
-  std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
-//  assert(0);
-}
-	
-#endif
-	InnerCGtimer.Stop();
-      
-	//Convert sol back to double and add to double prec solution
-	PrecChangeTimer.Start();
-	sol_d[0]=psi_d;
-	for(int i=1;i<nshift;i++)precisionChange(sol_d[i], sol_f[i]);
-      std::cout<<GridLogMessage<< "CGMultiShift: Checking solutions"<<std::endl;
-      // Check answers 
-      for(int s=0; s < nshift; s++) { 
-	RealD tmp1,tmp2;
-       Linop_d.HermOpAndNorm(sol_d[s],tmp_d,tmp1,tmp2);
-       axpy(tmp2_d,shifts.poles[s],sol_d[s],tmp_d);
-       axpy(tmp_d,-1.,src_d,tmp2_d);
-	std::cout<<GridLogMessage<<"CGMultiShift: shift["<<s<<"] true residual "<<std::sqrt(norm2(tmp_d)/norm2(src_d))<<std::endl;
-      }
-	PrecChangeTimer.Stop();
-      
-}
-    
-      //Final trial CG
- //     std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting final patch-up double-precision solve"<<std::endl;
-    
-      TotalTimer.Stop();
-      std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Total " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
-    }
-  };
-
-}
-
-#endif

lib/algorithms/iterative/ConjugateGradientShifted.h

@@ -1,168 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/ConjugateGradient.h
-
-    Copyright (C) 2015
-
-Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef GRID_CONJUGATE_GRADIENT_SHIFTED_H
-#define GRID_CONJUGATE_GRADIENT_SHIFTED_H
-
-namespace Grid {
-
-    /////////////////////////////////////////////////////////////
-    // Base classes for iterative processes based on operators
-    // single input vec, single output vec.
-    /////////////////////////////////////////////////////////////
-
-  template<class Field> 
-    class ConjugateGradientShifted : public OperatorFunction<Field> {
-public:                                                
-    bool ErrorOnNoConverge; //throw an assert when the CG fails to converge. Defaults true.
-    RealD   Tolerance;
-    Integer MaxIterations;
-    ConjugateGradientShifted(RealD tol,Integer maxit, bool err_on_no_conv = true) : Tolerance(tol), MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv) { 
-    };
-
-    void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi ){
-	(*this)(Linop,src,psi,NULL);
-    }
-
-    void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi, RealD *shift){
-
-      psi.checkerboard = src.checkerboard;
-      conformable(psi,src);
-
-      RealD cp,c,a,d,b,ssq,qq,b_pred;
-      
-      Field   p(src);
-      Field mmp(src);
-      Field   r(src);
-      
-      //Initial residual computation & set up
-      RealD guess = norm2(psi);
-      assert(std::isnan(guess)==0);
-
-      Linop.HermOpAndNorm(psi,mmp,d,b);
-	if(shift) axpy(mmp,*shift,psi,mmp);
-	RealD rn = norm2(psi);
-	if(shift) d += rn*(*shift);
-	RealD d2 = real(innerProduct(psi,mmp));
-	b= norm2(mmp);
-      RealD src_norm=norm2(src);
-      r= src-mmp;
-      p= r;
-      
-      a  =norm2(p);
-      cp =a;
-      ssq=norm2(src);
-
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient: guess "<<guess<<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:   src "<<ssq  <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:    mp "<<d    <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:   mmp "<<b    <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:  cp,r "<<cp   <<std::endl;
-      std::cout<<GridLogIterative <<std::setprecision(4)<< "ConjugateGradient:     p "<<a    <<std::endl;
-
-      RealD rsq =  Tolerance* Tolerance*ssq;
-      
-      //Check if guess is really REALLY good :)
-      if ( cp <= rsq ) {
-	return;
-      }
-      
-      std::cout<<GridLogIterative << std::setprecision(4)<< "ConjugateGradient: k=0 residual "<<cp<<" target "<<rsq<<std::endl;
-
-      GridStopWatch LinalgTimer;
-      GridStopWatch MatrixTimer;
-      GridStopWatch SolverTimer;
-
-      SolverTimer.Start();
-      int k;
-      for (k=1;k<=MaxIterations;k++){
-	
-	c=cp;
-
-	MatrixTimer.Start();
-	Linop.HermOpAndNorm(p,mmp,d,qq);
-	MatrixTimer.Stop();
-	LinalgTimer.Start();
-	if(shift) axpy(mmp,*shift,p,mmp);
-	RealD rn = norm2(p);
-	if(shift) d += rn*(*shift);
-	RealD d2 = real(innerProduct(p,mmp));
-	qq = norm2(mmp);
-      if (k%10==1) std::cout<< std::setprecision(4)<< "d:  "<<d<<" d2= "<<d2<<std::endl;
-
-	//	RealD    qqck = norm2(mmp);
-	//	ComplexD dck  = innerProduct(p,mmp);
-      
-	a      = c/d;
-	b_pred = a*(a*qq-d)/c;
-
-	cp = axpy_norm(r,-a,mmp,r);
-	b = cp/c;
-      if (k%10==1) std::cout<< std::setprecision(4)<<"k= "<<k<<" src:  "<<src_norm<<" r= "<<cp<<std::endl;
-	
-	// Fuse these loops ; should be really easy
-	psi= a*p+psi;
-	p  = p*b+r;
-	  
-	LinalgTimer.Stop();
-	std::cout<<GridLogIterative<<"ConjugateGradient: Iteration " <<k<<" residual "<<cp<< " target "<< rsq<<std::endl;
-	
-	// Stopping condition
-	if ( cp <= rsq ) { 
-	  
-	  SolverTimer.Stop();
-	  Linop.HermOpAndNorm(psi,mmp,d,qq);
-	  if(shift) mmp = mmp + (*shift) * psi;
-	  p=mmp-src;
-	  
-	  RealD mmpnorm = sqrt(norm2(mmp));
-	  RealD psinorm = sqrt(norm2(psi));
-	  RealD srcnorm = sqrt(norm2(src));
-	  RealD resnorm = sqrt(norm2(p));
-	  RealD true_residual = resnorm/srcnorm;
-
-	  std::cout<<GridLogMessage<<"ConjugateGradient: Converged on iteration " <<k
-		   <<" computed residual "<<sqrt(cp/ssq)
-		   <<" true residual "    <<true_residual
-		   <<" target "<<Tolerance<<std::endl;
-	  std::cout<<GridLogMessage<<"Time elapsed: Total "<< SolverTimer.Elapsed() << " Matrix  "<<MatrixTimer.Elapsed() << " Linalg "<<LinalgTimer.Elapsed();
-	  std::cout<<std::endl;
-	  
-	if(ErrorOnNoConverge)
-	  assert(true_residual/Tolerance < 1000.0);
-
-	  return;
-	}
-      }
-      std::cout<<GridLogMessage<<"ConjugateGradient did NOT converge"<<std::endl;
-//      assert(0);
-    }
-  };
-}
-#endif

lib/algorithms/iterative/ImplicitlyRestartedLanczos.h

@@ -31,16 +31,11 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <string.h> //memset
 #ifdef USE_LAPACK
-#ifdef USE_MKL
-#include<mkl_lapack.h>
-#else
 void LAPACK_dstegr(char *jobz, char *range, int *n, double *d, double *e,
                    double *vl, double *vu, int *il, int *iu, double *abstol,
                    int *m, double *w, double *z, int *ldz, int *isuppz,
                    double *work, int *lwork, int *iwork, int *liwork,
                    int *info);
-//#include <lapacke/lapacke.h>
-#endif
 #endif
 #include "DenseMatrix.h"
 #include "EigenSort.h"
@@ -64,16 +59,15 @@ public:
 
     int Nstop;   // Number of evecs checked for convergence
     int Nk;      // Number of converged sought
-    int Np;      // Np -- Number of spare vecs in kryloc space
+    int Np;      // Np -- Number of spare vecs in krylov space
     int Nm;      // Nm -- total number of vectors
 
-
-    RealD OrthoTime;
-
     RealD eresid;
 
     SortEigen<Field> _sort;
 
+//    GridCartesian &_fgrid;
+
     LinearOperatorBase<Field> &_Linop;
 
     OperatorFunction<Field>   &_poly;
@@ -130,23 +124,23 @@ public:
 
       GridBase *grid = evec[0]._grid;
       Field w(grid);
-      std::cout<<GridLogMessage << "RitzMatrix "<<std::endl;
+      std::cout << "RitzMatrix "<<std::endl;
       for(int i=0;i<k;i++){
 	_poly(_Linop,evec[i],w);
-	std::cout<<GridLogMessage << "["<<i<<"] ";
+	std::cout << "["<<i<<"] ";
 	for(int j=0;j<k;j++){
 	  ComplexD in = innerProduct(evec[j],w);
 	  if ( fabs((double)i-j)>1 ) { 
 	    if (abs(in) >1.0e-9 )  { 
-	      std::cout<<GridLogMessage<<"oops"<<std::endl;
+	      std::cout<<"oops"<<std::endl;
 	      abort();
 	    } else 
-	      std::cout<<GridLogMessage << " 0 ";
+	      std::cout << " 0 ";
 	  } else { 
-	    std::cout<<GridLogMessage << " "<<in<<" ";
+	    std::cout << " "<<in<<" ";
 	  }
 	}
-	std::cout<<GridLogMessage << std::endl;
+	std::cout << std::endl;
       }
     }
 
@@ -180,10 +174,10 @@ public:
       RealD beta = normalise(w); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
                                  // 7. vk+1 := wk/βk+1
 
-	std::cout<<GridLogMessage << "alpha = " << zalph << " beta "<<beta<<std::endl;
+//	std::cout << "alpha = " << zalph << " beta "<<beta<<std::endl;
       const RealD tiny = 1.0e-20;
       if ( beta < tiny ) { 
-	std::cout<<GridLogMessage << " beta is tiny "<<beta<<std::endl;
+	std::cout << " beta is tiny "<<beta<<std::endl;
      }
       lmd[k] = alph;
       lme[k]  = beta;
@@ -259,7 +253,6 @@ public:
     }
 
 #ifdef USE_LAPACK
-#define LAPACK_INT long long
     void diagonalize_lapack(DenseVector<RealD>& lmd,
 		     DenseVector<RealD>& lme, 
 		     int N1,
@@ -269,7 +262,7 @@ public:
   const int size = Nm;
 //  tevals.resize(size);
 //  tevecs.resize(size);
-  LAPACK_INT NN = N1;
+  int NN = N1;
   double evals_tmp[NN];
   double evec_tmp[NN][NN];
   memset(evec_tmp[0],0,sizeof(double)*NN*NN);
@@ -283,19 +276,19 @@ public:
         if (i==j) evals_tmp[i] = lmd[i];
         if (j==(i-1)) EE[j] = lme[j];
       }
-  LAPACK_INT evals_found;
+  int evals_found;
-  LAPACK_INT lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ;
+  int lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ;
-  LAPACK_INT liwork =  3+NN*10 ;
+  int liwork =  3+NN*10 ;
-  LAPACK_INT iwork[liwork];
+  int iwork[liwork];
   double work[lwork];
-  LAPACK_INT isuppz[2*NN];
+  int isuppz[2*NN];
   char jobz = 'V'; // calculate evals & evecs
   char range = 'I'; // calculate all evals
   //    char range = 'A'; // calculate all evals
   char uplo = 'U'; // refer to upper half of original matrix
   char compz = 'I'; // Compute eigenvectors of tridiagonal matrix
   int ifail[NN];
-  long long info;
+  int info;
 //  int total = QMP_get_number_of_nodes();
 //  int node = QMP_get_node_number();
 //  GridBase *grid = evec[0]._grid;
@@ -303,18 +296,14 @@ public:
   int node = grid->_processor;
   int interval = (NN/total)+1;
   double vl = 0.0, vu = 0.0;
-  LAPACK_INT il = interval*node+1 , iu = interval*(node+1);
+  int il = interval*node+1 , iu = interval*(node+1);
   if (iu > NN)  iu=NN;
   double tol = 0.0;
     if (1) {
       memset(evals_tmp,0,sizeof(double)*NN);
       if ( il <= NN){
         printf("total=%d node=%d il=%d iu=%d\n",total,node,il,iu);
-#ifdef USE_MKL
-        dstegr(&jobz, &range, &NN,
-#else
         LAPACK_dstegr(&jobz, &range, &NN,
-#endif
             (double*)DD, (double*)EE,
             &vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A'
             &tol, // tolerance
@@ -346,7 +335,6 @@ public:
       lmd [NN-1-i]=evals_tmp[i];
   }
 }
-#undef LAPACK_INT 
 #endif
 
 
@@ -377,14 +365,12 @@ public:
 //	diagonalize_lapack(lmd2,lme2,Nm2,Nm,Qt,grid);
 #endif
 
-      int Niter = 10000*N1;
+      int Niter = 100*N1;
       int kmin = 1;
       int kmax = N2;
       // (this should be more sophisticated)
 
-      for(int iter=0; ; ++iter){
+      for(int iter=0; iter<Niter; ++iter){
-      if ( (iter+1)%(100*N1)==0) 
-      std::cout<<GridLogMessage << "[QL method] Not converged - iteration "<<iter+1<<"\n";
 
 	// determination of 2x2 leading submatrix
 	RealD dsub = lmd[kmax-1]-lmd[kmax-2];
@@ -413,11 +399,11 @@ public:
         _sort.push(lmd3,N2);
         _sort.push(lmd2,N2);
          for(int k=0; k<N2; ++k){
-	    if (fabs(lmd2[k] - lmd3[k]) >SMALL)  std::cout<<GridLogMessage <<"lmd(qr) lmd(lapack) "<< k << ": " << lmd2[k] <<" "<< lmd3[k] <<std::endl;
+	    if (fabs(lmd2[k] - lmd3[k]) >SMALL)  std::cout <<"lmd(qr) lmd(lapack) "<< k << ": " << lmd2[k] <<" "<< lmd3[k] <<std::endl;
-//	    if (fabs(lme2[k] - lme[k]) >SMALL)  std::cout<<GridLogMessage <<"lme(qr)-lme(lapack) "<< k << ": " << lme2[k] - lme[k] <<std::endl;
+//	    if (fabs(lme2[k] - lme[k]) >SMALL)  std::cout <<"lme(qr)-lme(lapack) "<< k << ": " << lme2[k] - lme[k] <<std::endl;
 	  }
          for(int k=0; k<N1*N1; ++k){
-//	    if (fabs(Qt2[k] - Qt[k]) >SMALL)  std::cout<<GridLogMessage <<"Qt(qr)-Qt(lapack) "<< k << ": " << Qt2[k] - Qt[k] <<std::endl;
+//	    if (fabs(Qt2[k] - Qt[k]) >SMALL)  std::cout <<"Qt(qr)-Qt(lapack) "<< k << ": " << Qt2[k] - Qt[k] <<std::endl;
 	}
     }
 #endif
@@ -432,7 +418,7 @@ public:
 	  }
 	}
       }
-      std::cout<<GridLogMessage << "[QL method] Error - Too many iteration: "<<Niter<<"\n";
+      std::cout << "[QL method] Error - Too many iteration: "<<Niter<<"\n";
       abort();
     }
 
@@ -449,7 +435,6 @@ public:
 		       DenseVector<Field>& evec,
 		       int k)
     {
-      double t0=-usecond()/1e6;
       typedef typename Field::scalar_type MyComplex;
       MyComplex ip;
 
@@ -468,8 +453,6 @@ public:
 	w = w - ip * evec[j];
       }
       normalise(w);
-      t0+=usecond()/1e6;
-      OrthoTime +=t0;
     }
 
     void setUnit_Qt(int Nm, DenseVector<RealD> &Qt) {
@@ -503,10 +486,10 @@ until convergence
 	GridBase *grid = evec[0]._grid;
 	assert(grid == src._grid);
 
-	std::cout<<GridLogMessage << " -- Nk = " << Nk << " Np = "<< Np << std::endl;
+	std::cout << " -- Nk = " << Nk << " Np = "<< Np << std::endl;
-	std::cout<<GridLogMessage << " -- Nm = " << Nm << std::endl;
+	std::cout << " -- Nm = " << Nm << std::endl;
-	std::cout<<GridLogMessage << " -- size of eval   = " << eval.size() << std::endl;
+	std::cout << " -- size of eval   = " << eval.size() << std::endl;
-	std::cout<<GridLogMessage << " -- size of evec  = " << evec.size() << std::endl;
+	std::cout << " -- size of evec  = " << evec.size() << std::endl;
 	
 	assert(Nm == evec.size() && Nm == eval.size());
 	
@@ -517,7 +500,6 @@ until convergence
 	DenseVector<int>   Iconv(Nm);
 
 	DenseVector<Field>  B(Nm,grid); // waste of space replicating
-//	DenseVector<Field>  Btemp(Nm,grid); // waste of space replicating
 	
 	Field f(grid);
 	Field v(grid);
@@ -533,48 +515,35 @@ until convergence
 	// (uniform vector) Why not src??
 	//	evec[0] = 1.0;
 	evec[0] = src;
-	std:: cout<<GridLogMessage <<"norm2(src)= " << norm2(src)<<std::endl;
+	std:: cout <<"norm2(src)= " << norm2(src)<<std::endl;
 // << src._grid  << std::endl;
 	normalise(evec[0]);
-	std:: cout<<GridLogMessage <<"norm2(evec[0])= " << norm2(evec[0]) <<std::endl;
+	std:: cout <<"norm2(evec[0])= " << norm2(evec[0]) <<std::endl;
 // << evec[0]._grid << std::endl;
 	
 	// Initial Nk steps
-	OrthoTime=0.;
-	double t0=usecond()/1e6;
 	for(int k=0; k<Nk; ++k) step(eval,lme,evec,f,Nm,k);
-	double t1=usecond()/1e6;
+//	std:: cout <<"norm2(evec[1])= " << norm2(evec[1]) << std::endl;
-	std::cout<<GridLogMessage <<"IRL::Initial steps: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+//	std:: cout <<"norm2(evec[2])= " << norm2(evec[2]) << std::endl;
-	std::cout<<GridLogMessage <<"IRL::Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
-//	std:: cout<<GridLogMessage <<"norm2(evec[1])= " << norm2(evec[1]) << std::endl;
-//	std:: cout<<GridLogMessage <<"norm2(evec[2])= " << norm2(evec[2]) << std::endl;
 	RitzMatrix(evec,Nk);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL::RitzMatrix: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
 	for(int k=0; k<Nk; ++k){
-//	std:: cout<<GridLogMessage <<"eval " << k << " " <<eval[k] << std::endl;
+//	std:: cout <<"eval " << k << " " <<eval[k] << std::endl;
-//	std:: cout<<GridLogMessage <<"lme " << k << " " << lme[k] << std::endl;
+//	std:: cout <<"lme " << k << " " << lme[k] << std::endl;
 	}
 
 	// Restarting loop begins
 	for(int iter = 0; iter<Niter; ++iter){
 
-	  std::cout<<GridLogMessage<<"\n Restart iteration = "<< iter << std::endl;
+	  std::cout<<"\n Restart iteration = "<< iter << std::endl;
 
 	  // 
 	  // Rudy does a sort first which looks very different. Getting fed up with sorting out the algo defs.
 	  // We loop over 
 	  //
-	OrthoTime=0.;
 	  for(int k=Nk; k<Nm; ++k) step(eval,lme,evec,f,Nm,k);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL:: "<<Np <<" steps: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
-	std::cout<<GridLogMessage <<"IRL::Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
 	  f *= lme[Nm-1];
 
 	  RitzMatrix(evec,k2);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL:: RitzMatrix: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
 	  
 	  // getting eigenvalues
 	  for(int k=0; k<Nm; ++k){
@@ -583,27 +552,18 @@ until convergence
 	  }
 	  setUnit_Qt(Nm,Qt);
 	  diagonalize(eval2,lme2,Nm,Nm,Qt,grid);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL:: diagonalize: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
 
 	  // sorting
 	  _sort.push(eval2,Nm);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL:: eval sorting: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
 	  
 	  // Implicitly shifted QR transformations
 	  setUnit_Qt(Nm,Qt);
-	  for(int ip=0; ip<k2; ++ip){
-	std::cout<<GridLogMessage << "eval "<< ip << " "<< eval2[ip] << std::endl;
-	}
 	  for(int ip=k2; ip<Nm; ++ip){ 
-	std::cout<<GridLogMessage << "qr_decomp "<< ip << " "<< eval2[ip] << std::endl;
+	std::cout << "qr_decomp "<< ip << " "<< eval2[ip] << std::endl;
 	    qr_decomp(eval,lme,Nm,Nm,Qt,eval2[ip],k1,Nm);
 		
 	}
-	t1=usecond()/1e6;
+    
-	std::cout<<GridLogMessage <<"IRL::qr_decomp: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
-if (0) {  
 	  for(int i=0; i<(Nk+1); ++i) B[i] = 0.0;
 	  
 	  for(int j=k1-1; j<k2+1; ++j){
@@ -612,38 +572,14 @@ if (0) {
 	      B[j] += Qt[k+Nm*j] * evec[k];
 	    }
 	  }
-	t1=usecond()/1e6;
+	  for(int j=k1-1; j<k2+1; ++j) evec[j] = B[j];
-	std::cout<<GridLogMessage <<"IRL::QR Rotate: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
-}
-
-if (1) {
-	for(int i=0; i<(Nk+1); ++i) {
-		B[i] = 0.0;
-	  	B[i].checkerboard = evec[0].checkerboard;
-	}
-
-	int j_block = 24; int k_block=24;
-PARALLEL_FOR_LOOP
-	for(int ss=0;ss < grid->oSites();ss++){
-	for(int jj=k1-1; jj<k2+1; jj += j_block)
-	for(int kk=0; kk<Nm; kk += k_block)
-	for(int j=jj; (j<(k2+1)) && j<(jj+j_block); ++j){
-	for(int k=kk; (k<Nm) && k<(kk+k_block) ; ++k){
-	    B[j]._odata[ss] +=Qt[k+Nm*j] * evec[k]._odata[ss]; 
-	}
-	}
-	}
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL::QR rotation: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
-}
-	for(int j=k1-1; j<k2+1; ++j) evec[j] = B[j];
 
 	  // Compressed vector f and beta(k2)
 	  f *= Qt[Nm-1+Nm*(k2-1)];
 	  f += lme[k2-1] * evec[k2];
 	  beta_k = norm2(f);
 	  beta_k = sqrt(beta_k);
-	  std::cout<<GridLogMessage<<" beta(k) = "<<beta_k<<std::endl;
+	  std::cout<<" beta(k) = "<<beta_k<<std::endl;
 
 	  RealD betar = 1.0/beta_k;
 	  evec[k2] = betar * f;
@@ -656,10 +592,7 @@ PARALLEL_FOR_LOOP
 	  }
 	  setUnit_Qt(Nm,Qt);
 	  diagonalize(eval2,lme2,Nk,Nm,Qt,grid);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL::diagonalize: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
 	  
-if (0) {
 	  for(int k = 0; k<Nk; ++k) B[k]=0.0;
 	  
 	  for(int j = 0; j<Nk; ++j){
@@ -667,34 +600,12 @@ if (0) {
 	    B[j].checkerboard = evec[k].checkerboard;
 	      B[j] += Qt[k+j*Nm] * evec[k];
 	    }
-	    std::cout<<GridLogMessage << "norm(B["<<j<<"])="<<norm2(B[j])<<std::endl;
+//	    std::cout << "norm(B["<<j<<"])="<<norm2(B[j])<<std::endl;
 	  }
-	t1=usecond()/1e6;
+//	_sort.push(eval2,B,Nk);
-	std::cout<<GridLogMessage <<"IRL::Convergence rotation: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
-}
-if (1) {
-	for(int i=0; i<(Nk+1); ++i) {
-		B[i] = 0.0;
-	  	B[i].checkerboard = evec[0].checkerboard;
-	}
-
-	int j_block = 24; int k_block=24;
-PARALLEL_FOR_LOOP
-	for(int ss=0;ss < grid->oSites();ss++){
-	for(int jj=0; jj<Nk; jj += j_block)
-	for(int kk=0; kk<Nk; kk += k_block)
-	for(int j=jj; (j<Nk) && j<(jj+j_block); ++j){
-	for(int k=kk; (k<Nk) && k<(kk+k_block) ; ++k){
-	    B[j]._odata[ss] +=Qt[k+Nm*j] * evec[k]._odata[ss]; 
-	}
-	}
-	}
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL::convergence rotation : "<<t1-t0<< "seconds"<<std::endl; t0=t1;
-}
 
 	  Nconv = 0;
-	  //	  std::cout<<GridLogMessage << std::setiosflags(std::ios_base::scientific);
+	  //	  std::cout << std::setiosflags(std::ios_base::scientific);
 	  for(int i=0; i<Nk; ++i){
 
 //	    _poly(_Linop,B[i],v);
@@ -702,16 +613,14 @@ PARALLEL_FOR_LOOP
 	    
 	    RealD vnum = real(innerProduct(B[i],v)); // HermOp.
 	    RealD vden = norm2(B[i]);
-	    RealD vv0 = norm2(v);
 	    eval2[i] = vnum/vden;
 	    v -= eval2[i]*B[i];
 	    RealD vv = norm2(v);
 	    
 	    std::cout.precision(13);
-	    std::cout<<GridLogMessage << "[" << std::setw(3)<< std::setiosflags(std::ios_base::right) <<i<<"] ";
+	    std::cout << "[" << std::setw(3)<< std::setiosflags(std::ios_base::right) <<i<<"] ";
-	    std::cout<<"eval = "<<std::setw(25)<< std::setiosflags(std::ios_base::left)<< eval2[i];
+	    std::cout << "eval = "<<std::setw(25)<< std::setiosflags(std::ios_base::left)<< eval2[i];
-	    std::cout<<"|H B[i] - eval[i]B[i]|^2 "<< std::setw(25)<< std::setiosflags(std::ios_base::right)<< vv;
+	    std::cout <<" |H B[i] - eval[i]B[i]|^2 "<< std::setw(25)<< std::setiosflags(std::ios_base::right)<< vv<< std::endl;
-	    std::cout<<" "<< vnum/(sqrt(vden)*sqrt(vv0)) << std::endl;
 	    
 	// change the criteria as evals are supposed to be sorted, all evals smaller(larger) than Nstop should have converged
 	    if((vv<eresid*eresid) && (i == Nconv) ){
@@ -720,19 +629,17 @@ PARALLEL_FOR_LOOP
 	    }
 
 	  }  // i-loop end
-	  //	  std::cout<<GridLogMessage << std::resetiosflags(std::ios_base::scientific);
+	  //	  std::cout << std::resetiosflags(std::ios_base::scientific);
-	t1=usecond()/1e6;
-	std::cout<<GridLogMessage <<"IRL::convergence testing: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
 
 
-	  std::cout<<GridLogMessage<<" #modes converged: "<<Nconv<<std::endl;
+	  std::cout<<" #modes converged: "<<Nconv<<std::endl;
 
 	  if( Nconv>=Nstop ){
 	    goto converged;
 	  }
 	} // end of iter loop
 	
-	std::cout<<GridLogMessage<<"\n NOT converged.\n";
+	std::cout<<"\n NOT converged.\n";
 	abort();
 	
       converged:
@@ -745,10 +652,10 @@ PARALLEL_FOR_LOOP
        }
       _sort.push(eval,evec,Nconv);
 
-      std::cout<<GridLogMessage << "\n Converged\n Summary :\n";
+      std::cout << "\n Converged\n Summary :\n";
-      std::cout<<GridLogMessage << " -- Iterations  = "<< Nconv  << "\n";
+      std::cout << " -- Iterations  = "<< Nconv  << "\n";
-      std::cout<<GridLogMessage << " -- beta(k)     = "<< beta_k << "\n";
+      std::cout << " -- beta(k)     = "<< beta_k << "\n";
-      std::cout<<GridLogMessage << " -- Nconv       = "<< Nconv  << "\n";
+      std::cout << " -- Nconv       = "<< Nconv  << "\n";
      }
 
     /////////////////////////////////////////////////
@@ -771,25 +678,25 @@ PARALLEL_FOR_LOOP
 	}
       }
 
-      std::cout<<GridLogMessage<<"Lanczos_Factor start/end " <<start <<"/"<<end<<std::endl;
+      std::cout<<"Lanczos_Factor start/end " <<start <<"/"<<end<<std::endl;
 
       // Starting from scratch, bq[0] contains a random vector and |bq[0]| = 1
       int first;
       if(start == 0){
 
-	std::cout<<GridLogMessage << "start == 0\n"; //TESTING
+	std::cout << "start == 0\n"; //TESTING
 
 	_poly(_Linop,bq[0],bf);
 
 	alpha = real(innerProduct(bq[0],bf));//alpha =  bq[0]^dag A bq[0]
 
-	std::cout<<GridLogMessage << "alpha = " << alpha << std::endl;
+	std::cout << "alpha = " << alpha << std::endl;
 	
 	bf = bf - alpha * bq[0];  //bf =  A bq[0] - alpha bq[0]
 
 	H[0][0]=alpha;
 
-	std::cout<<GridLogMessage << "Set H(0,0) to " << H[0][0] << std::endl;
+	std::cout << "Set H(0,0) to " << H[0][0] << std::endl;
 
 	first = 1;
 
@@ -809,19 +716,19 @@ PARALLEL_FOR_LOOP
 
 	beta = 0;sqbt = 0;
 
-	std::cout<<GridLogMessage << "cont is true so setting beta to zero\n";
+	std::cout << "cont is true so setting beta to zero\n";
 
       }	else {
 
 	beta = norm2(bf);
 	sqbt = sqrt(beta);
 
-	std::cout<<GridLogMessage << "beta = " << beta << std::endl;
+	std::cout << "beta = " << beta << std::endl;
       }
 
       for(int j=first;j<end;j++){
 
-	std::cout<<GridLogMessage << "Factor j " << j <<std::endl;
+	std::cout << "Factor j " << j <<std::endl;
 
 	if(cont){ // switches to factoring; understand start!=0 and initial bf value is right.
 	  bq[j] = bf; cont = false;
@@ -844,7 +751,7 @@ PARALLEL_FOR_LOOP
 
 	beta = fnorm;
 	sqbt = sqrt(beta);
-	std::cout<<GridLogMessage << "alpha = " << alpha << " fnorm = " << fnorm << '\n';
+	std::cout << "alpha = " << alpha << " fnorm = " << fnorm << '\n';
 
 	///Iterative refinement of orthogonality V = [ bq[0]  bq[1]  ...  bq[M] ]
 	int re = 0;
@@ -879,8 +786,8 @@ PARALLEL_FOR_LOOP
 	  bck = sqrt( nmbex );
 	  re++;
 	}
-	std::cout<<GridLogMessage << "Iteratively refined orthogonality, changes alpha\n";
+	std::cout << "Iteratively refined orthogonality, changes alpha\n";
-	if(re > 1) std::cout<<GridLogMessage << "orthagonality refined " << re << " times" <<std::endl;
+	if(re > 1) std::cout << "orthagonality refined " << re << " times" <<std::endl;
 	H[j][j]=alpha;
       }
 
@@ -895,13 +802,11 @@ PARALLEL_FOR_LOOP
 
     void ImplicitRestart(int TM, DenseVector<RealD> &evals,  DenseVector<DenseVector<RealD> > &evecs, DenseVector<Field> &bq, Field &bf, int cont)
     {
-      std::cout<<GridLogMessage << "ImplicitRestart begin. Eigensort starting\n";
+      std::cout << "ImplicitRestart begin. Eigensort starting\n";
 
       DenseMatrix<RealD> H; Resize(H,Nm,Nm);
 
-#ifndef USE_LAPACK
       EigenSort(evals, evecs);
-#endif
 
       ///Assign shifts
       int K=Nk;
@@ -924,15 +829,15 @@ PARALLEL_FOR_LOOP
       /// Shifted H defines a new K step Arnoldi factorization
       RealD  beta = H[ff][ff-1]; 
       RealD  sig  = Q[TM - 1][ff - 1];
-      std::cout<<GridLogMessage << "beta = " << beta << " sig = " << real(sig) <<std::endl;
+      std::cout << "beta = " << beta << " sig = " << real(sig) <<std::endl;
 
-      std::cout<<GridLogMessage << "TM = " << TM << " ";
+      std::cout << "TM = " << TM << " ";
-      std::cout<<GridLogMessage << norm2(bq[0]) << " -- before" <<std::endl;
+      std::cout << norm2(bq[0]) << " -- before" <<std::endl;
 
       /// q -> q Q
       times_real(bq, Q, TM);
 
-      std::cout<<GridLogMessage << norm2(bq[0]) << " -- after " << ff <<std::endl;
+      std::cout << norm2(bq[0]) << " -- after " << ff <<std::endl;
       bf =  beta* bq[ff] + sig* bf;
 
       /// Do the rest of the factorization
@@ -956,7 +861,7 @@ PARALLEL_FOR_LOOP
       int ff = Lanczos_Factor(0, M, cont, bq,bf,H); // 0--M to begin with
 
       if(ff < M) {
-	std::cout<<GridLogMessage << "Krylov: aborting ff "<<ff <<" "<<M<<std::endl;
+	std::cout << "Krylov: aborting ff "<<ff <<" "<<M<<std::endl;
 	abort(); // Why would this happen?
       }
 
@@ -965,7 +870,7 @@ PARALLEL_FOR_LOOP
 
       for(int it = 0; it < Niter && (converged < Nk); ++it) {
 
-	std::cout<<GridLogMessage << "Krylov: Iteration --> " << it << std::endl;
+	std::cout << "Krylov: Iteration --> " << it << std::endl;
 	int lock_num = lock ? converged : 0;
 	DenseVector<RealD> tevals(M - lock_num );
 	DenseMatrix<RealD> tevecs; Resize(tevecs,M - lock_num,M - lock_num);
@@ -981,7 +886,7 @@ PARALLEL_FOR_LOOP
       Wilkinson<RealD>(H, evals, evecs, small); 
       //      Check();
 
-      std::cout<<GridLogMessage << "Done  "<<std::endl;
+      std::cout << "Done  "<<std::endl;
 
     }
 
@@ -1046,7 +951,7 @@ PARALLEL_FOR_LOOP
 		  DenseVector<RealD> &tevals, DenseVector<DenseVector<RealD> > &tevecs, 
 		  int lock, int converged)
     {
-      std::cout<<GridLogMessage << "Converged " << converged << " so far." << std::endl;
+      std::cout << "Converged " << converged << " so far." << std::endl;
       int lock_num = lock ? converged : 0;
       int M = Nm;
 
@@ -1061,9 +966,7 @@ PARALLEL_FOR_LOOP
       RealD small=1.0e-16;
       Wilkinson<RealD>(AH, tevals, tevecs, small);
 
-#ifndef USE_LAPACK
       EigenSort(tevals, tevecs);
-#endif
 
       RealD resid_nrm=  norm2(bf);
 
@@ -1074,7 +977,7 @@ PARALLEL_FOR_LOOP
 	RealD diff = 0;
 	diff = abs( tevecs[i][Nm - 1 - lock_num] ) * resid_nrm;
 
-	std::cout<<GridLogMessage << "residual estimate " << SS-1-i << " " << diff << " of (" << tevals[i] << ")" << std::endl;
+	std::cout << "residual estimate " << SS-1-i << " " << diff << " of (" << tevals[i] << ")" << std::endl;
 
 	if(diff < converged) {
 
@@ -1090,13 +993,13 @@ PARALLEL_FOR_LOOP
 	    lock_num++;
 	  }
 	  converged++;
-	  std::cout<<GridLogMessage << " converged on eval " << converged << " of " << Nk << std::endl;
+	  std::cout << " converged on eval " << converged << " of " << Nk << std::endl;
 	} else {
 	  break;
 	}
       }
 #endif
-      std::cout<<GridLogMessage << "Got " << converged << " so far " <<std::endl;	
+      std::cout << "Got " << converged << " so far " <<std::endl;	
     }
 
     ///Check
@@ -1105,9 +1008,7 @@ PARALLEL_FOR_LOOP
 
       DenseVector<RealD> goodval(this->get);
 
-#ifndef USE_LAPACK
       EigenSort(evals,evecs);
-#endif
 
       int NM = Nm;
 
@@ -1179,10 +1080,10 @@ say con = 2
 **/
 
 template<class T>
-static void Lock(DenseMatrix<T> &H, 	///Hess mtx	
+static void Lock(DenseMatrix<T> &H, 	// Hess mtx	
-		 DenseMatrix<T> &Q, 	///Lock Transform
+		 DenseMatrix<T> &Q, 	// Lock Transform
-		 T val, 		///value to be locked
+		 T val, 		// value to be locked
-		 int con, 	///number already locked
+		 int con, 	// number already locked
 		 RealD small,
 		 int dfg,
 		 bool herm)

lib/cartesian/Cartesian_base.h

@@ -77,7 +77,7 @@ public:
     // GridCartesian / GridRedBlackCartesian
     ////////////////////////////////////////////////////////////////
     virtual int CheckerBoarded(int dim)=0;
-    virtual int CheckerBoard(const std::vector<int> &site)=0;
+    virtual int CheckerBoard(std::vector<int> &site)=0;
     virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
     virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
     virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0;

lib/cartesian/Cartesian_full.h

@@ -49,7 +49,7 @@ public:
     virtual int CheckerBoarded(int dim){
       return 0;
     }
-    virtual int CheckerBoard(const std::vector<int> &site){
+    virtual int CheckerBoard(std::vector<int> &site){
         return 0;
     }
     virtual int CheckerBoardDestination(int cb,int shift,int dim){

lib/cartesian/Cartesian_red_black.h

@@ -49,7 +49,7 @@ public:
       if( dim==_checker_dim) return 1;
       else return 0;
     }
-    virtual int CheckerBoard(const std::vector<int> &site){
+    virtual int CheckerBoard(std::vector<int> &site){
       int linear=0;
       assert(site.size()==_ndimension);
       for(int d=0;d<_ndimension;d++){ 

lib/communicator/Communicator_base.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include "Grid.h"
 namespace Grid {
 
 ///////////////////////////////////////////////////////////////
@@ -65,6 +65,7 @@ const std::vector<int> & CartesianCommunicator::ThisProcessorCoor(void) { return
 const std::vector<int> & CartesianCommunicator::ProcessorGrid(void)     { return _processors; };
 int                      CartesianCommunicator::ProcessorCount(void)    { return _Nprocessors; };
 
+
 ////////////////////////////////////////////////////////////////////////////////
 // very VERY rarely (Log, serial RNG) we need world without a grid
 ////////////////////////////////////////////////////////////////////////////////
@@ -89,11 +90,11 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
 #if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L)
 
 void CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						       void *xmit,
+                                                       void *xmit,
-						       int xmit_to_rank,
+                                                       int xmit_to_rank,
-						       void *recv,
+                                                       void *recv,
-						       int recv_from_rank,
+                                                       int recv_from_rank,
-						       int bytes)
+                                                       int bytes)
 {
   SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
 }

lib/communicator/Communicator_base.h

@@ -68,6 +68,8 @@ class CartesianCommunicator {
   static MPI_Comm communicator_world;
          MPI_Comm communicator;
   typedef MPI_Request CommsRequest_t;
+  static char name[MPI_MAX_PROCESSOR_NAME]; // processing node physical name
+  static int length; 
 #else 
   typedef int CommsRequest_t;
 #endif
@@ -149,6 +151,7 @@ class CartesianCommunicator {
   const std::vector<int> & ProcessorGrid(void)     ;
   int                      ProcessorCount(void)    ;
 
+  void        		 PrintRankInfo(void)     ;
   ////////////////////////////////////////////////////////////////////////////////
   // very VERY rarely (Log, serial RNG) we need world without a grid
   ////////////////////////////////////////////////////////////////////////////////

lib/communicator/Communicator_mpi.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include "Grid.h"
 #include <mpi.h>
 
 namespace Grid {
@@ -35,6 +35,8 @@ namespace Grid {
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 MPI_Comm CartesianCommunicator::communicator_world;
+char CartesianCommunicator::name[MPI_MAX_PROCESSOR_NAME]; // processing node physical name
+int CartesianCommunicator::length; 
 
 // Should error check all MPI calls.
 void CartesianCommunicator::Init(int *argc, char ***argv) {
@@ -44,6 +46,8 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     MPI_Init(argc,argv);
   }
   MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
+
+  MPI_Get_processor_name(name, &length);
   ShmInitGeneric();
 }
 
@@ -206,5 +210,10 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
   assert(ierr==0);
 }
 
+void CartesianCommunicator::PrintRankInfo(){
+    std::cout << "Grid: Rank "<< _processor << "  -  Physical node name: " << name << std::endl;
 }
 
+
+}// end of namespace
+

lib/communicator/Communicator_mpi3.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include "Grid.h"
 #include <mpi.h>
 
 namespace Grid {
@@ -576,5 +576,10 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
   assert(ierr==0);
 }
 
+  void CartesianCommunicator::PrintRankInfo(){
+    std::cout << "Grid: Rank "<< _processor << "  -  Physical node name: " << name << std::endl;
+  }
+
+  
 }
 

lib/communicator/Communicator_mpi3_leader.cc

@@ -869,6 +869,9 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) {
   return NULL;
 }
 
+  void CartesianCommunicator::PrintRankInfo(){
+    std::cout << "Grid: Rank "<< _processor << "  -  Physical node name: " << name << std::endl;
+  }
 
 };
 

lib/communicator/Communicator_none.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include "Grid.h"
 namespace Grid {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -37,6 +37,11 @@ void CartesianCommunicator::Init(int *argc, char *** arv)
   ShmInitGeneric();
 }
 
+void CartesianCommunicator::PrintRankInfo(){
+    std::cout << GridLogMessage << "No Rank Info available" << std::endl;
+}
+
+
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _processors = processors;
@@ -60,10 +65,10 @@ void CartesianCommunicator::GlobalSum(uint64_t &){}
 void CartesianCommunicator::GlobalSumVector(double *,int N){}
 
 void CartesianCommunicator::SendRecvPacket(void *xmit,
-					   void *recv,
+                                           void *recv,
-					   int xmit_to_rank,
+                                           int xmit_to_rank,
-					   int recv_from_rank,
+                                           int recv_from_rank,
-					   int bytes)
+                                           int bytes)
 {
   assert(0);
 }
@@ -71,19 +76,19 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
 
 // Basic Halo comms primitive -- should never call in single node
 void CartesianCommunicator::SendToRecvFrom(void *xmit,
-					   int dest,
+                                           int dest,
-					   void *recv,
+                                           void *recv,
-					   int from,
+                                           int from,
-					   int bytes)
+                                           int bytes)
 {
   assert(0);
 }
 void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						void *xmit,
+                                                void *xmit,
-						int dest,
+                                                int dest,
-						void *recv,
+                                                void *recv,
-						int from,
+                                                int from,
-						int bytes)
+                                                int bytes)
 {
   assert(0);
 }

lib/communicator/Communicator_shmem.cc

@@ -25,7 +25,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include "Grid.h"
 #include <mpp/shmem.h>
 
 namespace Grid {
@@ -333,5 +333,9 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
   }
 }
 
+  void CartesianCommunicator::PrintRankInfo(){
+    std::cout << GridLogMessage << "SHMEM: Rank Info not implemented yet" << std::endl;
+  }
+
 }
 

lib/lattice/Lattice_peekpoke.h

@@ -86,7 +86,7 @@ PARALLEL_FOR_LOOP
     // Poke a scalar object into the SIMD array
     //////////////////////////////////////////////////////
     template<class vobj,class sobj>
-    void pokeSite(const sobj &s,Lattice<vobj> &l,const std::vector<int> &site){
+    void pokeSite(const sobj &s,Lattice<vobj> &l,std::vector<int> &site){
 
       GridBase *grid=l._grid;
 
@@ -120,7 +120,7 @@ PARALLEL_FOR_LOOP
     // Peek a scalar object from the SIMD array
     //////////////////////////////////////////////////////////
     template<class vobj,class sobj>
-      void peekSite(sobj &s,const Lattice<vobj> &l,const std::vector<int> &site){
+      void peekSite(sobj &s,const Lattice<vobj> &l,std::vector<int> &site){
         
       GridBase *grid=l._grid;
 

lib/lattice/Lattice_reduction.h

@@ -31,6 +31,87 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define GRID_LATTICE_REDUCTION_H
 
 namespace Grid {
+
+template <class T>
+class ReproducibilityState {
+ public:
+  typedef typename T::vector_type vector_type;
+  typedef std::vector<vector_type, alignedAllocator<vector_type> >  sum_type;
+  unsigned int  n_call;
+  bool          do_check;
+  bool          enable_reprocheck;
+  bool          success;
+  unsigned int  interval; 
+  std::vector<sum_type> th_states;
+
+  void reset_counter() { n_call = 0; }
+  void reset() {
+    th_states.clear();
+    do_check          = false;
+    enable_reprocheck = false;
+    success           = true;
+    n_call            = 0;
+  };
+
+  ReproducibilityState():interval(1) { 
+    reset(); 
+  }
+
+  void check(GridBase* grid, sum_type &sumarray){
+    ///////////////////////  Reproducibility section, not threaded on purpouse
+    if (enable_reprocheck) {
+      if (do_check && (n_call % interval) == 0) {
+        for (int thread = 0; thread < sumarray.size(); thread++) {
+          int words = sizeof(sumarray[thread])/sizeof(unsigned char);
+          unsigned char xors[words];
+          bitwise_xor(sumarray[thread], th_states[n_call][thread],xors);
+          // OR all words
+          unsigned char res = 0;
+          for (int w = 0; w < words; w++) res = res | xors[w];
+
+          Grid_unquiesce_nodes();
+          int rank = 0;
+          while (rank < grid->_Nprocessors){
+            if (rank == grid->ThisRank() ){
+              if ( res ) {
+                std::cout << "Reproducibility failure report" << std::endl;
+                grid->PrintRankInfo();
+                std::cout << "Call: "<< n_call << " Thread: " << thread << std::endl;
+                std::cout << "Size of states: " << th_states.size() << std::endl;
+                std::cout << std::setprecision(GRID_REAL_DIGITS+1) << std::scientific;
+                std::cout << "Saved partial sum  : " << th_states[n_call][thread] << std::endl;
+                std::cout << "Current partial sum: " << sumarray[thread] << std::endl;
+                std::cout << "Saved state "  << std::endl; show_binaryrep(th_states[n_call][thread]);
+                std::cout << "Current state" << std::endl; show_binaryrep(sumarray[thread]);
+                std::cout << "XOR result"    << std::endl; show_binaryrep(xors, words);
+              		//std::cout << std::defaultfloat;  //not supported by some compilers
+                std::cout << std::setprecision(6);
+                success = false; 
+              }
+            }
+            rank++;
+            grid->Barrier();
+          }
+          Grid_quiesce_nodes();                         
+        }
+
+      } else if (!do_check)
+      {
+        std::cout << GridLogDebug << "Saving thread state for inner product. Call n. " 
+        << n_call << std::endl;
+        th_states.resize(n_call+1);
+        th_states[n_call].resize(grid->SumArraySize());
+          th_states[n_call] = sumarray;  // save threads state
+          //n_call++;
+        }
+        n_call++;
+      }
+    }
+
+};
+
+
+
 #ifdef GRID_WARN_SUBOPTIMAL
 #warning "Optimisation alert all these reduction loops are NOT threaded "
 #endif     
@@ -39,12 +120,25 @@ namespace Grid {
     // Deterministic Reduction operations
     ////////////////////////////////////////////////////////////////////////////////////////////////////
   template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
-    ComplexD nrm = innerProduct(arg,arg);
+        ReproducibilityState<vobj> repr;
-    return std::real(nrm); 
+        ComplexD nrm = innerProduct(arg, arg, repr);
-  }
+        return std::real(nrm); 
+    }
+
+    template <class vobj>
+    inline RealD norm2(const Lattice<vobj> &arg, ReproducibilityState<vobj>& rep) {
+      ComplexD nrm = innerProduct(arg, arg, rep);
+      return std::real(nrm);
+    }
 
     template<class vobj>
-    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) 
+    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right){
+      ReproducibilityState<vobj> repr;
+      return innerProduct(left, right, repr);
+    } 
+    
+    template<class vobj>
+    inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right, ReproducibilityState<vobj>& repr) 
     {
       typedef typename vobj::scalar_type scalar_type;
       typedef typename vobj::vector_type vector_type;
@@ -54,24 +148,28 @@ namespace Grid {
 
       std::vector<vector_type,alignedAllocator<vector_type> > sumarray(grid->SumArraySize());
       for(int i=0;i<grid->SumArraySize();i++){
-	sumarray[i]=zero;
+        sumarray[i]=zero;
       }
 
-PARALLEL_FOR_LOOP
+      PARALLEL_FOR_LOOP_STATIC //request statically scheduled threads for reproducibility
       for(int thr=0;thr<grid->SumArraySize();thr++){
-	int nwork, mywork, myoff;
+        int nwork, mywork, myoff;
-	GridThread::GetWork(left._grid->oSites(),thr,mywork,myoff);
+        GridThread::GetWork(left._grid->oSites(),thr,mywork,myoff);
         
-	decltype(innerProduct(left._odata[0],right._odata[0])) vnrm=zero; // private to thread; sub summation
+        decltype(innerProduct(left._odata[0],right._odata[0])) vnrm = zero; // private to thread; sub summation
         for(int ss=myoff;ss<mywork+myoff; ss++){
-	  vnrm = vnrm + innerProduct(left._odata[ss],right._odata[ss]);
+          vnrm = vnrm + innerProduct(left._odata[ss],right._odata[ss]);
-	}
+        }
-	sumarray[thr]=TensorRemove(vnrm) ;
+        sumarray[thr]=TensorRemove(vnrm) ;
       }
       
+      /////////// Reproducibility
+      repr.check(grid, sumarray);
+      ///////////////////////////
+
       vector_type vvnrm; vvnrm=zero;  // sum across threads
       for(int i=0;i<grid->SumArraySize();i++){
-	vvnrm = vvnrm+sumarray[i];
+        vvnrm = vvnrm+sumarray[i];
       } 
       nrm = Reduce(vvnrm);// sum across simd
       right._grid->GlobalSum(nrm);
@@ -79,26 +177,26 @@ PARALLEL_FOR_LOOP
     }
 
     template<class Op,class T1>
-      inline auto sum(const LatticeUnaryExpression<Op,T1> & expr)
+    inline auto sum(const LatticeUnaryExpression<Op,T1> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second))))::scalar_object
+    ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second))))::scalar_object
     {
       return sum(closure(expr));
     }
 
     template<class Op,class T1,class T2>
-      inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
+    inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),eval(0,std::get<1>(expr.second))))::scalar_object
+    ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),eval(0,std::get<1>(expr.second))))::scalar_object
     {
       return sum(closure(expr));
     }
 
 
     template<class Op,class T1,class T2,class T3>
-      inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
+    inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
-      ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),
+    ->typename decltype(expr.first.func(eval(0,std::get<0>(expr.second)),
-				 eval(0,std::get<1>(expr.second)),
+     eval(0,std::get<1>(expr.second)),
-				 eval(0,std::get<2>(expr.second))
+     eval(0,std::get<2>(expr.second))
-				 ))::scalar_object
+     ))::scalar_object
     {
       return sum(closure(expr));
     }
@@ -111,24 +209,24 @@ PARALLEL_FOR_LOOP
 
       std::vector<vobj,alignedAllocator<vobj> > sumarray(grid->SumArraySize());
       for(int i=0;i<grid->SumArraySize();i++){
-	sumarray[i]=zero;
+        sumarray[i]=zero;
       }
 
-PARALLEL_FOR_LOOP
+      PARALLEL_FOR_LOOP
       for(int thr=0;thr<grid->SumArraySize();thr++){
-	int nwork, mywork, myoff;
+        int nwork, mywork, myoff;
-	GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
+        GridThread::GetWork(grid->oSites(),thr,mywork,myoff);
 
-	vobj vvsum=zero;
+        vobj vvsum=zero;
         for(int ss=myoff;ss<mywork+myoff; ss++){
-	  vvsum = vvsum + arg._odata[ss];
+          vvsum = vvsum + arg._odata[ss];
-	}
+        }
-	sumarray[thr]=vvsum;
+        sumarray[thr]=vvsum;
       }
 
       vobj vsum=zero;  // sum across threads
       for(int i=0;i<grid->SumArraySize();i++){
-	vsum = vsum+sumarray[i];
+        vsum = vsum+sumarray[i];
       } 
 
       typedef typename vobj::scalar_object sobj;
@@ -138,7 +236,7 @@ PARALLEL_FOR_LOOP
       extract(vsum,buf);
 
       for(int i=0;i<Nsimd;i++) ssum = ssum + buf[i];
-      arg._grid->GlobalSum(ssum);
+        arg._grid->GlobalSum(ssum);
 
       return ssum;
     }
@@ -146,23 +244,23 @@ PARALLEL_FOR_LOOP
 
 
 template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,std::vector<typename vobj::scalar_object> &result,int orthogdim)
-{
+    {
-  typedef typename vobj::scalar_object sobj;
+      typedef typename vobj::scalar_object sobj;
-  GridBase  *grid = Data._grid;
+      GridBase  *grid = Data._grid;
-  assert(grid!=NULL);
+      assert(grid!=NULL);
 
   // FIXME
   // std::cout<<GridLogMessage<<"WARNING ! SliceSum is unthreaded "<<grid->SumArraySize()<<" threads "<<std::endl;
 
-  const int    Nd = grid->_ndimension;
+      const int    Nd = grid->_ndimension;
-  const int Nsimd = grid->Nsimd();
+      const int Nsimd = grid->Nsimd();
 
-  assert(orthogdim >= 0);
+      assert(orthogdim >= 0);
-  assert(orthogdim < Nd);
+      assert(orthogdim < Nd);
 
-  int fd=grid->_fdimensions[orthogdim];
+      int fd=grid->_fdimensions[orthogdim];
-  int ld=grid->_ldimensions[orthogdim];
+      int ld=grid->_ldimensions[orthogdim];
-  int rd=grid->_rdimensions[orthogdim];
+      int rd=grid->_rdimensions[orthogdim];
 
   std::vector<vobj,alignedAllocator<vobj> > lvSum(rd); // will locally sum vectors first
   std::vector<sobj> lsSum(ld,zero); // sum across these down to scalars

lib/lattice/Lattice_rng.h

@@ -30,7 +30,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define GRID_LATTICE_RNG_H
 
 #include <random>
-#include <Grid/sitmo_rng/sitmo_prng_engine.hpp>
 
 namespace Grid {
 
@@ -115,14 +114,10 @@ namespace Grid {
     typedef uint64_t      RngStateType;
     typedef std::ranlux48 RngEngine;
     static const int RngStateCount = 15;
-#elif RNG_MT19937 
+#else
     typedef std::mt19937 RngEngine;
     typedef uint32_t     RngStateType;
     static const int     RngStateCount = std::mt19937::state_size;
-#elif RNG_SITMO
-    typedef sitmo::prng_engine 	RngEngine;
-    typedef uint64_t    	RngStateType;
-    static const int    	RngStateCount = 4;
 #endif
     std::vector<RngEngine>                             _generators;
     std::vector<std::uniform_real_distribution<RealD>> _uniform;

lib/lattice/Lattice_transfer.h

@@ -386,7 +386,6 @@ void InsertSlice(Lattice<vobj> &lowDim,Lattice<vobj> & higherDim,int slice, int
   }
 
   // the above should guarantee that the operations are local
-  // Guido: check the threading here
   //PARALLEL_FOR_LOOP
   for(int idx=0;idx<lg->lSites();idx++){
     std::vector<int> lcoor(nl);

lib/pugixml/pugixml.cc

@@ -14,7 +14,7 @@
 #ifndef SOURCE_PUGIXML_CPP
 #define SOURCE_PUGIXML_CPP
 
-#include <Grid/pugixml/pugixml.h>
+#include <pugixml/pugixml.h>
 
 #include <stdlib.h>
 #include <stdio.h>

lib/qcd/action/fermion/CayleyFermion5D.cc

@@ -29,8 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Eigen/Dense>
+#include <Grid.h>
-#include <Grid/Grid.h>
 
 
 namespace Grid {
@@ -49,19 +48,18 @@ namespace QCD {
 		   FourDimGrid,
  	 	   FourDimRedBlackGrid,_M5,p),
    mass(_mass)
- { 
+ { }
- }
 
 template<class Impl>  
 void CayleyFermion5D<Impl>::Dminus(const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
+  FermionField tmp(psi._grid);
 
-  FermionField tmp_f(this->FermionGrid());
+  this->DW(psi,tmp,DaggerNo);
-  this->DW(psi,tmp_f,DaggerNo);
 
   for(int s=0;s<Ls;s++){
-    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp_f,s,s);// chi = (1-c[s] D_W) psi
+    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp,s,s);// chi = (1-c[s] D_W) psi
   }
 }
 
@@ -89,8 +87,8 @@ template<class Impl> void CayleyFermion5D<Impl>::CayleyReport(void)
     std::cout << GridLogMessage << "CayleyFermion5D Number of MooeeInv Calls     : " << MooeeInvCalls   << std::endl;
     std::cout << GridLogMessage << "CayleyFermion5D ComputeTime/Calls            : " << MooeeInvTime / MooeeInvCalls << " us" << std::endl;
 
-    // Flops = MADD * Ls *Ls *4dvol * spin/colour/complex
+    // Flops = 9*12*Ls*vol/2
-    RealD mflops = 2.0*24*this->Ls*volume*MooeeInvCalls/MooeeInvTime/2; // 2 for red black counting
+    RealD mflops = 9.0*12*volume*MooeeInvCalls/MooeeInvTime/2; // 2 for red black counting
     std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
     std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
   }
@@ -112,12 +110,12 @@ template<class Impl>
 void CayleyFermion5D<Impl>::DminusDag(const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
+  FermionField tmp(psi._grid);
 
-  FermionField tmp_f(this->FermionGrid());
+  this->DW(psi,tmp,DaggerYes);
-  this->DW(psi,tmp_f,DaggerYes);
 
   for(int s=0;s<Ls;s++){
-    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp_f,s,s);// chi = (1-c[s] D_W) psi
+    axpby_ssp(chi,Coeff_t(1.0),psi,-cs[s],tmp,s,s);// chi = (1-c[s] D_W) psi
   }
 }
 template<class Impl>  
@@ -140,7 +138,6 @@ void CayleyFermion5D<Impl>::Meooe5D    (const FermionField &psi, FermionField &D
   lower[0]   =-mass*lower[0];
   M5D(psi,psi,Din,lower,diag,upper);
 }
-// FIXME Redunant with the above routine; check this and eliminate
 template<class Impl> void CayleyFermion5D<Impl>::Meo5D     (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
@@ -262,33 +259,36 @@ template<class Impl>
 void CayleyFermion5D<Impl>::Meooe       (const FermionField &psi, FermionField &chi)
 {
   int Ls=this->Ls;
+  FermionField tmp(psi._grid);
 
-  Meooe5D(psi,this->tmp()); 
+  Meooe5D(psi,tmp); 
 
   if ( psi.checkerboard == Odd ) {
-    this->DhopEO(this->tmp(),chi,DaggerNo);
+    this->DhopEO(tmp,chi,DaggerNo);
   } else {
-    this->DhopOE(this->tmp(),chi,DaggerNo);
+    this->DhopOE(tmp,chi,DaggerNo);
   }
 }
 
 template<class Impl>
 void CayleyFermion5D<Impl>::MeooeDag    (const FermionField &psi, FermionField &chi)
 {
+  FermionField tmp(psi._grid);
   // Apply 4d dslash
   if ( psi.checkerboard == Odd ) {
-    this->DhopEO(psi,this->tmp(),DaggerYes);
+    this->DhopEO(psi,tmp,DaggerYes);
   } else {
-    this->DhopOE(psi,this->tmp(),DaggerYes);
+    this->DhopOE(psi,tmp,DaggerYes);
   }
-  MeooeDag5D(this->tmp(),chi); 
+  MeooeDag5D(tmp,chi); 
 }
 
 template<class Impl>
 void  CayleyFermion5D<Impl>::Mdir (const FermionField &psi, FermionField &chi,int dir,int disp){
-  Meo5D(psi,this->tmp());
+  FermionField tmp(psi._grid);
+  Meo5D(psi,tmp);
   // Apply 4d dslash fragment
-  this->DhopDir(this->tmp(),chi,dir,disp);
+  this->DhopDir(tmp,chi,dir,disp);
 }
 // force terms; five routines; default to Dhop on diagonal
 template<class Impl>
@@ -459,91 +459,9 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
     for(int j=0;j<Ls-1;j++) delta_d *= cee[j]/bee[j];
     dee[Ls-1] += delta_d;
   }  
-
-  int inv=1;
-  this->MooeeInternalCompute(0,inv,MatpInv,MatmInv);
-  this->MooeeInternalCompute(1,inv,MatpInvDag,MatmInvDag);
-
 }
 
 
-template<class Impl>
-void CayleyFermion5D<Impl>::MooeeInternalCompute(int dag, int inv,
-						 Vector<iSinglet<Simd> > & Matp,
-						 Vector<iSinglet<Simd> > & Matm)
-{
-  int Ls=this->Ls;
-
-  GridBase *grid = this->FermionRedBlackGrid();
-  int LLs = grid->_rdimensions[0];
-
-  if ( LLs == Ls ) return; // Not vectorised in 5th direction
-
-  Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
-  Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
-  
-  for(int s=0;s<Ls;s++){
-    Pplus(s,s) = bee[s];
-    Pminus(s,s)= bee[s];
-  }
-  
-  for(int s=0;s<Ls-1;s++){
-    Pminus(s,s+1) = -cee[s];
-  }
-  
-  for(int s=0;s<Ls-1;s++){
-    Pplus(s+1,s) = -cee[s+1];
-  }
-  Pplus (0,Ls-1) = mass*cee[0];
-  Pminus(Ls-1,0) = mass*cee[Ls-1];
-  
-  Eigen::MatrixXcd PplusMat ;
-  Eigen::MatrixXcd PminusMat;
-  
-  if ( inv ) {
-    PplusMat =Pplus.inverse();
-    PminusMat=Pminus.inverse();
-  } else { 
-    PplusMat =Pplus;
-    PminusMat=Pminus;
-  }
-  
-  if(dag){
-    PplusMat.adjointInPlace();
-    PminusMat.adjointInPlace();
-  }
-  
-  typedef typename SiteHalfSpinor::scalar_type scalar_type;
-  const int Nsimd=Simd::Nsimd();
-  Matp.resize(Ls*LLs);
-  Matm.resize(Ls*LLs);
-
-  for(int s2=0;s2<Ls;s2++){
-  for(int s1=0;s1<LLs;s1++){
-    int istride = LLs;
-    int ostride = 1;
-    Simd Vp;
-    Simd Vm;
-    scalar_type *sp = (scalar_type *)&Vp;
-    scalar_type *sm = (scalar_type *)&Vm;
-    for(int l=0;l<Nsimd;l++){
-      if ( switcheroo<Coeff_t>::iscomplex() ) {
-	sp[l] = PplusMat (l*istride+s1*ostride,s2);
-	sm[l] = PminusMat(l*istride+s1*ostride,s2);
-      } else { 
-      // if real
-	scalar_type tmp;
-	tmp = PplusMat (l*istride+s1*ostride,s2);
-	sp[l] = scalar_type(tmp.real(),tmp.real());
-	tmp = PminusMat(l*istride+s1*ostride,s2);
-	sm[l] = scalar_type(tmp.real(),tmp.real());
-      }
-    }
-    Matp[LLs*s2+s1] = Vp;
-    Matm[LLs*s2+s1] = Vm;
-  }}
-}
-
 
   FermOpTemplateInstantiate(CayleyFermion5D);
   GparityFermOpTemplateInstantiate(CayleyFermion5D);

lib/qcd/action/fermion/CayleyFermion5D.h

@@ -33,31 +33,6 @@ namespace Grid {
 
   namespace QCD {
 
-     template<typename T> struct switcheroo   {  
-       static inline int iscomplex()  { return 0; } 
-
-       template<class vec>
-       static inline vec mult(vec a, vec b) {
-	 return real_mult(a,b);
-       }
-     };
-     template<> struct switcheroo<ComplexD> {  
-       static inline int iscomplex()  { return 1; } 
-
-       template<class vec>
-       static inline vec mult(vec a, vec b) {
-	 return a*b;
-       }
-     };
-     template<> struct switcheroo<ComplexF> {  
-       static inline int iscomplex()  { return 1; } 
-       template<class vec>
-       static inline vec mult(vec a, vec b) {
-	 return a*b;
-       }
-     };
-
-
     template<class Impl>
     class CayleyFermion5D : public WilsonFermion5D<Impl>
     {
@@ -100,19 +75,7 @@ namespace Grid {
 		  std::vector<Coeff_t> &lower,
 		  std::vector<Coeff_t> &diag,
 		  std::vector<Coeff_t> &upper);
-
       void MooeeInternal(const FermionField &in, FermionField &out,int dag,int inv);
-      void MooeeInternalCompute(int dag, int inv, Vector<iSinglet<Simd> > & Matp, Vector<iSinglet<Simd> > & Matm);
-
-      void MooeeInternalAsm(const FermionField &in, FermionField &out,
-			    int LLs, int site,
-			    Vector<iSinglet<Simd> > &Matp,
-			    Vector<iSinglet<Simd> > &Matm);
-      void MooeeInternalZAsm(const FermionField &in, FermionField &out,
-			    int LLs, int site,
-			    Vector<iSinglet<Simd> > &Matp,
-			    Vector<iSinglet<Simd> > &Matm);
-
 
       virtual void   Instantiatable(void)=0;
 
@@ -149,12 +112,6 @@ namespace Grid {
       std::vector<Coeff_t> ueem;    
       std::vector<Coeff_t> dee;    
 
-      // Matrices of 5d ee inverse params
-      Vector<iSinglet<Simd> >  MatpInv;
-      Vector<iSinglet<Simd> >  MatmInv;
-      Vector<iSinglet<Simd> >  MatpInvDag;
-      Vector<iSinglet<Simd> >  MatmInvDag;
-
       // Constructors
       CayleyFermion5D(GaugeField &_Umu,
 		      GridCartesian         &FiveDimGrid,

lib/qcd/action/fermion/CayleyFermion5Dcache.cc

@@ -29,7 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 
 namespace Grid {

lib/qcd/action/fermion/CayleyFermion5Ddense.cc

@@ -30,7 +30,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 
 #include <Grid/Eigen/Dense>
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 
 namespace Grid {

lib/qcd/action/fermion/CayleyFermion5Dssp.cc

@@ -29,7 +29,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 
 namespace Grid {

lib/qcd/action/fermion/CayleyFermion5Dvec.cc

@@ -29,12 +29,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 
-
+#include <Grid/Eigen/Dense>
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 
 namespace Grid {
-namespace QCD {  /*
+namespace QCD {
+  /*
    * Dense matrix versions of routines
    */
 template<class Impl>
@@ -125,6 +126,7 @@ PARALLEL_FOR_LOOP
       for(int v=0;v<LLs;v++){
 
 	vprefetch(psi[ss+v+LLs]);
+	//	vprefetch(phi[ss+v+LLs]);
 
 	int vp= (v==LLs-1) ? 0     : v+1;
 	int vm= (v==0    ) ? LLs-1 : v-1;
@@ -143,6 +145,9 @@ PARALLEL_FOR_LOOP
 	Simd hm_11 = psi[ss+vm]()(1)(1); 
 	Simd hm_12 = psi[ss+vm]()(1)(2); 
 
+	//	if ( ss==0) std::cout << " hp_00 " <<hp_00<<std::endl;
+	//	if ( ss==0) std::cout << " hm_00 " <<hm_00<<std::endl;
+
 	if ( vp<=v ) {
 	  hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
 	  hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
@@ -160,20 +165,42 @@ PARALLEL_FOR_LOOP
 	  hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
 	}
 
-	// Can force these to real arithmetic and save 2x.
+	/*
-	Simd p_00  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_00); 
+	if ( ss==0) std::cout << " dphi_00 " <<d[v]()()() * phi[ss+v]()(0)(0) <<std::endl;
-	Simd p_01  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_01); 
+	if ( ss==0) std::cout << " dphi_10 " <<d[v]()()() * phi[ss+v]()(1)(0) <<std::endl;
-	Simd p_02  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_02); 
+	if ( ss==0) std::cout << " dphi_20 " <<d[v]()()() * phi[ss+v]()(2)(0) <<std::endl;
-	Simd p_10  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_10); 
+	if ( ss==0) std::cout << " dphi_30 " <<d[v]()()() * phi[ss+v]()(3)(0) <<std::endl;
-	Simd p_11  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_11); 
+	*/	
-	Simd p_12  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_12); 
+	Simd p_00  = d[v]()()() * phi[ss+v]()(0)(0)  + l[v]()()()*hm_00; 
-	Simd p_20  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_00); 
+	Simd p_01  = d[v]()()() * phi[ss+v]()(0)(1)  + l[v]()()()*hm_01; 
-	Simd p_21  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_01); 
+	Simd p_02  = d[v]()()() * phi[ss+v]()(0)(2)  + l[v]()()()*hm_02; 
-	Simd p_22  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_02);  
+	Simd p_10  = d[v]()()() * phi[ss+v]()(1)(0)  + l[v]()()()*hm_10; 
-	Simd p_30  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_10); 
+	Simd p_11  = d[v]()()() * phi[ss+v]()(1)(1)  + l[v]()()()*hm_11; 
-	Simd p_31  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_11); 
+	Simd p_12  = d[v]()()() * phi[ss+v]()(1)(2)  + l[v]()()()*hm_12; 
-	Simd p_32  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_12); 
+	Simd p_20  = d[v]()()() * phi[ss+v]()(2)(0)  + u[v]()()()*hp_00; 
+	Simd p_21  = d[v]()()() * phi[ss+v]()(2)(1)  + u[v]()()()*hp_01; 
+	Simd p_22  = d[v]()()() * phi[ss+v]()(2)(2)  + u[v]()()()*hp_02;  
+	Simd p_30  = d[v]()()() * phi[ss+v]()(3)(0)  + u[v]()()()*hp_10; 
+	Simd p_31  = d[v]()()() * phi[ss+v]()(3)(1)  + u[v]()()()*hp_11; 
+	Simd p_32  = d[v]()()() * phi[ss+v]()(3)(2)  + u[v]()()()*hp_12; 
 
+	
+	//	if ( ss==0){
+	/*
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(0)(0) << " bad "<<p_00<<" diff "<<chi[ss+v]()(0)(0)-p_00<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(0)(1) << " bad "<<p_01<<" diff "<<chi[ss+v]()(0)(1)-p_01<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(0)(2) << " bad "<<p_02<<" diff "<<chi[ss+v]()(0)(2)-p_02<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(1)(0) << " bad "<<p_10<<" diff "<<chi[ss+v]()(1)(0)-p_10<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(1)(1) << " bad "<<p_11<<" diff "<<chi[ss+v]()(1)(1)-p_11<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(1)(2) << " bad "<<p_12<<" diff "<<chi[ss+v]()(1)(2)-p_12<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(2)(0) << " bad "<<p_20<<" diff "<<chi[ss+v]()(2)(0)-p_20<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(2)(1) << " bad "<<p_21<<" diff "<<chi[ss+v]()(2)(1)-p_21<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(2)(2) << " bad "<<p_22<<" diff "<<chi[ss+v]()(2)(2)-p_22<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(3)(0) << " bad "<<p_30<<" diff "<<chi[ss+v]()(3)(0)-p_30<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(3)(1) << " bad "<<p_31<<" diff "<<chi[ss+v]()(3)(1)-p_31<<std::endl;
+	std::cout << ss<<" "<< v<< " good "<< chi[ss+v]()(3)(2) << " bad "<<p_32<<" diff "<<chi[ss+v]()(3)(2)-p_32<<std::endl;
+	}
+	*/
 	vstream(chi[ss+v]()(0)(0),p_00);
 	vstream(chi[ss+v]()(0)(1),p_01);
 	vstream(chi[ss+v]()(0)(2),p_02);
@@ -234,7 +261,7 @@ void CayleyFermion5D<Impl>::M5Ddag(const FermionField &psi,
   M5Dtime-=usecond();
 PARALLEL_FOR_LOOP
   for(int ss=0;ss<grid->oSites();ss+=LLs){ // adds LLs
-#if 0
+
     alignas(64) SiteHalfSpinor hp;
     alignas(64) SiteHalfSpinor hm;
     alignas(64) SiteSpinor fp;
@@ -260,504 +287,9 @@ PARALLEL_FOR_LOOP
       chi[ss+v] = chi[ss+v]     +l[v]*fm;
 
     }
-#else
-      for(int v=0;v<LLs;v++){
-
-	vprefetch(psi[ss+v+LLs]);
-
-	int vp= (v==LLs-1) ? 0     : v+1;
-	int vm= (v==0    ) ? LLs-1 : v-1;
-	
-	Simd hp_00 = psi[ss+vp]()(0)(0); 
-	Simd hp_01 = psi[ss+vp]()(0)(1); 
-	Simd hp_02 = psi[ss+vp]()(0)(2); 
-	Simd hp_10 = psi[ss+vp]()(1)(0); 
-	Simd hp_11 = psi[ss+vp]()(1)(1); 
-	Simd hp_12 = psi[ss+vp]()(1)(2); 
-	
-	Simd hm_00 = psi[ss+vm]()(2)(0); 
-	Simd hm_01 = psi[ss+vm]()(2)(1); 
-	Simd hm_02 = psi[ss+vm]()(2)(2); 
-	Simd hm_10 = psi[ss+vm]()(3)(0); 
-	Simd hm_11 = psi[ss+vm]()(3)(1); 
-	Simd hm_12 = psi[ss+vm]()(3)(2); 
-
-	if ( vp<=v ) {
-	  hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
-	  hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
-	  hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
-	  hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
-	  hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
-	  hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
-	}
-	if ( vm>=v ) {
-	  hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
-	  hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
-	  hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
-	  hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
-	  hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
-	  hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
-	}
-
-	Simd p_00  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_00); 
-	Simd p_01  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_01); 
-	Simd p_02  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_02); 
-	Simd p_10  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_10); 
-	Simd p_11  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_11); 
-	Simd p_12  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2))  + switcheroo<Coeff_t>::mult(u[v]()()(),hp_12); 
-
-	Simd p_20  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_00); 
-	Simd p_21  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_01); 
-	Simd p_22  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_02);  
-	Simd p_30  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_10); 
-	Simd p_31  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_11); 
-	Simd p_32  = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2))  + switcheroo<Coeff_t>::mult(l[v]()()(),hm_12); 
-
-	vstream(chi[ss+v]()(0)(0),p_00);
-	vstream(chi[ss+v]()(0)(1),p_01);
-	vstream(chi[ss+v]()(0)(2),p_02);
-	vstream(chi[ss+v]()(1)(0),p_10);
-	vstream(chi[ss+v]()(1)(1),p_11);
-	vstream(chi[ss+v]()(1)(2),p_12);
-	vstream(chi[ss+v]()(2)(0),p_20);
-	vstream(chi[ss+v]()(2)(1),p_21);
-	vstream(chi[ss+v]()(2)(2),p_22);
-	vstream(chi[ss+v]()(3)(0),p_30);
-	vstream(chi[ss+v]()(3)(1),p_31);
-	vstream(chi[ss+v]()(3)(2),p_32);
-      }
-#endif
   }
   M5Dtime+=usecond();
 }
-
-
-#ifdef AVX512 
-#include <simd/Intel512common.h>
-#include <simd/Intel512avx.h>
-#include <simd/Intel512single.h>
-#endif 
-
-template<class Impl>
-void CayleyFermion5D<Impl>::MooeeInternalAsm(const FermionField &psi, FermionField &chi,
-					     int LLs, int site,
-					     Vector<iSinglet<Simd> > &Matp,
-					     Vector<iSinglet<Simd> > &Matm)
-{
-#ifndef AVX512
-  {
-  SiteHalfSpinor BcastP;
-  SiteHalfSpinor BcastM;
-  SiteHalfSpinor SiteChiP;
-  SiteHalfSpinor SiteChiM;
-
-  // Ls*Ls * 2 * 12 * vol flops
-  for(int s1=0;s1<LLs;s1++){ 
-    for(int s2=0;s2<LLs;s2++){ 
-      for(int  l=0; l<Simd::Nsimd();l++){ // simd lane
-
-        int s=s2+l*LLs;
-	int lex=s2+LLs*site;
-	
-	if ( s2==0 && l==0) {
-	  SiteChiP=zero;
-	  SiteChiM=zero;
-	}
-	
-	for(int sp=0;sp<2;sp++){
-        for(int co=0;co<Nc;co++){
-	  vbroadcast(BcastP()(sp  )(co),psi[lex]()(sp)(co),l);
-	}}
-	for(int sp=0;sp<2;sp++){
-        for(int co=0;co<Nc;co++){
-	  vbroadcast(BcastM()(sp  )(co),psi[lex]()(sp+2)(co),l);
-	}}
-
-	for(int sp=0;sp<2;sp++){
-        for(int co=0;co<Nc;co++){
-	  SiteChiP()(sp)(co)=real_madd(Matp[LLs*s+s1]()()(),BcastP()(sp)(co),SiteChiP()(sp)(co)); // 1100 us.
-	  SiteChiM()(sp)(co)=real_madd(Matm[LLs*s+s1]()()(),BcastM()(sp)(co),SiteChiM()(sp)(co)); // each found by commenting out
-	}}
-
-    }}
-    {
-      int lex = s1+LLs*site;
-      for(int sp=0;sp<2;sp++){
-      for(int co=0;co<Nc;co++){
-	vstream(chi[lex]()(sp)(co), SiteChiP()(sp)(co));
-	vstream(chi[lex]()(sp+2)(co), SiteChiM()(sp)(co));
-      }}
-    }
-  }
-
-  }
-#else
-  {
-  // pointers
-    //  MASK_REGS;
-#define Chi_00 %%zmm1
-#define Chi_01 %%zmm2
-#define Chi_02 %%zmm3
-#define Chi_10 %%zmm4
-#define Chi_11 %%zmm5
-#define Chi_12 %%zmm6
-#define Chi_20 %%zmm7
-#define Chi_21 %%zmm8
-#define Chi_22 %%zmm9
-#define Chi_30 %%zmm10
-#define Chi_31 %%zmm11
-#define Chi_32 %%zmm12
-
-#define BCAST0   %%zmm13
-#define BCAST1   %%zmm14
-#define BCAST2   %%zmm15
-#define BCAST3   %%zmm16
-#define BCAST4   %%zmm17
-#define BCAST5   %%zmm18
-#define BCAST6   %%zmm19
-#define BCAST7   %%zmm20
-#define BCAST8   %%zmm21
-#define BCAST9   %%zmm22
-#define BCAST10  %%zmm23
-#define BCAST11  %%zmm24
-
-  int incr=LLs*LLs*sizeof(iSinglet<Simd>);
-  for(int s1=0;s1<LLs;s1++){ 
-    for(int s2=0;s2<LLs;s2++){ 
-      int lex=s2+LLs*site;
-      uint64_t a0 = (uint64_t)&Matp[LLs*s2+s1]; // should be cacheable
-      uint64_t a1 = (uint64_t)&Matm[LLs*s2+s1];
-      uint64_t a2 = (uint64_t)&psi[lex];
-      for(int  l=0; l<Simd::Nsimd();l++){ // simd lane
-	if ( (s2+l)==0 ) {
-	  asm (
-  	           VPREFETCH1(0,%2)  	     VPREFETCH1(0,%1)
-  	           VPREFETCH1(12,%2)  	     VPREFETCH1(13,%2)
-  	           VPREFETCH1(14,%2)  	     VPREFETCH1(15,%2)         
-		   VBCASTCDUP(0,%2,BCAST0)   
-		   VBCASTCDUP(1,%2,BCAST1)   
-		   VBCASTCDUP(2,%2,BCAST2)   
-		   VBCASTCDUP(3,%2,BCAST3)   
-		   VBCASTCDUP(4,%2,BCAST4)     VMULMEM (0,%0,BCAST0,Chi_00)
-		   VBCASTCDUP(5,%2,BCAST5)     VMULMEM (0,%0,BCAST1,Chi_01)
-		   VBCASTCDUP(6,%2,BCAST6)     VMULMEM (0,%0,BCAST2,Chi_02)
-		   VBCASTCDUP(7,%2,BCAST7)     VMULMEM (0,%0,BCAST3,Chi_10)
-		   VBCASTCDUP(8,%2,BCAST8)     VMULMEM (0,%0,BCAST4,Chi_11)
-		   VBCASTCDUP(9,%2,BCAST9)     VMULMEM (0,%0,BCAST5,Chi_12)
-		   VBCASTCDUP(10,%2,BCAST10)   VMULMEM (0,%1,BCAST6,Chi_20)
-		   VBCASTCDUP(11,%2,BCAST11)   VMULMEM (0,%1,BCAST7,Chi_21)
-		   VMULMEM (0,%1,BCAST8,Chi_22)         
-		   VMULMEM (0,%1,BCAST9,Chi_30)
-		   VMULMEM (0,%1,BCAST10,Chi_31)       
-		   VMULMEM (0,%1,BCAST11,Chi_32)
-		   : : "r" (a0), "r" (a1), "r" (a2)  );
-	} else { 
-	  asm (
-		   VBCASTCDUP(0,%2,BCAST0)   VMADDMEM (0,%0,BCAST0,Chi_00)
-		   VBCASTCDUP(1,%2,BCAST1)   VMADDMEM (0,%0,BCAST1,Chi_01)
-		   VBCASTCDUP(2,%2,BCAST2)   VMADDMEM (0,%0,BCAST2,Chi_02)
-		   VBCASTCDUP(3,%2,BCAST3)   VMADDMEM (0,%0,BCAST3,Chi_10)
-		   VBCASTCDUP(4,%2,BCAST4)   VMADDMEM (0,%0,BCAST4,Chi_11)
-		   VBCASTCDUP(5,%2,BCAST5)   VMADDMEM (0,%0,BCAST5,Chi_12)
-		   VBCASTCDUP(6,%2,BCAST6)   VMADDMEM (0,%1,BCAST6,Chi_20)
-		   VBCASTCDUP(7,%2,BCAST7)   VMADDMEM (0,%1,BCAST7,Chi_21)
-		   VBCASTCDUP(8,%2,BCAST8)   VMADDMEM (0,%1,BCAST8,Chi_22)
-		   VBCASTCDUP(9,%2,BCAST9)   VMADDMEM (0,%1,BCAST9,Chi_30)
-		   VBCASTCDUP(10,%2,BCAST10)  VMADDMEM (0,%1,BCAST10,Chi_31)
-		   VBCASTCDUP(11,%2,BCAST11)  VMADDMEM (0,%1,BCAST11,Chi_32) 
-		   : : "r" (a0), "r" (a1), "r" (a2)  );
-	}
-	a0 = a0+incr;
-	a1 = a1+incr;
-	a2 = a2+sizeof(Simd::scalar_type);
-      }}
-    {
-      int lexa = s1+LLs*site;
-      asm (
-	       VSTORE(0,%0,Chi_00) VSTORE(1 ,%0,Chi_01)  VSTORE(2 ,%0,Chi_02)		
-	       VSTORE(3,%0,Chi_10) VSTORE(4 ,%0,Chi_11)  VSTORE(5 ,%0,Chi_12)		
-	       VSTORE(6,%0,Chi_20) VSTORE(7 ,%0,Chi_21)  VSTORE(8 ,%0,Chi_22)		
-	       VSTORE(9,%0,Chi_30) VSTORE(10,%0,Chi_31)  VSTORE(11,%0,Chi_32)		
-	       : : "r" ((uint64_t)&chi[lexa]) : "memory" );
-
-    }
-  }
-  }
-#undef Chi_00
-#undef Chi_01
-#undef Chi_02
-#undef Chi_10
-#undef Chi_11
-#undef Chi_12
-#undef Chi_20
-#undef Chi_21
-#undef Chi_22
-#undef Chi_30
-#undef Chi_31
-#undef Chi_32
-
-#undef BCAST0
-#undef BCAST1
-#undef BCAST2
-#undef BCAST3
-#undef BCAST4
-#undef BCAST5
-#undef BCAST6
-#undef BCAST7
-#undef BCAST8
-#undef BCAST9
-#undef BCAST10
-#undef BCAST11
-#endif
-};
-
-  // Z-mobius version
-template<class Impl>
-void CayleyFermion5D<Impl>::MooeeInternalZAsm(const FermionField &psi, FermionField &chi,
-					     int LLs, int site, Vector<iSinglet<Simd> > &Matp, Vector<iSinglet<Simd> > &Matm)
-{
-#ifndef AVX512
-  {
-  SiteHalfSpinor BcastP;
-  SiteHalfSpinor BcastM;
-  SiteHalfSpinor SiteChiP;
-  SiteHalfSpinor SiteChiM;
-
-  // Ls*Ls * 2 * 12 * vol flops
-  for(int s1=0;s1<LLs;s1++){ 
-    for(int s2=0;s2<LLs;s2++){ 
-      for(int  l=0; l<Simd::Nsimd();l++){ // simd lane
-
-        int s=s2+l*LLs;
-	int lex=s2+LLs*site;
-	
-	if ( s2==0 && l==0) {
-	  SiteChiP=zero;
-	  SiteChiM=zero;
-	}
-	
-	for(int sp=0;sp<2;sp++){
-        for(int co=0;co<Nc;co++){
-	  vbroadcast(BcastP()(sp  )(co),psi[lex]()(sp)(co),l);
-	}}
-	for(int sp=0;sp<2;sp++){
-        for(int co=0;co<Nc;co++){
-	  vbroadcast(BcastM()(sp  )(co),psi[lex]()(sp+2)(co),l);
-	}}
-
-	for(int sp=0;sp<2;sp++){
-        for(int co=0;co<Nc;co++){
-	  SiteChiP()(sp)(co)=SiteChiP()(sp)(co)+ Matp[LLs*s+s1]()()()*BcastP()(sp)(co); 
-	  SiteChiM()(sp)(co)=SiteChiM()(sp)(co)+ Matm[LLs*s+s1]()()()*BcastM()(sp)(co); 
-	}}
-
-
-    }}
-    {
-      int lex = s1+LLs*site;
-      for(int sp=0;sp<2;sp++){
-      for(int co=0;co<Nc;co++){
-	vstream(chi[lex]()(sp)(co), SiteChiP()(sp)(co));
-	vstream(chi[lex]()(sp+2)(co), SiteChiM()(sp)(co));
-      }}
-    }
-  }
-
-  }
-#else
-  {
-  // pointers
-  //  MASK_REGS;
-#define Chi_00 %zmm0
-#define Chi_01 %zmm1
-#define Chi_02 %zmm2
-#define Chi_10 %zmm3
-#define Chi_11 %zmm4
-#define Chi_12 %zmm5
-#define Chi_20 %zmm6
-#define Chi_21 %zmm7
-#define Chi_22 %zmm8
-#define Chi_30 %zmm9
-#define Chi_31 %zmm10
-#define Chi_32 %zmm11
-#define pChi_00 %%zmm0
-#define pChi_01 %%zmm1
-#define pChi_02 %%zmm2
-#define pChi_10 %%zmm3
-#define pChi_11 %%zmm4
-#define pChi_12 %%zmm5
-#define pChi_20 %%zmm6
-#define pChi_21 %%zmm7
-#define pChi_22 %%zmm8
-#define pChi_30 %%zmm9
-#define pChi_31 %%zmm10
-#define pChi_32 %%zmm11
-
-#define BCAST_00   %zmm12
-#define  SHUF_00   %zmm13
-#define BCAST_01   %zmm14
-#define  SHUF_01   %zmm15
-#define BCAST_02   %zmm16
-#define  SHUF_02   %zmm17
-#define BCAST_10   %zmm18
-#define  SHUF_10   %zmm19
-#define BCAST_11   %zmm20
-#define  SHUF_11   %zmm21
-#define BCAST_12   %zmm22
-#define  SHUF_12   %zmm23
-
-#define Mp  %zmm24
-#define Mps %zmm25
-#define Mm  %zmm26
-#define Mms %zmm27
-#define N 8
-  int incr=LLs*LLs*sizeof(iSinglet<Simd>);
-  for(int s1=0;s1<LLs;s1++){ 
-    for(int s2=0;s2<LLs;s2++){ 
-      int lex=s2+LLs*site;
-      uint64_t a0 = (uint64_t)&Matp[LLs*s2+s1]; // should be cacheable
-      uint64_t a1 = (uint64_t)&Matm[LLs*s2+s1];
-      uint64_t a2 = (uint64_t)&psi[lex];
-      for(int  l=0; l<Simd::Nsimd();l++){ // simd lane
-	if ( (s2+l)==0 ) {
-	  LOAD64(%r8,a0);
-	  LOAD64(%r9,a1);
-	  LOAD64(%r10,a2);
-	  asm (
-	       VLOAD(0,%r8,Mp)// i r
-	       VLOAD(0,%r9,Mm)
-	       VSHUF(Mp,Mps)  // r i 
-	       VSHUF(Mm,Mms)
-	       VPREFETCH1(12,%r10)  	     VPREFETCH1(13,%r10)
-	       VPREFETCH1(14,%r10)  	     VPREFETCH1(15,%r10)         
-
-	       VMULIDUP(0*N,%r10,Mps,Chi_00)
-	       VMULIDUP(1*N,%r10,Mps,Chi_01)
-	       VMULIDUP(2*N,%r10,Mps,Chi_02)
-	       VMULIDUP(3*N,%r10,Mps,Chi_10)
-	       VMULIDUP(4*N,%r10,Mps,Chi_11)
-	       VMULIDUP(5*N,%r10,Mps,Chi_12)
-
-	       VMULIDUP(6*N ,%r10,Mms,Chi_20)
-	       VMULIDUP(7*N ,%r10,Mms,Chi_21)
-	       VMULIDUP(8*N ,%r10,Mms,Chi_22)
-	       VMULIDUP(9*N ,%r10,Mms,Chi_30)
-	       VMULIDUP(10*N,%r10,Mms,Chi_31)
-	       VMULIDUP(11*N,%r10,Mms,Chi_32)
-
-	       VMADDSUBRDUP(0*N,%r10,Mp,Chi_00)
-	       VMADDSUBRDUP(1*N,%r10,Mp,Chi_01)
-	       VMADDSUBRDUP(2*N,%r10,Mp,Chi_02)
-	       VMADDSUBRDUP(3*N,%r10,Mp,Chi_10)
-	       VMADDSUBRDUP(4*N,%r10,Mp,Chi_11)
-	       VMADDSUBRDUP(5*N,%r10,Mp,Chi_12)
-
-	       VMADDSUBRDUP(6*N ,%r10,Mm,Chi_20)
-	       VMADDSUBRDUP(7*N ,%r10,Mm,Chi_21)
-	       VMADDSUBRDUP(8*N ,%r10,Mm,Chi_22)
-	       VMADDSUBRDUP(9*N ,%r10,Mm,Chi_30)
-	       VMADDSUBRDUP(10*N,%r10,Mm,Chi_31)
-	       VMADDSUBRDUP(11*N,%r10,Mm,Chi_32)
-	       );
-	} else { 
-	  LOAD64(%r8,a0);
-	  LOAD64(%r9,a1);
-	  LOAD64(%r10,a2);
-	  asm (
-	       VLOAD(0,%r8,Mp)
-	       VSHUF(Mp,Mps)
-
-	       VLOAD(0,%r9,Mm)
-	       VSHUF(Mm,Mms)
-
-	       VMADDSUBIDUP(0*N,%r10,Mps,Chi_00) //  Mri * Pii +- Cir
-	       VMADDSUBIDUP(1*N,%r10,Mps,Chi_01)
-	       VMADDSUBIDUP(2*N,%r10,Mps,Chi_02)
-	       VMADDSUBIDUP(3*N,%r10,Mps,Chi_10)
-	       VMADDSUBIDUP(4*N,%r10,Mps,Chi_11)
-	       VMADDSUBIDUP(5*N,%r10,Mps,Chi_12)
-
-	       VMADDSUBIDUP(6 *N,%r10,Mms,Chi_20)
-	       VMADDSUBIDUP(7 *N,%r10,Mms,Chi_21)
-	       VMADDSUBIDUP(8 *N,%r10,Mms,Chi_22)
-	       VMADDSUBIDUP(9 *N,%r10,Mms,Chi_30)
-	       VMADDSUBIDUP(10*N,%r10,Mms,Chi_31)
-	       VMADDSUBIDUP(11*N,%r10,Mms,Chi_32)
-
-	       VMADDSUBRDUP(0*N,%r10,Mp,Chi_00) //  Cir = Mir * Prr +- ( Mri * Pii +- Cir) 
-	       VMADDSUBRDUP(1*N,%r10,Mp,Chi_01) //  Ci = MiPr + Ci + MrPi ;    Cr = MrPr - ( MiPi - Cr)
-	       VMADDSUBRDUP(2*N,%r10,Mp,Chi_02)
-	       VMADDSUBRDUP(3*N,%r10,Mp,Chi_10)
-	       VMADDSUBRDUP(4*N,%r10,Mp,Chi_11)
-	       VMADDSUBRDUP(5*N,%r10,Mp,Chi_12)
-
-	       VMADDSUBRDUP(6 *N,%r10,Mm,Chi_20)
-	       VMADDSUBRDUP(7 *N,%r10,Mm,Chi_21)
-	       VMADDSUBRDUP(8 *N,%r10,Mm,Chi_22)
-	       VMADDSUBRDUP(9 *N,%r10,Mm,Chi_30)
-	       VMADDSUBRDUP(10*N,%r10,Mm,Chi_31)
-	       VMADDSUBRDUP(11*N,%r10,Mm,Chi_32)
-	       );
-	}
-	a0 = a0+incr;
-	a1 = a1+incr;
-	a2 = a2+sizeof(Simd::scalar_type);
-      }}
-    {
-      int lexa = s1+LLs*site;
-      /*
-      SiteSpinor tmp;
-      asm (
-	       VSTORE(0,%0,pChi_00) VSTORE(1 ,%0,pChi_01)  VSTORE(2 ,%0,pChi_02)		
-	       VSTORE(3,%0,pChi_10) VSTORE(4 ,%0,pChi_11)  VSTORE(5 ,%0,pChi_12)		
-	       VSTORE(6,%0,pChi_20) VSTORE(7 ,%0,pChi_21)  VSTORE(8 ,%0,pChi_22)		
-	       VSTORE(9,%0,pChi_30) VSTORE(10,%0,pChi_31)  VSTORE(11,%0,pChi_32)		
-	       : : "r" ((uint64_t)&tmp) : "memory" );
-      */
-
-      asm (
-	       VSTORE(0,%0,pChi_00) VSTORE(1 ,%0,pChi_01)  VSTORE(2 ,%0,pChi_02)		
-	       VSTORE(3,%0,pChi_10) VSTORE(4 ,%0,pChi_11)  VSTORE(5 ,%0,pChi_12)		
-	       VSTORE(6,%0,pChi_20) VSTORE(7 ,%0,pChi_21)  VSTORE(8 ,%0,pChi_22)		
-	       VSTORE(9,%0,pChi_30) VSTORE(10,%0,pChi_31)  VSTORE(11,%0,pChi_32)		
-	       : : "r" ((uint64_t)&chi[lexa]) : "memory" );
-
-      //      if ( 1 || (site==0) ) { 
-      //	std::cout<<site << " s1 "<<s1<<"\n\t"<<tmp << "\n't" << chi[lexa] <<"\n\t"<<tmp-chi[lexa]<<std::endl;
-      //      }
-    }
-  }
-  }
-#undef Chi_00
-#undef Chi_01
-#undef Chi_02
-#undef Chi_10
-#undef Chi_11
-#undef Chi_12
-#undef Chi_20
-#undef Chi_21
-#undef Chi_22
-#undef Chi_30
-#undef Chi_31
-#undef Chi_32
-
-#undef BCAST0
-#undef BCAST1
-#undef BCAST2
-#undef BCAST3
-#undef BCAST4
-#undef BCAST5
-#undef BCAST6
-#undef BCAST7
-#undef BCAST8
-#undef BCAST9
-#undef BCAST10
-#undef BCAST11
-
-#endif
-};
-
-
 template<class Impl>
 void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv)
 {
@@ -767,41 +299,108 @@ void CayleyFermion5D<Impl>::MooeeInternal(const FermionField &psi, FermionField
 
   chi.checkerboard=psi.checkerboard;
   
-  Vector<iSinglet<Simd> >  Matp;
+  Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
-  Vector<iSinglet<Simd> >  Matm;
+  Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
-  Vector<iSinglet<Simd> >  *_Matp;
-  Vector<iSinglet<Simd> >  *_Matm;
   
-  //  MooeeInternalCompute(dag,inv,Matp,Matm);
+  for(int s=0;s<Ls;s++){
-  if ( inv && dag ) { 
+    Pplus(s,s) = bee[s];
-    _Matp = &MatpInvDag;
+    Pminus(s,s)= bee[s];
-    _Matm = &MatmInvDag;
   }
-  if ( inv && (!dag) ) { 
+  
-    _Matp = &MatpInv;
+  for(int s=0;s<Ls-1;s++){
-    _Matm = &MatmInv;
+    Pminus(s,s+1) = -cee[s];
   }
-  if ( !inv ) {
+  
-    MooeeInternalCompute(dag,inv,Matp,Matm);
+  for(int s=0;s<Ls-1;s++){
-    _Matp = &Matp;
+    Pplus(s+1,s) = -cee[s+1];
-    _Matm = &Matm;
+  }
+  Pplus (0,Ls-1) = mass*cee[0];
+  Pminus(Ls-1,0) = mass*cee[Ls-1];
+  
+  Eigen::MatrixXcd PplusMat ;
+  Eigen::MatrixXcd PminusMat;
+  
+  if ( inv ) {
+    PplusMat =Pplus.inverse();
+    PminusMat=Pminus.inverse();
+  } else { 
+    PplusMat =Pplus;
+    PminusMat=Pminus;
+  }
+  
+  if(dag){
+    PplusMat.adjointInPlace();
+    PminusMat.adjointInPlace();
+  }
+  
+  typedef typename SiteHalfSpinor::scalar_type scalar_type;
+  const int Nsimd=Simd::Nsimd();
+  Vector<iSinglet<Simd> > Matp(Ls*LLs);
+  Vector<iSinglet<Simd> > Matm(Ls*LLs);
+
+  for(int s2=0;s2<Ls;s2++){
+  for(int s1=0;s1<LLs;s1++){
+    int istride = LLs;
+    int ostride = 1;
+      Simd Vp;
+      Simd Vm;
+      scalar_type *sp = (scalar_type *)&Vp;
+      scalar_type *sm = (scalar_type *)&Vm;
+      for(int l=0;l<Nsimd;l++){
+	sp[l] = PplusMat (l*istride+s1*ostride ,s2);
+	sm[l] = PminusMat(l*istride+s1*ostride,s2);
+      }
+      Matp[LLs*s2+s1] = Vp;
+      Matm[LLs*s2+s1] = Vm;
+    }
   }
-  assert(_Matp->size()==Ls*LLs);
   
   MooeeInvCalls++;
   MooeeInvTime-=usecond();
+  // Dynamic allocate on stack to get per thread without serialised heap acces
+#pragma omp parallel  
+  {
 
-  if ( switcheroo<Coeff_t>::iscomplex() ) {
+    Vector<SiteHalfSpinor> SitePplus(LLs);
-  PARALLEL_FOR_LOOP
+    Vector<SiteHalfSpinor> SitePminus(LLs);
-    for(auto site=0;site<vol;site++){
+    Vector<SiteHalfSpinor> SiteChiP(LLs);
-      MooeeInternalZAsm(psi,chi,LLs,site,*_Matp,*_Matm);
+    Vector<SiteHalfSpinor> SiteChiM(LLs);
+    Vector<SiteSpinor>     SiteChi(LLs);
+
+    SiteHalfSpinor BcastP;
+    SiteHalfSpinor BcastM;
+
+#pragma omp for 
+  for(auto site=0;site<vol;site++){
+
+    for(int s=0;s<LLs;s++){
+      int lex = s+LLs*site;
+      spProj5p(SitePplus[s] ,psi[lex]);
+      spProj5m(SitePminus[s],psi[lex]);
+      SiteChiP[s]=zero;
+      SiteChiM[s]=zero;
     }
-  } else { 
+      
-  PARALLEL_FOR_LOOP
+    int s=0;
-    for(auto site=0;site<vol;site++){
+    for(int  l=0; l<Simd::Nsimd();l++){ // simd lane
-      MooeeInternalAsm(psi,chi,LLs,site,*_Matp,*_Matm);
+      for(int s2=0;s2<LLs;s2++){ // Column loop of right hand side
+	vbroadcast(BcastP,SitePplus [s2],l);
+	vbroadcast(BcastM,SitePminus[s2],l);
+	for(int s1=0;s1<LLs;s1++){ // Column loop of reduction variables
+	  SiteChiP[s1]=SiteChiP[s1]+Matp[LLs*s+s1]*BcastP;
+	  SiteChiM[s1]=SiteChiM[s1]+Matm[LLs*s+s1]*BcastM;
+	}
+      s++;
+    }}
+
+    for(int s=0;s<LLs;s++){
+      int lex = s+LLs*site;
+      spRecon5p(SiteChi[s],SiteChiP[s]);
+      accumRecon5m(SiteChi[s],SiteChiM[s]);
+      chi[lex] = SiteChi[s]*0.5;
     }
   }
+  }
   MooeeInvTime+=usecond();
 }
 
@@ -815,5 +414,4 @@ template void CayleyFermion5D<DomainWallVec5dImplD>::MooeeInternal(const Fermion
 template void CayleyFermion5D<ZDomainWallVec5dImplF>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 template void CayleyFermion5D<ZDomainWallVec5dImplD>::MooeeInternal(const FermionField &psi, FermionField &chi,int dag, int inv);
 
-
 }}

lib/qcd/action/fermion/ContinuedFractionFermion5D.cc

@@ -26,7 +26,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 namespace Grid {
   namespace QCD {

lib/qcd/action/fermion/FermionOperator.h

@@ -48,8 +48,6 @@ namespace Grid {
 
       FermionOperator(const ImplParams &p= ImplParams()) : Impl(p) {};
 
-      virtual FermionField &tmp(void) = 0;
-
       GridBase * Grid(void)   { return FermionGrid(); };   // this is all the linalg routines need to know
       GridBase * RedBlackGrid(void) { return FermionRedBlackGrid(); };
 

lib/qcd/action/fermion/FermionOperatorImpl.h

@@ -48,10 +48,8 @@ namespace QCD {
   //    typedef typename XXX         GaugeField;
   //    typedef typename XXX      GaugeActField;
   //    typedef typename XXX       FermionField;
-  //    typedef typename XXX    PropagatorField;
   //    typedef typename XXX  DoubledGaugeField;
   //    typedef typename XXX         SiteSpinor;
-  //    typedef typename XXX     SitePropagator;
   //    typedef typename XXX     SiteHalfSpinor;	
   //    typedef typename XXX         Compressor;	
   //
@@ -97,15 +95,13 @@ namespace QCD {
   
 #define INHERIT_FIMPL_TYPES(Impl)\
   typedef typename Impl::FermionField           FermionField;		\
-  typedef typename Impl::PropagatorField     PropagatorField;		\
   typedef typename Impl::DoubledGaugeField DoubledGaugeField;		\
   typedef typename Impl::SiteSpinor               SiteSpinor;		\
-  typedef typename Impl::SitePropagator       SitePropagator;		\
   typedef typename Impl::SiteHalfSpinor       SiteHalfSpinor;		\
   typedef typename Impl::Compressor               Compressor;		\
   typedef typename Impl::StencilImpl             StencilImpl;		\
-  typedef typename Impl::ImplParams               ImplParams;	        \
+  typedef typename Impl::ImplParams ImplParams;				\
-  typedef typename Impl::Coeff_t                     Coeff_t;           \
+  typedef typename Impl::Coeff_t       Coeff_t;
   
 #define INHERIT_IMPL_TYPES(Base) \
   INHERIT_GIMPL_TYPES(Base)	 \
@@ -131,17 +127,14 @@ namespace QCD {
     INHERIT_GIMPL_TYPES(Gimpl);
       
     template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Dimension>, Ns> >;
-    template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Dimension>, Ns> >;
     template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Dimension>, Nhs> >;
     template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Dimension> >, Nds>;
     
     typedef iImplSpinor<Simd>            SiteSpinor;
-    typedef iImplPropagator<Simd>        SitePropagator;
     typedef iImplHalfSpinor<Simd>        SiteHalfSpinor;
     typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
     
     typedef Lattice<SiteSpinor>            FermionField;
-    typedef Lattice<SitePropagator>        PropagatorField;
     typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
     
     typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;
@@ -223,17 +216,14 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
   INHERIT_GIMPL_TYPES(Gimpl);
   
   template <typename vtype> using iImplSpinor            = iScalar<iVector<iVector<vtype, Nrepresentation>, Ns> >;
-  template <typename vtype> using iImplPropagator        = iScalar<iMatrix<iMatrix<vtype, Nrepresentation>, Ns> >;
   template <typename vtype> using iImplHalfSpinor        = iScalar<iVector<iVector<vtype, Nrepresentation>, Nhs> >;
   template <typename vtype> using iImplDoubledGaugeField = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>;
   template <typename vtype> using iImplGaugeField        = iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nd>;
   template <typename vtype> using iImplGaugeLink         = iScalar<iScalar<iMatrix<vtype, Nrepresentation> > >;
   
   typedef iImplSpinor<Simd> SiteSpinor;
-  typedef iImplPropagator<Simd> SitePropagator;
   typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
   typedef Lattice<SiteSpinor> FermionField;
-  typedef Lattice<SitePropagator> PropagatorField;
   
   // Make the doubled gauge field a *scalar*
   typedef iImplDoubledGaugeField<typename Simd::scalar_type>  SiteDoubledGaugeField;  // This is a scalar
@@ -325,17 +315,14 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
  INHERIT_GIMPL_TYPES(Gimpl);
       
  template <typename vtype> using iImplSpinor            = iVector<iVector<iVector<vtype, Nrepresentation>, Ns>, Ngp>;
- template <typename vtype> using iImplPropagator        = iVector<iMatrix<iMatrix<vtype, Nrepresentation>, Ns>, Ngp >;
  template <typename vtype> using iImplHalfSpinor        = iVector<iVector<iVector<vtype, Nrepresentation>, Nhs>, Ngp>;
  template <typename vtype> using iImplDoubledGaugeField = iVector<iVector<iScalar<iMatrix<vtype, Nrepresentation> >, Nds>, Ngp>;
       
  typedef iImplSpinor<Simd> SiteSpinor;
- typedef iImplPropagator<Simd> SitePropagator;
  typedef iImplHalfSpinor<Simd> SiteHalfSpinor;
  typedef iImplDoubledGaugeField<Simd> SiteDoubledGaugeField;
  
  typedef Lattice<SiteSpinor> FermionField;
- typedef Lattice<SitePropagator> PropagatorField;
  typedef Lattice<SiteDoubledGaugeField> DoubledGaugeField;
  
  typedef WilsonCompressor<SiteHalfSpinor, SiteSpinor> Compressor;

lib/qcd/action/fermion/PartialFractionFermion5D.cc

@@ -26,7 +26,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 namespace Grid {
   namespace QCD {
 

lib/qcd/action/fermion/WilsonFermion.cc

@@ -29,7 +29,7 @@ See the full license in the file "LICENSE" in the top level distribution
 directory
 *************************************************************************************/
 /*  END LEGAL */
-#include <Grid/Grid.h>
+#include <Grid.h>
 
 namespace Grid {
 namespace QCD {
@@ -61,9 +61,7 @@ WilsonFermion<Impl>::WilsonFermion(GaugeField &_Umu, GridCartesian &Fgrid,
       LebesgueEvenOdd(_cbgrid),
       Umu(&Fgrid),
       UmuEven(&Hgrid),
-      UmuOdd(&Hgrid),
+      UmuOdd(&Hgrid) {
-      _tmp(&Hgrid)
-{
   // Allocate the required comms buffer
   ImportGauge(_Umu);
 }
@@ -149,11 +147,11 @@ void WilsonFermion<Impl>::MeooeDag(const FermionField &in, FermionField &out) {
 
     typedef Lattice<iSinglet<vector_type> > LatComplex;
 
-    Gamma::Algebra Gmu [] = {
+    Gamma::GammaMatrix Gmu [] = {
-      Gamma::Algebra::GammaX,
+      Gamma::GammaX,
-      Gamma::Algebra::GammaY,
+      Gamma::GammaY,
-      Gamma::Algebra::GammaZ,
+      Gamma::GammaZ,
-      Gamma::Algebra::GammaT
+      Gamma::GammaT
     };
 
     std::vector<int> latt_size   = _grid->_fdimensions;

lib/qcd/action/fermion/WilsonFermion.h

@@ -58,9 +58,6 @@ class WilsonFermion : public WilsonKernels<Impl>, public WilsonFermionStatic {
   GridBase *FermionGrid(void) { return _grid; }
   GridBase *FermionRedBlackGrid(void) { return _cbgrid; }
 
-  FermionField _tmp;
-  FermionField &tmp(void) { return _tmp; }
-
   //////////////////////////////////////////////////////////////////
   // override multiply; cut number routines if pass dagger argument
   // and also make interface more uniformly consistent