Made checkerboard choice in staggered preconditioned solvers switchable

.gitignore

@@ -93,7 +93,6 @@ build*/*
 *.xcodeproj/*
 build.sh
 .vscode
-*.code-workspace
 
 # Eigen source #
 ################
@@ -124,8 +123,3 @@ make-bin-BUCK.sh
 lib/qcd/spin/gamma-gen/*.h
 lib/qcd/spin/gamma-gen/*.cc
 
-# vs code editor files #
-########################
-.vscode/
-.vscode/settings.json
-settings.json

.travis.yml

@@ -44,4 +44,3 @@ script:
     - make -j4
     - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - make check
-

TODO

@@ -1,36 +1,20 @@
 TODO:
 ---------------
 
-Code item work list
-
-a) namespaces & indentation
- GRID_BEGIN_NAMESPACE();
- GRID_END_NAMESPACE();
--- delete QCD namespace
-
-b) GPU branch
-- start branch
-- Increase Macro use in core library support; prepare for change
-- Audit volume of "device" code
-- Virtual function audit
-- Start port once Nvidia box is up
-- Cut down volume of code for first port? How?
-
-Physics item work list:
+Large item work list:
 
 1)- BG/Q port and check ; Andrew says ok.
-2)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
-3)- Physical propagator interface
-4)- Multigrid Wilson and DWF, compare to other Multigrid implementations
-5)- HDCR resume
+2)- Christoph's local basis expansion Lanczos
+--
+3a)- RNG I/O in ILDG/SciDAC (minor)
+3b)- Precision conversion and sort out localConvert      <-- partial/easy
+3c)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
+4)- Physical propagator interface
+5)- Conserved currents
+6)- Multigrid Wilson and DWF, compare to other Multigrid implementations
+7)- HDCR resume
 
-----------------------------
 Recent DONE 
--- RNG I/O in ILDG/SciDAC (minor) 
--- Precision conversion and sort out localConvert      <-- partial/easy
--- Conserved currents (Andrew)
--- Split grid
--- Christoph's local basis expansion Lanczos
 -- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O ; <-- DONE ; bmark cori
 -- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE
 -- GaugeFix into central location                      <-- DONE

benchmarks/Benchmark_comms.cc

@@ -106,7 +106,7 @@ int main (int argc, char ** argv)
       for(int i=0;i<Nloop;i++){
       double start=usecond();
 
-	std::vector<CommsRequest_t> requests;
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
@@ -202,7 +202,7 @@ int main (int argc, char ** argv)
 	    int recv_from_rank;
 	    
 	    {
-	      std::vector<CommsRequest_t> requests;
+	      std::vector<CartesianCommunicator::CommsRequest_t> requests;
 	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	      Grid.SendToRecvFromBegin(requests,
 				       (void *)&xbuf[mu][0],
@@ -215,7 +215,7 @@ int main (int argc, char ** argv)
 
 	    comm_proc = mpi_layout[mu]-1;
 	    {
-	      std::vector<CommsRequest_t> requests;
+	      std::vector<CartesianCommunicator::CommsRequest_t> requests;
 	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	      Grid.SendToRecvFromBegin(requests,
 				       (void *)&xbuf[mu+4][0],
@@ -290,7 +290,7 @@ int main (int argc, char ** argv)
 	dbytes=0;
 	ncomm=0;
 
-	std::vector<CommsRequest_t> requests;
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 
 	for(int mu=0;mu<4;mu++){
 	
@@ -383,7 +383,7 @@ int main (int argc, char ** argv)
       for(int i=0;i<Nloop;i++){
 	double start=usecond();
 
-	std::vector<CommsRequest_t> requests;
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 	dbytes=0;
 	ncomm=0;
 	for(int mu=0;mu<4;mu++){
@@ -481,7 +481,7 @@ int main (int argc, char ** argv)
       for(int i=0;i<Nloop;i++){
 	double start=usecond();
 
-	std::vector<CommsRequest_t> requests;
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
 	dbytes=0;
 	ncomm=0;
 

bootstrap.sh

@@ -3,7 +3,9 @@
 EIGEN_URL='http://bitbucket.org/eigen/eigen/get/3.3.3.tar.bz2'
 
 echo "-- deploying Eigen source..."
-wget ${EIGEN_URL} --no-check-certificate && ./scripts/update_eigen.sh `basename ${EIGEN_URL}` && rm `basename ${EIGEN_URL}`
+wget ${EIGEN_URL} --no-check-certificate
+./scripts/update_eigen.sh `basename ${EIGEN_URL}`
+#rm `basename ${EIGEN_URL}`
 
 echo '-- generating Make.inc files...'
 ./scripts/filelist

configure.ac

@@ -337,11 +337,15 @@ case ${ac_PRECISION} in
 esac
 
 ######################  Shared memory allocation technique under MPI3
-AC_ARG_ENABLE([shm],[AC_HELP_STRING([--enable-shm=shmopen|hugetlbfs],
+AC_ARG_ENABLE([shm],[AC_HELP_STRING([--enable-shm=shmget|shmopen|hugetlbfs],
               [Select SHM allocation technique])],[ac_SHM=${enable_shm}],[ac_SHM=shmopen])
 
 case ${ac_SHM} in
 
+     shmget)
+     AC_DEFINE([GRID_MPI3_SHMGET],[1],[GRID_MPI3_SHMGET] )
+     ;;
+
      shmopen)
      AC_DEFINE([GRID_MPI3_SHMOPEN],[1],[GRID_MPI3_SHMOPEN] )
      ;;
@@ -363,7 +367,7 @@ AC_ARG_ENABLE([shmpath],[AC_HELP_STRING([--enable-shmpath=path],
 AC_DEFINE_UNQUOTED([GRID_SHM_PATH],["$ac_SHMPATH"],[Path to a hugetlbfs filesystem for MMAPing])
 
 ############### communication type selection
-AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto],
+AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem],
               [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
 
 case ${ac_COMMS} in
@@ -371,10 +375,22 @@ case ${ac_COMMS} in
         AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
         comms_type='none'
      ;;
-     mpi*)
+     mpi3*)
         AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
         comms_type='mpi3'
      ;;
+     mpit)
+        AC_DEFINE([GRID_COMMS_MPIT],[1],[GRID_COMMS_MPIT] )
+        comms_type='mpit'
+     ;;
+     mpi*)
+        AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
+        comms_type='mpi'
+     ;;
+     shmem)
+        AC_DEFINE([GRID_COMMS_SHMEM],[1],[GRID_COMMS_SHMEM] )
+        comms_type='shmem'
+     ;;
      *)
         AC_MSG_ERROR([${ac_COMMS} unsupported --enable-comms option]);
      ;;

extras/Hadrons/Application.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Application.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -42,7 +43,6 @@ using namespace Hadrons;
 // constructors ////////////////////////////////////////////////////////////////
 Application::Application(void)
 {
-    initLogger();
     LOG(Message) << "Modules available:" << std::endl;
     auto list = ModuleFactory::getInstance().getBuilderList();
     for (auto &m: list)
@@ -73,6 +73,12 @@ Application::Application(const std::string parameterFileName)
     parameterFileName_ = parameterFileName;
 }
 
+// environment shortcut ////////////////////////////////////////////////////////
+Environment & Application::env(void) const
+{
+    return Environment::getInstance();
+}
+
 // access //////////////////////////////////////////////////////////////////////
 void Application::setPar(const Application::GlobalPar &par)
 {
@@ -88,13 +94,14 @@ const Application::GlobalPar & Application::getPar(void)
 // execute /////////////////////////////////////////////////////////////////////
 void Application::run(void)
 {
-    if (!parameterFileName_.empty() and (vm().getNModule() == 0))
+    if (!parameterFileName_.empty() and (env().getNModule() == 0))
     {
         parseParameterFile(parameterFileName_);
     }
-    vm().printContent();
-    env().printContent();
-    schedule();
+    if (!scheduled_)
+    {
+        schedule();
+    }
     printSchedule();
     configLoop();
 }
@@ -117,20 +124,12 @@ void Application::parseParameterFile(const std::string parameterFileName)
     LOG(Message) << "Building application from '" << parameterFileName << "'..." << std::endl;
     read(reader, "parameters", par);
     setPar(par);
-    if (!push(reader, "modules"))
-    {
-        HADRON_ERROR(Parsing, "Cannot open node 'modules' in parameter file '" 
-                              + parameterFileName + "'");
-    }
-    if (!push(reader, "module"))
-    {
-        HADRON_ERROR(Parsing, "Cannot open node 'modules/module' in parameter file '" 
-                              + parameterFileName + "'");
-    }
+    push(reader, "modules");
+    push(reader, "module");
     do
     {
         read(reader, "id", id);
-        vm().createModule(id.name, id.type, reader);
+        env().createModule(id.name, id.type, reader);
     } while (reader.nextElement("module"));
     pop(reader);
     pop(reader);
@@ -140,7 +139,7 @@ void Application::saveParameterFile(const std::string parameterFileName)
 {
     XmlWriter          writer(parameterFileName);
     ObjectId           id;
-    const unsigned int nMod = vm().getNModule();
+    const unsigned int nMod = env().getNModule();
     
     LOG(Message) << "Saving application to '" << parameterFileName << "'..." << std::endl;
     write(writer, "parameters", getPar());
@@ -148,10 +147,10 @@ void Application::saveParameterFile(const std::string parameterFileName)
     for (unsigned int i = 0; i < nMod; ++i)
     {
         push(writer, "module");
-        id.name = vm().getModuleName(i);
-        id.type = vm().getModule(i)->getRegisteredName();
+        id.name = env().getModuleName(i);
+        id.type = env().getModule(i)->getRegisteredName();
         write(writer, "id", id);
-        vm().getModule(i)->saveParameters(writer, "options");
+        env().getModule(i)->saveParameters(writer, "options");
         pop(writer);
     }
     pop(writer);
@@ -159,13 +158,95 @@ void Application::saveParameterFile(const std::string parameterFileName)
 }
 
 // schedule computation ////////////////////////////////////////////////////////
+#define MEM_MSG(size)\
+sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
+
+#define DEFINE_MEMPEAK \
+GeneticScheduler<unsigned int>::ObjFunc memPeak = \
+[this](const std::vector<unsigned int> &program)\
+{\
+    unsigned int memPeak;\
+    bool         msg;\
+    \
+    msg = HadronsLogMessage.isActive();\
+    HadronsLogMessage.Active(false);\
+    env().dryRun(true);\
+    memPeak = env().executeProgram(program);\
+    env().dryRun(false);\
+    env().freeAll();\
+    HadronsLogMessage.Active(true);\
+    \
+    return memPeak;\
+}
+
 void Application::schedule(void)
 {
-    if (!scheduled_ and !loadedSchedule_)
+    DEFINE_MEMPEAK;
+    
+    // build module dependency graph
+    LOG(Message) << "Building module graph..." << std::endl;
+    auto graph = env().makeModuleGraph();
+    auto con = graph.getConnectedComponents();
+    
+    // constrained topological sort using a genetic algorithm
+    LOG(Message) << "Scheduling computation..." << std::endl;
+    LOG(Message) << "               #module= " << graph.size() << std::endl;
+    LOG(Message) << "       population size= " << par_.genetic.popSize << std::endl;
+    LOG(Message) << "       max. generation= " << par_.genetic.maxGen << std::endl;
+    LOG(Message) << "  max. cst. generation= " << par_.genetic.maxCstGen << std::endl;
+    LOG(Message) << "         mutation rate= " << par_.genetic.mutationRate << std::endl;
+    
+    unsigned int                               k = 0, gen, prevPeak, nCstPeak = 0;
+    std::random_device                         rd;
+    GeneticScheduler<unsigned int>::Parameters par;
+    
+    par.popSize      = par_.genetic.popSize;
+    par.mutationRate = par_.genetic.mutationRate;
+    par.seed         = rd();
+    memPeak_         = 0;
+    CartesianCommunicator::BroadcastWorld(0, &(par.seed), sizeof(par.seed));
+    for (unsigned int i = 0; i < con.size(); ++i)
     {
-        program_   = vm().schedule(par_.genetic);
-        scheduled_ = true;
+        GeneticScheduler<unsigned int> scheduler(con[i], memPeak, par);
+        
+        gen = 0;
+        do
+        {
+            LOG(Debug) << "Generation " << gen << ":" << std::endl;
+            scheduler.nextGeneration();
+            if (gen != 0)
+            {
+                if (prevPeak == scheduler.getMinValue())
+                {
+                    nCstPeak++;
+                }
+                else
+                {
+                    nCstPeak = 0;
+                }
+            }
+            
+            prevPeak = scheduler.getMinValue();
+            if (gen % 10 == 0)
+            {
+                LOG(Iterative) << "Generation " << gen << ": "
+                               << MEM_MSG(scheduler.getMinValue()) << std::endl;
+            }
+            
+            gen++;
+        } while ((gen < par_.genetic.maxGen)
+                 and (nCstPeak < par_.genetic.maxCstGen));
+        auto &t = scheduler.getMinSchedule();
+        if (scheduler.getMinValue() > memPeak_)
+        {
+            memPeak_ = scheduler.getMinValue();
+        }
+        for (unsigned int j = 0; j < t.size(); ++j)
+        {
+            program_.push_back(t[j]);
+        }
     }
+    scheduled_ = true;
 }
 
 void Application::saveSchedule(const std::string filename)
@@ -175,19 +256,21 @@ void Application::saveSchedule(const std::string filename)
     
     if (!scheduled_)
     {
-        HADRON_ERROR(Definition, "Computation not scheduled");
+        HADRON_ERROR("Computation not scheduled");
     }
     LOG(Message) << "Saving current schedule to '" << filename << "'..."
                  << std::endl;
     for (auto address: program_)
     {
-        program.push_back(vm().getModuleName(address));
+        program.push_back(env().getModuleName(address));
     }
     write(writer, "schedule", program);
 }
 
 void Application::loadSchedule(const std::string filename)
 {
+    DEFINE_MEMPEAK;
+    
     TextReader               reader(filename);
     std::vector<std::string> program;
     
@@ -197,24 +280,24 @@ void Application::loadSchedule(const std::string filename)
     program_.clear();
     for (auto &name: program)
     {
-        program_.push_back(vm().getModuleAddress(name));
+        program_.push_back(env().getModuleAddress(name));
     }
-    loadedSchedule_ = true;
+    scheduled_ = true;
+    memPeak_   = memPeak(program_);
 }
 
 void Application::printSchedule(void)
 {
     if (!scheduled_)
     {
-        HADRON_ERROR(Definition, "Computation not scheduled");
+        HADRON_ERROR("Computation not scheduled");
     }
-    auto peak = vm().memoryNeeded(program_);
-    LOG(Message) << "Schedule (memory needed: " << sizeString(peak) << "):"
+    LOG(Message) << "Schedule (memory peak: " << MEM_MSG(memPeak_) << "):"
                  << std::endl;
     for (unsigned int i = 0; i < program_.size(); ++i)
     {
         LOG(Message) << std::setw(4) << i + 1 << ": "
-                     << vm().getModuleName(program_[i]) << std::endl;
+                     << env().getModuleName(program_[i]) << std::endl;
     }
 }
 
@@ -227,8 +310,8 @@ void Application::configLoop(void)
     {
         LOG(Message) << BIG_SEP << " Starting measurement for trajectory " << t
                      << " " << BIG_SEP << std::endl;
-        vm().setTrajectory(t);
-        vm().executeProgram(program_);
+        env().setTrajectory(t);
+        env().executeProgram(program_);
     }
     LOG(Message) << BIG_SEP << " End of measurement " << BIG_SEP << std::endl;
     env().freeAll();

extras/Hadrons/Application.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Application.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -30,7 +31,8 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #define Hadrons_Application_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/VirtualMachine.hpp>
+#include <Grid/Hadrons/Environment.hpp>
+#include <Grid/Hadrons/ModuleFactory.hpp>
 #include <Grid/Hadrons/Modules.hpp>
 
 BEGIN_HADRONS_NAMESPACE
@@ -49,13 +51,25 @@ public:
                                         unsigned int, end,
                                         unsigned int, step);
     };
+    class GeneticPar: Serializable
+    {
+    public:
+        GeneticPar(void):
+            popSize{20}, maxGen{1000}, maxCstGen{100}, mutationRate{.1} {};
+    public:
+        GRID_SERIALIZABLE_CLASS_MEMBERS(GeneticPar,
+                                        unsigned int, popSize,
+                                        unsigned int, maxGen,
+                                        unsigned int, maxCstGen,
+                                        double      , mutationRate);
+    };
     class GlobalPar: Serializable
     {
     public:
         GRID_SERIALIZABLE_CLASS_MEMBERS(GlobalPar,
-                                        TrajRange,                  trajCounter,
-                                        VirtualMachine::GeneticPar, genetic,
-                                        std::string,                seed);
+                                        TrajRange,   trajCounter,
+                                        GeneticPar,  genetic,
+                                        std::string, seed);
     };
 public:
     // constructors
@@ -86,15 +100,14 @@ public:
     void configLoop(void);
 private:
     // environment shortcut
-    DEFINE_ENV_ALIAS;
-    // virtual machine shortcut
-    DEFINE_VM_ALIAS;
+    Environment & env(void) const;
 private:
-    long unsigned int       locVol_;
-    std::string             parameterFileName_{""};
-    GlobalPar               par_;
-    VirtualMachine::Program program_;
-    bool                    scheduled_{false}, loadedSchedule_{false};
+    long unsigned int         locVol_;
+    std::string               parameterFileName_{""};
+    GlobalPar                 par_;
+    std::vector<unsigned int> program_;
+    Environment::Size         memPeak_;
+    bool                      scheduled_{false};
 };
 
 /******************************************************************************
@@ -104,16 +117,14 @@ private:
 template <typename M>
 void Application::createModule(const std::string name)
 {
-    vm().createModule<M>(name);
-    scheduled_ = false;
+    env().createModule<M>(name);
 }
 
 template <typename M>
 void Application::createModule(const std::string name,
                                const typename M::Par &par)
 {
-    vm().createModule<M>(name, par);
-    scheduled_ = false;
+    env().createModule<M>(name, par);
 }
 
 END_HADRONS_NAMESPACE

extras/Hadrons/Environment.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Environment.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -34,9 +35,6 @@ using namespace Grid;
 using namespace QCD;
 using namespace Hadrons;
 
-#define ERROR_NO_ADDRESS(address)\
-HADRON_ERROR(Definition, "no object with address " + std::to_string(address));
-
 /******************************************************************************
  *                       Environment implementation                           *
  ******************************************************************************/
@@ -58,6 +56,28 @@ Environment::Environment(void)
     rng4d_.reset(new GridParallelRNG(grid4d_.get()));
 }
 
+// dry run /////////////////////////////////////////////////////////////////////
+void Environment::dryRun(const bool isDry)
+{
+    dryRun_ = isDry;
+}
+
+bool Environment::isDryRun(void) const
+{
+    return dryRun_;
+}
+
+// trajectory number ///////////////////////////////////////////////////////////
+void Environment::setTrajectory(const unsigned int traj)
+{
+    traj_ = traj;
+}
+
+unsigned int Environment::getTrajectory(void) const
+{
+    return traj_;
+}
+
 // grids ///////////////////////////////////////////////////////////////////////
 void Environment::createGrid(const unsigned int Ls)
 {
@@ -85,7 +105,7 @@ GridCartesian * Environment::getGrid(const unsigned int Ls) const
     }
     catch(std::out_of_range &)
     {
-        HADRON_ERROR(Definition, "no grid with Ls= " + std::to_string(Ls));
+        HADRON_ERROR("no grid with Ls= " << Ls);
     }
 }
 
@@ -104,7 +124,7 @@ GridRedBlackCartesian * Environment::getRbGrid(const unsigned int Ls) const
     }
     catch(std::out_of_range &)
     {
-        HADRON_ERROR(Definition, "no red-black 5D grid with Ls= " + std::to_string(Ls));
+        HADRON_ERROR("no red-black 5D grid with Ls= " << Ls);
     }
 }
 
@@ -123,11 +143,6 @@ int Environment::getDim(const unsigned int mu) const
     return dim_[mu];
 }
 
-unsigned long int Environment::getLocalVolume(void) const
-{
-    return locVol_;
-}
-
 // random number generator /////////////////////////////////////////////////////
 void Environment::setSeed(const std::vector<int> &seed)
 {
@@ -139,6 +154,291 @@ GridParallelRNG * Environment::get4dRng(void) const
     return rng4d_.get();
 }
 
+// module management ///////////////////////////////////////////////////////////
+void Environment::pushModule(Environment::ModPt &pt)
+{
+    std::string name = pt->getName();
+    
+    if (!hasModule(name))
+    {
+        std::vector<unsigned int> inputAddress;
+        unsigned int              address;
+        ModuleInfo                m;
+        
+        m.data = std::move(pt);
+        m.type = typeIdPt(*m.data.get());
+        m.name = name;
+        auto input  = m.data->getInput();
+        for (auto &in: input)
+        {
+            if (!hasObject(in))
+            {
+                addObject(in , -1);
+            }
+            m.input.push_back(objectAddress_[in]);
+        }
+        auto output = m.data->getOutput();
+        module_.push_back(std::move(m));
+        address              = static_cast<unsigned int>(module_.size() - 1);
+        moduleAddress_[name] = address;
+        for (auto &out: output)
+        {
+            if (!hasObject(out))
+            {
+                addObject(out, address);
+            }
+            else
+            {
+                if (object_[objectAddress_[out]].module < 0)
+                {
+                    object_[objectAddress_[out]].module = address;
+                }
+                else
+                {
+                    HADRON_ERROR("object '" + out
+                                 + "' is already produced by module '"
+                                 + module_[object_[getObjectAddress(out)].module].name
+                                 + "' (while pushing module '" + name + "')");
+                }
+            }
+        }
+    }
+    else
+    {
+        HADRON_ERROR("module '" + name + "' already exists");
+    }
+}
+
+unsigned int Environment::getNModule(void) const
+{
+    return module_.size();
+}
+
+void Environment::createModule(const std::string name, const std::string type,
+                               XmlReader &reader)
+{
+    auto &factory = ModuleFactory::getInstance();
+    auto pt       = factory.create(type, name);
+    
+    pt->parseParameters(reader, "options");
+    pushModule(pt);
+}
+
+ModuleBase * Environment::getModule(const unsigned int address) const
+{
+    if (hasModule(address))
+    {
+        return module_[address].data.get();
+    }
+    else
+    {
+        HADRON_ERROR("no module with address " + std::to_string(address));
+    }
+}
+
+ModuleBase * Environment::getModule(const std::string name) const
+{
+    return getModule(getModuleAddress(name));
+}
+
+unsigned int Environment::getModuleAddress(const std::string name) const
+{
+    if (hasModule(name))
+    {
+        return moduleAddress_.at(name);
+    }
+    else
+    {
+        HADRON_ERROR("no module with name '" + name + "'");
+    }
+}
+
+std::string Environment::getModuleName(const unsigned int address) const
+{
+    if (hasModule(address))
+    {
+        return module_[address].name;
+    }
+    else
+    {
+        HADRON_ERROR("no module with address " + std::to_string(address));
+    }
+}
+
+std::string Environment::getModuleType(const unsigned int address) const
+{
+    if (hasModule(address))
+    {
+        return typeName(module_[address].type);
+    }
+    else
+    {
+        HADRON_ERROR("no module with address " + std::to_string(address));
+    }
+}
+
+std::string Environment::getModuleType(const std::string name) const
+{
+    return getModuleType(getModuleAddress(name));
+}
+
+std::string Environment::getModuleNamespace(const unsigned int address) const
+{
+    std::string type = getModuleType(address), ns;
+    
+    auto pos2 = type.rfind("::");
+    auto pos1 = type.rfind("::", pos2 - 2);
+    
+    return type.substr(pos1 + 2, pos2 - pos1 - 2);
+}
+
+std::string Environment::getModuleNamespace(const std::string name) const
+{
+    return getModuleNamespace(getModuleAddress(name));
+}
+
+bool Environment::hasModule(const unsigned int address) const
+{
+    return (address < module_.size());
+}
+
+bool Environment::hasModule(const std::string name) const
+{
+    return (moduleAddress_.find(name) != moduleAddress_.end());
+}
+
+Graph<unsigned int> Environment::makeModuleGraph(void) const
+{
+    Graph<unsigned int> moduleGraph;
+    
+    for (unsigned int i = 0; i < module_.size(); ++i)
+    {
+        moduleGraph.addVertex(i);
+        for (auto &j: module_[i].input)
+        {
+            moduleGraph.addEdge(object_[j].module, i);
+        }
+    }
+    
+    return moduleGraph;
+}
+
+#define BIG_SEP "==============="
+#define SEP     "---------------"
+#define MEM_MSG(size)\
+sizeString((size)*locVol_) << " (" << sizeString(size)  << "/site)"
+
+Environment::Size
+Environment::executeProgram(const std::vector<unsigned int> &p)
+{
+    Size                                memPeak = 0, sizeBefore, sizeAfter;
+    std::vector<std::set<unsigned int>> freeProg;
+    bool                                continueCollect, nothingFreed;
+    
+    // build garbage collection schedule
+    freeProg.resize(p.size());
+    for (unsigned int i = 0; i < object_.size(); ++i)
+    {
+        auto pred = [i, this](const unsigned int j)
+        {
+            auto &in = module_[j].input;
+            auto it  = std::find(in.begin(), in.end(), i);
+            
+            return (it != in.end()) or (j == object_[i].module);
+        };
+        auto it = std::find_if(p.rbegin(), p.rend(), pred);
+        if (it != p.rend())
+        {
+            freeProg[p.rend() - it - 1].insert(i);
+        }
+    }
+    
+    // program execution
+    for (unsigned int i = 0; i < p.size(); ++i)
+    {
+        // execute module
+        if (!isDryRun())
+        {
+            LOG(Message) << SEP << " Measurement step " << i+1 << "/"
+                         << p.size() << " (module '" << module_[p[i]].name
+                         << "') " << SEP << std::endl;
+        }
+        (*module_[p[i]].data)();
+        sizeBefore = getTotalSize();
+        // print used memory after execution
+        if (!isDryRun())
+        {
+            LOG(Message) << "Allocated objects: " << MEM_MSG(sizeBefore)
+                         << std::endl;
+        }
+        if (sizeBefore > memPeak)
+        {
+            memPeak = sizeBefore;
+        }
+        // garbage collection for step i
+        if (!isDryRun())
+        {
+            LOG(Message) << "Garbage collection..." << std::endl;
+        }
+        nothingFreed = true;
+        do
+        {
+            continueCollect = false;
+            auto toFree = freeProg[i];
+            for (auto &j: toFree)
+            {
+                // continue garbage collection while there are still
+                // objects without owners
+                continueCollect = continueCollect or !hasOwners(j);
+                if(freeObject(j))
+                {
+                    // if an object has been freed, remove it from
+                    // the garbage collection schedule
+                    freeProg[i].erase(j);
+                    nothingFreed = false;
+                }
+            }
+        } while (continueCollect);
+        // any remaining objects in step i garbage collection schedule
+        // is scheduled for step i + 1
+        if (i + 1 < p.size())
+        {
+            for (auto &j: freeProg[i])
+            {
+                freeProg[i + 1].insert(j);
+            }
+        }
+        // print used memory after garbage collection if necessary
+        if (!isDryRun())
+        {
+            sizeAfter = getTotalSize();
+            if (sizeBefore != sizeAfter)
+            {
+                LOG(Message) << "Allocated objects: " << MEM_MSG(sizeAfter)
+                             << std::endl;
+            }
+            else
+            {
+                LOG(Message) << "Nothing to free" << std::endl;
+            }
+        }
+    }
+    
+    return memPeak;
+}
+
+Environment::Size Environment::executeProgram(const std::vector<std::string> &p)
+{
+    std::vector<unsigned int> pAddress;
+    
+    for (auto &n: p)
+    {
+        pAddress.push_back(getModuleAddress(n));
+    }
+    
+    return executeProgram(pAddress);
+}
+
 // general memory management ///////////////////////////////////////////////////
 void Environment::addObject(const std::string name, const int moduleAddress)
 {
@@ -148,25 +448,46 @@ void Environment::addObject(const std::string name, const int moduleAddress)
         
         info.name   = name;
         info.module = moduleAddress;
-        info.data   = nullptr;
         object_.push_back(std::move(info));
         objectAddress_[name] = static_cast<unsigned int>(object_.size() - 1);
     }
     else
     {
-        HADRON_ERROR(Definition, "object '" + name + "' already exists");
+        HADRON_ERROR("object '" + name + "' already exists");
     }
 }
 
-void Environment::setObjectModule(const unsigned int objAddress,
-                                  const int modAddress)
+void Environment::registerObject(const unsigned int address,
+                                 const unsigned int size, const unsigned int Ls)
 {
-    object_[objAddress].module = modAddress;
+    if (!hasRegisteredObject(address))
+    {
+        if (hasObject(address))
+        {
+            object_[address].size         = size;
+            object_[address].Ls           = Ls;
+            object_[address].isRegistered = true;
+        }
+        else
+        {
+            HADRON_ERROR("no object with address " + std::to_string(address));
+        }
+    }
+    else
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " already registered");
+    }
 }
 
-unsigned int Environment::getMaxAddress(void) const
+void Environment::registerObject(const std::string name,
+                                 const unsigned int size, const unsigned int Ls)
 {
-    return object_.size();
+    if (!hasObject(name))
+    {
+        addObject(name);
+    }
+    registerObject(getObjectAddress(name), size, Ls);
 }
 
 unsigned int Environment::getObjectAddress(const std::string name) const
@@ -177,7 +498,7 @@ unsigned int Environment::getObjectAddress(const std::string name) const
     }
     else
     {
-        HADRON_ERROR(Definition, "no object with name '" + name + "'");
+        HADRON_ERROR("no object with name '" + name + "'");
     }
 }
 
@@ -189,13 +510,13 @@ std::string Environment::getObjectName(const unsigned int address) const
     }
     else
     {
-        ERROR_NO_ADDRESS(address);
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 
 std::string Environment::getObjectType(const unsigned int address) const
 {
-    if (hasObject(address))
+    if (hasRegisteredObject(address))
     {
         if (object_[address].type)
         {
@@ -206,9 +527,14 @@ std::string Environment::getObjectType(const unsigned int address) const
             return "<no type>";
         }
     }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " exists but is not registered");
+    }
     else
     {
-        ERROR_NO_ADDRESS(address);
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 
@@ -219,13 +545,18 @@ std::string Environment::getObjectType(const std::string name) const
 
 Environment::Size Environment::getObjectSize(const unsigned int address) const
 {
-    if (hasObject(address))
+    if (hasRegisteredObject(address))
     {
         return object_[address].size;
     }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " exists but is not registered");
+    }
     else
     {
-        ERROR_NO_ADDRESS(address);
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 
@@ -234,24 +565,7 @@ Environment::Size Environment::getObjectSize(const std::string name) const
     return getObjectSize(getObjectAddress(name));
 }
 
-Environment::Storage Environment::getObjectStorage(const unsigned int address) const
-{
-    if (hasObject(address))
-    {
-        return object_[address].storage;
-    }
-    else
-    {
-        ERROR_NO_ADDRESS(address);
-    }
-}
-
-Environment::Storage Environment::getObjectStorage(const std::string name) const
-{
-    return getObjectStorage(getObjectAddress(name));
-}
-
-int Environment::getObjectModule(const unsigned int address) const
+unsigned int Environment::getObjectModule(const unsigned int address) const
 {
     if (hasObject(address))
     {
@@ -259,24 +573,29 @@ int Environment::getObjectModule(const unsigned int address) const
     }
     else
     {
-        ERROR_NO_ADDRESS(address);
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 
-int Environment::getObjectModule(const std::string name) const
+unsigned int Environment::getObjectModule(const std::string name) const
 {
     return getObjectModule(getObjectAddress(name));
 }
 
 unsigned int Environment::getObjectLs(const unsigned int address) const
 {
-    if (hasObject(address))
+    if (hasRegisteredObject(address))
     {
         return object_[address].Ls;
     }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address)
+                     + " exists but is not registered");
+    }
     else
     {
-        ERROR_NO_ADDRESS(address);
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 
@@ -297,6 +616,30 @@ bool Environment::hasObject(const std::string name) const
     return ((it != objectAddress_.end()) and hasObject(it->second));
 }
 
+bool Environment::hasRegisteredObject(const unsigned int address) const
+{
+    if (hasObject(address))
+    {
+        return object_[address].isRegistered;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::hasRegisteredObject(const std::string name) const
+{
+    if (hasObject(name))
+    {
+        return hasRegisteredObject(getObjectAddress(name));
+    }
+    else
+    {
+        return false;
+    }
+}
+
 bool Environment::hasCreatedObject(const unsigned int address) const
 {
     if (hasObject(address))
@@ -337,27 +680,92 @@ Environment::Size Environment::getTotalSize(void) const
     
     for (auto &o: object_)
     {
-        size += o.size;
+        if (o.isRegistered)
+        {
+            size += o.size;
+        }
     }
     
     return size;
 }
 
-void Environment::freeObject(const unsigned int address)
+void Environment::addOwnership(const unsigned int owner,
+                               const unsigned int property)
 {
-    if (hasCreatedObject(address))
+    if (hasObject(property))
     {
-        LOG(Message) << "Destroying object '" << object_[address].name
-                     << "'" << std::endl;
+        object_[property].owners.insert(owner);
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(property));
+    }
+    if (hasObject(owner))
+    {
+        object_[owner].properties.insert(property);
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(owner));
     }
-    object_[address].size = 0;
-    object_[address].type = nullptr;
-    object_[address].data.reset(nullptr);
 }
 
-void Environment::freeObject(const std::string name)
+void Environment::addOwnership(const std::string owner,
+                               const std::string property)
 {
-    freeObject(getObjectAddress(name));
+    addOwnership(getObjectAddress(owner), getObjectAddress(property));
+}
+
+bool Environment::hasOwners(const unsigned int address) const
+{
+    
+    if (hasObject(address))
+    {
+        return (!object_[address].owners.empty());
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+bool Environment::hasOwners(const std::string name) const
+{
+    return hasOwners(getObjectAddress(name));
+}
+
+bool Environment::freeObject(const unsigned int address)
+{
+    if (!hasOwners(address))
+    {
+        if (!isDryRun() and object_[address].isRegistered)
+        {
+            LOG(Message) << "Destroying object '" << object_[address].name
+                         << "'" << std::endl;
+        }
+        for (auto &p: object_[address].properties)
+        {
+            object_[p].owners.erase(address);
+        }
+        object_[address].size         = 0;
+        object_[address].Ls           = 0;
+        object_[address].isRegistered = false;
+        object_[address].type         = nullptr;
+        object_[address].owners.clear();
+        object_[address].properties.clear();
+        object_[address].data.reset(nullptr);
+        
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+bool Environment::freeObject(const std::string name)
+{
+    return freeObject(getObjectAddress(name));
 }
 
 void Environment::freeAll(void)
@@ -368,24 +776,18 @@ void Environment::freeAll(void)
     }
 }
 
-void Environment::protectObjects(const bool protect)
+void Environment::printContent(void)
 {
-    protect_ = protect;
-}
-
-bool Environment::objectsProtected(void) const
-{
-    return protect_;
-}
-
-// print environment content ///////////////////////////////////////////////////
-void Environment::printContent(void) const
-{
-    LOG(Debug) << "Objects: " << std::endl;
+    LOG(Message) << "Modules: " << std::endl;
+    for (unsigned int i = 0; i < module_.size(); ++i)
+    {
+        LOG(Message) << std::setw(4) << i << ": "
+                     << getModuleName(i) << std::endl;
+    }
+    LOG(Message) << "Objects: " << std::endl;
     for (unsigned int i = 0; i < object_.size(); ++i)
     {
-        LOG(Debug) << std::setw(4) << i << ": "
-                   << getObjectName(i) << " ("
-                   << sizeString(getObjectSize(i)) << ")" << std::endl;
+        LOG(Message) << std::setw(4) << i << ": "
+                     << getObjectName(i) << std::endl;
     }
 }

extras/Hadrons/Environment.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Environment.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -30,12 +31,20 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #define Hadrons_Environment_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
+#include <Grid/Hadrons/Graph.hpp>
+
+#ifndef SITE_SIZE_TYPE
+#define SITE_SIZE_TYPE unsigned int
+#endif
 
 BEGIN_HADRONS_NAMESPACE
 
 /******************************************************************************
  *                         Global environment                                 *
  ******************************************************************************/
+// forward declaration of Module
+class ModuleBase;
+
 class Object
 {
 public:
@@ -57,78 +66,123 @@ private:
     std::unique_ptr<T> objPt_{nullptr};
 };
 
-#define DEFINE_ENV_ALIAS \
-inline Environment & env(void) const\
-{\
-    return Environment::getInstance();\
-}
-
 class Environment
 {
     SINGLETON(Environment);
 public:
     typedef SITE_SIZE_TYPE                         Size;
+    typedef std::unique_ptr<ModuleBase>            ModPt;
     typedef std::unique_ptr<GridCartesian>         GridPt;
     typedef std::unique_ptr<GridRedBlackCartesian> GridRbPt;
     typedef std::unique_ptr<GridParallelRNG>       RngPt;
-    enum class Storage {object, cache, temporary};
+    typedef std::unique_ptr<LatticeBase>           LatticePt;
 private:
+    struct ModuleInfo
+    {
+        const std::type_info      *type{nullptr};
+        std::string               name;
+        ModPt                     data{nullptr};
+        std::vector<unsigned int> input;
+    };
     struct ObjInfo
     {
         Size                    size{0};
-        Storage                 storage{Storage::object};
         unsigned int            Ls{0};
+        bool                    isRegistered{false};
         const std::type_info    *type{nullptr};
         std::string             name;
         int                     module{-1};
+        std::set<unsigned int>  owners, properties;
         std::unique_ptr<Object> data{nullptr};
     };
 public:
+    // dry run
+    void                    dryRun(const bool isDry);
+    bool                    isDryRun(void) const;
+    // trajectory number
+    void                    setTrajectory(const unsigned int traj);
+    unsigned int            getTrajectory(void) const;
     // grids
     void                    createGrid(const unsigned int Ls);
     GridCartesian *         getGrid(const unsigned int Ls = 1) const;
     GridRedBlackCartesian * getRbGrid(const unsigned int Ls = 1) const;
     std::vector<int>        getDim(void) const;
     int                     getDim(const unsigned int mu) const;
-    unsigned long int       getLocalVolume(void) const;
     unsigned int            getNd(void) const;
     // random number generator
     void                    setSeed(const std::vector<int> &seed);
     GridParallelRNG *       get4dRng(void) const;
+    // module management
+    void                    pushModule(ModPt &pt);
+    template <typename M>
+    void                    createModule(const std::string name);
+    template <typename M>
+    void                    createModule(const std::string name,
+                                         const typename M::Par &par);
+    void                    createModule(const std::string name,
+                                         const std::string type,
+                                         XmlReader &reader);
+    unsigned int            getNModule(void) const;
+    ModuleBase *            getModule(const unsigned int address) const;
+    ModuleBase *            getModule(const std::string name) const;
+    template <typename M>
+    M *                     getModule(const unsigned int address) const;
+    template <typename M>
+    M *                     getModule(const std::string name) const;
+    unsigned int            getModuleAddress(const std::string name) const;
+    std::string             getModuleName(const unsigned int address) const;
+    std::string             getModuleType(const unsigned int address) const;
+    std::string             getModuleType(const std::string name) const;
+    std::string             getModuleNamespace(const unsigned int address) const;
+    std::string             getModuleNamespace(const std::string name) const;
+    bool                    hasModule(const unsigned int address) const;
+    bool                    hasModule(const std::string name) const;
+    Graph<unsigned int>     makeModuleGraph(void) const;
+    Size                    executeProgram(const std::vector<unsigned int> &p);
+    Size                    executeProgram(const std::vector<std::string> &p);
     // general memory management
     void                    addObject(const std::string name,
                                       const int moduleAddress = -1);
-    template <typename B, typename T, typename ... Ts>
-    void                    createDerivedObject(const std::string name,
-                                                const Environment::Storage storage,
-                                                const unsigned int Ls,
-                                                Ts && ... args);
-    template <typename T, typename ... Ts>
-    void                    createObject(const std::string name,
-                                         const Environment::Storage storage,
-                                         const unsigned int Ls,
-                                         Ts && ... args);
-    void                    setObjectModule(const unsigned int objAddress,
-                                            const int modAddress);
+    void                    registerObject(const unsigned int address,
+                                           const unsigned int size,
+                                           const unsigned int Ls = 1);
+    void                    registerObject(const std::string name,
+                                           const unsigned int size,
+                                           const unsigned int Ls = 1);
+    template <typename T>
+    unsigned int            lattice4dSize(void) const;
+    template <typename T>
+    void                    registerLattice(const unsigned int address,
+                                            const unsigned int Ls = 1);
+    template <typename T>
+    void                    registerLattice(const std::string name,
+                                            const unsigned int Ls = 1);
+    template <typename T>
+    void                    setObject(const unsigned int address, T *object);
+    template <typename T>
+    void                    setObject(const std::string name, T *object);
     template <typename T>
     T *                     getObject(const unsigned int address) const;
     template <typename T>
     T *                     getObject(const std::string name) const;
-    unsigned int            getMaxAddress(void) const;
+    template <typename T>
+    T *                     createLattice(const unsigned int address);
+    template <typename T>
+    T *                     createLattice(const std::string name);
     unsigned int            getObjectAddress(const std::string name) const;
     std::string             getObjectName(const unsigned int address) const;
     std::string             getObjectType(const unsigned int address) const;
     std::string             getObjectType(const std::string name) const;
     Size                    getObjectSize(const unsigned int address) const;
     Size                    getObjectSize(const std::string name) const;
-    Storage                 getObjectStorage(const unsigned int address) const;
-    Storage                 getObjectStorage(const std::string name) const;
-    int                     getObjectModule(const unsigned int address) const;
-    int                     getObjectModule(const std::string name) const;
+    unsigned int            getObjectModule(const unsigned int address) const;
+    unsigned int            getObjectModule(const std::string name) const;
     unsigned int            getObjectLs(const unsigned int address) const;
     unsigned int            getObjectLs(const std::string name) const;
     bool                    hasObject(const unsigned int address) const;
     bool                    hasObject(const std::string name) const;
+    bool                    hasRegisteredObject(const unsigned int address) const;
+    bool                    hasRegisteredObject(const std::string name) const;
     bool                    hasCreatedObject(const unsigned int address) const;
     bool                    hasCreatedObject(const std::string name) const;
     bool                    isObject5d(const unsigned int address) const;
@@ -138,17 +192,20 @@ public:
     template <typename T>
     bool                    isObjectOfType(const std::string name) const;
     Environment::Size       getTotalSize(void) const;
-    void                    freeObject(const unsigned int address);
-    void                    freeObject(const std::string name);
+    void                    addOwnership(const unsigned int owner,
+                                         const unsigned int property);
+    void                    addOwnership(const std::string owner,
+                                         const std::string property);
+    bool                    hasOwners(const unsigned int address) const;
+    bool                    hasOwners(const std::string name) const;
+    bool                    freeObject(const unsigned int address);
+    bool                    freeObject(const std::string name);
     void                    freeAll(void);
-    void                    protectObjects(const bool protect);
-    bool                    objectsProtected(void) const;
-    // print environment content
-    void                    printContent(void) const;
+    void                    printContent(void);
 private:
     // general
-    unsigned long int                      locVol_;
-    bool                                   protect_{true};
+    bool                                   dryRun_{false};
+    unsigned int                           traj_, locVol_;
     // grids
     std::vector<int>                       dim_;
     GridPt                                 grid4d_;
@@ -158,6 +215,11 @@ private:
     unsigned int                           nd_;
     // random number generator
     RngPt                                  rng4d_;
+    // module and related maps
+    std::vector<ModuleInfo>                module_;
+    std::map<std::string, unsigned int>    moduleAddress_;
+    // lattice store
+    std::map<unsigned int, LatticePt>      lattice_;
     // object store
     std::vector<ObjInfo>                   object_;
     std::map<std::string, unsigned int>    objectAddress_;
@@ -194,85 +256,116 @@ void Holder<T>::reset(T *pt)
 /******************************************************************************
  *                     Environment template implementation                    *
  ******************************************************************************/
-// general memory management ///////////////////////////////////////////////////
-template <typename B, typename T, typename ... Ts>
-void Environment::createDerivedObject(const std::string name,
-                                      const Environment::Storage storage,
-                                      const unsigned int Ls,
-                                      Ts && ... args)
+// module management ///////////////////////////////////////////////////////////
+template <typename M>
+void Environment::createModule(const std::string name)
 {
-    if (!hasObject(name))
+    ModPt pt(new M(name));
+    
+    pushModule(pt);
+}
+
+template <typename M>
+void Environment::createModule(const std::string name,
+                               const typename M::Par &par)
+{
+    ModPt pt(new M(name));
+    
+    static_cast<M *>(pt.get())->setPar(par);
+    pushModule(pt);
+}
+
+template <typename M>
+M * Environment::getModule(const unsigned int address) const
+{
+    if (auto *pt = dynamic_cast<M *>(getModule(address)))
     {
-        addObject(name);
+        return pt;
     }
-    
-    unsigned int address = getObjectAddress(name);
-    
-    if (!object_[address].data or !objectsProtected())
+    else
     {
-        MemoryStats memStats;
-    
-        if (!MemoryProfiler::stats)
-        {
-            MemoryProfiler::stats = &memStats;
-        }
-        size_t initMem           = MemoryProfiler::stats->currentlyAllocated;
-        object_[address].storage = storage;
-        object_[address].Ls      = Ls;
-        object_[address].data.reset(new Holder<B>(new T(std::forward<Ts>(args)...)));
-        object_[address].size    = MemoryProfiler::stats->maxAllocated - initMem;
-        object_[address].type    = &typeid(T);
-        if (MemoryProfiler::stats == &memStats)
-        {
-            MemoryProfiler::stats = nullptr;
-        }
-    }
-    // object already exists, no error if it is a cache, error otherwise
-    else if ((object_[address].storage != Storage::cache) or 
-             (object_[address].storage != storage)        or
-             (object_[address].name    != name)           or
-             (object_[address].type    != &typeid(T)))
-    {
-        HADRON_ERROR(Definition, "object '" + name + "' already allocated");
+        HADRON_ERROR("module '" + module_[address].name
+                     + "' does not have type " + typeid(M).name()
+                     + "(object type: " + getModuleType(address) + ")");
     }
 }
 
-template <typename T, typename ... Ts>
-void Environment::createObject(const std::string name, 
-                               const Environment::Storage storage,
-                               const unsigned int Ls,
-                               Ts && ... args)
+template <typename M>
+M * Environment::getModule(const std::string name) const
 {
-    createDerivedObject<T, T>(name, storage, Ls, std::forward<Ts>(args)...);
+    return getModule<M>(getModuleAddress(name));
+}
+
+template <typename T>
+unsigned int Environment::lattice4dSize(void) const
+{
+    return sizeof(typename T::vector_object)/getGrid()->Nsimd();
+}
+
+template <typename T>
+void Environment::registerLattice(const unsigned int address,
+                                  const unsigned int Ls)
+{
+    createGrid(Ls);
+    registerObject(address, Ls*lattice4dSize<T>(), Ls);
+}
+
+template <typename T>
+void Environment::registerLattice(const std::string name, const unsigned int Ls)
+{
+    createGrid(Ls);
+    registerObject(name, Ls*lattice4dSize<T>(), Ls);
+}
+
+template <typename T>
+void Environment::setObject(const unsigned int address, T *object)
+{
+    if (hasRegisteredObject(address))
+    {
+        object_[address].data.reset(new Holder<T>(object));
+        object_[address].type = &typeid(T);
+    }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address) +
+                     " exists but is not registered");
+    }
+    else
+    {
+        HADRON_ERROR("no object with address " + std::to_string(address));
+    }
+}
+
+template <typename T>
+void Environment::setObject(const std::string name, T *object)
+{
+    setObject(getObjectAddress(name), object);
 }
 
 template <typename T>
 T * Environment::getObject(const unsigned int address) const
 {
-    if (hasObject(address))
+    if (hasRegisteredObject(address))
     {
-        if (hasCreatedObject(address))
+        if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
         {
-            if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
-            {
-                return h->getPt();
-            }
-            else
-            {
-                HADRON_ERROR(Definition, "object with address " + std::to_string(address) +
-                            " does not have type '" + typeName(&typeid(T)) +
-                            "' (has type '" + getObjectType(address) + "')");
-            }
+            return h->getPt();
         }
         else
         {
-            HADRON_ERROR(Definition, "object with address " + std::to_string(address) +
-                         " is empty");
+            HADRON_ERROR("object with address " + std::to_string(address) +
+                         " does not have type '" + typeName(&typeid(T)) +
+                         "' (has type '" + getObjectType(address) + "')");
         }
     }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address) +
+                     " exists but is not registered");
+    }
     else
     {
-        HADRON_ERROR(Definition, "no object with address " + std::to_string(address));
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 
@@ -282,10 +375,26 @@ T * Environment::getObject(const std::string name) const
     return getObject<T>(getObjectAddress(name));
 }
 
+template <typename T>
+T * Environment::createLattice(const unsigned int address)
+{
+    GridCartesian *g = getGrid(getObjectLs(address));
+    
+    setObject(address, new T(g));
+    
+    return getObject<T>(address);
+}
+
+template <typename T>
+T * Environment::createLattice(const std::string name)
+{
+    return createLattice<T>(getObjectAddress(name));
+}
+
 template <typename T>
 bool Environment::isObjectOfType(const unsigned int address) const
 {
-    if (hasObject(address))
+    if (hasRegisteredObject(address))
     {
         if (auto h = dynamic_cast<Holder<T> *>(object_[address].data.get()))
         {
@@ -296,9 +405,14 @@ bool Environment::isObjectOfType(const unsigned int address) const
             return false;
         }
     }
+    else if (hasObject(address))
+    {
+        HADRON_ERROR("object with address " + std::to_string(address) +
+                     " exists but is not registered");
+    }
     else
     {
-        HADRON_ERROR(Definition, "no object with address " + std::to_string(address));
+        HADRON_ERROR("no object with address " + std::to_string(address));
     }
 }
 

extras/Hadrons/Exceptions.cc

@@ -1,57 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Exceptions.cc
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Exceptions.hpp>
-
-#ifndef ERR_SUFF
-#define ERR_SUFF " (" + loc + ")"
-#endif
-
-#define CONST_EXC(name, init) \
-name::name(std::string msg, std::string loc)\
-:init\
-{}
-
-using namespace Grid;
-using namespace Hadrons;
-using namespace Exceptions;
-
-// logic errors
-CONST_EXC(Logic, logic_error(msg + ERR_SUFF))
-CONST_EXC(Definition, Logic("definition error: " + msg, loc))
-CONST_EXC(Implementation, Logic("implementation error: " + msg, loc))
-CONST_EXC(Range, Logic("range error: " + msg, loc))
-CONST_EXC(Size, Logic("size error: " + msg, loc))
-// runtime errors
-CONST_EXC(Runtime, runtime_error(msg + ERR_SUFF))
-CONST_EXC(Argument, Runtime("argument error: " + msg, loc))
-CONST_EXC(Io, Runtime("IO error: " + msg, loc))
-CONST_EXC(Memory, Runtime("memory error: " + msg, loc))
-CONST_EXC(Parsing, Runtime("parsing error: " + msg, loc))
-CONST_EXC(Program, Runtime("program error: " + msg, loc))
-CONST_EXC(System, Runtime("system error: " + msg, loc))

extras/Hadrons/Exceptions.hpp

@@ -1,72 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Exceptions.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_Exceptions_hpp_
-#define Hadrons_Exceptions_hpp_
-
-#include <stdexcept>
-#ifndef Hadrons_Global_hpp_
-#include <Grid/Hadrons/Global.hpp>
-#endif
-
-#define SRC_LOC std::string(__FUNCTION__) + " at " + std::string(__FILE__) + ":"\
-                + std::to_string(__LINE__)
-#define HADRON_ERROR(exc, msg)\
-LOG(Error) << msg << std::endl;\
-throw(Exceptions::exc(msg, SRC_LOC));
-
-#define DECL_EXC(name, base) \
-class name: public base\
-{\
-public:\
-    name(std::string msg, std::string loc);\
-}
-
-BEGIN_HADRONS_NAMESPACE
-
-namespace Exceptions
-{
-    // logic errors
-    DECL_EXC(Logic, std::logic_error);
-    DECL_EXC(Definition, Logic);
-    DECL_EXC(Implementation, Logic);
-    DECL_EXC(Range, Logic);
-    DECL_EXC(Size, Logic);
-    // runtime errors
-    DECL_EXC(Runtime, std::runtime_error);
-    DECL_EXC(Argument, Runtime);
-    DECL_EXC(Io, Runtime);
-    DECL_EXC(Memory, Runtime);
-    DECL_EXC(Parsing, Runtime);
-    DECL_EXC(Program, Runtime);
-    DECL_EXC(System, Runtime);
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_Exceptions_hpp_

extras/Hadrons/Factory.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Factory.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -94,7 +95,7 @@ std::unique_ptr<T> Factory<T>::create(const std::string type,
     }
     catch (std::out_of_range &)
     {
-        HADRON_ERROR(Argument, "object of type '" + type + "' unknown");
+        HADRON_ERROR("object of type '" + type + "' unknown");
     }
     
     return func(name);

extras/Hadrons/GeneticScheduler.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/GeneticScheduler.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -37,13 +38,13 @@ BEGIN_HADRONS_NAMESPACE
 /******************************************************************************
  *                   Scheduler based on a genetic algorithm                   *
  ******************************************************************************/
-template <typename V, typename T>
+template <typename T>
 class GeneticScheduler
 {
 public:
-    typedef std::vector<T>                 Gene;
-    typedef std::pair<Gene *, Gene *>      GenePair;
-    typedef std::function<V(const Gene &)> ObjFunc;
+    typedef std::vector<T>                   Gene;
+    typedef std::pair<Gene *, Gene *>        GenePair;
+    typedef std::function<int(const Gene &)> ObjFunc;
     struct Parameters
     {
         double       mutationRate;
@@ -64,7 +65,7 @@ public:
     void benchmarkCrossover(const unsigned int nIt);
     // print population
     friend std::ostream & operator<<(std::ostream &out,
-                                     const GeneticScheduler<V, T> &s)
+                                     const GeneticScheduler<T> &s)
     {
         out << "[";
         for (auto &p: s.population_)
@@ -86,19 +87,19 @@ private:
     void     mutation(Gene &m, const Gene &c);
     
 private:
-    Graph<T>               &graph_;
-    const ObjFunc          &func_;
-    const Parameters       par_;
-    std::multimap<V, Gene> population_;
-    std::mt19937           gen_;
+    Graph<T>                 &graph_;
+    const ObjFunc            &func_;
+    const Parameters         par_;
+    std::multimap<int, Gene> population_;
+    std::mt19937             gen_;
 };
 
 /******************************************************************************
  *                       template implementation                              *
  ******************************************************************************/
 // constructor /////////////////////////////////////////////////////////////////
-template <typename V, typename T>
-GeneticScheduler<V, T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
+template <typename T>
+GeneticScheduler<T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
                                       const Parameters &par)
 : graph_(graph)
 , func_(func)
@@ -108,22 +109,22 @@ GeneticScheduler<V, T>::GeneticScheduler(Graph<T> &graph, const ObjFunc &func,
 }
 
 // access //////////////////////////////////////////////////////////////////////
-template <typename V, typename T>
-const typename GeneticScheduler<V, T>::Gene &
-GeneticScheduler<V, T>::getMinSchedule(void)
+template <typename T>
+const typename GeneticScheduler<T>::Gene &
+GeneticScheduler<T>::getMinSchedule(void)
 {
     return population_.begin()->second;
 }
 
-template <typename V, typename T>
-int GeneticScheduler<V, T>::getMinValue(void)
+template <typename T>
+int GeneticScheduler<T>::getMinValue(void)
 {
     return population_.begin()->first;
 }
 
 // breed a new generation //////////////////////////////////////////////////////
-template <typename V, typename T>
-void GeneticScheduler<V, T>::nextGeneration(void)
+template <typename T>
+void GeneticScheduler<T>::nextGeneration(void)
 {
     // random initialization of the population if necessary
     if (population_.size() != par_.popSize)
@@ -157,8 +158,8 @@ void GeneticScheduler<V, T>::nextGeneration(void)
 }
 
 // evolution steps /////////////////////////////////////////////////////////////
-template <typename V, typename T>
-void GeneticScheduler<V, T>::initPopulation(void)
+template <typename T>
+void GeneticScheduler<T>::initPopulation(void)
 {
     population_.clear();
     for (unsigned int i = 0; i < par_.popSize; ++i)
@@ -169,8 +170,8 @@ void GeneticScheduler<V, T>::initPopulation(void)
     }
 }
 
-template <typename V, typename T>
-void GeneticScheduler<V, T>::doCrossover(void)
+template <typename T>
+void GeneticScheduler<T>::doCrossover(void)
 {
     auto p = selectPair();
     Gene &p1 = *(p.first), &p2 = *(p.second);
@@ -184,8 +185,8 @@ void GeneticScheduler<V, T>::doCrossover(void)
     }
 }
 
-template <typename V, typename T>
-void GeneticScheduler<V, T>::doMutation(void)
+template <typename T>
+void GeneticScheduler<T>::doMutation(void)
 {
     std::uniform_real_distribution<double>      mdis(0., 1.);
     std::uniform_int_distribution<unsigned int> pdis(0, population_.size() - 1);
@@ -205,35 +206,40 @@ void GeneticScheduler<V, T>::doMutation(void)
 }
 
 // genetic operators ///////////////////////////////////////////////////////////
-template <typename V, typename T>
-typename GeneticScheduler<V, T>::GenePair GeneticScheduler<V, T>::selectPair(void)
+template <typename T>
+typename GeneticScheduler<T>::GenePair GeneticScheduler<T>::selectPair(void)
 {
     std::vector<double> prob;
     unsigned int        ind;
     Gene                *p1, *p2;
-    const double        max = population_.rbegin()->first;
     
-
     for (auto &c: population_)
     {
-        prob.push_back(std::exp((c.first-1.)/max));
-    }        
-    std::discrete_distribution<unsigned int> dis1(prob.begin(), prob.end());
-    auto rIt = population_.begin();
-    ind = dis1(gen_);
-    std::advance(rIt, ind);
-    p1 = &(rIt->second);
-    prob[ind] = 0.;
-    std::discrete_distribution<unsigned int> dis2(prob.begin(), prob.end());
-    rIt = population_.begin();
-    std::advance(rIt, dis2(gen_));
-    p2 = &(rIt->second);
+        prob.push_back(1./c.first);
+    }
+    do
+    {
+        double probCpy;
+        
+        std::discrete_distribution<unsigned int> dis1(prob.begin(), prob.end());
+        auto rIt = population_.begin();
+        ind = dis1(gen_);
+        std::advance(rIt, ind);
+        p1 = &(rIt->second);
+        probCpy   = prob[ind];
+        prob[ind] = 0.;
+        std::discrete_distribution<unsigned int> dis2(prob.begin(), prob.end());
+        rIt = population_.begin();
+        std::advance(rIt, dis2(gen_));
+        p2 = &(rIt->second);
+        prob[ind] = probCpy;
+    } while (p1 == p2);
     
     return std::make_pair(p1, p2);
 }
 
-template <typename V, typename T>
-void GeneticScheduler<V, T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
+template <typename T>
+void GeneticScheduler<T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
                                     const Gene &p2)
 {
     Gene                                        buf;
@@ -267,8 +273,8 @@ void GeneticScheduler<V, T>::crossover(Gene &c1, Gene &c2, const Gene &p1,
     }
 }
 
-template <typename V, typename T>
-void GeneticScheduler<V, T>::mutation(Gene &m, const Gene &c)
+template <typename T>
+void GeneticScheduler<T>::mutation(Gene &m, const Gene &c)
 {
     Gene                                        buf;
     std::uniform_int_distribution<unsigned int> dis(0, c.size() - 1);
@@ -297,8 +303,8 @@ void GeneticScheduler<V, T>::mutation(Gene &m, const Gene &c)
     }
 }
 
-template <typename V, typename T>
-void GeneticScheduler<V, T>::benchmarkCrossover(const unsigned int nIt)
+template <typename T>
+void GeneticScheduler<T>::benchmarkCrossover(const unsigned int nIt)
 {
     Gene   p1, p2, c1, c2;
     double neg = 0., eq = 0., pos = 0., total;

extras/Hadrons/Global.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Global.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -38,19 +39,31 @@ HadronsLogger Hadrons::HadronsLogMessage(1,"Message");
 HadronsLogger Hadrons::HadronsLogIterative(1,"Iterative");
 HadronsLogger Hadrons::HadronsLogDebug(1,"Debug");
 
-void Hadrons::initLogger(void)
+// pretty size formatting //////////////////////////////////////////////////////
+std::string Hadrons::sizeString(long unsigned int bytes)
+
 {
-    auto w = std::string("Hadrons").length();
-    GridLogError.setTopWidth(w);
-    GridLogWarning.setTopWidth(w);
-    GridLogMessage.setTopWidth(w);
-    GridLogIterative.setTopWidth(w);
-    GridLogDebug.setTopWidth(w);
-    HadronsLogError.Active(GridLogError.isActive());
-    HadronsLogWarning.Active(GridLogWarning.isActive());
-    HadronsLogMessage.Active(GridLogMessage.isActive());
-    HadronsLogIterative.Active(GridLogIterative.isActive());
-    HadronsLogDebug.Active(GridLogDebug.isActive());
+    constexpr unsigned int bufSize = 256;
+    const char             *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
+    char                   buf[256];
+    long unsigned int      s     = 0;
+    double                 count = bytes;
+    
+    while (count >= 1024 && s < 7)
+    {
+        s++;
+        count /= 1024;
+    }
+    if (count - floor(count) == 0.0)
+    {
+        snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
+    }
+    else
+    {
+        snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
+    }
+    
+    return std::string(buf);
 }
 
 // type utilities //////////////////////////////////////////////////////////////
@@ -67,10 +80,3 @@ std::string Hadrons::typeName(const std::type_info *info)
     
     return name;
 }
-
-// default writers/readers /////////////////////////////////////////////////////
-#ifdef HAVE_HDF5
-const std::string Hadrons::resultFileExt = "h5";
-#else
-const std::string Hadrons::resultFileExt = "xml";
-#endif

extras/Hadrons/Global.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Global.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -35,10 +35,6 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #include <Grid/Grid.h>
 #include <cxxabi.h>
 
-#ifndef SITE_SIZE_TYPE
-#define SITE_SIZE_TYPE size_t
-#endif
-
 #define BEGIN_HADRONS_NAMESPACE \
 namespace Grid {\
 using namespace QCD;\
@@ -61,19 +57,17 @@ using Grid::operator<<;
 #ifndef SIMPL
 #define SIMPL ScalarImplCR
 #endif
-#ifndef GIMPL
-#define GIMPL GimplTypesR
-#endif
 
 BEGIN_HADRONS_NAMESPACE
 
 // type aliases
 #define FERM_TYPE_ALIASES(FImpl, suffix)\
-typedef FermionOperator<FImpl>                        FMat##suffix;            \
-typedef typename FImpl::FermionField                  FermionField##suffix;    \
-typedef typename FImpl::PropagatorField               PropagatorField##suffix; \
-typedef typename FImpl::SitePropagator::scalar_object SitePropagator##suffix;  \
-typedef std::vector<SitePropagator##suffix>           SlicedPropagator##suffix;
+typedef FermionOperator<FImpl>                       FMat##suffix;             \
+typedef typename FImpl::FermionField                 FermionField##suffix;     \
+typedef typename FImpl::PropagatorField              PropagatorField##suffix;  \
+typedef typename FImpl::SitePropagator               SitePropagator##suffix;   \
+typedef std::vector<typename FImpl::SitePropagator::scalar_object>             \
+                                                     SlicedPropagator##suffix;
 
 #define GAUGE_TYPE_ALIASES(FImpl, suffix)\
 typedef typename FImpl::DoubledGaugeField DoubledGaugeField##suffix;
@@ -87,8 +81,7 @@ typedef std::function<void(FermionField##suffix &,\
                       const FermionField##suffix &)> SolverFn##suffix;
 
 #define SINK_TYPE_ALIASES(suffix)\
-typedef std::function<SlicedPropagator##suffix\
-                      (const PropagatorField##suffix &)> SinkFn##suffix;
+typedef std::function<SlicedPropagator##suffix(const PropagatorField##suffix &)> SinkFn##suffix;
 
 #define FGS_TYPE_ALIASES(FImpl, suffix)\
 FERM_TYPE_ALIASES(FImpl, suffix)\
@@ -104,6 +97,11 @@ public:
 };
 
 #define LOG(channel) std::cout << HadronsLog##channel
+#define HADRON_ERROR(msg)\
+LOG(Error) << msg << " (" << __FUNCTION__ << " at " << __FILE__ << ":"\
+           << __LINE__ << ")" << std::endl;\
+abort();
+
 #define DEBUG_VAR(var) LOG(Debug) << #var << "= " << (var) << std::endl;
 
 extern HadronsLogger HadronsLogError;
@@ -112,8 +110,6 @@ extern HadronsLogger HadronsLogMessage;
 extern HadronsLogger HadronsLogIterative;
 extern HadronsLogger HadronsLogDebug;
 
-void initLogger(void);
-
 // singleton pattern
 #define SINGLETON(name)\
 public:\
@@ -139,6 +135,9 @@ public:\
 private:\
     name(void) = default;
 
+// pretty size formating
+std::string sizeString(long unsigned int bytes);
+
 // type utilities
 template <typename T>
 const std::type_info * typeIdPt(const T &x)
@@ -167,21 +166,14 @@ std::string typeName(void)
 }
 
 // default writers/readers
-extern const std::string resultFileExt;
-
 #ifdef HAVE_HDF5
-typedef Hdf5Reader ResultReader;
-typedef Hdf5Writer ResultWriter;
+typedef Hdf5Reader CorrReader;
+typedef Hdf5Writer CorrWriter;
 #else
-typedef XmlReader ResultReader;
-typedef XmlWriter ResultWriter;
+typedef XmlReader CorrReader;
+typedef XmlWriter CorrWriter;
 #endif
 
-#define RESULT_FILE_NAME(name) \
-name + "." + std::to_string(vm().getTrajectory()) + "." + resultFileExt
-
 END_HADRONS_NAMESPACE
 
-#include <Grid/Hadrons/Exceptions.hpp>
-
 #endif // Hadrons_Global_hpp_

extras/Hadrons/Graph.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Graph.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -184,7 +185,7 @@ void Graph<T>::removeVertex(const T &value)
     }
     else
     {
-        HADRON_ERROR(Range, "vertex does not exists");
+        HADRON_ERROR("vertex " << value << " does not exists");
     }
 
     // remove all edges containing the vertex
@@ -213,7 +214,7 @@ void Graph<T>::removeEdge(const Edge &e)
     }
     else
     {
-        HADRON_ERROR(Range, "edge does not exists");
+        HADRON_ERROR("edge "  << e << " does not exists");
     }
 }
 
@@ -259,7 +260,7 @@ void Graph<T>::mark(const T &value, const bool doMark)
     }
     else
     {
-        HADRON_ERROR(Range, "vertex does not exists");
+        HADRON_ERROR("vertex " << value << " does not exists");
     }
 }
 
@@ -297,7 +298,7 @@ bool Graph<T>::isMarked(const T &value) const
     }
     else
     {
-        HADRON_ERROR(Range, "vertex does not exists");
+        HADRON_ERROR("vertex " << value << " does not exists");
         
         return false;
     }
@@ -429,7 +430,7 @@ std::vector<T> Graph<T>::getAdjacentVertices(const T &value) const
     {
         return ((e.first == value) or (e.second == value));
     };
-    auto eIt = std::find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
     
     while (eIt != edgeSet_.end())
     {
@@ -441,7 +442,7 @@ std::vector<T> Graph<T>::getAdjacentVertices(const T &value) const
         {
             adjacentVertex.push_back((*eIt).first);
         }
-        eIt = std::find_if(++eIt, edgeSet_.end(), pred);
+        eIt = find_if(++eIt, edgeSet_.end(), pred);
     }
     
     return adjacentVertex;
@@ -457,12 +458,12 @@ std::vector<T> Graph<T>::getChildren(const T &value) const
     {
         return (e.first == value);
     };
-    auto eIt = std::find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
     
     while (eIt != edgeSet_.end())
     {
         child.push_back((*eIt).second);
-        eIt = std::find_if(++eIt, edgeSet_.end(), pred);
+        eIt = find_if(++eIt, edgeSet_.end(), pred);
     }
     
     return child;
@@ -478,12 +479,12 @@ std::vector<T> Graph<T>::getParents(const T &value) const
     {
         return (e.second == value);
     };
-    auto eIt = std::find_if(edgeSet_.begin(), edgeSet_.end(), pred);
+    auto eIt = find_if(edgeSet_.begin(), edgeSet_.end(), pred);
     
     while (eIt != edgeSet_.end())
     {
         parent.push_back((*eIt).first);
-        eIt = std::find_if(++eIt, edgeSet_.end(), pred);
+        eIt = find_if(++eIt, edgeSet_.end(), pred);
     }
     
     return parent;
@@ -543,7 +544,7 @@ std::vector<T> Graph<T>::topoSort(void)
     {
         if (tmpMarked.at(v))
         {
-            HADRON_ERROR(Range, "cannot topologically sort a cyclic graph");
+            HADRON_ERROR("cannot topologically sort a cyclic graph");
         }
         if (!isMarked(v))
         {
@@ -602,7 +603,7 @@ std::vector<T> Graph<T>::topoSort(Gen &gen)
     {
         if (tmpMarked.at(v))
         {
-            HADRON_ERROR(Range, "cannot topologically sort a cyclic graph");
+            HADRON_ERROR("cannot topologically sort a cyclic graph");
         }
         if (!isMarked(v))
         {

extras/Hadrons/HadronsXmlRun.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/HadronsXmlRun.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -54,6 +55,12 @@ int main(int argc, char *argv[])
     
     // initialization
     Grid_init(&argc, &argv);
+    HadronsLogError.Active(GridLogError.isActive());
+    HadronsLogWarning.Active(GridLogWarning.isActive());
+    HadronsLogMessage.Active(GridLogMessage.isActive());
+    HadronsLogIterative.Active(GridLogIterative.isActive());
+    HadronsLogDebug.Active(GridLogDebug.isActive());
+    LOG(Message) << "Grid initialized" << std::endl;
     
     // execution
     Application application(parameterFileName);

extras/Hadrons/HadronsXmlSchedule.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/HadronsXmlSchedule.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -48,6 +49,12 @@ int main(int argc, char *argv[])
     
     // initialization
     Grid_init(&argc, &argv);
+    HadronsLogError.Active(GridLogError.isActive());
+    HadronsLogWarning.Active(GridLogWarning.isActive());
+    HadronsLogMessage.Active(GridLogMessage.isActive());
+    HadronsLogIterative.Active(GridLogIterative.isActive());
+    HadronsLogDebug.Active(GridLogDebug.isActive());
+    LOG(Message) << "Grid initialized" << std::endl;
     
     // execution
     Application application;

extras/Hadrons/Makefile.am

@@ -7,24 +7,20 @@ libHadrons_a_SOURCES = \
     $(modules_cc)      \
     Application.cc     \
     Environment.cc     \
-	Exceptions.cc      \
     Global.cc          \
-    Module.cc		   \
-	VirtualMachine.cc
+    Module.cc
 libHadrons_adir = $(pkgincludedir)/Hadrons
 nobase_libHadrons_a_HEADERS = \
 	$(modules_hpp)            \
 	Application.hpp           \
 	Environment.hpp           \
-	Exceptions.hpp            \
 	Factory.hpp               \
 	GeneticScheduler.hpp      \
 	Global.hpp                \
 	Graph.hpp                 \
 	Module.hpp                \
 	Modules.hpp               \
-	ModuleFactory.hpp         \
-	VirtualMachine.hpp
+	ModuleFactory.hpp
 
 HadronsXmlRun_SOURCES = HadronsXmlRun.cc
 HadronsXmlRun_LDADD   = libHadrons.a -lGrid

extras/Hadrons/Module.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Module.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -38,6 +39,7 @@ using namespace Hadrons;
 // constructor /////////////////////////////////////////////////////////////////
 ModuleBase::ModuleBase(const std::string name)
 : name_(name)
+, env_(Environment::getInstance())
 {}
 
 // access //////////////////////////////////////////////////////////////////////
@@ -46,10 +48,15 @@ std::string ModuleBase::getName(void) const
     return name_;
 }
 
+Environment & ModuleBase::env(void) const
+{
+    return env_;
+}
+
 // get factory registration name if available
 std::string ModuleBase::getRegisteredName(void)
 {
-    HADRON_ERROR(Definition, "module '" + getName() + "' has no registered type"
+    HADRON_ERROR("module '" + getName() + "' has a type not registered"
                  + " in the factory");
 }
 
@@ -57,5 +64,8 @@ std::string ModuleBase::getRegisteredName(void)
 void ModuleBase::operator()(void)
 {
     setup();
-    execute();
+    if (!env().isDryRun())
+    {
+        execute();
+    }
 }

extras/Hadrons/Module.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Module.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -30,7 +31,7 @@ See the full license in the file "LICENSE" in the top level distribution directo
 #define Hadrons_Module_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/VirtualMachine.hpp>
+#include <Grid/Hadrons/Environment.hpp>
 
 BEGIN_HADRONS_NAMESPACE
 
@@ -86,56 +87,6 @@ public:\
 static ns##mod##ModuleRegistrar ns##mod##ModuleRegistrarInstance;
 
 #define ARG(...) __VA_ARGS__
-#define MACRO_REDIRECT(arg1, arg2, arg3, macro, ...) macro
-
-#define envGet(type, name)\
-*env().template getObject<type>(name)
-
-#define envGetTmp(type, var)\
-type &var = *env().template getObject<type>(getName() + "_tmp_" + #var)
-
-#define envHasType(type, name)\
-env().template isObjectOfType<type>(name)
-
-#define envCreate(type, name, Ls, ...)\
-env().template createObject<type>(name, Environment::Storage::object, Ls, __VA_ARGS__)
-
-#define envCreateDerived(base, type, name, Ls, ...)\
-env().template createDerivedObject<base, type>(name, Environment::Storage::object, Ls, __VA_ARGS__)
-
-#define envCreateLat4(type, name)\
-envCreate(type, name, 1, env().getGrid())
-
-#define envCreateLat5(type, name, Ls)\
-envCreate(type, name, Ls, env().getGrid(Ls))
-
-#define envCreateLat(...)\
-MACRO_REDIRECT(__VA_ARGS__, envCreateLat5, envCreateLat4)(__VA_ARGS__)
-
-#define envCache(type, name, Ls, ...)\
-env().template createObject<type>(name, Environment::Storage::cache, Ls, __VA_ARGS__)
-
-#define envCacheLat4(type, name)\
-envCache(type, name, 1, env().getGrid())
-
-#define envCacheLat5(type, name, Ls)\
-envCache(type, name, Ls, env().getGrid(Ls))
-
-#define envCacheLat(...)\
-MACRO_REDIRECT(__VA_ARGS__, envCacheLat5, envCacheLat4)(__VA_ARGS__)
-
-#define envTmp(type, name, Ls, ...)\
-env().template createObject<type>(getName() + "_tmp_" + name,         \
-                                  Environment::Storage::temporary, Ls, __VA_ARGS__)
-
-#define envTmpLat4(type, name)\
-envTmp(type, name, 1, env().getGrid())
-
-#define envTmpLat5(type, name, Ls)\
-envTmp(type, name, Ls, env().getGrid(Ls))
-
-#define envTmpLat(...)\
-MACRO_REDIRECT(__VA_ARGS__, envTmpLat5, envTmpLat4)(__VA_ARGS__)
 
 /******************************************************************************
  *                            Module class                                    *
@@ -150,30 +101,23 @@ public:
     virtual ~ModuleBase(void) = default;
     // access
     std::string getName(void) const;
+    Environment &env(void) const;
     // get factory registration name if available
     virtual std::string getRegisteredName(void);
     // dependencies/products
     virtual std::vector<std::string> getInput(void) = 0;
-    virtual std::vector<std::string> getReference(void)
-    {
-        return std::vector<std::string>(0);
-    };
     virtual std::vector<std::string> getOutput(void) = 0;
     // parse parameters
     virtual void parseParameters(XmlReader &reader, const std::string name) = 0;
     virtual void saveParameters(XmlWriter &writer, const std::string name) = 0;
     // setup
     virtual void setup(void) {};
-    virtual void execute(void) = 0;
     // execution
     void operator()(void);
-protected:
-    // environment shortcut
-    DEFINE_ENV_ALIAS;
-    // virtual machine shortcut
-    DEFINE_VM_ALIAS;
+    virtual void execute(void) = 0;
 private:
     std::string name_;
+    Environment &env_;
 };
 
 // derived class, templating the parameter class

extras/Hadrons/ModuleFactory.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/ModuleFactory.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 

extras/Hadrons/Modules.hpp

@@ -1,64 +1,25 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
-#include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
-#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
-#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WardIdentity.hpp>
-#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
-#include <Grid/Hadrons/Modules/MFermion/GaugeProp.hpp>
-#include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
-#include <Grid/Hadrons/Modules/MSource/Point.hpp>
-#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
-#include <Grid/Hadrons/Modules/MSource/Z2.hpp>
-#include <Grid/Hadrons/Modules/MSource/SeqConserved.hpp>
-#include <Grid/Hadrons/Modules/MSink/Smear.hpp>
-#include <Grid/Hadrons/Modules/MSink/Point.hpp>
-#include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
-#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
-#include <Grid/Hadrons/Modules/MGauge/StochEm.hpp>
-#include <Grid/Hadrons/Modules/MGauge/FundtoHirep.hpp>
-#include <Grid/Hadrons/Modules/MUtilities/TestSeqGamma.hpp>
-#include <Grid/Hadrons/Modules/MUtilities/TestSeqConserved.hpp>
-#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
-#include <Grid/Hadrons/Modules/MScalar/FreeProp.hpp>
-#include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
-#include <Grid/Hadrons/Modules/MScalar/ChargedProp.hpp>
 #include <Grid/Hadrons/Modules/MAction/DWF.hpp>
 #include <Grid/Hadrons/Modules/MAction/Wilson.hpp>
-#include <Grid/Hadrons/Modules/MAction/WilsonClover.hpp>
-#include <Grid/Hadrons/Modules/MScalarSUN/TrMag.hpp>
-#include <Grid/Hadrons/Modules/MScalarSUN/TwoPoint.hpp>
-#include <Grid/Hadrons/Modules/MScalarSUN/TrPhi.hpp>
-#include <Grid/Hadrons/Modules/MIO/LoadNersc.hpp>
-#include <Grid/Hadrons/Modules/MIO/LoadBinary.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Baryon.hpp>
+#include <Grid/Hadrons/Modules/MContraction/DiscLoop.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Gamma3pt.hpp>
+#include <Grid/Hadrons/Modules/MContraction/Meson.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonian.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp>
+#include <Grid/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp>
+#include <Grid/Hadrons/Modules/MFermion/GaugeProp.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Random.hpp>
+#include <Grid/Hadrons/Modules/MGauge/StochEm.hpp>
+#include <Grid/Hadrons/Modules/MGauge/Unit.hpp>
+#include <Grid/Hadrons/Modules/MLoop/NoiseLoop.hpp>
+#include <Grid/Hadrons/Modules/MScalar/ChargedProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/FreeProp.hpp>
+#include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
+#include <Grid/Hadrons/Modules/MSink/Point.hpp>
+#include <Grid/Hadrons/Modules/MSolver/RBPrecCG.hpp>
+#include <Grid/Hadrons/Modules/MSource/Point.hpp>
+#include <Grid/Hadrons/Modules/MSource/SeqGamma.hpp>
+#include <Grid/Hadrons/Modules/MSource/Wall.hpp>
+#include <Grid/Hadrons/Modules/MSource/Z2.hpp>

extras/Hadrons/Modules/MAction/DWF.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MAction/DWF.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -65,7 +65,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -104,29 +103,35 @@ std::vector<std::string> TDWF<FImpl>::getOutput(void)
 template <typename FImpl>
 void TDWF<FImpl>::setup(void)
 {
-    LOG(Message) << "Setting up domain wall fermion matrix with m= "
-                 << par().mass << ", M5= " << par().M5 << " and Ls= "
-                 << par().Ls << " using gauge field '" << par().gauge << "'"
-                 << std::endl;
-    LOG(Message) << "Fermion boundary conditions: " << par().boundary
-                 << std::endl;
-                 
-    env().createGrid(par().Ls);
-    auto &U    = envGet(LatticeGaugeField, par().gauge);
-    auto &g4   = *env().getGrid();
-    auto &grb4 = *env().getRbGrid();
-    auto &g5   = *env().getGrid(par().Ls);
-    auto &grb5 = *env().getRbGrid(par().Ls);
-    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
-    typename DomainWallFermion<FImpl>::ImplParams implParams(boundary);
-    envCreateDerived(FMat, DomainWallFermion<FImpl>, getName(), par().Ls, U, g5,
-                     grb5, g4, grb4, par().mass, par().M5, implParams);
+    unsigned int size;
+    
+    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
+    env().registerObject(getName(), size, par().Ls);
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TDWF<FImpl>::execute(void)
-{}
+{
+    LOG(Message) << "Setting up domain wall fermion matrix with m= "
+                 << par().mass << ", M5= " << par().M5 << " and Ls= "
+                 << par().Ls << " using gauge field '" << par().gauge << "'"
+                 << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary 
+                 << std::endl;
+    env().createGrid(par().Ls);
+    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
+    auto &g4     = *env().getGrid();
+    auto &grb4   = *env().getRbGrid();
+    auto &g5     = *env().getGrid(par().Ls);
+    auto &grb5   = *env().getRbGrid(par().Ls);
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename DomainWallFermion<FImpl>::ImplParams implParams(boundary);
+    FMat *fMatPt = new DomainWallFermion<FImpl>(U, g5, grb5, g4, grb4,
+                                                par().mass, par().M5,
+                                                implParams);
+    env().setObject(getName(), fMatPt);
+}
 
 END_MODULE_NAMESPACE
 

extras/Hadrons/Modules/MAction/Wilson.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MAction/Wilson.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -63,7 +63,6 @@ public:
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -102,24 +101,29 @@ std::vector<std::string> TWilson<FImpl>::getOutput(void)
 template <typename FImpl>
 void TWilson<FImpl>::setup(void)
 {
-    LOG(Message) << "Setting up TWilson fermion matrix with m= " << par().mass
-                 << " using gauge field '" << par().gauge << "'" << std::endl;
-    LOG(Message) << "Fermion boundary conditions: " << par().boundary
-                 << std::endl;
-                 
-    auto &U      = envGet(LatticeGaugeField, par().gauge);
-    auto &grid   = *env().getGrid();
-    auto &gridRb = *env().getRbGrid();
-    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
-    typename WilsonFermion<FImpl>::ImplParams implParams(boundary);
-    envCreateDerived(FMat, WilsonFermion<FImpl>, getName(), 1, U, grid, gridRb,
-                     par().mass, implParams);
+    unsigned int size;
+    
+    size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
+    env().registerObject(getName(), size);
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TWilson<FImpl>::execute()
-{}
+{
+    LOG(Message) << "Setting up TWilson fermion matrix with m= " << par().mass
+                 << " using gauge field '" << par().gauge << "'" << std::endl;
+    LOG(Message) << "Fermion boundary conditions: " << par().boundary 
+                 << std::endl;
+    auto &U      = *env().template getObject<LatticeGaugeField>(par().gauge);
+    auto &grid   = *env().getGrid();
+    auto &gridRb = *env().getRbGrid();
+    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
+    typename WilsonFermion<FImpl>::ImplParams implParams(boundary);
+    FMat *fMatPt = new WilsonFermion<FImpl>(U, grid, gridRb, par().mass,
+                                            implParams);
+    env().setObject(getName(), fMatPt);
+}
 
 END_MODULE_NAMESPACE
 

extras/Hadrons/Modules/MAction/WilsonClover.hpp

@@ -1,153 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MAction/Wilson.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MAction_WilsonClover_hpp_
-#define Hadrons_MAction_WilsonClover_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                            TWilson quark action                            *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MAction)
-
-class WilsonCloverPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonCloverPar,
-                                    std::string, gauge,
-                                    double     , mass,
-				                    double     , csw_r,
-				                    double     , csw_t,
-				                    WilsonAnisotropyCoefficients ,clover_anisotropy,
-                                    std::string, boundary
-				    );
-};
-
-template <typename FImpl>
-class TWilsonClover: public Module<WilsonCloverPar>
-{
-public:
-    FGS_TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TWilsonClover(const std::string name);
-    // destructor
-    virtual ~TWilsonClover(void) = default;
-    // dependencies/products
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(WilsonClover, TWilsonClover<FIMPL>, MAction);
-
-/******************************************************************************
- *                     TWilsonClover template implementation                        *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TWilsonClover<FImpl>::TWilsonClover(const std::string name)
-: Module<WilsonCloverPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TWilsonClover<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().gauge};
-
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TWilsonClover<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TWilsonClover<FImpl>::setup(void)
-{
-    //unsigned int size;
-
-    // size = 2*env().template lattice4dSize<typename FImpl::DoubledGaugeField>();
-    // env().registerObject(getName(), size);
-
-
-    LOG(Message) << "Setting up TWilsonClover fermion matrix with m= " << par().mass
-                 << " using gauge field '" << par().gauge << "'" << std::endl;
-    LOG(Message) << "Fermion boundary conditions: " << par().boundary 
-                 << std::endl;
-    LOG(Message) << "Clover term csw_r: " << par().csw_r
-                 << " csw_t: " << par().csw_t
-                 << std::endl;
-    auto &U      = envGet(LatticeGaugeField, par().gauge);
-    auto &grid   = *env().getGrid();
-    auto &gridRb = *env().getRbGrid();
-    std::vector<Complex> boundary = strToVec<Complex>(par().boundary);
-    typename WilsonCloverFermion<FImpl>::ImplParams implParams(boundary);
-    envCreateDerived(FMat, WilsonCloverFermion<FImpl>, getName(), 1, U, grid, gridRb, par().mass,
-						  par().csw_r,
-						  par().csw_t,
-					      par().clover_anisotropy,
-						  implParams); 
-
-
-    //FMat *fMatPt = new WilsonCloverFermion<FImpl>(U, grid, gridRb, par().mass,
-	//					  par().csw_r,
-	//					  par().csw_t,
-	//				      par().clover_anisotropy,
-	//					  implParams);
-    //env().setObject(getName(), fMatPt);
-
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TWilsonClover<FImpl>::execute()
-{
-
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_WilsonClover_hpp_

extras/Hadrons/Modules/MContraction/Baryon.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/Baryon.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -72,9 +72,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
-    // setup
-    virtual void setup(void);
     // execution
     virtual void execute(void);
 };
@@ -102,18 +99,11 @@ std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getInput(void)
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 std::vector<std::string> TBaryon<FImpl1, FImpl2, FImpl3>::getOutput(void)
 {
-    std::vector<std::string> out = {};
+    std::vector<std::string> out = {getName()};
     
     return out;
 }
 
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2, typename FImpl3>
-void TBaryon<FImpl1, FImpl2, FImpl3>::setup(void)
-{
-    envTmpLat(LatticeComplex, "c");
-}
-
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
@@ -122,12 +112,12 @@ void TBaryon<FImpl1, FImpl2, FImpl3>::execute(void)
                  << " quarks '" << par().q1 << "', '" << par().q2 << "', and '"
                  << par().q3 << "'" << std::endl;
     
-    ResultWriter writer(RESULT_FILE_NAME(par().output));
-    auto       &q1 = envGet(PropagatorField1, par().q1);
-    auto       &q2 = envGet(PropagatorField2, par().q2);
-    auto       &q3 = envGet(PropagatorField3, par().q2);
-    envGetTmp(LatticeComplex, c);
-    Result     result;
+    CorrWriter             writer(par().output);
+    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q2);
+    LatticeComplex        c(env().getGrid());
+    Result                result;
     
     // FIXME: do contractions
     

extras/Hadrons/Modules/MContraction/DiscLoop.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/DiscLoop.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -69,7 +68,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -99,7 +97,7 @@ std::vector<std::string> TDiscLoop<FImpl>::getInput(void)
 template <typename FImpl>
 std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
 {
-    std::vector<std::string> out = {};
+    std::vector<std::string> out = {getName()};
     
     return out;
 }
@@ -108,7 +106,7 @@ std::vector<std::string> TDiscLoop<FImpl>::getOutput(void)
 template <typename FImpl>
 void TDiscLoop<FImpl>::setup(void)
 {
-    envTmpLat(LatticeComplex, "c");
+    
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -119,13 +117,13 @@ void TDiscLoop<FImpl>::execute(void)
                  << "' using '" << par().q_loop << "' with " << par().gamma 
                  << " insertion." << std::endl;
 
-    ResultWriter          writer(RESULT_FILE_NAME(par().output));
-    auto                  &q_loop = envGet(PropagatorField, par().q_loop);
+    CorrWriter            writer(par().output);
+    PropagatorField       &q_loop = *env().template getObject<PropagatorField>(par().q_loop);
+    LatticeComplex        c(env().getGrid());
     Gamma                 gamma(par().gamma);
     std::vector<TComplex> buf;
     Result                result;
 
-    envGetTmp(LatticeComplex, c);
     c = trace(gamma*q_loop);
     sliceSum(c, buf, Tp);
 

extras/Hadrons/Modules/MContraction/Gamma3pt.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/Gamma3pt.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -52,14 +51,6 @@ BEGIN_HADRONS_NAMESPACE
  *                   q1
  *
  *      trace(g5*q1*adj(q2)*g5*gamma*q3)
- * 
- *  options:
- *   - q1: sink smeared propagator, source at i
- *   - q2: propagator, source at i
- *   - q3: propagator, source at f
- *   - gamma: gamma matrix to insert
- *   - tSnk: sink position for propagator q1.
- *
  */
 
 /******************************************************************************
@@ -75,7 +66,6 @@ public:
                                     std::string,    q2,
                                     std::string,    q3,
                                     Gamma::Algebra, gamma,
-                                    unsigned int,   tSnk,
                                     std::string,    output);
 };
 
@@ -100,7 +90,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -130,7 +119,7 @@ std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getInput(void)
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
 {
-    std::vector<std::string> out = {};
+    std::vector<std::string> out = {getName()};
     
     return out;
 }
@@ -139,7 +128,7 @@ std::vector<std::string> TGamma3pt<FImpl1, FImpl2, FImpl3>::getOutput(void)
 template <typename FImpl1, typename FImpl2, typename FImpl3>
 void TGamma3pt<FImpl1, FImpl2, FImpl3>::setup(void)
 {
-    envTmpLat(LatticeComplex, "c");
+    
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -151,22 +140,17 @@ void TGamma3pt<FImpl1, FImpl2, FImpl3>::execute(void)
                  << par().q3 << "', with " << par().gamma << " insertion." 
                  << std::endl;
 
-    // Initialise variables. q2 and q3 are normal propagators, q1 may be 
-    // sink smeared.
-    ResultWriter          writer(RESULT_FILE_NAME(par().output));
-    auto                  &q1 = envGet(SlicedPropagator1, par().q1);
-    auto                  &q2 = envGet(PropagatorField2, par().q2);
-    auto                  &q3 = envGet(PropagatorField2, par().q3);
+    CorrWriter            writer(par().output);
+    PropagatorField1      &q1 = *env().template getObject<PropagatorField1>(par().q1);
+    PropagatorField2      &q2 = *env().template getObject<PropagatorField2>(par().q2);
+    PropagatorField3      &q3 = *env().template getObject<PropagatorField3>(par().q3);
+    LatticeComplex        c(env().getGrid());
     Gamma                 g5(Gamma::Algebra::Gamma5);
     Gamma                 gamma(par().gamma);
     std::vector<TComplex> buf;
     Result                result;
-    
-    // Extract relevant timeslice of sinked propagator q1, then contract &
-    // sum over all spacial positions of gamma insertion.
-    SitePropagator1 q1Snk = q1[par().tSnk];
-    envGetTmp(LatticeComplex, c);
-    c = trace(g5*q1Snk*adj(q2)*(g5*gamma)*q3);
+
+    c = trace(g5*q1*adj(q2)*(g5*gamma)*q3);
     sliceSum(c, buf, Tp);
 
     result.gamma = par().gamma;

extras/Hadrons/Modules/MContraction/Meson.hpp

@@ -4,10 +4,12 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/Meson.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
+Copyright (C) 2017
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+        Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -49,7 +51,8 @@ BEGIN_HADRONS_NAMESPACE
            in a sequence (e.g. "<Gamma5 Gamma5><Gamma5 GammaT>").
 
            Special values: "all" - perform all possible contractions.
- - sink: module to compute the sink to use in contraction (string).
+ - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0."),
+        given as multiples of (2*pi) / L.
 */
 
 /******************************************************************************
@@ -95,9 +98,6 @@ public:
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
     virtual void parseGammaString(std::vector<GammaPair> &gammaList);
-protected:
-    // execution
-    virtual void setup(void);
     // execution
     virtual void execute(void);
 };
@@ -125,7 +125,7 @@ std::vector<std::string> TMeson<FImpl1, FImpl2>::getInput(void)
 template <typename FImpl1, typename FImpl2>
 std::vector<std::string> TMeson<FImpl1, FImpl2>::getOutput(void)
 {
-    std::vector<std::string> output = {};
+    std::vector<std::string> output = {getName()};
     
     return output;
 }
@@ -151,15 +151,9 @@ void TMeson<FImpl1, FImpl2>::parseGammaString(std::vector<GammaPair> &gammaList)
     {
         // Parse individual contractions from input string.
         gammaList = strToVec<GammaPair>(par().gammas);
-    } 
+    }
 }
 
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl1, typename FImpl2>
-void TMeson<FImpl1, FImpl2>::setup(void)
-{
-    envTmpLat(LatticeComplex, "c");
-}
 
 // execution ///////////////////////////////////////////////////////////////////
 #define mesonConnected(q1, q2, gSnk, gSrc) \
@@ -171,8 +165,8 @@ void TMeson<FImpl1, FImpl2>::execute(void)
     LOG(Message) << "Computing meson contractions '" << getName() << "' using"
                  << " quarks '" << par().q1 << "' and '" << par().q2 << "'"
                  << std::endl;
-
-    ResultWriter           writer(RESULT_FILE_NAME(par().output));
+    
+    CorrWriter             writer(par().output);
     std::vector<TComplex>  buf;
     std::vector<Result>    result;
     Gamma                  g5(Gamma::Algebra::Gamma5);
@@ -187,11 +181,11 @@ void TMeson<FImpl1, FImpl2>::execute(void)
         result[i].gamma_src = gammaList[i].second;
         result[i].corr.resize(nt);
     }
-    if (envHasType(SlicedPropagator1, par().q1) and
-        envHasType(SlicedPropagator2, par().q2))
+    if (env().template isObjectOfType<SlicedPropagator1>(par().q1) and
+        env().template isObjectOfType<SlicedPropagator2>(par().q2))
     {
-        auto &q1 = envGet(SlicedPropagator1, par().q1);
-        auto &q2 = envGet(SlicedPropagator2, par().q2);
+        SlicedPropagator1 &q1 = *env().template getObject<SlicedPropagator1>(par().q1);
+        SlicedPropagator2 &q2 = *env().template getObject<SlicedPropagator2>(par().q2);
         
         LOG(Message) << "(propagator already sinked)" << std::endl;
         for (unsigned int i = 0; i < result.size(); ++i)
@@ -207,10 +201,10 @@ void TMeson<FImpl1, FImpl2>::execute(void)
     }
     else
     {
-        auto &q1 = envGet(PropagatorField1, par().q1);
-        auto &q2 = envGet(PropagatorField2, par().q2);
+        PropagatorField1 &q1   = *env().template getObject<PropagatorField1>(par().q1);
+        PropagatorField2 &q2   = *env().template getObject<PropagatorField2>(par().q2);
+        LatticeComplex   c(env().getGrid());
         
-        envGetTmp(LatticeComplex, c);
         LOG(Message) << "(using sink '" << par().sink << "')" << std::endl;
         for (unsigned int i = 0; i < result.size(); ++i)
         {
@@ -218,17 +212,18 @@ void TMeson<FImpl1, FImpl2>::execute(void)
             Gamma       gSrc(gammaList[i].second);
             std::string ns;
                 
-            ns = vm().getModuleNamespace(env().getObjectModule(par().sink));
+            ns = env().getModuleNamespace(env().getObjectModule(par().sink));
             if (ns == "MSource")
             {
-                PropagatorField1 &sink = envGet(PropagatorField1, par().sink);
+                PropagatorField1 &sink =
+                    *env().template getObject<PropagatorField1>(par().sink);
                 
                 c = trace(mesonConnected(q1, q2, gSnk, gSrc)*sink);
                 sliceSum(c, buf, Tp);
             }
             else if (ns == "MSink")
             {
-                SinkFnScalar &sink = envGet(SinkFnScalar, par().sink);
+                SinkFnScalar &sink = *env().template getObject<SinkFnScalar>(par().sink);
                 
                 c   = trace(mesonConnected(q1, q2, gSnk, gSrc));
                 buf = sink(c);

extras/Hadrons/Modules/MContraction/WardIdentity.hpp

@@ -1,224 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MContraction/WardIdentity.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MContraction_WardIdentity_hpp_
-#define Hadrons_MContraction_WardIdentity_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/*
-  Ward Identity contractions
- -----------------------------
- 
- * options:
- - q:          propagator, 5D if available (string)
- - action:     action module used for propagator solution (string)
- - mass:       mass of quark (double)
- - test_axial: whether or not to test PCAC relation.
-*/
-
-/******************************************************************************
- *                              WardIdentity                                  *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MContraction)
-
-class WardIdentityPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(WardIdentityPar,
-                                    std::string, q,
-                                    std::string, action,
-                                    double,      mass,
-                                    bool,        test_axial);
-};
-
-template <typename FImpl>
-class TWardIdentity: public Module<WardIdentityPar>
-{
-public:
-    FERM_TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TWardIdentity(const std::string name);
-    // destructor
-    virtual ~TWardIdentity(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-protected:
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-private:
-    unsigned int Ls_;
-};
-
-MODULE_REGISTER_NS(WardIdentity, TWardIdentity<FIMPL>, MContraction);
-
-/******************************************************************************
- *                     TWardIdentity implementation                           *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TWardIdentity<FImpl>::TWardIdentity(const std::string name)
-: Module<WardIdentityPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TWardIdentity<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().q, par().action};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TWardIdentity<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TWardIdentity<FImpl>::setup(void)
-{
-    Ls_ = env().getObjectLs(par().q);
-    if (Ls_ != env().getObjectLs(par().action))
-    {
-        HADRON_ERROR(Size, "Ls mismatch between quark action and propagator");
-    }
-    envTmpLat(PropagatorField, "tmp");
-    envTmpLat(PropagatorField, "vector_WI");
-    if (par().test_axial)
-    {
-        envTmpLat(PropagatorField, "psi");
-        envTmpLat(LatticeComplex,  "PP");
-        envTmpLat(LatticeComplex,  "axial_defect");
-        envTmpLat(LatticeComplex,  "PJ5q");
-    }
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TWardIdentity<FImpl>::execute(void)
-{
-    LOG(Message) << "Performing Ward Identity checks for quark '" << par().q
-                 << "'." << std::endl;
-
-    auto  &q   = envGet(PropagatorField, par().q);
-    auto  &act = envGet(FMat, par().action);
-    Gamma g5(Gamma::Algebra::Gamma5);
-
-    // Compute D_mu V_mu, D here is backward derivative.
-    envGetTmp(PropagatorField, tmp);
-    envGetTmp(PropagatorField, vector_WI);
-    vector_WI    = zero;
-    for (unsigned int mu = 0; mu < Nd; ++mu)
-    {
-        act.ContractConservedCurrent(q, q, tmp, Current::Vector, mu);
-        tmp -= Cshift(tmp, mu, -1);
-        vector_WI += tmp;
-    }
-
-    // Test ward identity D_mu V_mu = 0;
-    LOG(Message) << "Vector Ward Identity check Delta_mu V_mu = " 
-                 << norm2(vector_WI) << std::endl;
-
-    if (par().test_axial)
-    {
-        envGetTmp(PropagatorField, psi);
-        envGetTmp(LatticeComplex, PP);
-        envGetTmp(LatticeComplex, axial_defect);
-        envGetTmp(LatticeComplex, PJ5q);
-        std::vector<TComplex> axial_buf;
-
-        // Compute <P|D_mu A_mu>, D is backwards derivative.
-        axial_defect = zero;
-        for (unsigned int mu = 0; mu < Nd; ++mu)
-        {
-            act.ContractConservedCurrent(q, q, tmp, Current::Axial, mu);
-            tmp -= Cshift(tmp, mu, -1);
-            axial_defect += trace(g5*tmp);
-        }
-
-        // Get <P|J5q> for 5D (zero for 4D) and <P|P>.
-        PJ5q = zero;
-        if (Ls_ > 1)
-        {
-            // <P|P>
-            ExtractSlice(tmp, q, 0, 0);
-            psi  = 0.5 * (tmp - g5*tmp);
-            ExtractSlice(tmp, q, Ls_ - 1, 0);
-            psi += 0.5 * (tmp + g5*tmp);
-            PP = trace(adj(psi)*psi);
-
-            // <P|5Jq>
-            ExtractSlice(tmp, q, Ls_/2 - 1, 0);
-            psi  = 0.5 * (tmp + g5*tmp);
-            ExtractSlice(tmp, q, Ls_/2, 0);
-            psi += 0.5 * (tmp - g5*tmp);
-            PJ5q = trace(adj(psi)*psi);
-        }
-        else
-        {
-            PP = trace(adj(q)*q);
-        }
-
-        // Test ward identity <P|D_mu A_mu> = 2m<P|P> + 2<P|J5q>
-        LOG(Message) << "|D_mu A_mu|^2 = " << norm2(axial_defect) << std::endl;
-        LOG(Message) << "|PP|^2        = " << norm2(PP) << std::endl;
-        LOG(Message) << "|PJ5q|^2      = " << norm2(PJ5q) << std::endl;
-        LOG(Message) << "Axial Ward Identity defect Delta_mu A_mu = "
-                     << norm2(axial_defect) << std::endl;
-    
-        // Axial defect by timeslice.
-        axial_defect -= 2.*(par().mass*PP + PJ5q);
-        LOG(Message) << "Check Axial defect by timeslice" << std::endl;
-        sliceSum(axial_defect, axial_buf, Tp);
-        for (int t = 0; t < axial_buf.size(); ++t)
-        {
-            LOG(Message) << "t = " << t << ": " 
-                         << TensorRemove(axial_buf[t]) << std::endl;
-        }
-    }
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_WardIdentity_hpp_

extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonian.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -77,7 +76,6 @@ public:
                                     std::string, q2,
                                     std::string, q3,
                                     std::string, q4,
-                                    unsigned int, tSnk,
                                     std::string, output);
 };
 
@@ -101,13 +99,11 @@ public:\
     /* dependency relation */ \
     virtual std::vector<std::string> getInput(void);\
     virtual std::vector<std::string> getOutput(void);\
-public:\
-    std::vector<std::string> VA_label = {"V", "A"};\
-protected:\
     /* setup */ \
     virtual void setup(void);\
     /* execution */ \
     virtual void execute(void);\
+    std::vector<std::string> VA_label = {"V", "A"};\
 };\
 MODULE_REGISTER_NS(modname, T##modname, MContraction);
 

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -55,8 +54,6 @@ using namespace MContraction;
  * 
  * S: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1]*q4*gL[mu][p_2])
  * E: trace(q3*g5*q1*adj(q2)*g5*gL[mu][p_1])*trace(q4*gL[mu][p_2])
- * 
- * Note q1 must be sink smeared.
  */
 
 /******************************************************************************
@@ -77,7 +74,7 @@ std::vector<std::string> TWeakHamiltonianEye::getInput(void)
 
 std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
 {
-    std::vector<std::string> out = {};
+    std::vector<std::string> out = {getName()};
     
     return out;
 }
@@ -85,15 +82,7 @@ std::vector<std::string> TWeakHamiltonianEye::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TWeakHamiltonianEye::setup(void)
 {
-    unsigned int ndim = env().getNd();
 
-    envTmpLat(LatticeComplex,  "expbuf");
-    envTmpLat(PropagatorField, "tmp1");
-    envTmpLat(LatticeComplex,  "tmp2");
-    envTmp(std::vector<PropagatorField>, "S_body", 1, ndim, PropagatorField(env().getGrid()));
-    envTmp(std::vector<PropagatorField>, "S_loop", 1, ndim, PropagatorField(env().getGrid()));
-    envTmp(std::vector<LatticeComplex>,  "E_body", 1, ndim, LatticeComplex(env().getGrid()));
-    envTmp(std::vector<LatticeComplex>,  "E_loop", 1, ndim, LatticeComplex(env().getGrid()));
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -104,31 +93,28 @@ void TWeakHamiltonianEye::execute(void)
                  << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
                  << "'." << std::endl;
 
-    ResultWriter           writer(RESULT_FILE_NAME(par().output));
-    auto                   &q1 = envGet(SlicedPropagator, par().q1);
-    auto                   &q2 = envGet(PropagatorField, par().q2);
-    auto                   &q3 = envGet(PropagatorField, par().q3);
-    auto                   &q4 = envGet(PropagatorField, par().q4);
-    Gamma                  g5  = Gamma(Gamma::Algebra::Gamma5);
-    std::vector<TComplex>  corrbuf;
-    std::vector<Result>    result(n_eye_diag);
-    unsigned int ndim    = env().getNd();
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
+    std::vector<TComplex> corrbuf;
+    std::vector<Result>   result(n_eye_diag);
+    unsigned int ndim   = env().getNd();
 
-    envGetTmp(LatticeComplex,               expbuf); 
-    envGetTmp(PropagatorField,              tmp1);
-    envGetTmp(LatticeComplex,               tmp2);
-    envGetTmp(std::vector<PropagatorField>, S_body);
-    envGetTmp(std::vector<PropagatorField>, S_loop);
-    envGetTmp(std::vector<LatticeComplex>,  E_body);
-    envGetTmp(std::vector<LatticeComplex>,  E_loop);
-
-    // Get sink timeslice of q1.
-    SitePropagator q1Snk = q1[par().tSnk];
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> S_body(ndim, tmp1);
+    std::vector<PropagatorField> S_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  E_body(ndim, tmp2);
+    std::vector<LatticeComplex>  E_loop(ndim, tmp2);
 
     // Setup for S-type contractions.
     for (int mu = 0; mu < ndim; ++mu)
     {
-        S_body[mu] = MAKE_SE_BODY(q1Snk, q2, q3, GammaL(Gamma::gmu[mu]));
+        S_body[mu] = MAKE_SE_BODY(q1, q2, q3, GammaL(Gamma::gmu[mu]));
         S_loop[mu] = MAKE_SE_LOOP(q4, GammaL(Gamma::gmu[mu]));
     }
 

extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianEye.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -77,7 +76,7 @@ std::vector<std::string> TWeakHamiltonianNonEye::getInput(void)
 
 std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
 {
-    std::vector<std::string> out = {};
+    std::vector<std::string> out = {getName()};
     
     return out;
 }
@@ -85,15 +84,7 @@ std::vector<std::string> TWeakHamiltonianNonEye::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TWeakHamiltonianNonEye::setup(void)
 {
-    unsigned int ndim = env().getNd();
 
-    envTmpLat(LatticeComplex,  "expbuf");
-    envTmpLat(PropagatorField, "tmp1");
-    envTmpLat(LatticeComplex,  "tmp2");
-    envTmp(std::vector<PropagatorField>, "C_i_side_loop", 1, ndim, PropagatorField(env().getGrid()));
-    envTmp(std::vector<PropagatorField>, "C_f_side_loop", 1, ndim, PropagatorField(env().getGrid()));
-    envTmp(std::vector<LatticeComplex>,  "W_i_side_loop", 1, ndim, LatticeComplex(env().getGrid()));
-    envTmp(std::vector<LatticeComplex>,  "W_f_side_loop", 1, ndim, LatticeComplex(env().getGrid()));
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -104,23 +95,23 @@ void TWeakHamiltonianNonEye::execute(void)
                  << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
                  << "'." << std::endl;
     
-    ResultWriter          writer(RESULT_FILE_NAME(par().output));
-    auto                  &q1 = envGet(PropagatorField, par().q1);
-    auto                  &q2 = envGet(PropagatorField, par().q2);
-    auto                  &q3 = envGet(PropagatorField, par().q3);
-    auto                  &q4 = envGet(PropagatorField, par().q4);
-    Gamma                 g5  = Gamma(Gamma::Algebra::Gamma5);
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
     std::vector<TComplex> corrbuf;
     std::vector<Result>   result(n_noneye_diag); 
-    unsigned int          ndim = env().getNd();
+    unsigned int ndim   = env().getNd();
 
-    envGetTmp(LatticeComplex,               expbuf); 
-    envGetTmp(PropagatorField,              tmp1);
-    envGetTmp(LatticeComplex,               tmp2);
-    envGetTmp(std::vector<PropagatorField>, C_i_side_loop);
-    envGetTmp(std::vector<PropagatorField>, C_f_side_loop);
-    envGetTmp(std::vector<LatticeComplex>,  W_i_side_loop);
-    envGetTmp(std::vector<LatticeComplex>,  W_f_side_loop);
+    PropagatorField              tmp1(env().getGrid());
+    LatticeComplex               tmp2(env().getGrid());
+    std::vector<PropagatorField> C_i_side_loop(ndim, tmp1);
+    std::vector<PropagatorField> C_f_side_loop(ndim, tmp1);
+    std::vector<LatticeComplex>  W_i_side_loop(ndim, tmp2);
+    std::vector<LatticeComplex>  W_f_side_loop(ndim, tmp2);
 
     // Setup for C-type contractions.
     for (int mu = 0; mu < ndim; ++mu)

extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakHamiltonianNonEye.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -79,7 +78,7 @@ std::vector<std::string> TWeakNeutral4ptDisc::getInput(void)
 
 std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
 {
-    std::vector<std::string> out = {};
+    std::vector<std::string> out = {getName()};
     
     return out;
 }
@@ -87,13 +86,7 @@ std::vector<std::string> TWeakNeutral4ptDisc::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TWeakNeutral4ptDisc::setup(void)
 {
-    unsigned int ndim = env().getNd();
 
-    envTmpLat(LatticeComplex,  "expbuf");
-    envTmpLat(PropagatorField, "tmp");
-    envTmpLat(LatticeComplex,  "curr");
-    envTmp(std::vector<PropagatorField>, "meson", 1, ndim, PropagatorField(env().getGrid()));
-    envTmp(std::vector<PropagatorField>, "loop", 1, ndim,  PropagatorField(env().getGrid()));
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -104,21 +97,21 @@ void TWeakNeutral4ptDisc::execute(void)
                  << par().q2 << ", '" << par().q3 << "' and '" << par().q4 
                  << "'." << std::endl;
 
-    ResultWriter          writer(RESULT_FILE_NAME(par().output));
-    auto                  &q1 = envGet(PropagatorField, par().q1);
-    auto                  &q2 = envGet(PropagatorField, par().q2);
-    auto                  &q3 = envGet(PropagatorField, par().q3);
-    auto                  &q4 = envGet(PropagatorField, par().q4);
-    Gamma                 g5  = Gamma(Gamma::Algebra::Gamma5);
+    CorrWriter             writer(par().output);
+    PropagatorField &q1 = *env().template getObject<PropagatorField>(par().q1);
+    PropagatorField &q2 = *env().template getObject<PropagatorField>(par().q2);
+    PropagatorField &q3 = *env().template getObject<PropagatorField>(par().q3);
+    PropagatorField &q4 = *env().template getObject<PropagatorField>(par().q4);
+    Gamma g5            = Gamma(Gamma::Algebra::Gamma5);
+    LatticeComplex        expbuf(env().getGrid());
     std::vector<TComplex> corrbuf;
     std::vector<Result>   result(n_neut_disc_diag);
-    unsigned int          ndim = env().getNd();
+    unsigned int ndim   = env().getNd();
 
-    envGetTmp(LatticeComplex,               expbuf); 
-    envGetTmp(PropagatorField,              tmp);
-    envGetTmp(LatticeComplex,               curr);
-    envGetTmp(std::vector<PropagatorField>, meson);
-    envGetTmp(std::vector<PropagatorField>, loop);
+    PropagatorField              tmp(env().getGrid());
+    std::vector<PropagatorField> meson(ndim, tmp);
+    std::vector<PropagatorField> loop(ndim, tmp);
+    LatticeComplex               curr(env().getGrid());
 
     // Setup for type 1 contractions.
     for (int mu = 0; mu < ndim; ++mu)

extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MContraction/WeakNeutral4ptDisc.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson    <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

extras/Hadrons/Modules/MFermion/GaugeProp.hpp

@@ -1,32 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MFermion/GaugeProp.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
 #ifndef Hadrons_MFermion_GaugeProp_hpp_
 #define Hadrons_MFermion_GaugeProp_hpp_
 
@@ -36,27 +7,6 @@ See the full license in the file "LICENSE" in the top level distribution directo
 
 BEGIN_HADRONS_NAMESPACE
 
-/******************************************************************************
- * 5D -> 4D and 4D -> 5D conversions.                                         *
- ******************************************************************************/
-template<class vobj> // Note that 5D object is modified.
-inline void make_4D(Lattice<vobj> &in_5d, Lattice<vobj> &out_4d, int Ls)
-{
-    axpby_ssp_pminus(in_5d, 0., in_5d, 1., in_5d, 0, 0);
-    axpby_ssp_pplus(in_5d, 1., in_5d, 1., in_5d, 0, Ls-1);
-    ExtractSlice(out_4d, in_5d, 0, 0);
-}
-
-template<class vobj>
-inline void make_5D(Lattice<vobj> &in_4d, Lattice<vobj> &out_5d, int Ls)
-{
-    out_5d = zero;
-    InsertSlice(in_4d, out_5d, 0, 0);
-    InsertSlice(in_4d, out_5d, Ls-1, 0);
-    axpby_ssp_pplus(out_5d, 0., out_5d, 1., out_5d, 0, 0);
-    axpby_ssp_pminus(out_5d, 0., out_5d, 1., out_5d, Ls-1, Ls-1);
-}
-
 /******************************************************************************
  *                                GaugeProp                                   *
  ******************************************************************************/
@@ -83,7 +33,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -94,6 +43,7 @@ private:
 };
 
 MODULE_REGISTER_NS(GaugeProp, TGaugeProp<FIMPL>, MFermion);
+
 /******************************************************************************
  *                      TGaugeProp implementation                             *
  ******************************************************************************/
@@ -125,13 +75,10 @@ template <typename FImpl>
 void TGaugeProp<FImpl>::setup(void)
 {
     Ls_ = env().getObjectLs(par().solver);
-    envCreateLat(PropagatorField, getName());
-    envTmpLat(FermionField, "source", Ls_);
-    envTmpLat(FermionField, "sol", Ls_);
-    envTmpLat(FermionField, "tmp");
+    env().template registerLattice<PropagatorField>(getName());
     if (Ls_ > 1)
     {
-        envCreateLat(PropagatorField, getName() + "_5d", Ls_);
+        env().template registerLattice<PropagatorField>(getName() + "_5d", Ls_);
     }
 }
 
@@ -140,34 +87,41 @@ template <typename FImpl>
 void TGaugeProp<FImpl>::execute(void)
 {
     LOG(Message) << "Computing quark propagator '" << getName() << "'"
-                 << std::endl;
+    << std::endl;
     
-    std::string propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
-    auto        &prop    = envGet(PropagatorField, propName);
-    auto        &fullSrc = envGet(PropagatorField, par().source);
-    auto        &solver  = envGet(SolverFn, par().solver);
+    FermionField    source(env().getGrid(Ls_)), sol(env().getGrid(Ls_)),
+    tmp(env().getGrid());
+    std::string     propName = (Ls_ == 1) ? getName() : (getName() + "_5d");
+    PropagatorField &prop    = *env().template createLattice<PropagatorField>(propName);
+    PropagatorField &fullSrc = *env().template getObject<PropagatorField>(par().source);
+    SolverFn        &solver  = *env().template getObject<SolverFn>(par().solver);
+    if (Ls_ > 1)
+    {
+        env().template createLattice<PropagatorField>(getName());
+    }
     
-    envGetTmp(FermionField, source);
-    envGetTmp(FermionField, sol);
-    envGetTmp(FermionField, tmp);
     LOG(Message) << "Inverting using solver '" << par().solver
-                 << "' on source '" << par().source << "'" << std::endl;
+    << "' on source '" << par().source << "'" << std::endl;
     for (unsigned int s = 0; s < Ns; ++s)
-      for (unsigned int c = 0; c < FImpl::Dimension; ++c)
+    for (unsigned int c = 0; c < Nc; ++c)
     {
         LOG(Message) << "Inversion for spin= " << s << ", color= " << c
-                     << std::endl;
+        << std::endl;
         // source conversion for 4D sources
         if (!env().isObject5d(par().source))
         {
             if (Ls_ == 1)
             {
-               PropToFerm<FImpl>(source, fullSrc, s, c);
+                PropToFerm(source, fullSrc, s, c);
             }
             else
             {
-                PropToFerm<FImpl>(tmp, fullSrc, s, c);
-                make_5D(tmp, source, Ls_);
+                source = zero;
+                PropToFerm(tmp, fullSrc, s, c);
+                InsertSlice(tmp, source, 0, 0);
+                InsertSlice(tmp, source, Ls_-1, 0);
+                axpby_ssp_pplus(source, 0., source, 1., source, 0, 0);
+                axpby_ssp_pminus(source, 0., source, 1., source, Ls_-1, Ls_-1);
             }
         }
         // source conversion for 5D sources
@@ -175,22 +129,26 @@ void TGaugeProp<FImpl>::execute(void)
         {
             if (Ls_ != env().getObjectLs(par().source))
             {
-                HADRON_ERROR(Size, "Ls mismatch between quark action and source");
+                HADRON_ERROR("Ls mismatch between quark action and source");
             }
             else
             {
-                PropToFerm<FImpl>(source, fullSrc, s, c);
+                PropToFerm(source, fullSrc, s, c);
             }
         }
         sol = zero;
         solver(sol, source);
-        FermToProp<FImpl>(prop, sol, s, c);
+        FermToProp(prop, sol, s, c);
         // create 4D propagators from 5D one if necessary
         if (Ls_ > 1)
         {
-            PropagatorField &p4d = envGet(PropagatorField, getName());
-            make_4D(sol, tmp, Ls_);
-            FermToProp<FImpl>(p4d, tmp, s, c);
+            PropagatorField &p4d =
+            *env().template getObject<PropagatorField>(getName());
+            
+            axpby_ssp_pminus(sol, 0., sol, 1., sol, 0, 0);
+            axpby_ssp_pplus(sol, 1., sol, 1., sol, 0, Ls_-1);
+            ExtractSlice(tmp, sol, 0, 0);
+            FermToProp(p4d, tmp, s, c);
         }
     }
 }

extras/Hadrons/Modules/MGauge/FundtoHirep.cc

@@ -1,75 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/FundtoHirep.cc
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/Modules/MGauge/FundtoHirep.hpp>
-
-using namespace Grid;
-using namespace Hadrons;
-using namespace MGauge;
-
-// constructor /////////////////////////////////////////////////////////////////
-template <class Rep>
-TFundtoHirep<Rep>::TFundtoHirep(const std::string name)
-: Module<FundtoHirepPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <class Rep>
-std::vector<std::string> TFundtoHirep<Rep>::getInput(void)
-{
-    std::vector<std::string> in;
-    return in;
-}
-
-template <class Rep>
-std::vector<std::string> TFundtoHirep<Rep>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename Rep>
-void TFundtoHirep<Rep>::setup(void)
-{
-    env().template registerLattice<typename Rep::LatticeField>(getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <class Rep>
-void TFundtoHirep<Rep>::execute(void)
-{
-    auto &U      = *env().template getObject<LatticeGaugeField>(par().gaugeconf);
-    LOG(Message) << "Transforming Representation" << std::endl;
-
-    Rep TargetRepresentation(U._grid);
-    TargetRepresentation.update_representation(U);
-
-   typename Rep::LatticeField &URep = *env().template createLattice<typename Rep::LatticeField>(getName());
-    URep = TargetRepresentation.U;
-}

extras/Hadrons/Modules/MGauge/FundtoHirep.hpp

@@ -1,77 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MGauge/FundtoHirep.hpp
-
-Copyright (C) 2015
-Copyright (C) 2016
-
-Author: David Preti <david.preti@to.infn.it>
-	Guido Cossu <guido.cossu@ed.ac.uk>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MGauge_FundtoHirep_hpp_
-#define Hadrons_MGauge_FundtoHirep_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         Load a NERSC configuration                         *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MGauge)
-
-class FundtoHirepPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(FundtoHirepPar,
-                                    std::string, gaugeconf);
-};
-
-template <class Rep>
-class TFundtoHirep: public Module<FundtoHirepPar>
-{
-public:
-    // constructor
-    TFundtoHirep(const std::string name);
-    // destructor
-    virtual ~TFundtoHirep(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    void setup(void);
-    // execution
-    void execute(void);
-};
-
-//MODULE_REGISTER_NS(FundtoAdjoint,   TFundtoHirep<AdjointRepresentation>, MGauge);
-//MODULE_REGISTER_NS(FundtoTwoIndexSym, TFundtoHirep<TwoIndexSymmetricRepresentation>, MGauge);
-//MODULE_REGISTER_NS(FundtoTwoIndexAsym, TFundtoHirep<TwoIndexAntiSymmetricRepresentation>, MGauge);
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_MGauge_FundtoHirep_hpp_

extras/Hadrons/Modules/MGauge/Load.cc

@@ -2,9 +2,10 @@
 
 Grid physics library, www.github.com/paboyle/Grid 
 
-Source file: extras/Hadrons/Modules/MIO/LoadNersc.cc
+Source file: extras/Hadrons/Modules/MGauge/Load.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -25,29 +26,30 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 See the full license in the file "LICENSE" in the top level distribution directory
 *************************************************************************************/
 /*  END LEGAL */
-#include <Grid/Hadrons/Modules/MIO/LoadNersc.hpp>
+
+#include <Grid/Hadrons/Modules/MGauge/Load.hpp>
 
 using namespace Grid;
 using namespace Hadrons;
-using namespace MIO;
+using namespace MGauge;
 
 /******************************************************************************
-*                       TLoadNersc implementation                             *
+*                           TLoad implementation                               *
 ******************************************************************************/
 // constructor /////////////////////////////////////////////////////////////////
-TLoadNersc::TLoadNersc(const std::string name)
-: Module<LoadNerscPar>(name)
+TLoad::TLoad(const std::string name)
+: Module<LoadPar>(name)
 {}
 
 // dependencies/products ///////////////////////////////////////////////////////
-std::vector<std::string> TLoadNersc::getInput(void)
+std::vector<std::string> TLoad::getInput(void)
 {
     std::vector<std::string> in;
     
     return in;
 }
 
-std::vector<std::string> TLoadNersc::getOutput(void)
+std::vector<std::string> TLoad::getOutput(void)
 {
     std::vector<std::string> out = {getName()};
     
@@ -55,21 +57,21 @@ std::vector<std::string> TLoadNersc::getOutput(void)
 }
 
 // setup ///////////////////////////////////////////////////////////////////////
-void TLoadNersc::setup(void)
+void TLoad::setup(void)
 {
-    envCreateLat(LatticeGaugeField, getName());
+    env().registerLattice<LatticeGaugeField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
-void TLoadNersc::execute(void)
+void TLoad::execute(void)
 {
-    FieldMetaData header;
-    std::string   fileName = par().file + "."
-                             + std::to_string(vm().getTrajectory());
+    FieldMetaData  header;
+    std::string fileName = par().file + "."
+                           + std::to_string(env().getTrajectory());
+    
     LOG(Message) << "Loading NERSC configuration from file '" << fileName
                  << "'" << std::endl;
-
-    auto &U = envGet(LatticeGaugeField, getName());
+    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
     NerscIO::readConfiguration(U, header, fileName);
     LOG(Message) << "NERSC header:" << std::endl;
     dump_meta_data(header, LOG(Message));

extras/Hadrons/Modules/MGauge/Load.hpp

@@ -2,9 +2,10 @@
 
 Grid physics library, www.github.com/paboyle/Grid 
 
-Source file: extras/Hadrons/Modules/MIO/LoadNersc.hpp
+Source file: extras/Hadrons/Modules/MGauge/Load.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -25,8 +26,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 See the full license in the file "LICENSE" in the top level distribution directory
 *************************************************************************************/
 /*  END LEGAL */
-#ifndef Hadrons_MIO_LoadNersc_hpp_
-#define Hadrons_MIO_LoadNersc_hpp_
+
+#ifndef Hadrons_MGauge_Load_hpp_
+#define Hadrons_MGauge_Load_hpp_
 
 #include <Grid/Hadrons/Global.hpp>
 #include <Grid/Hadrons/Module.hpp>
@@ -35,24 +37,24 @@ See the full license in the file "LICENSE" in the top level distribution directo
 BEGIN_HADRONS_NAMESPACE
 
 /******************************************************************************
- *                       Load a NERSC configuration                           *
+ *                         Load a NERSC configuration                         *
  ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MIO)
+BEGIN_MODULE_NAMESPACE(MGauge)
 
-class LoadNerscPar: Serializable
+class LoadPar: Serializable
 {
 public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadNerscPar,
+    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadPar,
                                     std::string, file);
 };
 
-class TLoadNersc: public Module<LoadNerscPar>
+class TLoad: public Module<LoadPar>
 {
 public:
     // constructor
-    TLoadNersc(const std::string name);
+    TLoad(const std::string name);
     // destructor
-    virtual ~TLoadNersc(void) = default;
+    virtual ~TLoad(void) = default;
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
@@ -62,10 +64,10 @@ public:
     virtual void execute(void);
 };
 
-MODULE_REGISTER_NS(LoadNersc, TLoadNersc, MIO);
+MODULE_REGISTER_NS(Load, TLoad, MGauge);
 
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE
 
-#endif // Hadrons_MIO_LoadNersc_hpp_
+#endif // Hadrons_MGauge_Load_hpp_

extras/Hadrons/Modules/MGauge/Random.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Random.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -43,9 +44,7 @@ TRandom::TRandom(const std::string name)
 // dependencies/products ///////////////////////////////////////////////////////
 std::vector<std::string> TRandom::getInput(void)
 {
-    std::vector<std::string> in;
-    
-    return in;
+    return std::vector<std::string>();
 }
 
 std::vector<std::string> TRandom::getOutput(void)
@@ -58,14 +57,13 @@ std::vector<std::string> TRandom::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TRandom::setup(void)
 {
-    envCreateLat(LatticeGaugeField, getName());
+    env().registerLattice<LatticeGaugeField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TRandom::execute(void)
 {
     LOG(Message) << "Generating random gauge configuration" << std::endl;
-    
-    auto &U = envGet(LatticeGaugeField, getName());
+    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
     SU3::HotConfiguration(*env().get4dRng(), U);
 }

extras/Hadrons/Modules/MGauge/Random.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Random.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -50,7 +51,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution

extras/Hadrons/Modules/MGauge/StochEm.cc

@@ -4,9 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/StochEm.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
-Author: Antonin Portelli <antonin.portelli@me.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -57,28 +57,32 @@ std::vector<std::string> TStochEm::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TStochEm::setup(void)
 {
-    if (!env().hasCreatedObject("_" + getName() + "_weight"))
+    if (!env().hasRegisteredObject("_" + getName() + "_weight"))
     {
-        envCacheLat(EmComp, "_" + getName() + "_weight");
+        env().registerLattice<EmComp>("_" + getName() + "_weight");
     }
-    envCreateLat(EmField, getName());
+    env().registerLattice<EmField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TStochEm::execute(void)
 {
-    LOG(Message) << "Generating stochatic EM potential..." << std::endl;
-
     PhotonR photon(par().gauge, par().zmScheme);
-    auto    &a = envGet(EmField, getName());
-    auto    &w = envGet(EmComp, "_" + getName() + "_weight");
+    EmField &a = *env().createLattice<EmField>(getName());
+    EmComp  *w;
     
     if (!env().hasCreatedObject("_" + getName() + "_weight"))
     {
         LOG(Message) << "Caching stochatic EM potential weight (gauge: "
                      << par().gauge << ", zero-mode scheme: "
                      << par().zmScheme << ")..." << std::endl;
-        photon.StochasticWeight(w);
+        w = env().createLattice<EmComp>("_" + getName() + "_weight");
+        photon.StochasticWeight(*w);
     }
-    photon.StochasticField(a, *env().get4dRng(), w);
+    else
+    {
+        w = env().getObject<EmComp>("_" + getName() + "_weight");
+    }
+    LOG(Message) << "Generating stochatic EM potential..." << std::endl;
+    photon.StochasticField(a, *env().get4dRng(), *w);
 }

extras/Hadrons/Modules/MGauge/StochEm.hpp

@@ -4,9 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/StochEm.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
-Author: Antonin Portelli <antonin.portelli@me.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -60,7 +60,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution

extras/Hadrons/Modules/MGauge/Unit.cc

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Unit.cc
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -56,14 +57,13 @@ std::vector<std::string> TUnit::getOutput(void)
 // setup ///////////////////////////////////////////////////////////////////////
 void TUnit::setup(void)
 {
-    envCreateLat(LatticeGaugeField, getName());
+    env().registerLattice<LatticeGaugeField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TUnit::execute(void)
 {
     LOG(Message) << "Creating unit gauge configuration" << std::endl;
-    
-    auto &U = envGet(LatticeGaugeField, getName());
+    LatticeGaugeField &U = *env().createLattice<LatticeGaugeField>(getName());
     SU3::ColdConfiguration(*env().get4dRng(), U);
 }

extras/Hadrons/Modules/MGauge/Unit.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MGauge/Unit.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -50,7 +51,6 @@ public:
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution

extras/Hadrons/Modules/MIO/LoadBinary.hpp

@@ -1,140 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MIO/LoadBinary.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef Hadrons_MIO_LoadBinary_hpp_
-#define Hadrons_MIO_LoadBinary_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                       Load a binary configurations                         *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MIO)
-
-class LoadBinaryPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(LoadBinaryPar,
-                                    std::string, file,
-                                    std::string, format);
-};
-
-template <typename Impl>
-class TLoadBinary: public Module<LoadBinaryPar>
-{
-public:
-    typedef typename Impl::Field                  Field;
-    typedef typename Impl::Simd                   Simd;
-    typedef typename Field::vector_object         vobj;
-    typedef typename vobj::scalar_object          sobj;
-    typedef typename sobj::DoublePrecision        sobj_double;
-    typedef BinarySimpleMunger<sobj_double, sobj> Munger;
-public:
-    // constructor
-    TLoadBinary(const std::string name);
-    // destructor
-    virtual ~TLoadBinary(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(LoadBinary, TLoadBinary<GIMPL>, MIO);
-MODULE_REGISTER_NS(LoadBinaryScalarSU2, TLoadBinary<ScalarNxNAdjImplR<2>>, MIO);
-MODULE_REGISTER_NS(LoadBinaryScalarSU3, TLoadBinary<ScalarNxNAdjImplR<3>>, MIO);
-MODULE_REGISTER_NS(LoadBinaryScalarSU4, TLoadBinary<ScalarNxNAdjImplR<4>>, MIO);
-MODULE_REGISTER_NS(LoadBinaryScalarSU5, TLoadBinary<ScalarNxNAdjImplR<5>>, MIO);
-MODULE_REGISTER_NS(LoadBinaryScalarSU6, TLoadBinary<ScalarNxNAdjImplR<6>>, MIO);
-
-/******************************************************************************
- *                         TLoadBinary implementation                         *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename Impl>
-TLoadBinary<Impl>::TLoadBinary(const std::string name)
-: Module<LoadBinaryPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename Impl>
-std::vector<std::string> TLoadBinary<Impl>::getInput(void)
-{
-    std::vector<std::string> in;
-    
-    return in;
-}
-
-template <typename Impl>
-std::vector<std::string> TLoadBinary<Impl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename Impl>
-void TLoadBinary<Impl>::setup(void)
-{
-    envCreateLat(Field, getName());
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename Impl>
-void TLoadBinary<Impl>::execute(void)
-{
-    Munger      munge;
-    uint32_t    nersc_csum, scidac_csuma, scidac_csumb;
-    auto        &U = envGet(Field, getName());
-    std::string filename = par().file + "."
-                           + std::to_string(vm().getTrajectory());
-
-    LOG(Message) << "Loading " << par().format 
-                 << " binary configuration from file '" << filename
-                 << "'" << std::endl;
-    BinaryIO::readLatticeObject<vobj, sobj_double>(U, filename, munge, 0, 
-                                                   par().format, nersc_csum,
-                                                   scidac_csuma, scidac_csumb);
-    LOG(Message) << "Checksums:" << std::endl;
-    LOG(Message) << "  NERSC    " << nersc_csum << std::endl;
-    LOG(Message) << "  SciDAC A " << scidac_csuma << std::endl;
-    LOG(Message) << "  SciDAC B " << scidac_csumb << std::endl;
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_MIO_LoadBinary_hpp_

extras/Hadrons/Modules/MLoop/NoiseLoop.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MLoop/NoiseLoop.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2016
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -75,7 +74,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -114,16 +112,16 @@ std::vector<std::string> TNoiseLoop<FImpl>::getOutput(void)
 template <typename FImpl>
 void TNoiseLoop<FImpl>::setup(void)
 {
-    envCreateLat(PropagatorField, getName());
+    env().template registerLattice<PropagatorField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TNoiseLoop<FImpl>::execute(void)
 {
-    auto &loop = envGet(PropagatorField, getName());
-    auto &q    = envGet(PropagatorField, par().q);
-    auto &eta  = envGet(PropagatorField, par().eta);
+    PropagatorField &loop = *env().template createLattice<PropagatorField>(getName());
+    PropagatorField &q    = *env().template getObject<PropagatorField>(par().q);
+    PropagatorField &eta  = *env().template getObject<PropagatorField>(par().eta);
     loop = q*adj(eta);
 }
 

extras/Hadrons/Modules/MScalar/ChargedProp.cc

@@ -1,31 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalar/ChargedProp.cc
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: James Harrison <jch1g10@soton.ac.uk>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
 #include <Grid/Hadrons/Modules/MScalar/ChargedProp.hpp>
 #include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
 
@@ -65,44 +37,90 @@ void TChargedProp::setup(void)
     {
         phaseName_.push_back("_shiftphase_" + std::to_string(mu));
     }
-    GFSrcName_ = getName() + "_DinvSrc";
-    fftName_   = getName() + "_fft";
-
-    freeMomPropDone_ = env().hasCreatedObject(freeMomPropName_);
-    GFSrcDone_       = env().hasCreatedObject(GFSrcName_);
-    phasesDone_      = env().hasCreatedObject(phaseName_[0]);
-    envCacheLat(ScalarField, freeMomPropName_);
-    for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+    GFSrcName_ = "_" + getName() + "_DinvSrc";
+    if (!env().hasRegisteredObject(freeMomPropName_))
     {
-        envCacheLat(ScalarField, phaseName_[mu]);
+        env().registerLattice<ScalarField>(freeMomPropName_);
     }
-    envCacheLat(ScalarField, GFSrcName_);
-    envCreateLat(ScalarField, getName());
-    envTmpLat(ScalarField, "buf");
-    envTmpLat(ScalarField, "result");
-    envTmpLat(ScalarField, "Amu");
-    envCache(FFT, fftName_, 1, env().getGrid());
+    if (!env().hasRegisteredObject(phaseName_[0]))
+    {
+        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+        {
+            env().registerLattice<ScalarField>(phaseName_[mu]);
+        }
+    }
+    if (!env().hasRegisteredObject(GFSrcName_))
+    {
+        env().registerLattice<ScalarField>(GFSrcName_);
+    }
+    env().registerLattice<ScalarField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TChargedProp::execute(void)
 {
     // CACHING ANALYTIC EXPRESSIONS
-    makeCaches();
+    ScalarField &source = *env().getObject<ScalarField>(par().source);
+    Complex     ci(0.0,1.0);
+    FFT         fft(env().getGrid());
+    
+    // cache free scalar propagator
+    if (!env().hasCreatedObject(freeMomPropName_))
+    {
+        LOG(Message) << "Caching momentum space free scalar propagator"
+                     << " (mass= " << par().mass << ")..." << std::endl;
+        freeMomProp_ = env().createLattice<ScalarField>(freeMomPropName_);
+        SIMPL::MomentumSpacePropagator(*freeMomProp_, par().mass);
+    }
+    else
+    {
+        freeMomProp_ = env().getObject<ScalarField>(freeMomPropName_);
+    }
+    // cache G*F*src
+    if (!env().hasCreatedObject(GFSrcName_))
+        
+    {
+        GFSrc_ = env().createLattice<ScalarField>(GFSrcName_);
+        fft.FFT_all_dim(*GFSrc_, source, FFT::forward);
+        *GFSrc_ = (*freeMomProp_)*(*GFSrc_);
+    }
+    else
+    {
+        GFSrc_ = env().getObject<ScalarField>(GFSrcName_);
+    }
+    // cache phases
+    if (!env().hasCreatedObject(phaseName_[0]))
+    {
+        std::vector<int> &l = env().getGrid()->_fdimensions;
+        
+        LOG(Message) << "Caching shift phases..." << std::endl;
+        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+        {
+            Real    twoPiL = M_PI*2./l[mu];
+            
+            phase_.push_back(env().createLattice<ScalarField>(phaseName_[mu]));
+            LatticeCoordinate(*(phase_[mu]), mu);
+            *(phase_[mu]) = exp(ci*twoPiL*(*(phase_[mu])));
+        }
+    }
+    else
+    {
+        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
+        {
+            phase_.push_back(env().getObject<ScalarField>(phaseName_[mu]));
+        }
+    }
 
     // PROPAGATOR CALCULATION
     LOG(Message) << "Computing charged scalar propagator"
                  << " (mass= " << par().mass
                  << ", charge= " << par().charge << ")..." << std::endl;
     
-    auto   &prop  = envGet(ScalarField, getName());
-    auto   &GFSrc = envGet(ScalarField, GFSrcName_);
-    auto   &G     = envGet(ScalarField, freeMomPropName_);
-    auto   &fft   = envGet(FFT, fftName_);
-    double q      = par().charge;
-    envGetTmp(ScalarField, result); 
-    envGetTmp(ScalarField, buf); 
-
+    ScalarField &prop   = *env().createLattice<ScalarField>(getName());
+    ScalarField buf(env().getGrid());
+    ScalarField &GFSrc = *GFSrc_, &G = *freeMomProp_;
+    double      q = par().charge;
+    
     // G*F*Src
     prop = GFSrc;
 
@@ -128,12 +146,12 @@ void TChargedProp::execute(void)
     if (!par().output.empty())
     {
         std::string           filename = par().output + "." +
-                                         std::to_string(vm().getTrajectory());
+                                         std::to_string(env().getTrajectory());
         
         LOG(Message) << "Saving zero-momentum projection to '"
                      << filename << "'..." << std::endl;
         
-        ResultWriter          writer(RESULT_FILE_NAME(par().output));
+        CorrWriter            writer(filename);
         std::vector<TComplex> vecBuf;
         std::vector<Complex>  result;
         
@@ -148,55 +166,15 @@ void TChargedProp::execute(void)
     }
 }
 
-void TChargedProp::makeCaches(void)
-{
-    auto &freeMomProp = envGet(ScalarField, freeMomPropName_);
-    auto &GFSrc       = envGet(ScalarField, GFSrcName_);
-    auto &fft         = envGet(FFT, fftName_);
-
-    if (!freeMomPropDone_)
-    {
-        LOG(Message) << "Caching momentum space free scalar propagator"
-                     << " (mass= " << par().mass << ")..." << std::endl;
-        SIMPL::MomentumSpacePropagator(freeMomProp, par().mass);
-    }
-    if (!GFSrcDone_)
-    {   
-        FFT  fft(env().getGrid());
-        auto &source = envGet(ScalarField, par().source);
-
-        LOG(Message) << "Caching G*F*src..." << std::endl;
-        fft.FFT_all_dim(GFSrc, source, FFT::forward);
-        GFSrc = freeMomProp*GFSrc;
-    }
-    if (!phasesDone_)
-    {
-        std::vector<int> &l = env().getGrid()->_fdimensions;
-        Complex          ci(0.0,1.0);
-        
-        LOG(Message) << "Caching shift phases..." << std::endl;
-        for (unsigned int mu = 0; mu < env().getNd(); ++mu)
-        {
-            Real twoPiL = M_PI*2./l[mu];
-            auto &phmu  = envGet(ScalarField, phaseName_[mu]);
-            
-            LatticeCoordinate(phmu, mu);
-            phmu = exp(ci*twoPiL*phmu);
-            phase_.push_back(&phmu);
-        }
-    }
-}
-
 void TChargedProp::momD1(ScalarField &s, FFT &fft)
 {
-    auto        &A = envGet(EmField, par().emField);
+    EmField     &A = *env().getObject<EmField>(par().emField);
+    ScalarField buf(env().getGrid()), result(env().getGrid()),
+                Amu(env().getGrid());
     Complex     ci(0.0,1.0);
 
-    envGetTmp(ScalarField, buf);
-    envGetTmp(ScalarField, result);
-    envGetTmp(ScalarField, Amu);
-
     result = zero;
+
     for (unsigned int mu = 0; mu < env().getNd(); ++mu)
     {
         Amu = peekLorentz(A, mu);
@@ -220,13 +198,12 @@ void TChargedProp::momD1(ScalarField &s, FFT &fft)
 
 void TChargedProp::momD2(ScalarField &s, FFT &fft)
 {
-    auto &A = envGet(EmField, par().emField);
-
-    envGetTmp(ScalarField, buf);
-    envGetTmp(ScalarField, result);
-    envGetTmp(ScalarField, Amu);
+    EmField     &A = *env().getObject<EmField>(par().emField);
+    ScalarField buf(env().getGrid()), result(env().getGrid()),
+                Amu(env().getGrid());
 
     result = zero;
+    
     for (unsigned int mu = 0; mu < env().getNd(); ++mu)
     {
         Amu = peekLorentz(A, mu);

extras/Hadrons/Modules/MScalar/ChargedProp.hpp

@@ -1,30 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalar/ChargedProp.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
 #ifndef Hadrons_MScalar_ChargedProp_hpp_
 #define Hadrons_MScalar_ChargedProp_hpp_
 
@@ -64,20 +37,19 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 private:
-    void makeCaches(void);
     void momD1(ScalarField &s, FFT &fft);
     void momD2(ScalarField &s, FFT &fft);
 private:
-    bool                       freeMomPropDone_, GFSrcDone_, phasesDone_;
-    std::string                freeMomPropName_, GFSrcName_, fftName_;
+    std::string                freeMomPropName_, GFSrcName_;
     std::vector<std::string>   phaseName_;
+    ScalarField                *freeMomProp_, *GFSrc_;
     std::vector<ScalarField *> phase_;
+    EmField                    *A;
 };
 
 MODULE_REGISTER_NS(ChargedProp, TChargedProp, MScalar);

extras/Hadrons/Modules/MScalar/FreeProp.cc

@@ -1,30 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalar/FreeProp.cc
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
 #include <Grid/Hadrons/Modules/MScalar/FreeProp.hpp>
 #include <Grid/Hadrons/Modules/MScalar/Scalar.hpp>
 
@@ -60,31 +33,38 @@ void TFreeProp::setup(void)
 {
     freeMomPropName_ = FREEMOMPROP(par().mass);
     
-    freePropDone_ = env().hasCreatedObject(freeMomPropName_);
-    envCacheLat(ScalarField, freeMomPropName_);
-    envCreateLat(ScalarField, getName());
+    if (!env().hasRegisteredObject(freeMomPropName_))
+    {
+        env().registerLattice<ScalarField>(freeMomPropName_);
+    }
+    env().registerLattice<ScalarField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 void TFreeProp::execute(void)
 {
-    auto &freeMomProp = envGet(ScalarField, freeMomPropName_);
-    auto &prop        = envGet(ScalarField, getName());
-    auto &source      = envGet(ScalarField, par().source);
+    ScalarField &prop   = *env().createLattice<ScalarField>(getName());
+    ScalarField &source = *env().getObject<ScalarField>(par().source);
+    ScalarField *freeMomProp;
 
-    if (!freePropDone_)
+    if (!env().hasCreatedObject(freeMomPropName_))
     {
         LOG(Message) << "Caching momentum space free scalar propagator"
                      << " (mass= " << par().mass << ")..." << std::endl;
-        SIMPL::MomentumSpacePropagator(freeMomProp, par().mass);
+        freeMomProp = env().createLattice<ScalarField>(freeMomPropName_);
+        SIMPL::MomentumSpacePropagator(*freeMomProp, par().mass);
+    }
+    else
+    {
+        freeMomProp = env().getObject<ScalarField>(freeMomPropName_);
     }
     LOG(Message) << "Computing free scalar propagator..." << std::endl;
-    SIMPL::FreePropagator(source, prop, freeMomProp);
+    SIMPL::FreePropagator(source, prop, *freeMomProp);
     
     if (!par().output.empty())
     {
         TextWriter            writer(par().output + "." +
-                                     std::to_string(vm().getTrajectory()));
+                                     std::to_string(env().getTrajectory()));
         std::vector<TComplex> buf;
         std::vector<Complex>  result;
         

extras/Hadrons/Modules/MScalar/FreeProp.hpp

@@ -1,30 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalar/FreeProp.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
 #ifndef Hadrons_MScalar_FreeProp_hpp_
 #define Hadrons_MScalar_FreeProp_hpp_
 
@@ -60,14 +33,12 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
 private:
     std::string freeMomPropName_;
-    bool        freePropDone_;
 };
 
 MODULE_REGISTER_NS(FreeProp, TFreeProp, MScalar);

extras/Hadrons/Modules/MScalar/Scalar.hpp

@@ -1,30 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalar/Scalar.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
 #ifndef Hadrons_Scalar_hpp_
 #define Hadrons_Scalar_hpp_
 

extras/Hadrons/Modules/MScalarSUN/TrMag.hpp

@@ -1,146 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalarSUN/TrMag.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef Hadrons_MScalarSUN_TrMag_hpp_
-#define Hadrons_MScalarSUN_TrMag_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                       Module to compute tr(mag^n)                          *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MScalarSUN)
-
-class TrMagPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(TrMagPar,
-                                    std::string,  field,
-                                    unsigned int, maxPow,
-                                    std::string,  output);
-};
-
-template <typename SImpl>
-class TTrMag: public Module<TrMagPar>
-{
-public:
-    typedef typename SImpl::Field        Field;
-    typedef typename SImpl::ComplexField ComplexField;
-    class Result: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
-                                        std::string, op,
-                                        Real,        value);
-    };
-public:
-    // constructor
-    TTrMag(const std::string name);
-    // destructor
-    virtual ~TTrMag(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(TrMagSU2, TTrMag<ScalarNxNAdjImplR<2>>, MScalarSUN);
-MODULE_REGISTER_NS(TrMagSU3, TTrMag<ScalarNxNAdjImplR<3>>, MScalarSUN);
-MODULE_REGISTER_NS(TrMagSU4, TTrMag<ScalarNxNAdjImplR<4>>, MScalarSUN);
-MODULE_REGISTER_NS(TrMagSU5, TTrMag<ScalarNxNAdjImplR<5>>, MScalarSUN);
-MODULE_REGISTER_NS(TrMagSU6, TTrMag<ScalarNxNAdjImplR<6>>, MScalarSUN);
-
-/******************************************************************************
- *                         TTrMag implementation                              *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename SImpl>
-TTrMag<SImpl>::TTrMag(const std::string name)
-: Module<TrMagPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename SImpl>
-std::vector<std::string> TTrMag<SImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().field};
-    
-    return in;
-}
-
-template <typename SImpl>
-std::vector<std::string> TTrMag<SImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename SImpl>
-void TTrMag<SImpl>::setup(void)
-{}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename SImpl>
-void TTrMag<SImpl>::execute(void)
-{
-    LOG(Message) << "Computing tr(mag^n) for n even up to " << par().maxPow
-                 << "..." << std::endl;
-
-    std::vector<Result> result;
-    ResultWriter        writer(RESULT_FILE_NAME(par().output));
-    auto                &phi = envGet(Field, par().field);
-
-    auto m2 = sum(phi), mn = m2;
-
-    m2 = -m2*m2;
-    mn = 1.;
-    for (unsigned int n = 2; n <= par().maxPow; n += 2)
-    {
-        Result r;
-
-        mn = mn*m2;
-        r.op    = "tr(mag^" + std::to_string(n) + ")";
-        r.value = TensorRemove(trace(mn)).real();
-        result.push_back(r);
-    }
-    write(writer, "trmag", result);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_MScalarSUN_TrMag_hpp_

extras/Hadrons/Modules/MScalarSUN/TrPhi.hpp

@@ -1,182 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalarSUN/TrPhi.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef Hadrons_MScalarSUN_TrPhi_hpp_
-#define Hadrons_MScalarSUN_TrPhi_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                         Module to compute tr(phi^n)                        *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MScalarSUN)
-
-class TrPhiPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(TrPhiPar,
-                                    std::string,  field,
-                                    unsigned int, maxPow,
-                                    std::string,  output);
-};
-
-template <typename SImpl>
-class TTrPhi: public Module<TrPhiPar>
-{
-public:
-    typedef typename SImpl::Field        Field;
-    typedef typename SImpl::ComplexField ComplexField;
-    class Result: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
-                                        std::string, op,
-                                        Real,        value);
-    };
-public:
-    // constructor
-    TTrPhi(const std::string name);
-    // destructor
-    virtual ~TTrPhi(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-private:
-    // output name generator
-    std::string outName(const unsigned int n);
-};
-
-MODULE_REGISTER_NS(TrPhiSU2, TTrPhi<ScalarNxNAdjImplR<2>>, MScalarSUN);
-MODULE_REGISTER_NS(TrPhiSU3, TTrPhi<ScalarNxNAdjImplR<3>>, MScalarSUN);
-MODULE_REGISTER_NS(TrPhiSU4, TTrPhi<ScalarNxNAdjImplR<4>>, MScalarSUN);
-MODULE_REGISTER_NS(TrPhiSU5, TTrPhi<ScalarNxNAdjImplR<5>>, MScalarSUN);
-MODULE_REGISTER_NS(TrPhiSU6, TTrPhi<ScalarNxNAdjImplR<6>>, MScalarSUN);
-
-/******************************************************************************
- *                          TTrPhi implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename SImpl>
-TTrPhi<SImpl>::TTrPhi(const std::string name)
-: Module<TrPhiPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename SImpl>
-std::vector<std::string> TTrPhi<SImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().field};
-    
-    return in;
-}
-
-template <typename SImpl>
-std::vector<std::string> TTrPhi<SImpl>::getOutput(void)
-{
-    std::vector<std::string> out;
-
-    for (unsigned int n = 2; n <= par().maxPow; n += 2)
-    {
-        out.push_back(outName(n));
-    }
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename SImpl>
-void TTrPhi<SImpl>::setup(void)
-{
-    if (par().maxPow < 2)
-    {
-        HADRON_ERROR(Size, "'maxPow' should be at least equal to 2");
-    }
-    envTmpLat(Field, "phi2");
-    envTmpLat(Field, "buf");
-    for (unsigned int n = 2; n <= par().maxPow; n += 2)
-    {
-        envCreateLat(ComplexField, outName(n));
-    }
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename SImpl>
-void TTrPhi<SImpl>::execute(void)
-{
-    LOG(Message) << "Computing tr(phi^n) for n even up to " << par().maxPow
-                 << "..." << std::endl; 
-
-    std::vector<Result> result;
-    auto                &phi = envGet(Field, par().field);
-
-    envGetTmp(Field, phi2);
-    envGetTmp(Field, buf);
-    buf  = 1.;
-    phi2 = -phi*phi; 
-    for (unsigned int n = 2; n <= par().maxPow; n += 2)
-    {
-        auto &phin = envGet(ComplexField, outName(n));
-
-        buf  = buf*phi2;
-        phin = trace(buf);
-        if (!par().output.empty())
-        {
-            Result r;
-
-            r.op    = "tr(phi^" + std::to_string(n) + ")";
-            r.value = TensorRemove(sum(phin)).real();
-            result.push_back(r);
-        }
-    }
-    if (result.size() > 0)
-    {
-        ResultWriter writer(RESULT_FILE_NAME(par().output));
-
-        write(writer, "trphi", result);
-    }
-}
-
-// output name generator ///////////////////////////////////////////////////////
-template <typename SImpl>
-std::string TTrPhi<SImpl>::outName(const unsigned int n)
-{
-    return getName() + "_" + std::to_string(n);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_MScalarSUN_TrPhi_hpp_

extras/Hadrons/Modules/MScalarSUN/TwoPoint.hpp

@@ -1,184 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MScalarSUN/TwoPoint.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-#ifndef Hadrons_MScalarSUN_TwoPoint_hpp_
-#define Hadrons_MScalarSUN_TwoPoint_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                 2-pt functions for a given set of operators                *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MScalarSUN)
-
-class TwoPointPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(TwoPointPar,
-                                    std::vector<std::string>, op,
-                                    std::string,              output);
-};
-
-template <typename SImpl>
-class TTwoPoint: public Module<TwoPointPar>
-{
-public:
-    typedef typename SImpl::Field        Field;
-    typedef typename SImpl::ComplexField ComplexField;
-    class Result: Serializable
-    {
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(Result,
-                                        std::string, sink,
-                                        std::string, source,
-                                        std::vector<Complex>, data);
-    };
-public:
-    // constructor
-    TTwoPoint(const std::string name);
-    // destructor
-    virtual ~TTwoPoint(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-private:
-    // make 2-pt function
-    template <class SinkSite, class SourceSite>
-    std::vector<Complex> makeTwoPoint(const std::vector<SinkSite>   &sink,
-                                      const std::vector<SourceSite> &source);
-};
-
-MODULE_REGISTER_NS(TwoPointSU2, TTwoPoint<ScalarNxNAdjImplR<2>>, MScalarSUN);
-MODULE_REGISTER_NS(TwoPointSU3, TTwoPoint<ScalarNxNAdjImplR<3>>, MScalarSUN);
-MODULE_REGISTER_NS(TwoPointSU4, TTwoPoint<ScalarNxNAdjImplR<4>>, MScalarSUN);
-MODULE_REGISTER_NS(TwoPointSU5, TTwoPoint<ScalarNxNAdjImplR<5>>, MScalarSUN);
-MODULE_REGISTER_NS(TwoPointSU6, TTwoPoint<ScalarNxNAdjImplR<6>>, MScalarSUN);
-
-/******************************************************************************
- *                 TTwoPoint implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename SImpl>
-TTwoPoint<SImpl>::TTwoPoint(const std::string name)
-: Module<TwoPointPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename SImpl>
-std::vector<std::string> TTwoPoint<SImpl>::getInput(void)
-{   
-    return par().op;
-}
-
-template <typename SImpl>
-std::vector<std::string> TTwoPoint<SImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {};
-
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename SImpl>
-void TTwoPoint<SImpl>::setup(void)
-{
-    const unsigned int nt = env().getDim().back();
-    envTmp(std::vector<std::vector<TComplex>>, "slicedOp", 1, par().op.size(), 
-           std::vector<TComplex>(nt));
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename SImpl>
-void TTwoPoint<SImpl>::execute(void)
-{
-    LOG(Message) << "Computing 2-point functions for operators:" << std::endl;
-    for (auto &o: par().op)
-    {
-        LOG(Message) << "  '" << o << "'" << std::endl;
-    }
-
-    ResultWriter        writer(RESULT_FILE_NAME(par().output));
-    const unsigned int  nd = env().getDim().size();
-    std::vector<Result> result;
-    
-    envGetTmp(std::vector<std::vector<TComplex>>, slicedOp);
-    for (unsigned int i = 0; i < par().op.size(); ++i)
-    {
-        auto &op = envGet(ComplexField, par().op[i]);
-
-        sliceSum(op, slicedOp[i], nd - 1);
-    }
-    for (unsigned int i = 0; i < par().op.size(); ++i)
-    for (unsigned int j = 0; j < par().op.size(); ++j)
-    {
-        Result r;
-
-        r.sink   = par().op[i];
-        r.source = par().op[j];
-        r.data   = makeTwoPoint(slicedOp[i], slicedOp[j]);
-        result.push_back(r);
-    }
-    write(writer, "twopt", result);
-}
-
-// make 2-pt function //////////////////////////////////////////////////////////
-template <class SImpl>
-template <class SinkSite, class SourceSite>
-std::vector<Complex> TTwoPoint<SImpl>::makeTwoPoint(
-                                  const std::vector<SinkSite>   &sink,
-                                  const std::vector<SourceSite> &source)
-{
-    assert(sink.size() == source.size());
-    
-    unsigned int         nt = sink.size();
-    std::vector<Complex> res(nt, 0.);
-    
-    for (unsigned int dt = 0; dt < nt; ++dt)
-    {
-        for (unsigned int t  = 0; t < nt; ++t)
-        {
-            res[dt] += TensorRemove(trace(sink[(t+dt)%nt]*source[t]));
-        }
-        res[dt] *= 1./static_cast<double>(nt);
-    }
-    
-    return res;
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_MScalarSUN_TwoPoint_hpp_

extras/Hadrons/Modules/MSink/Point.hpp

@@ -1,32 +1,3 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSink/Point.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
 #ifndef Hadrons_MSink_Point_hpp_
 #define Hadrons_MSink_Point_hpp_
 
@@ -62,14 +33,10 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
-private:
-    bool        hasPhase_{false}; 
-    std::string momphName_;
 };
 
 MODULE_REGISTER_NS(Point,       TPoint<FIMPL>,        MSink);
@@ -82,7 +49,6 @@ MODULE_REGISTER_NS(ScalarPoint, TPoint<ScalarImplCR>, MSink);
 template <typename FImpl>
 TPoint<FImpl>::TPoint(const std::string name)
 : Module<PointPar>(name)
-, momphName_ (name + "_momph")
 {}
 
 // dependencies/products ///////////////////////////////////////////////////////
@@ -106,37 +72,30 @@ std::vector<std::string> TPoint<FImpl>::getOutput(void)
 template <typename FImpl>
 void TPoint<FImpl>::setup(void)
 {
-    envTmpLat(LatticeComplex, "coor");
-    envCacheLat(LatticeComplex, momphName_);
-    envCreate(SinkFn, getName(), 1, nullptr);
+    unsigned int size;
+    
+    size = env().template lattice4dSize<LatticeComplex>();
+    env().registerObject(getName(), size);
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TPoint<FImpl>::execute(void)
-{   
+{
+    std::vector<Real> p = strToVec<Real>(par().mom);
+    LatticeComplex    ph(env().getGrid()), coor(env().getGrid());
+    Complex           i(0.0,1.0);
+    
     LOG(Message) << "Setting up point sink function for momentum ["
                  << par().mom << "]" << std::endl;
-
-    auto &ph = envGet(LatticeComplex, momphName_);
-    
-    if (!hasPhase_)
+    ph = zero;
+    for(unsigned int mu = 0; mu < env().getNd(); mu++)
     {
-        Complex           i(0.0,1.0);
-        std::vector<Real> p;
-
-        envGetTmp(LatticeComplex, coor);
-        p  = strToVec<Real>(par().mom);
-        ph = zero;
-        for(unsigned int mu = 0; mu < env().getNd(); mu++)
-        {
-            LatticeCoordinate(coor, mu);
-            ph = ph + (p[mu]/env().getGrid()->_fdimensions[mu])*coor;
-        }
-        ph = exp((Real)(2*M_PI)*i*ph);
-        hasPhase_ = true;
+        LatticeCoordinate(coor, mu);
+        ph = ph + (p[mu]/env().getGrid()->_fdimensions[mu])*coor;
     }
-    auto sink = [&ph](const PropagatorField &field)
+    ph = exp((Real)(2*M_PI)*i*ph);
+    auto sink = [ph](const PropagatorField &field)
     {
         SlicedPropagator res;
         PropagatorField  tmp = ph*field;
@@ -145,7 +104,7 @@ void TPoint<FImpl>::execute(void)
         
         return res;
     };
-    envGet(SinkFn, getName()) = sink;
+    env().setObject(getName(), new SinkFn(sink));
 }
 
 END_MODULE_NAMESPACE

extras/Hadrons/Modules/MSink/Smear.hpp

@@ -1,127 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSink/Smear.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MSink_Smear_hpp_
-#define Hadrons_MSink_Smear_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                                 Smear                                      *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MSink)
-
-class SmearPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(SmearPar,
-                                    std::string, q,
-                                    std::string, sink);
-};
-
-template <typename FImpl>
-class TSmear: public Module<SmearPar>
-{
-public:
-    FERM_TYPE_ALIASES(FImpl,);
-    SINK_TYPE_ALIASES();
-public:
-    // constructor
-    TSmear(const std::string name);
-    // destructor
-    virtual ~TSmear(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-protected:
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(Smear, TSmear<FIMPL>, MSink);
-
-/******************************************************************************
- *                          TSmear implementation                             *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TSmear<FImpl>::TSmear(const std::string name)
-: Module<SmearPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TSmear<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().q, par().sink};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TSmear<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TSmear<FImpl>::setup(void)
-{
-    envCreate(SlicedPropagator, getName(), 1, env().getDim(Tp));
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TSmear<FImpl>::execute(void)
-{
-    LOG(Message) << "Sink smearing propagator '" << par().q
-                 << "' using sink function '" << par().sink << "'."
-                 << std::endl;
-
-    auto &sink = envGet(SinkFn, par().sink);
-    auto &q    = envGet(PropagatorField, par().q);
-    auto &out  = envGet(SlicedPropagator, getName());
-    
-    out = sink(q);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_MSink_Smear_hpp_

extras/Hadrons/Modules/MSolver/RBPrecCG.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MSolver/RBPrecCG.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -60,9 +61,7 @@ public:
     virtual ~TRBPrecCG(void) = default;
     // dependencies/products
     virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getReference(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -84,19 +83,11 @@ TRBPrecCG<FImpl>::TRBPrecCG(const std::string name)
 template <typename FImpl>
 std::vector<std::string> TRBPrecCG<FImpl>::getInput(void)
 {
-    std::vector<std::string> in = {};
+    std::vector<std::string> in = {par().action};
     
     return in;
 }
 
-template <typename FImpl>
-std::vector<std::string> TRBPrecCG<FImpl>::getReference(void)
-{
-    std::vector<std::string> ref = {par().action};
-    
-    return ref;
-}
-
 template <typename FImpl>
 std::vector<std::string> TRBPrecCG<FImpl>::getOutput(void)
 {
@@ -109,12 +100,17 @@ std::vector<std::string> TRBPrecCG<FImpl>::getOutput(void)
 template <typename FImpl>
 void TRBPrecCG<FImpl>::setup(void)
 {
-    LOG(Message) << "setting up Schur red-black preconditioned CG for"
-                 << " action '" << par().action << "' with residual "
-                 << par().residual << std::endl;
+    auto Ls = env().getObjectLs(par().action);
+    
+    env().registerObject(getName(), 0, Ls);
+    env().addOwnership(getName(), par().action);
+}
 
-    auto Ls     = env().getObjectLs(par().action);
-    auto &mat   = envGet(FMat, par().action);
+// execution ///////////////////////////////////////////////////////////////////
+template <typename FImpl>
+void TRBPrecCG<FImpl>::execute(void)
+{
+    auto &mat   = *(env().template getObject<FMat>(par().action));
     auto solver = [&mat, this](FermionField &sol, const FermionField &source)
     {
         ConjugateGradient<FermionField>           cg(par().residual, 10000);
@@ -122,14 +118,13 @@ void TRBPrecCG<FImpl>::setup(void)
         
         schurSolver(mat, source, sol);
     };
-    envCreate(SolverFn, getName(), Ls, solver);
+    
+    LOG(Message) << "setting up Schur red-black preconditioned CG for"
+                 << " action '" << par().action << "' with residual "
+                 << par().residual << std::endl;
+    env().setObject(getName(), new SolverFn(solver));
 }
 
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TRBPrecCG<FImpl>::execute(void)
-{}
-
 END_MODULE_NAMESPACE
 
 END_HADRONS_NAMESPACE

extras/Hadrons/Modules/MSource/Point.hpp

@@ -4,10 +4,10 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MSource/Point.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -72,7 +72,6 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
@@ -112,20 +111,19 @@ std::vector<std::string> TPoint<FImpl>::getOutput(void)
 template <typename FImpl>
 void TPoint<FImpl>::setup(void)
 {
-    envCreateLat(PropagatorField, getName());
+    env().template registerLattice<PropagatorField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TPoint<FImpl>::execute(void)
 {
-    LOG(Message) << "Creating point source at position [" << par().position
-                << "]" << std::endl;
-
     std::vector<int> position = strToVec<int>(par().position);
-    auto             &src     = envGet(PropagatorField, getName());
-    SitePropagator   id;
+    typename SitePropagator::scalar_object id;
     
+    LOG(Message) << "Creating point source at position [" << par().position
+                 << "]" << std::endl;
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
     id  = 1.;
     src = zero;
     pokeSite(id, src, position);

extras/Hadrons/Modules/MSource/SeqConserved.hpp

@@ -1,160 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MSource/SeqConserved.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MSource_SeqConserved_hpp_
-#define Hadrons_MSource_SeqConserved_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/*
- 
- Sequential source
- -----------------------------
- * src_x = q_x * theta(x_3 - tA) * theta(tB - x_3) * J_mu * exp(i x.mom)
- 
- * options:
- - q: input propagator (string)
- - action: fermion action used for propagator q (string)
- - tA: begin timeslice (integer)
- - tB: end timesilce (integer)
- - curr_type: type of conserved current to insert (Current)
- - mu: Lorentz index of current to insert (integer)
- - mom: momentum insertion, space-separated float sequence (e.g ".1 .2 1. 0.")
- 
- */
-
-/******************************************************************************
- *                              SeqConserved                                  *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MSource)
-
-class SeqConservedPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(SeqConservedPar,
-                                    std::string,  q,
-                                    std::string,  action,
-                                    unsigned int, tA,
-                                    unsigned int, tB,
-                                    Current,      curr_type,
-                                    unsigned int, mu,
-                                    std::string,  mom);
-};
-
-template <typename FImpl>
-class TSeqConserved: public Module<SeqConservedPar>
-{
-public:
-    FERM_TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TSeqConserved(const std::string name);
-    // destructor
-    virtual ~TSeqConserved(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-protected:
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(SeqConserved, TSeqConserved<FIMPL>, MSource);
-
-/******************************************************************************
- *                      TSeqConserved implementation                          *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TSeqConserved<FImpl>::TSeqConserved(const std::string name)
-: Module<SeqConservedPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TSeqConserved<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().q, par().action};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TSeqConserved<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TSeqConserved<FImpl>::setup(void)
-{
-    auto Ls_ = env().getObjectLs(par().action);
-    envCreateLat(PropagatorField, getName(), Ls_);
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TSeqConserved<FImpl>::execute(void)
-{
-    if (par().tA == par().tB)
-    {
-        LOG(Message) << "Generating sequential source with conserved "
-                     << par().curr_type << " current insertion (mu = " 
-                     << par().mu << ") at " << "t = " << par().tA << std::endl;
-    }
-    else
-    {
-        LOG(Message) << "Generating sequential source with conserved "
-                     << par().curr_type << " current insertion (mu = " 
-                     << par().mu << ") for " << par().tA << " <= t <= " 
-                     << par().tB << std::endl;
-    }
-    auto &src = envGet(PropagatorField, getName());
-    auto &q   = envGet(PropagatorField, par().q);
-    auto &mat = envGet(FMat, par().action);
-
-    std::vector<Real> mom = strToVec<Real>(par().mom);
-    mat.SeqConservedCurrent(q, src, par().curr_type, par().mu, 
-                            mom, par().tA, par().tB);
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_SeqConserved_hpp_

extras/Hadrons/Modules/MSource/SeqGamma.hpp

@@ -4,10 +4,11 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MSource/SeqGamma.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
+Copyright (C) 2017
 
 Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -80,14 +81,10 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
-private:
-    bool        hasPhase_{false};
-    std::string momphName_, tName_;
 };
 
 MODULE_REGISTER_NS(SeqGamma, TSeqGamma<FIMPL>, MSource);
@@ -99,8 +96,6 @@ MODULE_REGISTER_NS(SeqGamma, TSeqGamma<FIMPL>, MSource);
 template <typename FImpl>
 TSeqGamma<FImpl>::TSeqGamma(const std::string name)
 : Module<SeqGammaPar>(name)
-, momphName_ (name + "_momph")
-, tName_ (name + "_t")
 {}
 
 // dependencies/products ///////////////////////////////////////////////////////
@@ -124,10 +119,7 @@ std::vector<std::string> TSeqGamma<FImpl>::getOutput(void)
 template <typename FImpl>
 void TSeqGamma<FImpl>::setup(void)
 {
-    envCreateLat(PropagatorField, getName());
-    envCacheLat(Lattice<iScalar<vInteger>>, tName_);
-    envCacheLat(LatticeComplex, momphName_);
-    envTmpLat(LatticeComplex, "coor");
+    env().template registerLattice<PropagatorField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -145,29 +137,23 @@ void TSeqGamma<FImpl>::execute(void)
                      << " sequential source for "
                      << par().tA << " <= t <= " << par().tB << std::endl;
     }
-    auto  &src = envGet(PropagatorField, getName());
-    auto  &q   = envGet(PropagatorField, par().q);
-    auto  &ph  = envGet(LatticeComplex, momphName_);
-    auto  &t   = envGet(Lattice<iScalar<vInteger>>, tName_);
-    Gamma g(par().gamma);
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    PropagatorField &q   = *env().template getObject<PropagatorField>(par().q);
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
+    Gamma                      g(par().gamma);
+    std::vector<Real>          p;
+    Complex                    i(0.0,1.0);
     
-    if (!hasPhase_)
+    p  = strToVec<Real>(par().mom);
+    ph = zero;
+    for(unsigned int mu = 0; mu < env().getNd(); mu++)
     {
-        Complex           i(0.0,1.0);
-        std::vector<Real> p;
-
-        envGetTmp(LatticeComplex, coor);
-        p  = strToVec<Real>(par().mom);
-        ph = zero;
-        for(unsigned int mu = 0; mu < env().getNd(); mu++)
-        {
-            LatticeCoordinate(coor, mu);
-            ph = ph + (p[mu]/env().getGrid()->_fdimensions[mu])*coor;
-        }
-        ph = exp((Real)(2*M_PI)*i*ph);
-        LatticeCoordinate(t, Tp);
-        hasPhase_ = true;
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
     }
+    ph = exp((Real)(2*M_PI)*i*ph);
+    LatticeCoordinate(t, Tp);
     src = where((t >= par().tA) and (t <= par().tB), ph*(g*q), 0.*q);
 }
 

extras/Hadrons/Modules/MSource/Wall.hpp

@@ -4,10 +4,9 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MSource/Wall.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2017
 
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
+Author: Andrew Lawson <andrew.lawson1991@gmail.com>
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -74,14 +73,10 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
-private:
-    bool        hasPhase_{false};
-    std::string momphName_, tName_;
 };
 
 MODULE_REGISTER_NS(Wall, TWall<FIMPL>, MSource);
@@ -93,15 +88,13 @@ MODULE_REGISTER_NS(Wall, TWall<FIMPL>, MSource);
 template <typename FImpl>
 TWall<FImpl>::TWall(const std::string name)
 : Module<WallPar>(name)
-, momphName_ (name + "_momph")
-, tName_ (name + "_t")
 {}
 
 // dependencies/products ///////////////////////////////////////////////////////
 template <typename FImpl>
 std::vector<std::string> TWall<FImpl>::getInput(void)
 {
-    std::vector<std::string> in = {};
+    std::vector<std::string> in;
     
     return in;
 }
@@ -118,7 +111,7 @@ std::vector<std::string> TWall<FImpl>::getOutput(void)
 template <typename FImpl>
 void TWall<FImpl>::setup(void)
 {
-    envCreateLat(PropagatorField, getName());
+    env().template registerLattice<PropagatorField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
@@ -128,28 +121,21 @@ void TWall<FImpl>::execute(void)
     LOG(Message) << "Generating wall source at t = " << par().tW 
                  << " with momentum " << par().mom << std::endl;
     
-    auto  &src = envGet(PropagatorField, getName());
-    auto  &ph  = envGet(LatticeComplex, momphName_);
-    auto  &t   = envGet(Lattice<iScalar<vInteger>>, tName_);
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             ph(env().getGrid()), coor(env().getGrid());
+    std::vector<Real>          p;
+    Complex                    i(0.0,1.0);
     
-    if (!hasPhase_)
+    p  = strToVec<Real>(par().mom);
+    ph = zero;
+    for(unsigned int mu = 0; mu < Nd; mu++)
     {
-        Complex           i(0.0,1.0);
-        std::vector<Real> p;
-
-        envGetTmp(LatticeComplex, coor);
-        p  = strToVec<Real>(par().mom);
-        ph = zero;
-        for(unsigned int mu = 0; mu < env().getNd(); mu++)
-        {
-            LatticeCoordinate(coor, mu);
-            ph = ph + (p[mu]/env().getGrid()->_fdimensions[mu])*coor;
-        }
-        ph = exp((Real)(2*M_PI)*i*ph);
-        LatticeCoordinate(t, Tp);
-        hasPhase_ = true;
+        LatticeCoordinate(coor, mu);
+        ph = ph + p[mu]*coor*((1./(env().getGrid()->_fdimensions[mu])));
     }
-
+    ph = exp((Real)(2*M_PI)*i*ph);
+    LatticeCoordinate(t, Tp);
     src = 1.;
     src = where((t == par().tW), src*ph, 0.*src);
 }

extras/Hadrons/Modules/MSource/Z2.hpp

@@ -4,7 +4,8 @@ Grid physics library, www.github.com/paboyle/Grid
 
 Source file: extras/Hadrons/Modules/MSource/Z2.hpp
 
-Copyright (C) 2015-2018
+Copyright (C) 2015
+Copyright (C) 2016
 
 Author: Antonin Portelli <antonin.portelli@me.com>
 
@@ -75,14 +76,10 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-protected:
     // setup
     virtual void setup(void);
     // execution
     virtual void execute(void);
-private:
-    bool        hasT_{false};
-    std::string tName_;
 };
 
 MODULE_REGISTER_NS(Z2,       TZ2<FIMPL>,        MSource);
@@ -95,7 +92,6 @@ MODULE_REGISTER_NS(ScalarZ2, TZ2<ScalarImplCR>, MSource);
 template <typename FImpl>
 TZ2<FImpl>::TZ2(const std::string name)
 : Module<Z2Par>(name)
-, tName_ (name + "_t")
 {}
 
 // dependencies/products ///////////////////////////////////////////////////////
@@ -119,36 +115,29 @@ std::vector<std::string> TZ2<FImpl>::getOutput(void)
 template <typename FImpl>
 void TZ2<FImpl>::setup(void)
 {
-    envCreateLat(PropagatorField, getName());
-    envCacheLat(Lattice<iScalar<vInteger>>, tName_);
-    envTmpLat(LatticeComplex, "eta");
+    env().template registerLattice<PropagatorField>(getName());
 }
 
 // execution ///////////////////////////////////////////////////////////////////
 template <typename FImpl>
 void TZ2<FImpl>::execute(void)
 {
+    Lattice<iScalar<vInteger>> t(env().getGrid());
+    LatticeComplex             eta(env().getGrid());
+    Complex                    shift(1., 1.);
+    
     if (par().tA == par().tB)
     {
         LOG(Message) << "Generating Z_2 wall source at t= " << par().tA
-                     << std::endl;
+        << std::endl;
     }
     else
     {
         LOG(Message) << "Generating Z_2 band for " << par().tA << " <= t <= "
-                     << par().tB << std::endl;
+        << par().tB << std::endl;
     }
-    
-    auto    &src = envGet(PropagatorField, getName());
-    auto    &t   = envGet(Lattice<iScalar<vInteger>>, tName_);
-    Complex shift(1., 1.);
-
-    if (!hasT_)
-    {
-        LatticeCoordinate(t, Tp);
-        hasT_ = true;
-    }
-    envGetTmp(LatticeComplex, eta);
+    PropagatorField &src = *env().template createLattice<PropagatorField>(getName());
+    LatticeCoordinate(t, Tp);
     bernoulli(*env().get4dRng(), eta);
     eta = (2.*eta - shift)*(1./::sqrt(2.));
     eta = where((t >= par().tA) and (t <= par().tB), eta, 0.*eta);

extras/Hadrons/Modules/MUtilities/TestSeqConserved.hpp

@@ -1,186 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MUtilities/TestSeqConserved.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MUtilities_TestSeqConserved_hpp_
-#define Hadrons_MUtilities_TestSeqConserved_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/*
-  Ward Identity contractions using sequential propagators.
- -----------------------------
- 
- * options:
- - q:      point source propagator, 5D if available (string)
- - qSeq:   result of sequential insertion of conserved current using q (string)
- - action: action used for computation of q (string)
- - origin: string giving point source origin of q (string)
- - t_J:    time at which sequential current is inserted (int)
- - mu:     Lorentz index of current inserted (int)
- - curr:   current type, e.g. vector/axial (Current)
-*/
-
-/******************************************************************************
- *                            TestSeqConserved                                *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MUtilities)
-
-class TestSeqConservedPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(TestSeqConservedPar,
-                                    std::string,  q,
-                                    std::string,  qSeq,
-                                    std::string,  action,
-                                    std::string,  origin,
-                                    unsigned int, t_J,
-                                    unsigned int, mu,
-                                    Current,      curr);
-};
-
-template <typename FImpl>
-class TTestSeqConserved: public Module<TestSeqConservedPar>
-{
-public:
-    FERM_TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TTestSeqConserved(const std::string name);
-    // destructor
-    virtual ~TTestSeqConserved(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-protected:
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(TestSeqConserved, TTestSeqConserved<FIMPL>, MUtilities);
-
-/******************************************************************************
- *                     TTestSeqConserved implementation                       *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TTestSeqConserved<FImpl>::TTestSeqConserved(const std::string name)
-: Module<TestSeqConservedPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TTestSeqConserved<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().q, par().qSeq, par().action};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TTestSeqConserved<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TTestSeqConserved<FImpl>::setup(void)
-{
-    auto Ls = env().getObjectLs(par().q);
-    if (Ls != env().getObjectLs(par().action))
-    {
-        HADRON_ERROR(Size, "Ls mismatch between quark action and propagator");
-    }
-    envTmpLat(PropagatorField, "tmp");
-    envTmpLat(LatticeComplex, "c");
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TTestSeqConserved<FImpl>::execute(void)
-{
-    // Check sequential insertion of current gives same result as conserved 
-    // current sink upon contraction. Assume q uses a point source.
-
-    auto                  &q    = envGet(PropagatorField, par().q);
-    auto                  &qSeq = envGet(PropagatorField, par().qSeq);
-    auto                  &act  = envGet(FMat, par().action);
-    Gamma                 g5(Gamma::Algebra::Gamma5);
-    Gamma::Algebra        gA = (par().curr == Current::Axial) ?
-                                  Gamma::Algebra::Gamma5 :
-                                  Gamma::Algebra::Identity;
-    Gamma                 g(gA);
-    SitePropagator        qSite;
-    Complex               test_S, test_V, check_S, check_V;
-    std::vector<TComplex> check_buf;
-    std::vector<int>      siteCoord;
-
-    envGetTmp(PropagatorField, tmp);
-    envGetTmp(LatticeComplex, c);
-    siteCoord = strToVec<int>(par().origin);
-    peekSite(qSite, qSeq, siteCoord);
-    test_S = trace(qSite*g);
-    test_V = trace(qSite*g*Gamma::gmu[par().mu]);
-    act.ContractConservedCurrent(q, q, tmp, par().curr, par().mu);
-    c = trace(tmp*g);
-    sliceSum(c, check_buf, Tp);
-    check_S = TensorRemove(check_buf[par().t_J]);
-
-    c = trace(tmp*g*Gamma::gmu[par().mu]);
-    sliceSum(c, check_buf, Tp);
-    check_V = TensorRemove(check_buf[par().t_J]);
-
-    LOG(Message) << "Test S  = " << abs(test_S)   << std::endl;
-    LOG(Message) << "Test V  = " << abs(test_V) << std::endl;
-    LOG(Message) << "Check S = " << abs(check_S) << std::endl;
-    LOG(Message) << "Check V = " << abs(check_V) << std::endl;
-
-    // Check difference = 0
-    check_S -= test_S;
-    check_V -= test_V;
-
-    LOG(Message) << "Consistency check for sequential conserved " 
-                 << par().curr << " current insertion: " << std::endl; 
-    LOG(Message) << "Diff S  = " << abs(check_S) << std::endl;
-    LOG(Message) << "Diff V  = " << abs(check_V) << std::endl;
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_TestSeqConserved_hpp_

extras/Hadrons/Modules/MUtilities/TestSeqGamma.hpp

@@ -1,150 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/Modules/MUtilities/TestSeqGamma.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-Author: Lanny91 <andrew.lawson@gmail.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_MUtilities_TestSeqGamma_hpp_
-#define Hadrons_MUtilities_TestSeqGamma_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Module.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-/******************************************************************************
- *                              TestSeqGamma                                  *
- ******************************************************************************/
-BEGIN_MODULE_NAMESPACE(MUtilities)
-
-class TestSeqGammaPar: Serializable
-{
-public:
-    GRID_SERIALIZABLE_CLASS_MEMBERS(TestSeqGammaPar,
-                                    std::string,    q,
-                                    std::string,    qSeq,
-                                    std::string,    origin,
-                                    Gamma::Algebra, gamma,
-                                    unsigned int,   t_g);
-};
-
-template <typename FImpl>
-class TTestSeqGamma: public Module<TestSeqGammaPar>
-{
-public:
-    FERM_TYPE_ALIASES(FImpl,);
-public:
-    // constructor
-    TTestSeqGamma(const std::string name);
-    // destructor
-    virtual ~TTestSeqGamma(void) = default;
-    // dependency relation
-    virtual std::vector<std::string> getInput(void);
-    virtual std::vector<std::string> getOutput(void);
-protected:
-    // setup
-    virtual void setup(void);
-    // execution
-    virtual void execute(void);
-};
-
-MODULE_REGISTER_NS(TestSeqGamma, TTestSeqGamma<FIMPL>, MUtilities);
-
-/******************************************************************************
- *                      TTestSeqGamma implementation                          *
- ******************************************************************************/
-// constructor /////////////////////////////////////////////////////////////////
-template <typename FImpl>
-TTestSeqGamma<FImpl>::TTestSeqGamma(const std::string name)
-: Module<TestSeqGammaPar>(name)
-{}
-
-// dependencies/products ///////////////////////////////////////////////////////
-template <typename FImpl>
-std::vector<std::string> TTestSeqGamma<FImpl>::getInput(void)
-{
-    std::vector<std::string> in = {par().q, par().qSeq};
-    
-    return in;
-}
-
-template <typename FImpl>
-std::vector<std::string> TTestSeqGamma<FImpl>::getOutput(void)
-{
-    std::vector<std::string> out = {getName()};
-    
-    return out;
-}
-
-// setup ///////////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TTestSeqGamma<FImpl>::setup(void)
-{
-    envTmpLat(LatticeComplex, "c");
-}
-
-// execution ///////////////////////////////////////////////////////////////////
-template <typename FImpl>
-void TTestSeqGamma<FImpl>::execute(void)
-{
-    auto                  &q    = envGet(PropagatorField, par().q);
-    auto                  &qSeq = envGet(PropagatorField, par().qSeq);
-    Gamma                 g5(Gamma::Algebra::Gamma5);
-    Gamma                 g(par().gamma);
-    SitePropagator        qSite;
-    Complex               test, check;
-    std::vector<TComplex> check_buf;
-    std::vector<int>      siteCoord;
-
-    // Check sequential insertion of gamma matrix gives same result as 
-    // insertion of gamma at sink upon contraction. Assume q uses a point 
-    // source.
-    
-    envGetTmp(LatticeComplex, c);
-    siteCoord = strToVec<int>(par().origin);
-    peekSite(qSite, qSeq, siteCoord);
-    test = trace(g*qSite);
-
-    c = trace(adj(g)*g5*adj(q)*g5*g*q);
-    sliceSum(c, check_buf, Tp);
-    check = TensorRemove(check_buf[par().t_g]);
-
-    LOG(Message) << "Seq Result = " << abs(test)  << std::endl;
-    LOG(Message) << "Reference  = " << abs(check) << std::endl;
-
-    // Check difference = 0
-    check -= test;
-
-    LOG(Message) << "Consistency check for sequential " << par().gamma  
-                 << " insertion = " << abs(check) << std::endl;
-}
-
-END_MODULE_NAMESPACE
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_TestSeqGamma_hpp_

extras/Hadrons/VirtualMachine.cc

@@ -1,622 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/VirtualMachine.cc
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#include <Grid/Hadrons/VirtualMachine.hpp>
-#include <Grid/Hadrons/GeneticScheduler.hpp>
-#include <Grid/Hadrons/ModuleFactory.hpp>
-
-using namespace Grid;
-using namespace QCD;
-using namespace Hadrons;
-
-/******************************************************************************
- *                      VirtualMachine implementation                         *
- ******************************************************************************/
-// trajectory counter //////////////////////////////////////////////////////////
-void VirtualMachine::setTrajectory(const unsigned int traj)
-{
-    traj_ = traj;
-}
-
-unsigned int VirtualMachine::getTrajectory(void) const
-{
-    return traj_;
-}
-
-// module management ///////////////////////////////////////////////////////////
-void VirtualMachine::pushModule(VirtualMachine::ModPt &pt)
-{
-    std::string name = pt->getName();
-    
-    if (!hasModule(name))
-    {
-        std::vector<unsigned int> inputAddress;
-        unsigned int              address;
-        ModuleInfo                m;
-        
-        // module registration -------------------------------------------------
-        m.data = std::move(pt);
-        m.type = typeIdPt(*m.data.get());
-        m.name = name;
-        // input dependencies
-        for (auto &in: m.data->getInput())
-        {
-            if (!env().hasObject(in))
-            {
-                // if object does not exist, add it with no creator module
-                env().addObject(in , -1);
-            }
-            m.input.push_back(env().getObjectAddress(in));
-        }
-        // reference dependencies
-        for (auto &ref: m.data->getReference())
-        {
-            if (!env().hasObject(ref))
-            {
-                // if object does not exist, add it with no creator module
-                env().addObject(ref , -1);
-            }
-            m.input.push_back(env().getObjectAddress(ref));
-        }
-        auto inCopy = m.input;
-        // if module has inputs with references, they need to be added as
-        // an input
-        for (auto &in: inCopy)
-        {
-            int inm = env().getObjectModule(in);
-
-            if (inm > 0)
-            {
-                if (getModule(inm)->getReference().size() > 0)
-                {
-                    for (auto &rin: getModule(inm)->getReference())
-                    {
-                        m.input.push_back(env().getObjectAddress(rin));
-                    }
-                }
-            }
-        }
-        module_.push_back(std::move(m));
-        address              = static_cast<unsigned int>(module_.size() - 1);
-        moduleAddress_[name] = address;
-        // connecting outputs to potential inputs ------------------------------
-        for (auto &out: getModule(address)->getOutput())
-        {
-            if (!env().hasObject(out))
-            {
-                // output does not exists, add it
-                env().addObject(out, address);
-            }
-            else
-            {
-                if (env().getObjectModule(env().getObjectAddress(out)) < 0)
-                {
-                    // output exists but without creator, correct it
-                    env().setObjectModule(env().getObjectAddress(out), address);
-                }
-                else
-                {
-                    // output already fully registered, error
-                    HADRON_ERROR(Definition, "object '" + out
-                                 + "' is already produced by module '"
-                                 + module_[env().getObjectModule(out)].name
-                                 + "' (while pushing module '" + name + "')");
-                }
-                if (getModule(address)->getReference().size() > 0)
-                {
-                    // module has references, dependency should be propagated
-                    // to children modules; find module with `out` as an input
-                    // and add references to their input
-                    auto pred = [this, out](const ModuleInfo &n)
-                    {
-                        auto &in = n.input;
-                        auto it  = std::find(in.begin(), in.end(), 
-                                             env().getObjectAddress(out));
-                        
-                        return (it != in.end());
-                    };
-                    auto it = std::find_if(module_.begin(), module_.end(), pred);
-                    while (it != module_.end())
-                    {
-                        for (auto &ref: getModule(address)->getReference())
-                        {
-                            it->input.push_back(env().getObjectAddress(ref));
-                        }
-                        it = std::find_if(++it, module_.end(), pred);
-                    }   
-                }
-            }
-        }
-        graphOutdated_         = true;
-        memoryProfileOutdated_ = true;
-    }
-    else
-    {
-        HADRON_ERROR(Definition, "module '" + name + "' already exists");
-    }
-}
-
-unsigned int VirtualMachine::getNModule(void) const
-{
-    return module_.size();
-}
-
-void VirtualMachine::createModule(const std::string name, const std::string type,
-                                  XmlReader &reader)
-{
-    auto &factory = ModuleFactory::getInstance();
-    auto pt       = factory.create(type, name);
-    
-    pt->parseParameters(reader, "options");
-    pushModule(pt);
-}
-
-ModuleBase * VirtualMachine::getModule(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return module_[address].data.get();
-    }
-    else
-    {
-        HADRON_ERROR(Definition, "no module with address " + std::to_string(address));
-    }
-}
-
-ModuleBase * VirtualMachine::getModule(const std::string name) const
-{
-    return getModule(getModuleAddress(name));
-}
-
-unsigned int VirtualMachine::getModuleAddress(const std::string name) const
-{
-    if (hasModule(name))
-    {
-        return moduleAddress_.at(name);
-    }
-    else
-    {
-        HADRON_ERROR(Definition, "no module with name '" + name + "'");
-    }
-}
-
-std::string VirtualMachine::getModuleName(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return module_[address].name;
-    }
-    else
-    {
-        HADRON_ERROR(Definition, "no module with address " + std::to_string(address));
-    }
-}
-
-std::string VirtualMachine::getModuleType(const unsigned int address) const
-{
-    if (hasModule(address))
-    {
-        return typeName(module_[address].type);
-    }
-    else
-    {
-        HADRON_ERROR(Definition, "no module with address " + std::to_string(address));
-    }
-}
-
-std::string VirtualMachine::getModuleType(const std::string name) const
-{
-    return getModuleType(getModuleAddress(name));
-}
-
-std::string VirtualMachine::getModuleNamespace(const unsigned int address) const
-{
-    std::string type = getModuleType(address), ns;
-    
-    auto pos2 = type.rfind("::");
-    auto pos1 = type.rfind("::", pos2 - 2);
-    
-    return type.substr(pos1 + 2, pos2 - pos1 - 2);
-}
-
-std::string VirtualMachine::getModuleNamespace(const std::string name) const
-{
-    return getModuleNamespace(getModuleAddress(name));
-}
-
-bool VirtualMachine::hasModule(const unsigned int address) const
-{
-    return (address < module_.size());
-}
-
-bool VirtualMachine::hasModule(const std::string name) const
-{
-    return (moduleAddress_.find(name) != moduleAddress_.end());
-}
-
-// print VM content ////////////////////////////////////////////////////////////
-void VirtualMachine::printContent(void) const
-{
-    LOG(Debug) << "Modules: " << std::endl;
-    for (unsigned int i = 0; i < module_.size(); ++i)
-    {
-        LOG(Debug) << std::setw(4) << i << ": "
-                   << getModuleName(i) << std::endl;
-    }
-}
-
-// module graph ////////////////////////////////////////////////////////////////
-Graph<unsigned int> VirtualMachine::getModuleGraph(void)
-{
-    if (graphOutdated_)
-    {
-        makeModuleGraph();
-        graphOutdated_ = false;
-    }
-
-    return graph_;
-}
-
-void VirtualMachine::makeModuleGraph(void)
-{
-    Graph<unsigned int> graph;
-    
-    // create vertices
-    for (unsigned int m = 0; m < module_.size(); ++m)
-    {
-        graph.addVertex(m);
-    }
-    // create edges
-    for (unsigned int m = 0; m < module_.size(); ++m)
-    {
-        for (auto &in: module_[m].input)
-        {
-            graph.addEdge(env().getObjectModule(in), m);
-        }
-    }
-    graph_ = graph;
-}
-
-// memory profile //////////////////////////////////////////////////////////////
-const VirtualMachine::MemoryProfile & VirtualMachine::getMemoryProfile(void)
-{
-    if (memoryProfileOutdated_)
-    {
-        makeMemoryProfile();
-        memoryProfileOutdated_ = false;
-    }
-
-    return profile_;
-}
-
-void VirtualMachine::makeMemoryProfile(void)
-{
-    bool protect = env().objectsProtected();
-    bool hmsg    = HadronsLogMessage.isActive();
-    bool gmsg    = GridLogMessage.isActive();
-    bool err     = HadronsLogError.isActive();
-    auto program = getModuleGraph().topoSort();
-
-    resetProfile();
-    profile_.module.resize(getNModule());
-    env().protectObjects(false);
-    GridLogMessage.Active(false);
-    HadronsLogMessage.Active(false);
-    HadronsLogError.Active(false);
-    for (auto it = program.rbegin(); it != program.rend(); ++it) 
-    {
-        auto a = *it;
-
-        if (profile_.module[a].empty())
-        {
-            LOG(Debug) << "Profiling memory for module '" << module_[a].name
-                       << "' (" << a << ")..." << std::endl;
-            memoryProfile(a);
-            env().freeAll();
-        }
-    }
-    env().protectObjects(protect);
-    GridLogMessage.Active(gmsg);
-    HadronsLogMessage.Active(hmsg);
-    HadronsLogError.Active(err);
-    LOG(Debug) << "Memory profile:" << std::endl;
-    LOG(Debug) << "----------------" << std::endl;
-    for (unsigned int a = 0; a < profile_.module.size(); ++a)
-    {
-        LOG(Debug) << getModuleName(a) << " (" << a << ")" << std::endl;
-        for (auto &o: profile_.module[a])
-        {
-            LOG(Debug) << "|__ " << env().getObjectName(o.first) << " ("
-                       << sizeString(o.second) << ")" << std::endl;
-        }
-        LOG(Debug) << std::endl;
-    }
-    LOG(Debug) << "----------------" << std::endl;
-}
-
-void VirtualMachine::resetProfile(void)
-{
-    profile_.module.clear();
-    profile_.object.clear();
-}
-
-void VirtualMachine::resizeProfile(void)
-{
-    if (env().getMaxAddress() > profile_.object.size())
-    {
-        MemoryPrint empty;
-
-        empty.size   = 0;
-        empty.module = -1;
-        profile_.object.resize(env().getMaxAddress(), empty);
-    }
-}
-
-void VirtualMachine::updateProfile(const unsigned int address)
-{
-    resizeProfile();
-    for (unsigned int a = 0; a < env().getMaxAddress(); ++a)
-    {
-        if (env().hasCreatedObject(a) and (profile_.object[a].module == -1))
-        {
-            profile_.object[a].size     = env().getObjectSize(a);
-            profile_.object[a].storage  = env().getObjectStorage(a);
-            profile_.object[a].module   = address;
-            profile_.module[address][a] = profile_.object[a].size;
-            if (env().getObjectModule(a) < 0)
-            {
-                env().setObjectModule(a, address);
-            }
-        }
-    }
-}
-
-void VirtualMachine::cleanEnvironment(void)
-{
-    resizeProfile();
-    for (unsigned int a = 0; a < env().getMaxAddress(); ++a)
-    {
-        if (env().hasCreatedObject(a) and (profile_.object[a].module == -1))
-        {
-            env().freeObject(a);
-        }
-    }
-}
-
-void VirtualMachine::memoryProfile(const unsigned int address)
-{
-    auto m = getModule(address);
-
-    LOG(Debug) << "Setting up module '" << m->getName() 
-               << "' (" << address << ")..." << std::endl;
-    try
-    {
-        m->setup();
-        updateProfile(address);
-    }
-    catch (Exceptions::Definition &)
-    {
-        cleanEnvironment();
-        for (auto &in: m->getInput())
-        {
-            memoryProfile(env().getObjectModule(in));
-        }
-        for (auto &ref: m->getReference())
-        {
-            memoryProfile(env().getObjectModule(ref));
-        }
-        m->setup();
-        updateProfile(address);
-    }
-}
-
-void VirtualMachine::memoryProfile(const std::string name)
-{
-    memoryProfile(getModuleAddress(name));
-}
-
-// garbage collector ///////////////////////////////////////////////////////////
-VirtualMachine::GarbageSchedule 
-VirtualMachine::makeGarbageSchedule(const Program &p) const
-{
-    GarbageSchedule freeProg;
-    
-    freeProg.resize(p.size());
-    for (unsigned int a = 0; a < env().getMaxAddress(); ++a)
-    {
-        if (env().getObjectStorage(a) == Environment::Storage::temporary)
-        {
-            auto it = std::find(p.begin(), p.end(), env().getObjectModule(a));
-
-            if (it != p.end())
-            {
-                freeProg[std::distance(p.begin(), it)].insert(a);
-            }
-        }
-        else if (env().getObjectStorage(a) == Environment::Storage::object)
-        {
-            auto pred = [a, this](const unsigned int b)
-            {
-                auto &in = module_[b].input;
-                auto it  = std::find(in.begin(), in.end(), a);
-                
-                return (it != in.end()) or (b == env().getObjectModule(a));
-            };
-            auto it = std::find_if(p.rbegin(), p.rend(), pred);
-            if (it != p.rend())
-            {
-                freeProg[std::distance(it, p.rend()) - 1].insert(a);
-            }
-        }
-    }
-
-    return freeProg;
-}
-
-// high-water memory function //////////////////////////////////////////////////
-VirtualMachine::Size VirtualMachine::memoryNeeded(const Program &p)
-{
-    const MemoryProfile &profile = getMemoryProfile();
-    GarbageSchedule     freep    = makeGarbageSchedule(p);
-    Size                current = 0, max = 0;
-
-    for (unsigned int i = 0; i < p.size(); ++i)
-    {
-        for (auto &o: profile.module[p[i]])
-        {
-            current += o.second;
-        }
-        max = std::max(current, max);
-        for (auto &o: freep[i])
-        {
-            current -= profile.object[o].size;
-        }
-    }
-
-    return max;
-}
-
-// genetic scheduler ///////////////////////////////////////////////////////////
-VirtualMachine::Program VirtualMachine::schedule(const GeneticPar &par)
-{
-    typedef GeneticScheduler<Size, unsigned int> Scheduler;
-
-    auto graph = getModuleGraph();
-
-    //constrained topological sort using a genetic algorithm
-    LOG(Message) << "Scheduling computation..." << std::endl;
-    LOG(Message) << "               #module= " << graph.size() << std::endl;
-    LOG(Message) << "       population size= " << par.popSize << std::endl;
-    LOG(Message) << "       max. generation= " << par.maxGen << std::endl;
-    LOG(Message) << "  max. cst. generation= " << par.maxCstGen << std::endl;
-    LOG(Message) << "         mutation rate= " << par.mutationRate << std::endl;
-    
-    unsigned int          k = 0, gen, prevPeak, nCstPeak = 0;
-    std::random_device    rd;
-    Scheduler::Parameters gpar;
-    
-    gpar.popSize      = par.popSize;
-    gpar.mutationRate = par.mutationRate;
-    gpar.seed         = rd();
-    CartesianCommunicator::BroadcastWorld(0, &(gpar.seed), sizeof(gpar.seed));
-    Scheduler::ObjFunc memPeak = [this](const Program &p)->Size
-    {
-        return memoryNeeded(p);
-    };
-    Scheduler scheduler(graph, memPeak, gpar);
-    gen = 0;
-    do
-    {
-        LOG(Debug) << "Generation " << gen << ":" << std::endl;
-        scheduler.nextGeneration();
-        if (gen != 0)
-        {
-            if (prevPeak == scheduler.getMinValue())
-            {
-                nCstPeak++;
-            }
-            else
-            {
-                nCstPeak = 0;
-            }
-        }
-        
-        prevPeak = scheduler.getMinValue();
-        if (gen % 10 == 0)
-        {
-            LOG(Iterative) << "Generation " << gen << ": "
-                           << sizeString(scheduler.getMinValue()) << std::endl;
-        }
-        
-        gen++;
-    } while ((gen < par.maxGen) and (nCstPeak < par.maxCstGen));
-    
-    return scheduler.getMinSchedule();
-}
-
-// general execution ///////////////////////////////////////////////////////////
-#define BIG_SEP "==============="
-#define SEP     "---------------"
-#define MEM_MSG(size) sizeString(size)
-
-void VirtualMachine::executeProgram(const Program &p) const
-{
-    Size            memPeak = 0, sizeBefore, sizeAfter;
-    GarbageSchedule freeProg;
-    
-    // build garbage collection schedule
-    LOG(Debug) << "Building garbage collection schedule..." << std::endl;
-    freeProg = makeGarbageSchedule(p);
-
-    // program execution
-    LOG(Debug) << "Executing program..." << std::endl;
-    for (unsigned int i = 0; i < p.size(); ++i)
-    {
-        // execute module
-        LOG(Message) << SEP << " Measurement step " << i + 1 << "/"
-                     << p.size() << " (module '" << module_[p[i]].name
-                     << "') " << SEP << std::endl;
-        (*module_[p[i]].data)();
-        sizeBefore = env().getTotalSize();
-        // print used memory after execution
-        LOG(Message) << "Allocated objects: " << MEM_MSG(sizeBefore)
-                     << std::endl;
-        if (sizeBefore > memPeak)
-        {
-            memPeak = sizeBefore;
-        }
-        // garbage collection for step i
-        LOG(Message) << "Garbage collection..." << std::endl;
-        for (auto &j: freeProg[i])
-        {
-            env().freeObject(j);
-        }
-        // print used memory after garbage collection if necessary
-        sizeAfter = env().getTotalSize();
-        if (sizeBefore != sizeAfter)
-        {
-            LOG(Message) << "Allocated objects: " << MEM_MSG(sizeAfter)
-                            << std::endl;
-        }
-        else
-        {
-            LOG(Message) << "Nothing to free" << std::endl;
-        }
-    }
-}
-
-void VirtualMachine::executeProgram(const std::vector<std::string> &p) const
-{
-    Program pAddress;
-    
-    for (auto &n: p)
-    {
-        pAddress.push_back(getModuleAddress(n));
-    }
-    executeProgram(pAddress);
-}

extras/Hadrons/VirtualMachine.hpp

@@ -1,207 +0,0 @@
-/*************************************************************************************
-
-Grid physics library, www.github.com/paboyle/Grid 
-
-Source file: extras/Hadrons/VirtualMachine.hpp
-
-Copyright (C) 2015-2018
-
-Author: Antonin Portelli <antonin.portelli@me.com>
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-See the full license in the file "LICENSE" in the top level distribution directory
-*************************************************************************************/
-/*  END LEGAL */
-
-#ifndef Hadrons_VirtualMachine_hpp_
-#define Hadrons_VirtualMachine_hpp_
-
-#include <Grid/Hadrons/Global.hpp>
-#include <Grid/Hadrons/Graph.hpp>
-#include <Grid/Hadrons/Environment.hpp>
-
-BEGIN_HADRONS_NAMESPACE
-
-#define DEFINE_VM_ALIAS \
-inline VirtualMachine & vm(void) const\
-{\
-    return VirtualMachine::getInstance();\
-}
-
-/******************************************************************************
- *                   Virtual machine for module execution                     *
- ******************************************************************************/
-// forward declaration of Module
-class ModuleBase;
-
-class VirtualMachine
-{
-    SINGLETON_DEFCTOR(VirtualMachine);
-public:
-    typedef SITE_SIZE_TYPE                      Size;
-    typedef std::unique_ptr<ModuleBase>         ModPt;
-    typedef std::vector<std::set<unsigned int>> GarbageSchedule;
-    typedef std::vector<unsigned int>           Program;
-    struct MemoryPrint
-    {
-        Size                 size;
-        Environment::Storage storage;
-        int                  module;
-    };
-    struct MemoryProfile
-    {
-        std::vector<std::map<unsigned int, Size>> module;
-        std::vector<MemoryPrint>                  object;
-    };
-    class GeneticPar: Serializable
-    {
-    public:
-        GeneticPar(void):
-            popSize{20}, maxGen{1000}, maxCstGen{100}, mutationRate{.1} {};
-    public:
-        GRID_SERIALIZABLE_CLASS_MEMBERS(GeneticPar,
-                                        unsigned int, popSize,
-                                        unsigned int, maxGen,
-                                        unsigned int, maxCstGen,
-                                        double      , mutationRate);
-    };
-private:
-    struct ModuleInfo
-    {
-        const std::type_info      *type{nullptr};
-        std::string               name;
-        ModPt                     data{nullptr};
-        std::vector<unsigned int> input;
-        size_t                    maxAllocated;
-    };
-public:
-    // trajectory counter
-    void                setTrajectory(const unsigned int traj);
-    unsigned int        getTrajectory(void) const;
-    // module management
-    void                pushModule(ModPt &pt);
-    template <typename M>
-    void                createModule(const std::string name);
-    template <typename M>
-    void                createModule(const std::string name,
-                                         const typename M::Par &par);
-    void                createModule(const std::string name,
-                                         const std::string type,
-                                         XmlReader &reader);
-    unsigned int        getNModule(void) const;
-    ModuleBase *        getModule(const unsigned int address) const;
-    ModuleBase *        getModule(const std::string name) const;
-    template <typename M>
-    M *                 getModule(const unsigned int address) const;
-    template <typename M>
-    M *                 getModule(const std::string name) const;
-    unsigned int        getModuleAddress(const std::string name) const;
-    std::string         getModuleName(const unsigned int address) const;
-    std::string         getModuleType(const unsigned int address) const;
-    std::string         getModuleType(const std::string name) const;
-    std::string         getModuleNamespace(const unsigned int address) const;
-    std::string         getModuleNamespace(const std::string name) const;
-    bool                hasModule(const unsigned int address) const;
-    bool                hasModule(const std::string name) const;
-    // print VM content
-    void                printContent(void) const;
-    // module graph (could be a const reference if topoSort was const)
-    Graph<unsigned int> getModuleGraph(void);
-    // memory profile
-    const MemoryProfile &getMemoryProfile(void);
-    // garbage collector
-    GarbageSchedule     makeGarbageSchedule(const Program &p) const;
-    // high-water memory function
-    Size                memoryNeeded(const Program &p);
-    // genetic scheduler
-    Program             schedule(const GeneticPar &par);
-    // general execution
-    void                executeProgram(const Program &p) const;
-    void                executeProgram(const std::vector<std::string> &p) const;
-private:
-    // environment shortcut
-    DEFINE_ENV_ALIAS;
-    // module graph
-    void makeModuleGraph(void);
-    // memory profile
-    void makeMemoryProfile(void);
-    void resetProfile(void);
-    void resizeProfile(void);
-    void updateProfile(const unsigned int address);
-    void cleanEnvironment(void);
-    void memoryProfile(const std::string name);
-    void memoryProfile(const unsigned int address);
-private:
-    // general
-    unsigned int                        traj_;
-    // module and related maps
-    std::vector<ModuleInfo>             module_;
-    std::map<std::string, unsigned int> moduleAddress_;
-    std::string                         currentModule_{""};
-    // module graph
-    bool                                graphOutdated_{true};
-    Graph<unsigned int>                 graph_;
-    // memory profile
-    bool                                memoryProfileOutdated_{true};
-    MemoryProfile                       profile_;
-};
-
-/******************************************************************************
- *                   VirtualMachine template implementation                   *
- ******************************************************************************/
-// module management ///////////////////////////////////////////////////////////
-template <typename M>
-void VirtualMachine::createModule(const std::string name)
-{
-    ModPt pt(new M(name));
-    
-    pushModule(pt);
-}
-
-template <typename M>
-void VirtualMachine::createModule(const std::string name,
-                               const typename M::Par &par)
-{
-    ModPt pt(new M(name));
-    
-    static_cast<M *>(pt.get())->setPar(par);
-    pushModule(pt);
-}
-
-template <typename M>
-M * VirtualMachine::getModule(const unsigned int address) const
-{
-    if (auto *pt = dynamic_cast<M *>(getModule(address)))
-    {
-        return pt;
-    }
-    else
-    {
-        HADRON_ERROR(Definition, "module '" + module_[address].name
-                     + "' does not have type " + typeid(M).name()
-                     + "(has type: " + getModuleType(address) + ")");
-    }
-}
-
-template <typename M>
-M * VirtualMachine::getModule(const std::string name) const
-{
-    return getModule<M>(getModuleAddress(name));
-}
-
-END_HADRONS_NAMESPACE
-
-#endif // Hadrons_VirtualMachine_hpp_

extras/Hadrons/modules.inc

@@ -1,52 +1,38 @@
 modules_cc =\
-  Modules/MScalar/ChargedProp.cc \
-  Modules/MScalar/FreeProp.cc \
   Modules/MContraction/WeakHamiltonianEye.cc \
-  Modules/MContraction/WeakNeutral4ptDisc.cc \
   Modules/MContraction/WeakHamiltonianNonEye.cc \
-  Modules/MGauge/Unit.cc \
-  Modules/MGauge/StochEm.cc \
+  Modules/MContraction/WeakNeutral4ptDisc.cc \
+  Modules/MGauge/Load.cc \
   Modules/MGauge/Random.cc \
-  Modules/MGauge/FundtoHirep.cc \
-  Modules/MScalar/FreeProp.cc \
+  Modules/MGauge/StochEm.cc \
+  Modules/MGauge/Unit.cc \
   Modules/MScalar/ChargedProp.cc \
-  Modules/MIO/LoadNersc.cc
+  Modules/MScalar/FreeProp.cc
 
 modules_hpp =\
-  Modules/MContraction/Baryon.hpp \
-  Modules/MContraction/Meson.hpp \
-  Modules/MContraction/WeakHamiltonian.hpp \
-  Modules/MContraction/WeakHamiltonianNonEye.hpp \
-  Modules/MContraction/DiscLoop.hpp \
-  Modules/MContraction/WeakNeutral4ptDisc.hpp \
-  Modules/MContraction/Gamma3pt.hpp \
-  Modules/MContraction/WardIdentity.hpp \
-  Modules/MContraction/WeakHamiltonianEye.hpp \
-  Modules/MFermion/GaugeProp.hpp \
-  Modules/MSource/SeqGamma.hpp \
-  Modules/MSource/Point.hpp \
-  Modules/MSource/Wall.hpp \
-  Modules/MSource/Z2.hpp \
-  Modules/MSource/SeqConserved.hpp \
-  Modules/MSink/Smear.hpp \
-  Modules/MSink/Point.hpp \
-  Modules/MSolver/RBPrecCG.hpp \
-  Modules/MGauge/Unit.hpp \
-  Modules/MGauge/Random.hpp \
-  Modules/MGauge/StochEm.hpp \
-  Modules/MGauge/FundtoHirep.hpp \
-  Modules/MUtilities/TestSeqGamma.hpp \
-  Modules/MUtilities/TestSeqConserved.hpp \
-  Modules/MLoop/NoiseLoop.hpp \
-  Modules/MScalar/FreeProp.hpp \
-  Modules/MScalar/Scalar.hpp \
-  Modules/MScalar/ChargedProp.hpp \
   Modules/MAction/DWF.hpp \
   Modules/MAction/Wilson.hpp \
-  Modules/MAction/WilsonClover.hpp \
-  Modules/MScalarSUN/TrMag.hpp \
-  Modules/MScalarSUN/TwoPoint.hpp \
-  Modules/MScalarSUN/TrPhi.hpp \
-  Modules/MIO/LoadNersc.hpp \
-  Modules/MIO/LoadBinary.hpp
+  Modules/MContraction/Baryon.hpp \
+  Modules/MContraction/DiscLoop.hpp \
+  Modules/MContraction/Gamma3pt.hpp \
+  Modules/MContraction/Meson.hpp \
+  Modules/MContraction/WeakHamiltonian.hpp \
+  Modules/MContraction/WeakHamiltonianEye.hpp \
+  Modules/MContraction/WeakHamiltonianNonEye.hpp \
+  Modules/MContraction/WeakNeutral4ptDisc.hpp \
+  Modules/MFermion/GaugeProp.hpp \
+  Modules/MGauge/Load.hpp \
+  Modules/MGauge/Random.hpp \
+  Modules/MGauge/StochEm.hpp \
+  Modules/MGauge/Unit.hpp \
+  Modules/MLoop/NoiseLoop.hpp \
+  Modules/MScalar/ChargedProp.hpp \
+  Modules/MScalar/FreeProp.hpp \
+  Modules/MScalar/Scalar.hpp \
+  Modules/MSink/Point.hpp \
+  Modules/MSolver/RBPrecCG.hpp \
+  Modules/MSource/Point.hpp \
+  Modules/MSource/SeqGamma.hpp \
+  Modules/MSource/Wall.hpp \
+  Modules/MSource/Z2.hpp
 

lib/Makefile.am

@@ -1,18 +1,28 @@
 extra_sources=
 extra_headers=
+if BUILD_COMMS_MPI
+  extra_sources+=communicator/Communicator_mpi.cc
+  extra_sources+=communicator/Communicator_base.cc
+endif
 
 if BUILD_COMMS_MPI3
   extra_sources+=communicator/Communicator_mpi3.cc
   extra_sources+=communicator/Communicator_base.cc
-  extra_sources+=communicator/SharedMemoryMPI.cc
-  extra_sources+=communicator/SharedMemory.cc
+endif
+
+if BUILD_COMMS_MPIT
+  extra_sources+=communicator/Communicator_mpit.cc
+  extra_sources+=communicator/Communicator_base.cc
+endif
+
+if BUILD_COMMS_SHMEM
+  extra_sources+=communicator/Communicator_shmem.cc
+  extra_sources+=communicator/Communicator_base.cc
 endif
 
 if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_none.cc
   extra_sources+=communicator/Communicator_base.cc
-  extra_sources+=communicator/SharedMemoryNone.cc
-  extra_sources+=communicator/SharedMemory.cc
 endif
 
 if BUILD_HDF5

lib/algorithms/Algorithms.h

@@ -39,6 +39,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/approx/MultiShiftFunction.h>
 #include <Grid/algorithms/approx/Forecast.h>
 
+#include <Grid/algorithms/densematrix/DenseMatrix.h>
+#include <Grid/algorithms/densematrix/Francis.h>
+#include <Grid/algorithms/densematrix/Householder.h>
+
 #include <Grid/algorithms/iterative/ConjugateGradient.h>
 #include <Grid/algorithms/iterative/ConjugateResidual.h>
 #include <Grid/algorithms/iterative/NormalEquations.h>
@@ -48,6 +52,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/iterative/BlockConjugateGradient.h>
 #include <Grid/algorithms/iterative/ConjugateGradientReliableUpdate.h>
 #include <Grid/algorithms/iterative/ImplicitlyRestartedLanczos.h>
+#include <Grid/algorithms/iterative/ImplicitlyRestartedLanczosCJ.h>
+#include <Grid/algorithms/iterative/SimpleLanczos.h>
 #include <Grid/algorithms/CoarsenedMatrix.h>
 #include <Grid/algorithms/FFT.h>
 

lib/algorithms/CoarsenedMatrix.h

@@ -103,32 +103,29 @@ namespace Grid {
     GridBase *CoarseGrid;
     GridBase *FineGrid;
     std::vector<Lattice<Fobj> > subspace;
-    int checkerboard;
 
-  Aggregation(GridBase *_CoarseGrid,GridBase *_FineGrid,int _checkerboard) : 
-    CoarseGrid(_CoarseGrid),
+    Aggregation(GridBase *_CoarseGrid,GridBase *_FineGrid) : 
+      CoarseGrid(_CoarseGrid),
       FineGrid(_FineGrid),
-      subspace(nbasis,_FineGrid),
-      checkerboard(_checkerboard)
+      subspace(nbasis,_FineGrid)
 	{
 	};
   
     void Orthogonalise(void){
       CoarseScalar InnerProd(CoarseGrid); 
-      std::cout << GridLogMessage <<" Gramm-Schmidt pass 1"<<std::endl;
       blockOrthogonalise(InnerProd,subspace);
-      std::cout << GridLogMessage <<" Gramm-Schmidt pass 2"<<std::endl;
-      blockOrthogonalise(InnerProd,subspace);
-      //      std::cout << GridLogMessage <<" Gramm-Schmidt checking orthogonality"<<std::endl;
-      //      CheckOrthogonal();
     } 
     void CheckOrthogonal(void){
       CoarseVector iProj(CoarseGrid); 
       CoarseVector eProj(CoarseGrid); 
+      Lattice<CComplex> pokey(CoarseGrid);
+
+      
       for(int i=0;i<nbasis;i++){
 	blockProject(iProj,subspace[i],subspace);
+
 	eProj=zero; 
-	parallel_for(int ss=0;ss<CoarseGrid->oSites();ss++){
+	for(int ss=0;ss<CoarseGrid->oSites();ss++){
 	  eProj._odata[ss](i)=CComplex(1.0);
 	}
 	eProj=eProj - iProj;
@@ -140,7 +137,6 @@ namespace Grid {
       blockProject(CoarseVec,FineVec,subspace);
     }
     void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
-      FineVec.checkerboard = subspace[0].checkerboard;
       blockPromote(CoarseVec,FineVec,subspace);
     }
     void CreateSubspaceRandom(GridParallelRNG &RNG){
@@ -151,7 +147,6 @@ namespace Grid {
       Orthogonalise();
     }
 
-    /*
     virtual void CreateSubspaceLanczos(GridParallelRNG  &RNG,LinearOperatorBase<FineField> &hermop,int nn=nbasis) 
     {
       // Run a Lanczos with sloppy convergence
@@ -200,7 +195,7 @@ namespace Grid {
 	  std::cout << GridLogMessage <<"subspace["<<b<<"] = "<<norm2(subspace[b])<<std::endl;
 	}
     }
-    */
+
     virtual void CreateSubspace(GridParallelRNG  &RNG,LinearOperatorBase<FineField> &hermop,int nn=nbasis) {
 
       RealD scale;

lib/algorithms/LinearOperator.h

@@ -8,6 +8,7 @@
 
 Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Chulwoo Jung <chulwoo@bnl.gov>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -183,13 +184,11 @@ namespace Grid {
       virtual  RealD Mpc      (const Field &in, Field &out) =0;
       virtual  RealD MpcDag   (const Field &in, Field &out) =0;
       virtual void MpcDagMpc(const Field &in, Field &out,RealD &ni,RealD &no) {
-      Field tmp(in._grid);
-      tmp.checkerboard = in.checkerboard;
+	Field tmp(in._grid);
 	ni=Mpc(in,tmp);
 	no=MpcDag(tmp,out);
       }
       virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
-      out.checkerboard = in.checkerboard;
 	MpcDagMpc(in,out,n1,n2);
       }
       virtual void HermOp(const Field &in, Field &out){
@@ -217,15 +216,13 @@ namespace Grid {
     public:
       SchurDiagMooeeOperator (Matrix &Mat): _Mat(Mat){};
       virtual  RealD Mpc      (const Field &in, Field &out) {
-      Field tmp(in._grid);
-      tmp.checkerboard = !in.checkerboard;
-	//std::cout <<"grid pointers: in._grid="<< in._grid << " out._grid=" << out._grid << "  _Mat.Grid=" << _Mat.Grid() << " _Mat.RedBlackGrid=" << _Mat.RedBlackGrid() << std::endl;
+	Field tmp(in._grid);
+//	std::cout <<"grid pointers: in._grid="<< in._grid << " out._grid=" << out._grid << "  _Mat.Grid=" << _Mat.Grid() << " _Mat.RedBlackGrid=" << _Mat.RedBlackGrid() << std::endl;
 
 	_Mat.Meooe(in,tmp);
 	_Mat.MooeeInv(tmp,out);
 	_Mat.Meooe(out,tmp);
 
-      //std::cout << "cb in " << in.checkerboard << "  cb out " << out.checkerboard << std::endl;
 	_Mat.Mooee(in,out);
 	return axpy_norm(out,-1.0,tmp,out);
       }
@@ -312,34 +309,20 @@ namespace Grid {
     public:
       SchurStaggeredOperator (Matrix &Mat): _Mat(Mat){};
       virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
-	GridLogIterative.TimingMode(1);
-	std::cout << GridLogIterative << " HermOpAndNorm "<<std::endl;
 	n2 = Mpc(in,out);
-	std::cout << GridLogIterative << " HermOpAndNorm.Mpc "<<std::endl;
 	ComplexD dot= innerProduct(in,out);
-	std::cout << GridLogIterative << " HermOpAndNorm.innerProduct "<<std::endl;
 	n1 = real(dot);
       }
       virtual void HermOp(const Field &in, Field &out){
-	std::cout << GridLogIterative << " HermOp "<<std::endl;
 	Mpc(in,out);
       }
       virtual  RealD Mpc      (const Field &in, Field &out) {
 	Field tmp(in._grid);
-	Field tmp2(in._grid);
-
-	std::cout << GridLogIterative << " HermOp.Mpc "<<std::endl;
+	_Mat.Meooe(in,tmp);
+	_Mat.MooeeInv(tmp,out);
+	_Mat.Meooe(out,tmp);
 	_Mat.Mooee(in,out);
-	_Mat.Mooee(out,tmp);
-	std::cout << GridLogIterative << " HermOp.MooeeMooee "<<std::endl;
-
-	_Mat.Meooe(in,out);
-	_Mat.Meooe(out,tmp2);
-	std::cout << GridLogIterative << " HermOp.MeooeMeooe "<<std::endl;
-
-	RealD nn=axpy_norm(out,-1.0,tmp2,tmp);
-	std::cout << GridLogIterative << " HermOp.axpy_norm "<<std::endl;
-	return nn;
+        return axpy_norm(out,-1.0,tmp,out);
       }
       virtual  RealD MpcDag   (const Field &in, Field &out){
 	return Mpc(in,out);
@@ -348,7 +331,130 @@ namespace Grid {
 	assert(0);// Never need with staggered
       }
     };
-    template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
+//    template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
+    template<class Matrix,class Field>
+//      class SchurStagOperator :  public LinearOperatorBase<Field> {
+      class SchurStagOperator :  public SchurOperatorBase<Field> {
+    protected:
+      Matrix &_Mat;
+    public:
+      SchurStagOperator (Matrix &Mat): _Mat(Mat){};
+      virtual  RealD Mpc      (const Field &in, Field &out) {
+	Field tmp(in._grid);
+	Field tmp2(in._grid);
+
+	_Mat.Mooee(in,out);
+	_Mat.Mooee(out,tmp);
+
+	_Mat.Meooe(in,out);
+	_Mat.Meooe(out,tmp2);
+
+	return axpy_norm(out,-1.0,tmp2,tmp);
+      }
+      virtual  RealD MpcDag   (const Field &in, Field &out){
+
+	return Mpc(in,out);
+      }
+#if 0
+      virtual void MpcDagMpc(const Field &in, Field &out,RealD &ni,RealD &no) {
+	Field tmp(in._grid);
+	ni=Mpc(in,tmp);
+	no=MpcDag(tmp,out);
+      }
+#endif
+      void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+	n2 = Mpc(in,out);
+	ComplexD dot = innerProduct(in,out);
+	n1 = real(dot);
+      }
+      void HermOp(const Field &in, Field &out){
+	RealD n1,n2;
+	HermOpAndNorm(in,out,n1,n2);
+      }
+      void Op     (const Field &in, Field &out){
+	Mpc(in,out);
+      }
+      void AdjOp     (const Field &in, Field &out){ 
+	MpcDag(in,out);
+      }
+      // Support for coarsening to a multigrid
+      void OpDiag (const Field &in, Field &out) {
+	assert(0); // must coarsen the unpreconditioned system
+      }
+      void OpDir  (const Field &in, Field &out,int dir,int disp) {
+	assert(0);
+      }
+
+    };
+
+#if 0
+  // This is specific to (Z)mobius fermions
+  template<class Matrix, class Field>
+    class KappaSimilarityTransform {
+  public:
+//    INHERIT_IMPL_TYPES(Matrix);
+    typedef typename Matrix::Coeff_t                     Coeff_t;
+    std::vector<Coeff_t> kappa, kappaDag, kappaInv, kappaInvDag;
+
+    KappaSimilarityTransform (Matrix &zmob) {
+      for (int i=0;i<(int)zmob.bs.size();i++) {
+	Coeff_t k = 1.0 / ( 2.0 * (zmob.bs[i] *(4 - zmob.M5) + 1.0) );
+	kappa.push_back( k );
+	kappaDag.push_back( conj(k) );
+	kappaInv.push_back( 1.0 / k );
+	kappaInvDag.push_back( 1.0 / conj(k) );
+      }
+    }
+
+  template<typename vobj>
+    void sscale(const Lattice<vobj>& in, Lattice<vobj>& out, Coeff_t* s) {
+    GridBase *grid=out._grid;
+    out.checkerboard = in.checkerboard;
+    assert(grid->_simd_layout[0] == 1); // should be fine for ZMobius for now
+    int Ls = grid->_rdimensions[0];
+    parallel_for(int ss=0;ss<grid->oSites();ss++){
+      vobj tmp = s[ss % Ls]*in._odata[ss];
+      vstream(out._odata[ss],tmp);
+    }
+  }
+
+  RealD sscale_norm(const Field& in, Field& out, Coeff_t* s) {
+    sscale(in,out,s);
+    return norm2(out);
+  }
+
+  virtual RealD M       (const Field& in, Field& out) { return sscale_norm(in,out,&kappa[0]);   }
+  virtual RealD MDag    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaDag[0]);}
+  virtual RealD MInv    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInv[0]);}
+  virtual RealD MInvDag (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInvDag[0]);}
+
+  };
+
+  template<class Matrix,class Field>
+    class SchurDiagTwoKappaOperator :  public SchurOperatorBase<Field> {
+  public:
+    KappaSimilarityTransform<Matrix, Field> _S;
+    SchurDiagTwoOperator<Matrix, Field> _Mat;
+
+    SchurDiagTwoKappaOperator (Matrix &Mat): _S(Mat), _Mat(Mat) {};
+
+    virtual  RealD Mpc      (const Field &in, Field &out) {
+      Field tmp(in._grid);
+
+      _S.MInv(in,out);
+      _Mat.Mpc(out,tmp);
+      return _S.M(tmp,out);
+
+    }
+    virtual  RealD MpcDag   (const Field &in, Field &out){
+      Field tmp(in._grid);
+
+      _S.MDag(in,out);
+      _Mat.MpcDag(out,tmp);
+      return _S.MInvDag(tmp,out);
+    }
+  };
+#endif
 
 
     /////////////////////////////////////////////////////////////
@@ -364,14 +470,6 @@ namespace Grid {
       virtual void operator() (const Field &in, Field &out) = 0;
     };
 
-    template<class Field> class IdentityLinearFunction : public LinearFunction<Field> {
-    public:
-      void operator() (const Field &in, Field &out){
-	out = in;
-      };
-    };
-
-
     /////////////////////////////////////////////////////////////
     // Base classes for Multishift solvers for operators
     /////////////////////////////////////////////////////////////
@@ -394,64 +492,6 @@ namespace Grid {
      };
     */
 
-  ////////////////////////////////////////////////////////////////////////////////////////////
-  // Hermitian operator Linear function and operator function
-  ////////////////////////////////////////////////////////////////////////////////////////////
-    template<class Field>
-      class HermOpOperatorFunction : public OperatorFunction<Field> {
-      void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
-	Linop.HermOp(in,out);
-      };
-    };
-
-    template<typename Field>
-      class PlainHermOp : public LinearFunction<Field> {
-    public:
-      LinearOperatorBase<Field> &_Linop;
-      
-      PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop) 
-      {}
-      
-      void operator()(const Field& in, Field& out) {
-	_Linop.HermOp(in,out);
-      }
-    };
-
-    template<typename Field>
-    class FunctionHermOp : public LinearFunction<Field> {
-    public:
-      OperatorFunction<Field>   & _poly;
-      LinearOperatorBase<Field> &_Linop;
-      
-      FunctionHermOp(OperatorFunction<Field> & poly,LinearOperatorBase<Field>& linop) 
-	: _poly(poly), _Linop(linop) {};
-      
-      void operator()(const Field& in, Field& out) {
-	_poly(_Linop,in,out);
-      }
-    };
-
-  template<class Field>
-  class Polynomial : public OperatorFunction<Field> {
-  private:
-    std::vector<RealD> Coeffs;
-  public:
-    Polynomial(std::vector<RealD> &_Coeffs) : Coeffs(_Coeffs) { };
-
-    // Implement the required interface
-    void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
-
-      Field AtoN(in._grid);
-      Field Mtmp(in._grid);
-      AtoN = in;
-      out = AtoN*Coeffs[0];
-      for(int n=1;n<Coeffs.size();n++){
-	Mtmp = AtoN;
-	Linop.HermOp(Mtmp,AtoN);
-	out=out+AtoN*Coeffs[n];
-      }
-    };
-  };
 
 }
 

lib/algorithms/approx/Chebyshev.h

@@ -34,12 +34,41 @@ Author: Christoph Lehner <clehner@bnl.gov>
 
 namespace Grid {
 
-struct ChebyParams : Serializable {
-  GRID_SERIALIZABLE_CLASS_MEMBERS(ChebyParams,
-				  RealD, alpha,  
-				  RealD, beta,   
-				  int, Npoly);
-};
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  // Simple general polynomial with user supplied coefficients
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  template<class Field>
+  class HermOpOperatorFunction : public OperatorFunction<Field> {
+    void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
+      Linop.HermOp(in,out);
+    };
+  };
+
+  template<class Field>
+  class Polynomial : public OperatorFunction<Field> {
+  private:
+    std::vector<RealD> Coeffs;
+  public:
+    Polynomial(std::vector<RealD> &_Coeffs) : Coeffs(_Coeffs) { };
+
+    // Implement the required interface
+    void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
+
+      Field AtoN(in._grid);
+      Field Mtmp(in._grid);
+      AtoN = in;
+      out = AtoN*Coeffs[0];
+//            std::cout <<"Poly in " <<norm2(in)<<" size "<< Coeffs.size()<<std::endl;
+//            std::cout <<"Coeffs[0]= "<<Coeffs[0]<< " 0 " <<norm2(out)<<std::endl;
+      for(int n=1;n<Coeffs.size();n++){
+	Mtmp = AtoN;
+	Linop.HermOp(Mtmp,AtoN);
+	out=out+AtoN*Coeffs[n];
+//            std::cout <<"Coeffs "<<n<<"= "<< Coeffs[n]<< " 0 " <<std::endl;
+//		std::cout << n<<" " <<norm2(out)<<std::endl;
+      }
+    };
+  };
 
   ////////////////////////////////////////////////////////////////////////////////////////////
   // Generic Chebyshev approximations
@@ -54,10 +83,8 @@ struct ChebyParams : Serializable {
 
   public:
     void csv(std::ostream &out){
-      RealD diff = hi-lo;
-      RealD delta = (hi-lo)*1.0e-9;
-      for (RealD x=lo; x<hi; x+=delta) {
-	delta*=1.1;
+	RealD diff = hi-lo;
+      for (RealD x=lo-0.2*diff; x<hi+0.2*diff; x+=(hi-lo)/1000) {
 	RealD f = approx(x);
 	out<< x<<" "<<f<<std::endl;
       }
@@ -73,7 +100,6 @@ struct ChebyParams : Serializable {
     };
 
     Chebyshev(){};
-    Chebyshev(ChebyParams p){ Init(p.alpha,p.beta,p.Npoly);};
     Chebyshev(RealD _lo,RealD _hi,int _order, RealD (* func)(RealD) ) {Init(_lo,_hi,_order,func);};
     Chebyshev(RealD _lo,RealD _hi,int _order) {Init(_lo,_hi,_order);};
 

lib/algorithms/densematrix/DenseMatrix.h

@@ -0,0 +1,137 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/DenseMatrix.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef GRID_DENSE_MATRIX_H
+#define GRID_DENSE_MATRIX_H
+
+namespace Grid {
+    /////////////////////////////////////////////////////////////
+    // Matrix untils
+    /////////////////////////////////////////////////////////////
+
+template<class T> using DenseVector = std::vector<T>;
+template<class T> using DenseMatrix = DenseVector<DenseVector<T> >;
+
+template<class T> void Size(DenseVector<T> & vec, int &N) 
+{ 
+  N= vec.size();
+}
+template<class T> void Size(DenseMatrix<T> & mat, int &N,int &M) 
+{ 
+  N= mat.size();
+  M= mat[0].size();
+}
+
+template<class T> void SizeSquare(DenseMatrix<T> & mat, int &N) 
+{ 
+  int M; Size(mat,N,M);
+  assert(N==M);
+}
+
+template<class T> void Resize(DenseVector<T > & mat, int N) { 
+  mat.resize(N);
+}
+template<class T> void Resize(DenseMatrix<T > & mat, int N, int M) { 
+  mat.resize(N);
+  for(int i=0;i<N;i++){
+    mat[i].resize(M);
+  }
+}
+template<class T> void Fill(DenseMatrix<T> & mat, T&val) { 
+  int N,M;
+  Size(mat,N,M);
+  for(int i=0;i<N;i++){
+  for(int j=0;j<M;j++){
+    mat[i][j] = val;
+  }}
+}
+
+/** Transpose of a matrix **/
+template<class T> DenseMatrix<T> Transpose(DenseMatrix<T> & mat){
+  int N,M;
+  Size(mat,N,M);
+  DenseMatrix<T> C; Resize(C,M,N);
+  for(int i=0;i<M;i++){
+  for(int j=0;j<N;j++){
+    C[i][j] = mat[j][i];
+  }} 
+  return C;
+}
+/** Set DenseMatrix to unit matrix **/
+template<class T> void Unity(DenseMatrix<T> &A){
+  int N;  SizeSquare(A,N);
+  for(int i=0;i<N;i++){
+    for(int j=0;j<N;j++){
+      if ( i==j ) A[i][j] = 1;
+      else        A[i][j] = 0;
+    } 
+  } 
+}
+
+/** Add C * I to matrix **/
+template<class T>
+void PlusUnit(DenseMatrix<T> & A,T c){
+  int dim;  SizeSquare(A,dim);
+  for(int i=0;i<dim;i++){A[i][i] = A[i][i] + c;} 
+}
+
+/** return the Hermitian conjugate of matrix **/
+template<class T>
+DenseMatrix<T> HermitianConj(DenseMatrix<T> &mat){
+
+  int dim; SizeSquare(mat,dim);
+
+  DenseMatrix<T> C; Resize(C,dim,dim);
+
+  for(int i=0;i<dim;i++){
+    for(int j=0;j<dim;j++){
+      C[i][j] = conj(mat[j][i]);
+    } 
+  } 
+  return C;
+}
+/**Get a square submatrix**/
+template <class T>
+DenseMatrix<T> GetSubMtx(DenseMatrix<T> &A,int row_st, int row_end, int col_st, int col_end)
+{
+  DenseMatrix<T> H; Resize(H,row_end - row_st,col_end-col_st);
+
+  for(int i = row_st; i<row_end; i++){
+  for(int j = col_st; j<col_end; j++){
+    H[i-row_st][j-col_st]=A[i][j];
+  }}
+  return H;
+}
+
+}
+
+#include "Householder.h"
+#include "Francis.h"
+
+#endif
+

lib/algorithms/densematrix/Francis.h

@@ -0,0 +1,525 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/Francis.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef FRANCIS_H
+#define FRANCIS_H
+
+#include <cstdlib>
+#include <string>
+#include <cmath>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <fstream>
+#include <complex>
+#include <algorithm>
+
+//#include <timer.h>
+//#include <lapacke.h>
+//#include <Eigen/Dense>
+
+namespace Grid {
+
+template <class T> int SymmEigensystem(DenseMatrix<T > &Ain, DenseVector<T> &evals, DenseMatrix<T> &evecs, RealD small);
+template <class T> int     Eigensystem(DenseMatrix<T > &Ain, DenseVector<T> &evals, DenseMatrix<T> &evecs, RealD small);
+
+/**
+  Find the eigenvalues of an upper hessenberg matrix using the Francis QR algorithm.
+H =
+      x  x  x  x  x  x  x  x  x
+      x  x  x  x  x  x  x  x  x
+      0  x  x  x  x  x  x  x  x
+      0  0  x  x  x  x  x  x  x
+      0  0  0  x  x  x  x  x  x
+      0  0  0  0  x  x  x  x  x
+      0  0  0  0  0  x  x  x  x
+      0  0  0  0  0  0  x  x  x
+      0  0  0  0  0  0  0  x  x
+Factorization is P T P^H where T is upper triangular (mod cc blocks) and P is orthagonal/unitary.
+**/
+template <class T>
+int QReigensystem(DenseMatrix<T> &Hin, DenseVector<T> &evals, DenseMatrix<T> &evecs, RealD small)
+{
+  DenseMatrix<T> H = Hin; 
+
+  int N ; SizeSquare(H,N);
+  int M = N;
+
+  Fill(evals,0);
+  Fill(evecs,0);
+
+  T s,t,x=0,y=0,z=0;
+  T u,d;
+  T apd,amd,bc;
+  DenseVector<T> p(N,0);
+  T nrm = Norm(H);    ///DenseMatrix Norm
+  int n, m;
+  int e = 0;
+  int it = 0;
+  int tot_it = 0;
+  int l = 0;
+  int r = 0;
+  DenseMatrix<T> P; Resize(P,N,N); Unity(P);
+  DenseVector<int> trows(N,0);
+
+  /// Check if the matrix is really hessenberg, if not abort
+  RealD sth = 0;
+  for(int j=0;j<N;j++){
+    for(int i=j+2;i<N;i++){
+      sth = abs(H[i][j]);
+      if(sth > small){
+	std::cout << "Non hessenberg H = " << sth << " > " << small << std::endl;
+	exit(1);
+      }
+    }
+  }
+
+  do{
+    std::cout << "Francis QR Step N = " << N << std::endl;
+    /** Check for convergence
+      x  x  x  x  x
+      0  x  x  x  x
+      0  0  x  x  x
+      0  0  x  x  x
+      0  0  0  0  x
+      for this matrix l = 4
+     **/
+    do{
+      l = Chop_subdiag(H,nrm,e,small);
+      r = 0;    ///May have converged on more than one eval
+      ///Single eval
+      if(l == N-1){
+        evals[e] = H[l][l];
+        N--; e++; r++; it = 0;
+      }
+      ///RealD eval
+      if(l == N-2){
+        trows[l+1] = 1;    ///Needed for UTSolve
+        apd = H[l][l] + H[l+1][l+1];
+        amd = H[l][l] - H[l+1][l+1];
+        bc =  (T)4.0*H[l+1][l]*H[l][l+1];
+        evals[e]   = (T)0.5*( apd + sqrt(amd*amd + bc) );
+        evals[e+1] = (T)0.5*( apd - sqrt(amd*amd + bc) );
+        N-=2; e+=2; r++; it = 0;
+      }
+    } while(r>0);
+
+    if(N ==0) break;
+
+    DenseVector<T > ck; Resize(ck,3);
+    DenseVector<T> v;   Resize(v,3);
+
+    for(int m = N-3; m >= l; m--){
+      ///Starting vector essentially random shift.
+      if(it%10 == 0 && N >= 3 && it > 0){
+        s = (T)1.618033989*( abs( H[N-1][N-2] ) + abs( H[N-2][N-3] ) );
+        t = (T)0.618033989*( abs( H[N-1][N-2] ) + abs( H[N-2][N-3] ) );
+        x = H[m][m]*H[m][m] + H[m][m+1]*H[m+1][m] - s*H[m][m] + t;
+        y = H[m+1][m]*(H[m][m] + H[m+1][m+1] - s);
+        z = H[m+1][m]*H[m+2][m+1];
+      }
+      ///Starting vector implicit Q theorem
+      else{
+        s = (H[N-2][N-2] + H[N-1][N-1]);
+        t = (H[N-2][N-2]*H[N-1][N-1] - H[N-2][N-1]*H[N-1][N-2]);
+        x = H[m][m]*H[m][m] + H[m][m+1]*H[m+1][m] - s*H[m][m] + t;
+        y = H[m+1][m]*(H[m][m] + H[m+1][m+1] - s);
+        z = H[m+1][m]*H[m+2][m+1];
+      }
+      ck[0] = x; ck[1] = y; ck[2] = z;
+
+      if(m == l) break;
+
+      /** Some stupid thing from numerical recipies, seems to work**/
+      // PAB.. for heaven's sake quote page, purpose, evidence it works.
+      //       what sort of comment is that!?!?!?
+      u=abs(H[m][m-1])*(abs(y)+abs(z));
+      d=abs(x)*(abs(H[m-1][m-1])+abs(H[m][m])+abs(H[m+1][m+1]));
+      if ((T)abs(u+d) == (T)abs(d) ){
+	l = m; break;
+      }
+
+      //if (u < small){l = m; break;}
+    }
+    if(it > 100000){
+     std::cout << "QReigensystem: bugger it got stuck after 100000 iterations" << std::endl;
+     std::cout << "got " << e << " evals " << l << " " << N << std::endl;
+      exit(1);
+    }
+    normalize(ck);    ///Normalization cancels in PHP anyway
+    T beta;
+    Householder_vector<T >(ck, 0, 2, v, beta);
+    Householder_mult<T >(H,v,beta,0,l,l+2,0);
+    Householder_mult<T >(H,v,beta,0,l,l+2,1);
+    ///Accumulate eigenvector
+    Householder_mult<T >(P,v,beta,0,l,l+2,1);
+    int sw = 0;      ///Are we on the last row?
+    for(int k=l;k<N-2;k++){
+      x = H[k+1][k];
+      y = H[k+2][k];
+      z = (T)0.0;
+      if(k+3 <= N-1){
+	z = H[k+3][k];
+      } else{
+	sw = 1; 
+	v[2] = (T)0.0;
+      }
+      ck[0] = x; ck[1] = y; ck[2] = z;
+      normalize(ck);
+      Householder_vector<T >(ck, 0, 2-sw, v, beta);
+      Householder_mult<T >(H,v, beta,0,k+1,k+3-sw,0);
+      Householder_mult<T >(H,v, beta,0,k+1,k+3-sw,1);
+      ///Accumulate eigenvector
+      Householder_mult<T >(P,v, beta,0,k+1,k+3-sw,1);
+    }
+    it++;
+    tot_it++;
+  }while(N > 1);
+  N = evals.size();
+  ///Annoying - UT solves in reverse order;
+  DenseVector<T> tmp; Resize(tmp,N);
+  for(int i=0;i<N;i++){
+    tmp[i] = evals[N-i-1];
+  } 
+  evals = tmp;
+  UTeigenvectors(H, trows, evals, evecs);
+  for(int i=0;i<evals.size();i++){evecs[i] = P*evecs[i]; normalize(evecs[i]);}
+  return tot_it;
+}
+
+template <class T>
+int my_Wilkinson(DenseMatrix<T> &Hin, DenseVector<T> &evals, DenseMatrix<T> &evecs, RealD small)
+{
+  /**
+  Find the eigenvalues of an upper Hessenberg matrix using the Wilkinson QR algorithm.
+  H =
+  x  x  0  0  0  0
+  x  x  x  0  0  0
+  0  x  x  x  0  0
+  0  0  x  x  x  0
+  0  0  0  x  x  x
+  0  0  0  0  x  x
+  Factorization is P T P^H where T is upper triangular (mod cc blocks) and P is orthagonal/unitary.  **/
+  return my_Wilkinson(Hin, evals, evecs, small, small);
+}
+
+template <class T>
+int my_Wilkinson(DenseMatrix<T> &Hin, DenseVector<T> &evals, DenseMatrix<T> &evecs, RealD small, RealD tol)
+{
+  int N; SizeSquare(Hin,N);
+  int M = N;
+
+  ///I don't want to modify the input but matricies must be passed by reference
+  //Scale a matrix by its "norm"
+  //RealD Hnorm = abs( Hin.LargestDiag() ); H =  H*(1.0/Hnorm);
+  DenseMatrix<T> H;  H = Hin;
+  
+  RealD Hnorm = abs(Norm(Hin));
+  H = H * (1.0 / Hnorm);
+
+  // TODO use openmp and memset
+  Fill(evals,0);
+  Fill(evecs,0);
+
+  T s, t, x = 0, y = 0, z = 0;
+  T u, d;
+  T apd, amd, bc;
+  DenseVector<T> p; Resize(p,N); Fill(p,0);
+
+  T nrm = Norm(H);    ///DenseMatrix Norm
+  int n, m;
+  int e = 0;
+  int it = 0;
+  int tot_it = 0;
+  int l = 0;
+  int r = 0;
+  DenseMatrix<T> P; Resize(P,N,N);
+  Unity(P);
+  DenseVector<int> trows(N, 0);
+  /// Check if the matrix is really symm tridiag
+  RealD sth = 0;
+  for(int j = 0; j < N; ++j)
+  {
+    for(int i = j + 2; i < N; ++i)
+    {
+      if(abs(H[i][j]) > tol || abs(H[j][i]) > tol)
+      {
+	std::cout << "Non Tridiagonal H(" << i << ","<< j << ") = |" << Real( real( H[j][i] ) ) << "| > " << tol << std::endl;
+	std::cout << "Warning tridiagonalize and call again" << std::endl;
+        // exit(1); // see what is going on
+        //return;
+      }
+    }
+  }
+
+  do{
+    do{
+      //Jasper
+      //Check if the subdiagonal term is small enough (<small)
+      //if true then it is converged.
+      //check start from H.dim - e - 1
+      //How to deal with more than 2 are converged?
+      //What if Chop_symm_subdiag return something int the middle?
+      //--------------
+      l = Chop_symm_subdiag(H,nrm, e, small);
+      r = 0;    ///May have converged on more than one eval
+      //Jasper
+      //In this case
+      // x  x  0  0  0  0
+      // x  x  x  0  0  0
+      // 0  x  x  x  0  0
+      // 0  0  x  x  x  0
+      // 0  0  0  x  x  0
+      // 0  0  0  0  0  x  <- l
+      //--------------
+      ///Single eval
+      if(l == N - 1)
+      {
+        evals[e] = H[l][l];
+        N--;
+        e++;
+        r++;
+        it = 0;
+      }
+      //Jasper
+      // x  x  0  0  0  0
+      // x  x  x  0  0  0
+      // 0  x  x  x  0  0
+      // 0  0  x  x  0  0
+      // 0  0  0  0  x  x  <- l
+      // 0  0  0  0  x  x
+      //--------------
+      ///RealD eval
+      if(l == N - 2)
+      {
+        trows[l + 1] = 1;    ///Needed for UTSolve
+        apd = H[l][l] + H[l + 1][ l + 1];
+        amd = H[l][l] - H[l + 1][l + 1];
+        bc =  (T) 4.0 * H[l + 1][l] * H[l][l + 1];
+        evals[e] = (T) 0.5 * (apd + sqrt(amd * amd + bc));
+        evals[e + 1] = (T) 0.5 * (apd - sqrt(amd * amd + bc));
+        N -= 2;
+        e += 2;
+        r++;
+        it = 0;
+      }
+    }while(r > 0);
+    //Jasper
+    //Already converged
+    //--------------
+    if(N == 0) break;
+
+    DenseVector<T> ck,v; Resize(ck,2); Resize(v,2);
+
+    for(int m = N - 3; m >= l; m--)
+    {
+      ///Starting vector essentially random shift.
+      if(it%10 == 0 && N >= 3 && it > 0)
+      {
+        t = abs(H[N - 1][N - 2]) + abs(H[N - 2][N - 3]);
+        x = H[m][m] - t;
+        z = H[m + 1][m];
+      } else {
+      ///Starting vector implicit Q theorem
+        d = (H[N - 2][N - 2] - H[N - 1][N - 1]) * (T) 0.5;
+        t =  H[N - 1][N - 1] - H[N - 1][N - 2] * H[N - 1][N - 2] 
+	  / (d + sign(d) * sqrt(d * d + H[N - 1][N - 2] * H[N - 1][N - 2]));
+        x = H[m][m] - t;
+        z = H[m + 1][m];
+      }
+      //Jasper
+      //why it is here????
+      //-----------------------
+      if(m == l)
+        break;
+
+      u = abs(H[m][m - 1]) * (abs(y) + abs(z));
+      d = abs(x) * (abs(H[m - 1][m - 1]) + abs(H[m][m]) + abs(H[m + 1][m + 1]));
+      if ((T)abs(u + d) == (T)abs(d))
+      {
+        l = m;
+        break;
+      }
+    }
+    //Jasper
+    if(it > 1000000)
+    {
+      std::cout << "Wilkinson: bugger it got stuck after 100000 iterations" << std::endl;
+      std::cout << "got " << e << " evals " << l << " " << N << std::endl;
+      exit(1);
+    }
+    //
+    T s, c;
+    Givens_calc<T>(x, z, c, s);
+    Givens_mult<T>(H, l, l + 1, c, -s, 0);
+    Givens_mult<T>(H, l, l + 1, c,  s, 1);
+    Givens_mult<T>(P, l, l + 1, c,  s, 1);
+    //
+    for(int k = l; k < N - 2; ++k)
+    {
+      x = H.A[k + 1][k];
+      z = H.A[k + 2][k];
+      Givens_calc<T>(x, z, c, s);
+      Givens_mult<T>(H, k + 1, k + 2, c, -s, 0);
+      Givens_mult<T>(H, k + 1, k + 2, c,  s, 1);
+      Givens_mult<T>(P, k + 1, k + 2, c,  s, 1);
+    }
+    it++;
+    tot_it++;
+  }while(N > 1);
+
+  N = evals.size();
+  ///Annoying - UT solves in reverse order;
+  DenseVector<T> tmp(N);
+  for(int i = 0; i < N; ++i)
+    tmp[i] = evals[N-i-1];
+  evals = tmp;
+  //
+  UTeigenvectors(H, trows, evals, evecs);
+  //UTSymmEigenvectors(H, trows, evals, evecs);
+  for(int i = 0; i < evals.size(); ++i)
+  {
+    evecs[i] = P * evecs[i];
+    normalize(evecs[i]);
+    evals[i] = evals[i] * Hnorm;
+  }
+  // // FIXME this is to test
+  // Hin.write("evecs3", evecs);
+  // Hin.write("evals3", evals);
+  // // check rsd
+  // for(int i = 0; i < M; i++) {
+  //   vector<T> Aevec = Hin * evecs[i];
+  //   RealD norm2(0.);
+  //   for(int j = 0; j < M; j++) {
+  //     norm2 += (Aevec[j] - evals[i] * evecs[i][j]) * (Aevec[j] - evals[i] * evecs[i][j]);
+  //   }
+  // }
+  return tot_it;
+}
+
+template <class T>
+void Hess(DenseMatrix<T > &A, DenseMatrix<T> &Q, int start){
+
+  /**
+  turn a matrix A =
+  x  x  x  x  x
+  x  x  x  x  x
+  x  x  x  x  x
+  x  x  x  x  x
+  x  x  x  x  x
+  into
+  x  x  x  x  x
+  x  x  x  x  x
+  0  x  x  x  x
+  0  0  x  x  x
+  0  0  0  x  x
+  with householder rotations
+  Slow.
+  */
+  int N ; SizeSquare(A,N);
+  DenseVector<T > p; Resize(p,N); Fill(p,0);
+
+  for(int k=start;k<N-2;k++){
+    //cerr << "hess" << k << std::endl;
+    DenseVector<T > ck,v; Resize(ck,N-k-1); Resize(v,N-k-1);
+    for(int i=k+1;i<N;i++){ck[i-k-1] = A(i,k);}  ///kth column
+    normalize(ck);    ///Normalization cancels in PHP anyway
+    T beta;
+    Householder_vector<T >(ck, 0, ck.size()-1, v, beta);  ///Householder vector
+    Householder_mult<T>(A,v,beta,start,k+1,N-1,0);  ///A -> PA
+    Householder_mult<T >(A,v,beta,start,k+1,N-1,1);  ///PA -> PAP^H
+    ///Accumulate eigenvector
+    Householder_mult<T >(Q,v,beta,start,k+1,N-1,1);  ///Q -> QP^H
+  }
+  /*for(int l=0;l<N-2;l++){
+    for(int k=l+2;k<N;k++){
+    A(0,k,l);
+    }
+    }*/
+}
+
+template <class T>
+void Tri(DenseMatrix<T > &A, DenseMatrix<T> &Q, int start){
+///Tridiagonalize a matrix
+  int N; SizeSquare(A,N);
+  Hess(A,Q,start);
+  /*for(int l=0;l<N-2;l++){
+    for(int k=l+2;k<N;k++){
+    A(0,l,k);
+    }
+    }*/
+}
+
+template <class T>
+void ForceTridiagonal(DenseMatrix<T> &A){
+///Tridiagonalize a matrix
+  int N ; SizeSquare(A,N);
+  for(int l=0;l<N-2;l++){
+    for(int k=l+2;k<N;k++){
+      A[l][k]=0;
+      A[k][l]=0;
+    }
+  }
+}
+
+template <class T>
+int my_SymmEigensystem(DenseMatrix<T > &Ain, DenseVector<T> &evals, DenseVector<DenseVector<T> > &evecs, RealD small){
+  ///Solve a symmetric eigensystem, not necessarily in tridiagonal form
+  int N; SizeSquare(Ain,N);
+  DenseMatrix<T > A; A = Ain;
+  DenseMatrix<T > Q; Resize(Q,N,N); Unity(Q);
+  Tri(A,Q,0);
+  int it = my_Wilkinson<T>(A, evals, evecs, small);
+  for(int k=0;k<N;k++){evecs[k] = Q*evecs[k];}
+  return it;
+}
+
+
+template <class T>
+int Wilkinson(DenseMatrix<T> &Ain, DenseVector<T> &evals, DenseVector<DenseVector<T> > &evecs, RealD small){
+  return my_Wilkinson(Ain, evals, evecs, small);
+}
+
+template <class T>
+int SymmEigensystem(DenseMatrix<T> &Ain, DenseVector<T> &evals, DenseVector<DenseVector<T> > &evecs, RealD small){
+  return my_SymmEigensystem(Ain, evals, evecs, small);
+}
+
+template <class T>
+int Eigensystem(DenseMatrix<T > &Ain, DenseVector<T> &evals, DenseVector<DenseVector<T> > &evecs, RealD small){
+///Solve a general eigensystem, not necessarily in tridiagonal form
+  int N = Ain.dim;
+  DenseMatrix<T > A(N); A = Ain;
+  DenseMatrix<T > Q(N);Q.Unity();
+  Hess(A,Q,0);
+  int it = QReigensystem<T>(A, evals, evecs, small);
+  for(int k=0;k<N;k++){evecs[k] = Q*evecs[k];}
+  return it;
+}
+
+}
+#endif

lib/algorithms/densematrix/Householder.h

@@ -0,0 +1,242 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/Householder.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef HOUSEHOLDER_H
+#define HOUSEHOLDER_H
+
+#define TIMER(A) std::cout << GridLogMessage << __FUNC__ << " file "<< __FILE__ <<" line " << __LINE__ << std::endl;
+#define ENTER()  std::cout << GridLogMessage << "ENTRY "<<__FUNC__ << " file "<< __FILE__ <<" line " << __LINE__ << std::endl;
+#define LEAVE()  std::cout << GridLogMessage << "EXIT  "<<__FUNC__ << " file "<< __FILE__ <<" line " << __LINE__ << std::endl;
+
+#include <cstdlib>
+#include <string>
+#include <cmath>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <fstream>
+#include <complex>
+#include <algorithm>
+
+namespace Grid {
+/** Comparison function for finding the max element in a vector **/
+template <class T> bool cf(T i, T j) { 
+  return abs(i) < abs(j); 
+}
+
+/** 
+	Calculate a real Givens angle 
+ **/
+template <class T> inline void Givens_calc(T y, T z, T &c, T &s){
+
+  RealD mz = (RealD)abs(z);
+  
+  if(mz==0.0){
+    c = 1; s = 0;
+  }
+  if(mz >= (RealD)abs(y)){
+    T t = -y/z;
+    s = (T)1.0 / sqrt ((T)1.0 + t * t);
+    c = s * t;
+  } else {
+    T t = -z/y;
+    c = (T)1.0 / sqrt ((T)1.0 + t * t);
+    s = c * t;
+  }
+}
+
+template <class T> inline void Givens_mult(DenseMatrix<T> &A,  int i, int k, T c, T s, int dir)
+{
+  int q ; SizeSquare(A,q);
+
+  if(dir == 0){
+    for(int j=0;j<q;j++){
+      T nu = A[i][j];
+      T w  = A[k][j];
+      A[i][j] = (c*nu + s*w);
+      A[k][j] = (-s*nu + c*w);
+    }
+  }
+
+  if(dir == 1){
+    for(int j=0;j<q;j++){
+      T nu = A[j][i];
+      T w  = A[j][k];
+      A[j][i] = (c*nu - s*w);
+      A[j][k] = (s*nu + c*w);
+    }
+  }
+}
+
+/**
+	from input = x;
+	Compute the complex Householder vector, v, such that
+	P = (I - b v transpose(v) )
+	b = 2/v.v
+
+	P | x |    | x | k = 0
+	| x |    | 0 | 
+	| x | =  | 0 |
+	| x |    | 0 | j = 3
+	| x |	   | x |
+
+	These are the "Unreduced" Householder vectors.
+
+ **/
+template <class T> inline void Householder_vector(DenseVector<T> input, int k, int j, DenseVector<T> &v, T &beta)
+{
+  int N ; Size(input,N);
+  T m = *max_element(input.begin() + k, input.begin() + j + 1, cf<T> );
+
+  if(abs(m) > 0.0){
+    T alpha = 0;
+
+    for(int i=k; i<j+1; i++){
+      v[i] = input[i]/m;
+      alpha = alpha + v[i]*conj(v[i]);
+    }
+    alpha = sqrt(alpha);
+    beta = (T)1.0/(alpha*(alpha + abs(v[k]) ));
+
+    if(abs(v[k]) > 0.0)  v[k] = v[k] + (v[k]/abs(v[k]))*alpha;
+    else                 v[k] = -alpha;
+  } else{
+    for(int i=k; i<j+1; i++){
+      v[i] = 0.0;
+    } 
+  }
+}
+
+/**
+	from input = x;
+	Compute the complex Householder vector, v, such that
+	P = (I - b v transpose(v) )
+	b = 2/v.v
+
+	Px = alpha*e_dir
+
+	These are the "Unreduced" Householder vectors.
+
+ **/
+
+template <class T> inline void Householder_vector(DenseVector<T> input, int k, int j, int dir, DenseVector<T> &v, T &beta)
+{
+  int N = input.size();
+  T m = *max_element(input.begin() + k, input.begin() + j + 1, cf);
+  
+  if(abs(m) > 0.0){
+    T alpha = 0;
+
+    for(int i=k; i<j+1; i++){
+      v[i] = input[i]/m;
+      alpha = alpha + v[i]*conj(v[i]);
+    }
+    
+    alpha = sqrt(alpha);
+    beta = 1.0/(alpha*(alpha + abs(v[dir]) ));
+	
+    if(abs(v[dir]) > 0.0) v[dir] = v[dir] + (v[dir]/abs(v[dir]))*alpha;
+    else                  v[dir] = -alpha;
+  }else{
+    for(int i=k; i<j+1; i++){
+      v[i] = 0.0;
+    } 
+  }
+}
+
+/**
+	Compute the product PA if trans = 0
+	AP if trans = 1
+	P = (I - b v transpose(v) )
+	b = 2/v.v
+	start at element l of matrix A
+	v is of length j - k + 1 of v are nonzero
+ **/
+
+template <class T> inline void Householder_mult(DenseMatrix<T> &A , DenseVector<T> v, T beta, int l, int k, int j, int trans)
+{
+  int N ; SizeSquare(A,N);
+
+  if(abs(beta) > 0.0){
+    for(int p=l; p<N; p++){
+      T s = 0;
+      if(trans==0){
+	for(int i=k;i<j+1;i++) s += conj(v[i-k])*A[i][p];
+	s *= beta;
+	for(int i=k;i<j+1;i++){ A[i][p] = A[i][p]-s*conj(v[i-k]);}
+      } else {
+	for(int i=k;i<j+1;i++){ s += conj(v[i-k])*A[p][i];}
+	s *= beta;
+	for(int i=k;i<j+1;i++){ A[p][i]=A[p][i]-s*conj(v[i-k]);}
+      }
+    }
+  }
+}
+
+/**
+	Compute the product PA if trans = 0
+	AP if trans = 1
+	P = (I - b v transpose(v) )
+	b = 2/v.v
+	start at element l of matrix A
+	v is of length j - k + 1 of v are nonzero
+	A is tridiagonal
+ **/
+template <class T> inline void Householder_mult_tri(DenseMatrix<T> &A , DenseVector<T> v, T beta, int l, int M, int k, int j, int trans)
+{
+  if(abs(beta) > 0.0){
+
+    int N ; SizeSquare(A,N);
+
+    DenseMatrix<T> tmp; Resize(tmp,N,N); Fill(tmp,0); 
+
+    T s;
+    for(int p=l; p<M; p++){
+      s = 0;
+      if(trans==0){
+	for(int i=k;i<j+1;i++) s = s + conj(v[i-k])*A[i][p];
+      }else{
+	for(int i=k;i<j+1;i++) s = s + v[i-k]*A[p][i];
+      }
+      s = beta*s;
+      if(trans==0){
+	for(int i=k;i<j+1;i++) tmp[i][p] = tmp(i,p) - s*v[i-k];
+      }else{
+	for(int i=k;i<j+1;i++) tmp[p][i] = tmp[p][i] - s*conj(v[i-k]);
+      }
+    }
+    for(int p=l; p<M; p++){
+      if(trans==0){
+	for(int i=k;i<j+1;i++) A[i][p] = A[i][p] + tmp[i][p];
+      }else{
+	for(int i=k;i<j+1;i++) A[p][i] = A[p][i] + tmp[p][i];
+      }
+    }
+  }
+}
+}
+#endif

lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h

@@ -0,0 +1,753 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Chulwoo Jung <chulwoo@bnl.gov>
+Author: Christoph Lehner <clehner@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef GRID_BIRL_H
+#define GRID_BIRL_H
+
+#include <string.h> //memset
+
+#include <zlib.h>
+#include <sys/stat.h>
+
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h>
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h>
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h>
+
+namespace Grid { 
+
+/////////////////////////////////////////////////////////////
+// Implicitly restarted lanczos
+/////////////////////////////////////////////////////////////
+
+ template<class Field> 
+ class BlockImplicitlyRestartedLanczos {
+
+    const RealD small = 1.0e-16;
+public:       
+    int lock;
+    int get;
+    int Niter;
+    int converged;
+
+    int Nminres; // Minimum number of restarts; only check for convergence after
+    int Nstop;   // Number of evecs checked for convergence
+    int Nk;      // Number of converged sought
+    int Np;      // Np -- Number of spare vecs in kryloc space
+    int Nm;      // Nm -- total number of vectors
+
+    int orth_period;
+
+    RealD OrthoTime;
+
+    RealD eresid, betastp;
+    SortEigen<Field> _sort;
+    LinearFunction<Field> &_HermOp;
+    LinearFunction<Field> &_HermOpTest;
+    /////////////////////////
+    // Constructor
+    /////////////////////////
+
+    BlockImplicitlyRestartedLanczos(
+			       LinearFunction<Field> & HermOp,
+			       LinearFunction<Field> & HermOpTest,
+			       int _Nstop, // sought vecs
+			       int _Nk, // sought vecs
+			       int _Nm, // spare vecs
+			       RealD _eresid, // resid in lmdue deficit 
+			       RealD _betastp, // if beta(k) < betastp: converged
+			       int _Niter, // Max iterations
+			       int _Nminres, int _orth_period = 1) :
+      _HermOp(HermOp),
+      _HermOpTest(HermOpTest),
+      Nstop(_Nstop),
+      Nk(_Nk),
+      Nm(_Nm),
+      eresid(_eresid),
+      betastp(_betastp),
+      Niter(_Niter),
+	Nminres(_Nminres),
+	orth_period(_orth_period)
+    { 
+      Np = Nm-Nk; assert(Np>0);
+    };
+
+    BlockImplicitlyRestartedLanczos(
+			       LinearFunction<Field> & HermOp,
+			       LinearFunction<Field> & HermOpTest,
+			       int _Nk, // sought vecs
+			       int _Nm, // spare vecs
+			       RealD _eresid, // resid in lmdue deficit 
+			       RealD _betastp, // if beta(k) < betastp: converged
+			       int _Niter, // Max iterations
+			       int _Nminres,
+			       int _orth_period = 1) : 
+      _HermOp(HermOp),
+      _HermOpTest(HermOpTest),
+      Nstop(_Nk),
+      Nk(_Nk),
+      Nm(_Nm),
+      eresid(_eresid),
+      betastp(_betastp),
+      Niter(_Niter),
+	Nminres(_Nminres),
+	orth_period(_orth_period)
+    { 
+      Np = Nm-Nk; assert(Np>0);
+    };
+
+
+/* Saad PP. 195
+1. Choose an initial vector v1 of 2-norm unity. Set β1 ≡ 0, v0 ≡ 0
+2. For k = 1,2,...,m Do:
+3. wk:=Avk−βkv_{k−1}      
+4. αk:=(wk,vk)       // 
+5. wk:=wk−αkvk       // wk orthog vk 
+6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
+7. vk+1 := wk/βk+1
+8. EndDo
+ */
+    void step(std::vector<RealD>& lmd,
+	      std::vector<RealD>& lme, 
+	      BasisFieldVector<Field>& evec,
+	      Field& w,int Nm,int k)
+    {
+      assert( k< Nm );
+
+      GridStopWatch gsw_op,gsw_o;
+
+      Field& evec_k = evec[k];
+
+      gsw_op.Start();
+      _HermOp(evec_k,w);
+      gsw_op.Stop();
+
+      if(k>0){
+	w -= lme[k-1] * evec[k-1];
+      }    
+
+      ComplexD zalph = innerProduct(evec_k,w); // 4. αk:=(wk,vk)
+      RealD     alph = real(zalph);
+
+      w = w - alph * evec_k;// 5. wk:=wk−αkvk
+
+      RealD beta = normalise(w); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
+                                 // 7. vk+1 := wk/βk+1
+
+      std::cout<<GridLogMessage << "alpha[" << k << "] = " << zalph << " beta[" << k << "] = "<<beta<<std::endl;
+      const RealD tiny = 1.0e-20;
+      if ( beta < tiny ) { 
+	std::cout<<GridLogMessage << " beta is tiny "<<beta<<std::endl;
+     }
+      lmd[k] = alph;
+      lme[k]  = beta;
+
+      gsw_o.Start();
+      if (k>0 && k % orth_period == 0) { 
+	orthogonalize(w,evec,k); // orthonormalise
+      }
+      gsw_o.Stop();
+
+      if(k < Nm-1) { 
+	evec[k+1] = w;
+      }
+
+      std::cout << GridLogMessage << "Timing: operator=" << gsw_op.Elapsed() <<
+	" orth=" << gsw_o.Elapsed() << std::endl;
+
+    }
+
+    void qr_decomp(std::vector<RealD>& lmd,
+		   std::vector<RealD>& lme,
+		   int Nk,
+		   int Nm,
+		   std::vector<RealD>& Qt,
+		   RealD Dsh, 
+		   int kmin,
+		   int kmax)
+    {
+      int k = kmin-1;
+      RealD x;
+
+      RealD Fden = 1.0/hypot(lmd[k]-Dsh,lme[k]);
+      RealD c = ( lmd[k] -Dsh) *Fden;
+      RealD s = -lme[k] *Fden;
+      
+      RealD tmpa1 = lmd[k];
+      RealD tmpa2 = lmd[k+1];
+      RealD tmpb  = lme[k];
+
+      lmd[k]   = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb;
+      lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb;
+      lme[k]   = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb;
+      x        =-s*lme[k+1];
+      lme[k+1] = c*lme[k+1];
+      
+      for(int i=0; i<Nk; ++i){
+	RealD Qtmp1 = Qt[i+Nm*k  ];
+	RealD Qtmp2 = Qt[i+Nm*(k+1)];
+	Qt[i+Nm*k    ] = c*Qtmp1 - s*Qtmp2;
+	Qt[i+Nm*(k+1)] = s*Qtmp1 + c*Qtmp2; 
+      }
+
+      // Givens transformations
+      for(int k = kmin; k < kmax-1; ++k){
+
+	RealD Fden = 1.0/hypot(x,lme[k-1]);
+	RealD c = lme[k-1]*Fden;
+	RealD s = - x*Fden;
+	
+	RealD tmpa1 = lmd[k];
+	RealD tmpa2 = lmd[k+1];
+	RealD tmpb  = lme[k];
+
+	lmd[k]   = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb;
+	lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb;
+	lme[k]   = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb;
+	lme[k-1] = c*lme[k-1] -s*x;
+
+	if(k != kmax-2){
+	  x = -s*lme[k+1];
+	  lme[k+1] = c*lme[k+1];
+	}
+
+	for(int i=0; i<Nk; ++i){
+	  RealD Qtmp1 = Qt[i+Nm*k    ];
+	  RealD Qtmp2 = Qt[i+Nm*(k+1)];
+	  Qt[i+Nm*k    ] = c*Qtmp1 -s*Qtmp2;
+	  Qt[i+Nm*(k+1)] = s*Qtmp1 +c*Qtmp2;
+	}
+      }
+    }
+
+#ifdef USE_LAPACK_IRL
+#define LAPACK_INT int
+//long long
+    void diagonalize_lapack(std::vector<RealD>& lmd,
+			    std::vector<RealD>& lme, 
+			    int N1,
+			    int N2,
+			    std::vector<RealD>& Qt,
+			    GridBase *grid){
+
+      std::cout << GridLogMessage << "diagonalize_lapack start\n";
+      GridStopWatch gsw;
+
+      const int size = Nm;
+      //  tevals.resize(size);
+      //  tevecs.resize(size);
+      LAPACK_INT NN = N1;
+      std::vector<double> evals_tmp(NN);
+      std::vector<double> evec_tmp(NN*NN);
+      memset(&evec_tmp[0],0,sizeof(double)*NN*NN);
+      //  double AA[NN][NN];
+      std::vector<double> DD(NN);
+      std::vector<double> EE(NN);
+      for (int i = 0; i< NN; i++)
+	for (int j = i - 1; j <= i + 1; j++)
+	  if ( j < NN && j >= 0 ) {
+	    if (i==j) DD[i] = lmd[i];
+	    if (i==j) evals_tmp[i] = lmd[i];
+	    if (j==(i-1)) EE[j] = lme[j];
+	  }
+      LAPACK_INT evals_found;
+      LAPACK_INT lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ;
+      LAPACK_INT liwork =  3+NN*10 ;
+      std::vector<LAPACK_INT> iwork(liwork);
+      std::vector<double> work(lwork);
+      std::vector<LAPACK_INT> isuppz(2*NN);
+      char jobz = 'V'; // calculate evals & evecs
+      char range = 'I'; // calculate all evals
+      //    char range = 'A'; // calculate all evals
+      char uplo = 'U'; // refer to upper half of original matrix
+      char compz = 'I'; // Compute eigenvectors of tridiagonal matrix
+      std::vector<int> ifail(NN);
+      LAPACK_INT info;
+      //  int total = QMP_get_number_of_nodes();
+      //  int node = QMP_get_node_number();
+      //  GridBase *grid = evec[0]._grid;
+      int total = grid->_Nprocessors;
+      int node = grid->_processor;
+      int interval = (NN/total)+1;
+      double vl = 0.0, vu = 0.0;
+      LAPACK_INT il = interval*node+1 , iu = interval*(node+1);
+      if (iu > NN)  iu=NN;
+      double tol = 0.0;
+      if (1) {
+	memset(&evals_tmp[0],0,sizeof(double)*NN);
+	if ( il <= NN){
+	  std::cout << GridLogMessage << "dstegr started" << std::endl; 
+	  gsw.Start();
+	  dstegr(&jobz, &range, &NN,
+		 (double*)&DD[0], (double*)&EE[0],
+		 &vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A'
+		 &tol, // tolerance
+		 &evals_found, &evals_tmp[0], (double*)&evec_tmp[0], &NN,
+		 &isuppz[0],
+		 &work[0], &lwork, &iwork[0], &liwork,
+		 &info);
+	  gsw.Stop();
+	  std::cout << GridLogMessage << "dstegr completed in " << gsw.Elapsed() << std::endl;
+	  for (int i = iu-1; i>= il-1; i--){
+	    evals_tmp[i] = evals_tmp[i - (il-1)];
+	    if (il>1) evals_tmp[i-(il-1)]=0.;
+	    for (int j = 0; j< NN; j++){
+	      evec_tmp[i*NN + j] = evec_tmp[(i - (il-1)) * NN + j];
+	      if (il>1) evec_tmp[(i-(il-1)) * NN + j]=0.;
+	    }
+	  }
+	}
+	{
+	  //        QMP_sum_double_array(evals_tmp,NN);
+	  //        QMP_sum_double_array((double *)evec_tmp,NN*NN);
+	  grid->GlobalSumVector(&evals_tmp[0],NN);
+	  grid->GlobalSumVector(&evec_tmp[0],NN*NN);
+	}
+      } 
+      // cheating a bit. It is better to sort instead of just reversing it, but the document of the routine says evals are sorted in increasing order. qr gives evals in decreasing order.
+      for(int i=0;i<NN;i++){
+	for(int j=0;j<NN;j++)
+	  Qt[(NN-1-i)*N2+j]=evec_tmp[i*NN + j];
+	lmd [NN-1-i]=evals_tmp[i];
+      }
+
+      std::cout << GridLogMessage << "diagonalize_lapack complete\n";
+    }
+#undef LAPACK_INT 
+#endif
+
+
+    void diagonalize(std::vector<RealD>& lmd,
+		     std::vector<RealD>& lme, 
+		     int N2,
+		     int N1,
+		     std::vector<RealD>& Qt,
+		     GridBase *grid)
+    {
+
+#ifdef USE_LAPACK_IRL
+    const int check_lapack=0; // just use lapack if 0, check against lapack if 1
+
+    if(!check_lapack)
+	return diagonalize_lapack(lmd,lme,N2,N1,Qt,grid);
+
+	std::vector <RealD> lmd2(N1);
+	std::vector <RealD> lme2(N1);
+	std::vector<RealD> Qt2(N1*N1);
+         for(int k=0; k<N1; ++k){
+	    lmd2[k] = lmd[k];
+	    lme2[k] = lme[k];
+	  }
+         for(int k=0; k<N1*N1; ++k)
+	Qt2[k] = Qt[k];
+
+//	diagonalize_lapack(lmd2,lme2,Nm2,Nm,Qt,grid);
+#endif
+
+      int Niter = 10000*N1;
+      int kmin = 1;
+      int kmax = N2;
+      // (this should be more sophisticated)
+
+      for(int iter=0; ; ++iter){
+      if ( (iter+1)%(100*N1)==0) 
+      std::cout<<GridLogMessage << "[QL method] Not converged - iteration "<<iter+1<<"\n";
+
+	// determination of 2x2 leading submatrix
+	RealD dsub = lmd[kmax-1]-lmd[kmax-2];
+	RealD dd = sqrt(dsub*dsub + 4.0*lme[kmax-2]*lme[kmax-2]);
+	RealD Dsh = 0.5*(lmd[kmax-2]+lmd[kmax-1] +dd*(dsub/fabs(dsub)));
+	// (Dsh: shift)
+	
+	// transformation
+	qr_decomp(lmd,lme,N2,N1,Qt,Dsh,kmin,kmax);
+	
+	// Convergence criterion (redef of kmin and kamx)
+	for(int j=kmax-1; j>= kmin; --j){
+	  RealD dds = fabs(lmd[j-1])+fabs(lmd[j]);
+	  if(fabs(lme[j-1])+dds > dds){
+	    kmax = j+1;
+	    goto continued;
+	  }
+	}
+	Niter = iter;
+#ifdef USE_LAPACK_IRL
+    if(check_lapack){
+	const double SMALL=1e-8;
+	diagonalize_lapack(lmd2,lme2,N2,N1,Qt2,grid);
+	std::vector <RealD> lmd3(N2);
+         for(int k=0; k<N2; ++k) lmd3[k]=lmd[k];
+        _sort.push(lmd3,N2);
+        _sort.push(lmd2,N2);
+         for(int k=0; k<N2; ++k){
+	    if (fabs(lmd2[k] - lmd3[k]) >SMALL)  std::cout<<GridLogMessage <<"lmd(qr) lmd(lapack) "<< k << ": " << lmd2[k] <<" "<< lmd3[k] <<std::endl;
+//	    if (fabs(lme2[k] - lme[k]) >SMALL)  std::cout<<GridLogMessage <<"lme(qr)-lme(lapack) "<< k << ": " << lme2[k] - lme[k] <<std::endl;
+	  }
+         for(int k=0; k<N1*N1; ++k){
+//	    if (fabs(Qt2[k] - Qt[k]) >SMALL)  std::cout<<GridLogMessage <<"Qt(qr)-Qt(lapack) "<< k << ": " << Qt2[k] - Qt[k] <<std::endl;
+	}
+    }
+#endif
+	return;
+
+      continued:
+	for(int j=0; j<kmax-1; ++j){
+	  RealD dds = fabs(lmd[j])+fabs(lmd[j+1]);
+	  if(fabs(lme[j])+dds > dds){
+	    kmin = j+1;
+	    break;
+	  }
+	}
+      }
+      std::cout<<GridLogMessage << "[QL method] Error - Too many iteration: "<<Niter<<"\n";
+      abort();
+    }
+
+#if 1
+    template<typename T>
+    static RealD normalise(T& v) 
+    {
+      RealD nn = norm2(v);
+      nn = sqrt(nn);
+      v = v * (1.0/nn);
+      return nn;
+    }
+
+    void orthogonalize(Field& w,
+		       BasisFieldVector<Field>& evec,
+		       int k)
+    {
+      double t0=-usecond()/1e6;
+
+      evec.orthogonalize(w,k);
+
+      normalise(w);
+      t0+=usecond()/1e6;
+      OrthoTime +=t0;
+    }
+
+    void setUnit_Qt(int Nm, std::vector<RealD> &Qt) {
+      for(int i=0; i<Qt.size(); ++i) Qt[i] = 0.0;
+      for(int k=0; k<Nm; ++k) Qt[k + k*Nm] = 1.0;
+    }
+
+/* Rudy Arthur's thesis pp.137
+------------------------
+Require: M > K P = M − K †
+Compute the factorization AVM = VM HM + fM eM 
+repeat
+  Q=I
+  for i = 1,...,P do
+    QiRi =HM −θiI Q = QQi
+    H M = Q †i H M Q i
+  end for
+  βK =HM(K+1,K) σK =Q(M,K)
+  r=vK+1βK +rσK
+  VK =VM(1:M)Q(1:M,1:K)
+  HK =HM(1:K,1:K)
+  →AVK =VKHK +fKe†K † Extend to an M = K + P step factorization AVM = VMHM + fMeM
+until convergence
+*/
+
+    void calc(std::vector<RealD>& eval,
+	      BasisFieldVector<Field>& evec,
+	      const Field& src,
+	      int& Nconv,
+	      bool reverse,
+	      int SkipTest)
+      {
+
+	GridBase *grid = evec._v[0]._grid;//evec.get(0 + evec_offset)._grid;
+	assert(grid == src._grid);
+
+	std::cout<<GridLogMessage << " -- Nk = " << Nk << " Np = "<< Np << std::endl;
+	std::cout<<GridLogMessage << " -- Nm = " << Nm << std::endl;
+	std::cout<<GridLogMessage << " -- size of eval   = " << eval.size() << std::endl;
+	std::cout<<GridLogMessage << " -- size of evec  = " << evec.size() << std::endl;
+	
+	assert(Nm <= evec.size() && Nm <= eval.size());
+
+	// quickly get an idea of the largest eigenvalue to more properly normalize the residuum
+	RealD evalMaxApprox = 0.0;
+	{
+	  auto src_n = src;
+	  auto tmp = src;
+	  const int _MAX_ITER_IRL_MEVAPP_ = 50;
+	  for (int i=0;i<_MAX_ITER_IRL_MEVAPP_;i++) {
+	    _HermOpTest(src_n,tmp);
+	    RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
+	    RealD vden = norm2(src_n);
+	    RealD na = vnum/vden;
+	    if (fabs(evalMaxApprox/na - 1.0) < 0.05)
+	      i=_MAX_ITER_IRL_MEVAPP_;
+	    evalMaxApprox = na;
+	    std::cout << GridLogMessage << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
+	    src_n = tmp;
+	  }
+	}
+	
+	std::vector<RealD> lme(Nm);  
+	std::vector<RealD> lme2(Nm);
+	std::vector<RealD> eval2(Nm);
+	std::vector<RealD> eval2_copy(Nm);
+	std::vector<RealD> Qt(Nm*Nm);
+
+
+	Field f(grid);
+	Field v(grid);
+  
+	int k1 = 1;
+	int k2 = Nk;
+
+	Nconv = 0;
+
+	RealD beta_k;
+  
+	// Set initial vector
+	evec[0] = src;
+	normalise(evec[0]);
+	std:: cout<<GridLogMessage <<"norm2(evec[0])= " << norm2(evec[0])<<std::endl;
+	
+	// Initial Nk steps
+	OrthoTime=0.;
+	double t0=usecond()/1e6;
+	for(int k=0; k<Nk; ++k) step(eval,lme,evec,f,Nm,k);
+	double t1=usecond()/1e6;
+	std::cout<<GridLogMessage <<"IRL::Initial steps: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	std::cout<<GridLogMessage <<"IRL::Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
+	t1=usecond()/1e6;
+
+	// Restarting loop begins
+	for(int iter = 0; iter<Niter; ++iter){
+	  
+	  std::cout<<GridLogMessage<<"\n Restart iteration = "<< iter << std::endl;
+	  
+	  // 
+	  // Rudy does a sort first which looks very different. Getting fed up with sorting out the algo defs.
+	  // We loop over 
+	  //
+	  OrthoTime=0.;
+	  for(int k=Nk; k<Nm; ++k) step(eval,lme,evec,f,Nm,k);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL:: "<<Np <<" steps: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  std::cout<<GridLogMessage <<"IRL::Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
+	  f *= lme[Nm-1];
+	  
+	  t1=usecond()/1e6;
+
+	  
+	  // getting eigenvalues
+	  for(int k=0; k<Nm; ++k){
+	    eval2[k] = eval[k+k1-1];
+	    lme2[k] = lme[k+k1-1];
+	  }
+	  setUnit_Qt(Nm,Qt);
+	  diagonalize(eval2,lme2,Nm,Nm,Qt,grid);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL:: diagonalize: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  
+	  // sorting
+	  eval2_copy = eval2;
+
+	  _sort.push(eval2,Nm);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL:: eval sorting: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  
+	  // Implicitly shifted QR transformations
+	  setUnit_Qt(Nm,Qt);
+	  for(int ip=0; ip<k2; ++ip){
+	    std::cout<<GridLogMessage << "eval "<< ip << " "<< eval2[ip] << std::endl;
+	  }
+
+	  for(int ip=k2; ip<Nm; ++ip){ 
+	    std::cout<<GridLogMessage << "qr_decomp "<< ip << " "<< eval2[ip] << std::endl;
+	    qr_decomp(eval,lme,Nm,Nm,Qt,eval2[ip],k1,Nm);
+	    
+	  }
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL::qr_decomp: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  assert(k2<Nm);
+	  
+
+	  assert(k2<Nm);
+	  assert(k1>0);
+	  evec.rotate(Qt,k1-1,k2+1,0,Nm,Nm);
+	  
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL::QR rotation: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  fflush(stdout);
+	  
+	  // Compressed vector f and beta(k2)
+	  f *= Qt[Nm-1+Nm*(k2-1)];
+	  f += lme[k2-1] * evec[k2];
+	  beta_k = norm2(f);
+	  beta_k = sqrt(beta_k);
+	  std::cout<<GridLogMessage<<" beta(k) = "<<beta_k<<std::endl;
+	  
+	  RealD betar = 1.0/beta_k;
+	  evec[k2] = betar * f;
+	  lme[k2-1] = beta_k;
+	  
+	  // Convergence test
+	  for(int k=0; k<Nm; ++k){    
+	    eval2[k] = eval[k];
+	    lme2[k] = lme[k];
+
+	    std::cout<<GridLogMessage << "eval2[" << k << "] = " << eval2[k] << std::endl;
+	  }
+	  setUnit_Qt(Nm,Qt);
+	  diagonalize(eval2,lme2,Nk,Nm,Qt,grid);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL::diagonalize: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  
+	  
+	  Nconv = 0;
+	  
+	  if (iter >= Nminres) {
+	    std::cout << GridLogMessage << "Rotation to test convergence " << std::endl;
+	    
+	    Field ev0_orig(grid);
+	    ev0_orig = evec[0];
+	    
+	    evec.rotate(Qt,0,Nk,0,Nk,Nm);
+	    
+	    {
+	      std::cout << GridLogMessage << "Test convergence" << std::endl;
+	      Field B(grid);
+	      
+	      for(int j = 0; j<Nk; j+=SkipTest){
+		B=evec[j];
+		//std::cout << "Checkerboard: " << evec[j].checkerboard << std::endl; 
+		B.checkerboard = evec[0].checkerboard;
+
+		_HermOpTest(B,v);
+		
+		RealD vnum = real(innerProduct(B,v)); // HermOp.
+		RealD vden = norm2(B);
+		RealD vv0 = norm2(v);
+		eval2[j] = vnum/vden;
+		v -= eval2[j]*B;
+		RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
+		std::cout.precision(13);
+		std::cout<<GridLogMessage << "[" << std::setw(3)<< std::setiosflags(std::ios_base::right) <<j<<"] "
+			 <<"eval = "<<std::setw(25)<< std::setiosflags(std::ios_base::left)<< eval2[j] << " (" << eval2_copy[j] << ")"
+			 <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25)<< std::setiosflags(std::ios_base::right)<< vv
+			 <<" "<< vnum/(sqrt(vden)*sqrt(vv0))
+			 << " norm(B["<<j<<"])="<< vden <<std::endl;
+		
+		// change the criteria as evals are supposed to be sorted, all evals smaller(larger) than Nstop should have converged
+		if((vv<eresid*eresid) && (j == Nconv) ){
+		  Nconv+=SkipTest;
+		}
+	      }
+	      
+	      // test if we converged, if so, terminate
+	      t1=usecond()/1e6;
+	      std::cout<<GridLogMessage <<"IRL::convergence testing: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	      
+	      std::cout<<GridLogMessage<<" #modes converged: "<<Nconv<<std::endl;
+	      
+	      if( Nconv>=Nstop || beta_k < betastp){
+		goto converged;
+	      }
+	      
+	      std::cout << GridLogMessage << "Rotate back" << std::endl;
+	      //B[j] +=Qt[k+_Nm*j] * _v[k]._odata[ss];
+	      {
+		Eigen::MatrixXd qm = Eigen::MatrixXd::Zero(Nk,Nk);
+		for (int k=0;k<Nk;k++)
+		  for (int j=0;j<Nk;j++)
+		    qm(j,k) = Qt[k+Nm*j];
+		GridStopWatch timeInv;
+		timeInv.Start();
+		Eigen::MatrixXd qmI = qm.inverse();
+		timeInv.Stop();
+		std::vector<RealD> QtI(Nm*Nm);
+		for (int k=0;k<Nk;k++)
+		  for (int j=0;j<Nk;j++)
+		    QtI[k+Nm*j] = qmI(j,k);
+		
+		RealD res_check_rotate_inverse = (qm*qmI - Eigen::MatrixXd::Identity(Nk,Nk)).norm(); // sqrt( |X|^2 )
+		assert(res_check_rotate_inverse < 1e-7);
+		evec.rotate(QtI,0,Nk,0,Nk,Nm);
+		
+		axpy(ev0_orig,-1.0,evec[0],ev0_orig);
+		std::cout << GridLogMessage << "Rotation done (in " << timeInv.Elapsed() << " = " << timeInv.useconds() << " us" <<
+		  ", error = " << res_check_rotate_inverse << 
+		  "); | evec[0] - evec[0]_orig | = " << ::sqrt(norm2(ev0_orig)) << std::endl;
+	      }
+	    }
+	  } else {
+	    std::cout << GridLogMessage << "iter < Nminres: do not yet test for convergence\n";
+	  } // end of iter loop
+	}
+
+	std::cout<<GridLogMessage<<"\n NOT converged.\n";
+	abort();
+	
+      converged:
+
+	if (SkipTest == 1) {
+	  eval = eval2;
+	} else {
+
+	  // test quickly
+	  for (int j=0;j<Nstop;j+=SkipTest) {
+	    std::cout<<GridLogMessage << "Eigenvalue[" << j << "] = " << eval2[j] << " (" << eval2_copy[j] << ")" << std::endl;
+	  }
+
+	  eval2_copy.resize(eval2.size());
+	  eval = eval2_copy;
+	}
+
+	evec.sortInPlace(eval,reverse);
+
+	{
+	  
+	 // test
+	 for (int j=0;j<Nstop;j++) {
+	   std::cout<<GridLogMessage << " |e[" << j << "]|^2 = " << norm2(evec[j]) << std::endl;
+	 }
+       }
+       
+       //_sort.push(eval,evec,Nconv);
+       //evec.sort(eval,Nconv);
+       
+       std::cout<<GridLogMessage << "\n Converged\n Summary :\n";
+       std::cout<<GridLogMessage << " -- Iterations  = "<< Nconv  << "\n";
+       std::cout<<GridLogMessage << " -- beta(k)     = "<< beta_k << "\n";
+       std::cout<<GridLogMessage << " -- Nconv       = "<< Nconv  << "\n";
+      }
+#endif
+
+    };
+
+}
+#endif
+

lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h

@@ -0,0 +1,163 @@
+namespace Grid { 
+
+template<class Field>
+class BasisFieldVector {
+ public:
+  int _Nm;
+
+  typedef typename Field::scalar_type Coeff_t;
+  typedef typename Field::vector_type vCoeff_t;
+  typedef typename Field::vector_object vobj;
+  typedef typename vobj::scalar_object sobj;
+
+  std::vector<Field> _v; // _Nfull vectors
+
+  void report(int n,GridBase* value) {
+
+    std::cout << GridLogMessage << "BasisFieldVector allocated:\n";
+    std::cout << GridLogMessage << " Delta N = " << n << "\n";
+    std::cout << GridLogMessage << " Size of full vectors (size) = " << 
+      ((double)n*sizeof(vobj)*value->oSites() / 1024./1024./1024.) << " GB\n";
+    std::cout << GridLogMessage << " Size = " << _v.size() << " Capacity = " << _v.capacity() << std::endl;
+
+    value->Barrier();
+
+    if (value->IsBoss()) {
+      system("cat /proc/meminfo");
+    }
+
+    value->Barrier();
+
+  }
+
+  BasisFieldVector(int Nm,GridBase* value) : _Nm(Nm), _v(Nm,value) {
+    report(Nm,value);
+  }
+  
+  ~BasisFieldVector() {
+  }
+
+  Field& operator[](int i) {
+    return _v[i];
+  }
+
+  void orthogonalize(Field& w, int k) {
+    for(int j=0; j<k; ++j){
+      Coeff_t ip = (Coeff_t)innerProduct(_v[j],w);
+      w = w - ip*_v[j];
+    }
+  }
+
+  void rotate(std::vector<RealD>& Qt,int j0, int j1, int k0,int k1,int Nm) {
+    
+    GridBase* grid = _v[0]._grid;
+      
+#pragma omp parallel
+    {
+      std::vector < vobj > B(Nm);
+      
+#pragma omp for
+      for(int ss=0;ss < grid->oSites();ss++){
+	for(int j=j0; j<j1; ++j) B[j]=0.;
+	
+	for(int j=j0; j<j1; ++j){
+	  for(int k=k0; k<k1; ++k){
+	    B[j] +=Qt[k+Nm*j] * _v[k]._odata[ss];
+	  }
+	}
+	for(int j=j0; j<j1; ++j){
+	  _v[j]._odata[ss] = B[j];
+	}
+      }
+    }
+
+  }
+
+  size_t size() const {
+    return _Nm;
+  }
+
+  void resize(int n) {
+    if (n > _Nm)
+      _v.reserve(n);
+    
+    _v.resize(n,_v[0]._grid);
+
+    if (n < _Nm)
+      _v.shrink_to_fit();
+
+    report(n - _Nm,_v[0]._grid);
+
+    _Nm = n;
+  }
+
+  std::vector<int> getIndex(std::vector<RealD>& sort_vals) {
+
+    std::vector<int> idx(sort_vals.size());
+    iota(idx.begin(), idx.end(), 0);
+
+    // sort indexes based on comparing values in v
+    sort(idx.begin(), idx.end(),
+	 [&sort_vals](int i1, int i2) {return ::fabs(sort_vals[i1]) < ::fabs(sort_vals[i2]);});
+
+    return idx;
+  }
+
+  void reorderInPlace(std::vector<RealD>& sort_vals, std::vector<int>& idx) {
+    GridStopWatch gsw;
+    gsw.Start();
+
+    int nswaps = 0;
+    for (size_t i=0;i<idx.size();i++) {
+      if (idx[i] != i) {
+
+	// find proper place (this could be done in logarithmic time, don't bother for now)
+	size_t j;
+	for (j=i;j<idx.size();j++)
+	  if (idx[j]==i)
+	    break;
+	assert(j!=idx.size());
+	
+	Field _t(_v[0]._grid);
+	_t = _v[idx[j]];
+	_v[idx[j]] = _v[idx[i]];
+	_v[idx[i]] = _t;
+
+	RealD _td = sort_vals[idx[j]];
+	sort_vals[idx[j]] = sort_vals[idx[i]];
+	sort_vals[idx[i]] = _td;
+
+	int _tt = idx[i];
+	idx[i] = idx[j];
+	idx[j] = _tt;
+	
+	nswaps++;
+      }
+    }
+
+    // sort values
+    gsw.Stop();
+    std::cout << GridLogMessage << "Sorted eigenspace in place in " << gsw.Elapsed() << " using " << nswaps << " swaps" << std::endl;
+  }
+
+  void sortInPlace(std::vector<RealD>& sort_vals, bool reverse) {
+
+    std::vector<int> idx = getIndex(sort_vals);
+    if (reverse)
+      std::reverse(idx.begin(), idx.end());
+
+    reorderInPlace(sort_vals,idx);
+
+  }
+
+  void deflate(const std::vector<RealD>& eval,const Field& src_orig,Field& result) {
+    result = zero;
+    int N = (int)_v.size();
+    for (int i=0;i<N;i++) {
+      Field& tmp = _v[i];
+      axpy(result,TensorRemove(innerProduct(tmp,src_orig)) / eval[i],tmp,result);
+    }
+  }
+
+ }; 
+}

lib/algorithms/iterative/ConjugateGradient.h

@@ -78,12 +78,12 @@ class ConjugateGradient : public OperatorFunction<Field> {
     cp = a;
     ssq = norm2(src);
 
-    std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: guess " << guess << std::endl;
-    std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient:   src " << ssq << std::endl;
-    std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient:    mp " << d << std::endl;
-    std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient:   mmp " << b << std::endl;
-    std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient:  cp,r " << cp << std::endl;
-    std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient:     p " << a << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient: guess " << guess << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:   src " << ssq << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:    mp " << d << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:   mmp " << b << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:  cp,r " << cp << std::endl;
+    std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradient:     p " << a << std::endl;
 
     RealD rsq = Tolerance * Tolerance * ssq;
 
@@ -92,7 +92,7 @@ class ConjugateGradient : public OperatorFunction<Field> {
       return;
     }
 
-    std::cout << GridLogIterative << std::setprecision(8)
+    std::cout << GridLogIterative << std::setprecision(4)
               << "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
 
     GridStopWatch LinalgTimer;

lib/algorithms/iterative/ConjugateGradientMixedPrec.h

@@ -60,6 +60,7 @@ namespace Grid {
     }
   
     void operator() (const FieldD &src_d_in, FieldD &sol_d){
+
       TotalInnerIterations = 0;
 	
       GridStopWatch TotalTimer;

lib/algorithms/iterative/ImplicitlyRestartedLanczos.h

@@ -7,9 +7,8 @@
     Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-Author: Chulwoo Jung <chulwoo@bnl.gov>
-Author: Christoph Lehner <clehner@bnl.gov>
+Author: Chulwoo Jung
+Author: Guido Cossu
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -28,288 +27,125 @@ Author: Christoph Lehner <clehner@bnl.gov>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-#ifndef GRID_BIRL_H
-#define GRID_BIRL_H
+#ifndef GRID_IRL_H
+#define GRID_IRL_H
 
 #include <string.h> //memset
-//#include <zlib.h>
-#include <sys/stat.h>
 
-namespace Grid { 
+namespace Grid {
 
-  ////////////////////////////////////////////////////////
-  // Move following 100 LOC to lattice/Lattice_basis.h
-  ////////////////////////////////////////////////////////
+  enum IRLdiagonalisation { 
+    IRLdiagonaliseWithDSTEGR,
+    IRLdiagonaliseWithQR,
+    IRLdiagonaliseWithEigen
+  };
+
+////////////////////////////////////////////////////////////////////////////////
+// Helper class for sorting the evalues AND evectors by Field
+// Use pointer swizzle on vectors
+////////////////////////////////////////////////////////////////////////////////
 template<class Field>
-void basisOrthogonalize(std::vector<Field> &basis,Field &w,int k) 
-{
-  for(int j=0; j<k; ++j){
-    auto ip = innerProduct(basis[j],w);
-    w = w - ip*basis[j];
-  }
-}
-
-template<class Field>
-void basisRotate(std::vector<Field> &basis,Eigen::MatrixXd& Qt,int j0, int j1, int k0,int k1,int Nm) 
-{
-  typedef typename Field::vector_object vobj;
-  GridBase* grid = basis[0]._grid;
-      
-  parallel_region
-  {
-    std::vector < vobj > B(Nm); // Thread private
-        
-    parallel_for_internal(int ss=0;ss < grid->oSites();ss++){
-      for(int j=j0; j<j1; ++j) B[j]=0.;
-      
-      for(int j=j0; j<j1; ++j){
-	for(int k=k0; k<k1; ++k){
-	  B[j] +=Qt(j,k) * basis[k]._odata[ss];
-	}
-      }
-      for(int j=j0; j<j1; ++j){
-	  basis[j]._odata[ss] = B[j];
-      }
-    }
-  }
-}
-
-// Extract a single rotated vector
-template<class Field>
-void basisRotateJ(Field &result,std::vector<Field> &basis,Eigen::MatrixXd& Qt,int j, int k0,int k1,int Nm) 
-{
-  typedef typename Field::vector_object vobj;
-  GridBase* grid = basis[0]._grid;
-
-  result.checkerboard = basis[0].checkerboard;
-  parallel_for(int ss=0;ss < grid->oSites();ss++){
-    vobj B = zero;
-    for(int k=k0; k<k1; ++k){
-      B +=Qt(j,k) * basis[k]._odata[ss];
-    }
-    result._odata[ss] = B;
-  }
-}
-
-template<class Field>
-void basisReorderInPlace(std::vector<Field> &_v,std::vector<RealD>& sort_vals, std::vector<int>& idx) 
-{
-  int vlen = idx.size();
-
-  assert(vlen>=1);
-  assert(vlen<=sort_vals.size());
-  assert(vlen<=_v.size());
-
-  for (size_t i=0;i<vlen;i++) {
-
-    if (idx[i] != i) {
-
-      //////////////////////////////////////
-      // idx[i] is a table of desired sources giving a permutation.
-      // Swap v[i] with v[idx[i]].
-      // Find  j>i for which _vnew[j] = _vold[i],
-      // track the move idx[j] => idx[i]
-      // track the move idx[i] => i
-      //////////////////////////////////////
-      size_t j;
-      for (j=i;j<idx.size();j++)
-	if (idx[j]==i)
-	  break;
-
-      assert(idx[i] > i);     assert(j!=idx.size());      assert(idx[j]==i);
-
-      std::swap(_v[i]._odata,_v[idx[i]]._odata); // should use vector move constructor, no data copy
-      std::swap(sort_vals[i],sort_vals[idx[i]]);
-
-      idx[j] = idx[i];
-      idx[i] = i;
-    }
-  }
-}
-
-inline std::vector<int> basisSortGetIndex(std::vector<RealD>& sort_vals) 
-{
-  std::vector<int> idx(sort_vals.size());
-  std::iota(idx.begin(), idx.end(), 0);
-
-  // sort indexes based on comparing values in v
-  std::sort(idx.begin(), idx.end(), [&sort_vals](int i1, int i2) {
-    return ::fabs(sort_vals[i1]) < ::fabs(sort_vals[i2]);
-  });
-  return idx;
-}
-
-template<class Field>
-void basisSortInPlace(std::vector<Field> & _v,std::vector<RealD>& sort_vals, bool reverse) 
-{
-  std::vector<int> idx = basisSortGetIndex(sort_vals);
-  if (reverse)
-    std::reverse(idx.begin(), idx.end());
+class SortEigen {
+ private:
+  static bool less_lmd(RealD left,RealD right){
+    return left > right;
+  }  
+  static bool less_pair(std::pair<RealD,Field const*>& left,
+                        std::pair<RealD,Field const*>& right){
+    return left.first > (right.first);
+  }  
   
-  basisReorderInPlace(_v,sort_vals,idx);
-}
+ public:
+  void push(std::vector<RealD>& lmd,std::vector<Field>& evec,int N) {
+    
+    ////////////////////////////////////////////////////////////////////////
+    // PAB: FIXME: VERY VERY VERY wasteful: takes a copy of the entire vector set.
+    //    : The vector reorder should be done by pointer swizzle somehow
+    ////////////////////////////////////////////////////////////////////////
+    std::vector<Field> cpy(lmd.size(),evec[0]._grid);
+    for(int i=0;i<lmd.size();i++) cpy[i] = evec[i];
+    
+    std::vector<std::pair<RealD, Field const*> > emod(lmd.size());    
 
-// PAB: faster to compute the inner products first then fuse loops.
-// If performance critical can improve.
-template<class Field>
-void basisDeflate(const std::vector<Field> &_v,const std::vector<RealD>& eval,const Field& src_orig,Field& result) {
-  result = zero;
-  assert(_v.size()==eval.size());
-  int N = (int)_v.size();
-  for (int i=0;i<N;i++) {
-    Field& tmp = _v[i];
-    axpy(result,TensorRemove(innerProduct(tmp,src_orig)) / eval[i],tmp,result);
+    for(int i=0;i<lmd.size();++i)  emod[i] = std::pair<RealD,Field const*>(lmd[i],&cpy[i]);
+
+    partial_sort(emod.begin(),emod.begin()+N,emod.end(),less_pair);
+
+    typename std::vector<std::pair<RealD, Field const*> >::iterator it = emod.begin();
+    for(int i=0;i<N;++i){
+      lmd[i]=it->first;
+      evec[i]=*(it->second);
+      ++it;
+    }
   }
-}
+  void push(std::vector<RealD>& lmd,int N) {
+    std::partial_sort(lmd.begin(),lmd.begin()+N,lmd.end(),less_lmd);
+  }
+  bool saturated(RealD lmd, RealD thrs) {
+    return fabs(lmd) > fabs(thrs);
+  }
+};
 
 /////////////////////////////////////////////////////////////
 // Implicitly restarted lanczos
 /////////////////////////////////////////////////////////////
-template<class Field> class ImplicitlyRestartedLanczosTester 
-{
- public:
-  virtual int TestConvergence(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox)=0;
-  virtual int ReconstructEval(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox)=0;
-};
-
-enum IRLdiagonalisation { 
-  IRLdiagonaliseWithDSTEGR,
-  IRLdiagonaliseWithQR,
-  IRLdiagonaliseWithEigen
-};
-
-template<class Field> class ImplicitlyRestartedLanczosHermOpTester  : public ImplicitlyRestartedLanczosTester<Field>
-{
- public:
-  LinearFunction<Field>       &_HermOp;
-  ImplicitlyRestartedLanczosHermOpTester(LinearFunction<Field> &HermOp) : _HermOp(HermOp)  {  };
-  int ReconstructEval(int j,RealD resid,Field &B, RealD &eval,RealD evalMaxApprox)
-  {
-    return TestConvergence(j,resid,B,eval,evalMaxApprox);
-  }
-  int TestConvergence(int j,RealD eresid,Field &B, RealD &eval,RealD evalMaxApprox)
-  {
-    Field v(B);
-    RealD eval_poly = eval;
-    // Apply operator
-    _HermOp(B,v);
-
-    RealD vnum = real(innerProduct(B,v)); // HermOp.
-    RealD vden = norm2(B);
-    RealD vv0  = norm2(v);
-    eval   = vnum/vden;
-    v -= eval*B;
-
-    RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
-
-    std::cout.precision(13);
-    std::cout<<GridLogIRL  << "[" << std::setw(3)<<j<<"] "
-	     <<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
-	     <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
-	     <<std::endl;
-
-    int conv=0;
-    if( (vv<eresid*eresid) ) conv = 1;
-
-    return conv;
-  }
-};
-
 template<class Field> 
 class ImplicitlyRestartedLanczos {
- private:
-  const RealD small = 1.0e-8;
-  int MaxIter;
-  int MinRestart; // Minimum number of restarts; only check for convergence after
-  int Nstop;   // Number of evecs checked for convergence
-  int Nk;      // Number of converged sought
-  //  int Np;      // Np -- Number of spare vecs in krylov space //  == Nm - Nk
-  int Nm;      // Nm -- total number of vectors
+
+private:       
+
+  int MaxIter;   // Max iterations
+  int Nstop;     // Number of evecs checked for convergence
+  int Nk;        // Number of converged sought
+  int Nm;        // Nm -- total number of vectors
+  RealD eresid;
   IRLdiagonalisation diagonalisation;
-  int orth_period;
-    
-  RealD OrthoTime;
-  RealD eresid, betastp;
-  ////////////////////////////////
+  ////////////////////////////////////
   // Embedded objects
-  ////////////////////////////////
-  LinearFunction<Field>       &_PolyOp;
-  LinearFunction<Field>       &_HermOp;
-  ImplicitlyRestartedLanczosTester<Field> &_Tester;
-  // Default tester provided (we need a ref to something in default case)
-  ImplicitlyRestartedLanczosHermOpTester<Field> SimpleTester;
+  ////////////////////////////////////
+           SortEigen<Field> _sort;
+  LinearOperatorBase<Field> &_Linop;
+    OperatorFunction<Field> &_poly;
+
   /////////////////////////
   // Constructor
   /////////////////////////
-  
 public:       
-  //////////////////////////////////////////////////////////////////
-  // PAB:
-  //////////////////////////////////////////////////////////////////
-  // Too many options  & knobs. 
-  // Eliminate:
-  //   orth_period
-  //   betastp
-  //   MinRestart
-  //
-  // Do we really need orth_period
-  // What is the theoretical basis & guarantees of betastp ?
-  // Nstop=Nk viable?
-  // MinRestart avoidable with new convergence test?
-  // Could cut to PolyOp, HermOp, Tester, Nk, Nm, resid, maxiter (+diagonalisation)
-  // HermOp could be eliminated if we dropped the Power method for max eval.
-  // -- also: The eval, eval2, eval2_copy stuff is still unnecessarily unclear
-  //////////////////////////////////////////////////////////////////
- ImplicitlyRestartedLanczos(LinearFunction<Field> & PolyOp,
-			    LinearFunction<Field> & HermOp,
-			    ImplicitlyRestartedLanczosTester<Field> & Tester,
-			    int _Nstop, // sought vecs
-			    int _Nk, // sought vecs
-			    int _Nm, // spare vecs
-			    RealD _eresid, // resid in lmdue deficit 
-			    int _MaxIter, // Max iterations
-			    RealD _betastp=0.0, // if beta(k) < betastp: converged
-			    int _MinRestart=1, int _orth_period = 1,
-			    IRLdiagonalisation _diagonalisation= IRLdiagonaliseWithEigen) :
-    SimpleTester(HermOp), _PolyOp(PolyOp),      _HermOp(HermOp), _Tester(Tester),
-    Nstop(_Nstop)  ,      Nk(_Nk),      Nm(_Nm),
-    eresid(_eresid),      betastp(_betastp),
-    MaxIter(_MaxIter)  ,      MinRestart(_MinRestart),
-    orth_period(_orth_period), diagonalisation(_diagonalisation)  { };
-
-    ImplicitlyRestartedLanczos(LinearFunction<Field> & PolyOp,
-			       LinearFunction<Field> & HermOp,
-			       int _Nstop, // sought vecs
-			       int _Nk, // sought vecs
-			       int _Nm, // spare vecs
-			       RealD _eresid, // resid in lmdue deficit 
-			       int _MaxIter, // Max iterations
-			       RealD _betastp=0.0, // if beta(k) < betastp: converged
-			       int _MinRestart=1, int _orth_period = 1,
-			       IRLdiagonalisation _diagonalisation= IRLdiagonaliseWithEigen) :
-    SimpleTester(HermOp),  _PolyOp(PolyOp),      _HermOp(HermOp), _Tester(SimpleTester),
-    Nstop(_Nstop)  ,      Nk(_Nk),      Nm(_Nm),
-    eresid(_eresid),      betastp(_betastp),
-    MaxIter(_MaxIter)  ,      MinRestart(_MinRestart),
-    orth_period(_orth_period), diagonalisation(_diagonalisation)  { };
+ ImplicitlyRestartedLanczos(LinearOperatorBase<Field> &Linop, // op
+			    OperatorFunction<Field> & poly,   // polynomial
+			    int _Nstop, // really sought vecs
+			    int _Nk,    // sought vecs
+			    int _Nm,    // total vecs
+			    RealD _eresid, // resid in lmd deficit 
+			    int _MaxIter,  // Max iterations
+			    IRLdiagonalisation _diagonalisation= IRLdiagonaliseWithEigen ) :
+    _Linop(Linop),    _poly(poly),
+      Nstop(_Nstop), Nk(_Nk), Nm(_Nm),
+      eresid(_eresid),  MaxIter(_MaxIter),
+      diagonalisation(_diagonalisation)
+      { };
 
   ////////////////////////////////
   // Helpers
   ////////////////////////////////
-  template<typename T>  static RealD normalise(T& v) 
+  static RealD normalise(Field& v) 
   {
     RealD nn = norm2(v);
     nn = sqrt(nn);
     v = v * (1.0/nn);
     return nn;
   }
-
-  void orthogonalize(Field& w, std::vector<Field>& evec,int k)
+  
+  void orthogonalize(Field& w, std::vector<Field>& evec, int k)
   {
-    OrthoTime-=usecond()/1e6;
-    basisOrthogonalize(evec,w,k);
+    typedef typename Field::scalar_type MyComplex;
+    MyComplex ip;
+    
+    for(int j=0; j<k; ++j){
+      ip = innerProduct(evec[j],w); 
+      w = w - ip * evec[j];
+    }
     normalise(w);
-    OrthoTime+=usecond()/1e6;
   }
 
 /* Rudy Arthur's thesis pp.137
@@ -329,238 +165,184 @@ repeat
   →AVK =VKHK +fKe†K † Extend to an M = K + P step factorization AVM = VMHM + fMeM
 until convergence
 */
-  void calc(std::vector<RealD>& eval, std::vector<Field>& evec,  const Field& src, int& Nconv, bool reverse=false)
+  void calc(std::vector<RealD>& eval,  std::vector<Field>& evec, const Field& src, int& Nconv)
   {
-    GridBase *grid = src._grid;
-    assert(grid == evec[0]._grid);
     
-    GridLogIRL.TimingMode(1);
-    std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
-    std::cout << GridLogIRL <<" ImplicitlyRestartedLanczos::calc() starting iteration 0 /  "<< MaxIter<< std::endl;
-    std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
-    std::cout << GridLogIRL <<" -- seek   Nk    = " << Nk    <<" vectors"<< std::endl;
-    std::cout << GridLogIRL <<" -- accept Nstop = " << Nstop <<" vectors"<< std::endl;
-    std::cout << GridLogIRL <<" -- total  Nm    = " << Nm    <<" vectors"<< std::endl;
-    std::cout << GridLogIRL <<" -- size of eval = " << eval.size() << std::endl;
-    std::cout << GridLogIRL <<" -- size of evec = " << evec.size() << std::endl;
+    GridBase *grid = evec[0]._grid;
+    assert(grid == src._grid);
+    
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
+    std::cout << GridLogMessage <<" ImplicitlyRestartedLanczos::calc() starting iteration 0 /  "<< MaxIter<< std::endl;
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
+    std::cout << GridLogMessage <<" -- seek   Nk    = " << Nk    <<" vectors"<< std::endl;
+    std::cout << GridLogMessage <<" -- accept Nstop = " << Nstop <<" vectors"<< std::endl;
+    std::cout << GridLogMessage <<" -- total  Nm    = " << Nm    <<" vectors"<< std::endl;
+    std::cout << GridLogMessage <<" -- size of eval = " << eval.size() << std::endl;
+    std::cout << GridLogMessage <<" -- size of evec = " << evec.size() << std::endl;
     if ( diagonalisation == IRLdiagonaliseWithDSTEGR ) {
-      std::cout << GridLogIRL << "Diagonalisation is DSTEGR "<<std::endl;
+      std::cout << GridLogMessage << "Diagonalisation is DSTEGR "<<std::endl;
     } else if ( diagonalisation == IRLdiagonaliseWithQR ) { 
-      std::cout << GridLogIRL << "Diagonalisation is QR "<<std::endl;
+      std::cout << GridLogMessage << "Diagonalisation is QR "<<std::endl;
     }  else if ( diagonalisation == IRLdiagonaliseWithEigen ) { 
-      std::cout << GridLogIRL << "Diagonalisation is Eigen "<<std::endl;
+      std::cout << GridLogMessage << "Diagonalisation is Eigen "<<std::endl;
     }
-    std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
-	
-    assert(Nm <= evec.size() && Nm <= eval.size());
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
     
-    // quickly get an idea of the largest eigenvalue to more properly normalize the residuum
-    RealD evalMaxApprox = 0.0;
-    {
-      auto src_n = src;
-      auto tmp = src;
-      const int _MAX_ITER_IRL_MEVAPP_ = 50;
-      for (int i=0;i<_MAX_ITER_IRL_MEVAPP_;i++) {
-	normalise(src_n);
-	_HermOp(src_n,tmp);
-	RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
-	RealD vden = norm2(src_n);
-	RealD na = vnum/vden;
-	if (fabs(evalMaxApprox/na - 1.0) < 0.05)
-	  i=_MAX_ITER_IRL_MEVAPP_;
-	evalMaxApprox = na;
-	std::cout << GridLogIRL << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
-	src_n = tmp;
-      }
-    }
+    assert(Nm == evec.size() && Nm == eval.size());
 	
     std::vector<RealD> lme(Nm);  
     std::vector<RealD> lme2(Nm);
     std::vector<RealD> eval2(Nm);
-    std::vector<RealD> eval2_copy(Nm);
-    Eigen::MatrixXd Qt = Eigen::MatrixXd::Zero(Nm,Nm);
 
+    Eigen::MatrixXd    Qt = Eigen::MatrixXd::Zero(Nm,Nm);
+
+    std::vector<int>   Iconv(Nm);
+    std::vector<Field>  B(Nm,grid); // waste of space replicating
+    
     Field f(grid);
     Field v(grid);
+    
     int k1 = 1;
     int k2 = Nk;
-    RealD beta_k;
-
+    
     Nconv = 0;
+    
+    RealD beta_k;
   
     // Set initial vector
     evec[0] = src;
+    std::cout << GridLogMessage <<"norm2(src)= " << norm2(src)<<std::endl;
+    
     normalise(evec[0]);
-	
+    std::cout << GridLogMessage <<"norm2(evec[0])= " << norm2(evec[0]) <<std::endl;
+    
     // Initial Nk steps
-    OrthoTime=0.;
     for(int k=0; k<Nk; ++k) step(eval,lme,evec,f,Nm,k);
-    std::cout<<GridLogIRL <<"Initial "<< Nk <<"steps done "<<std::endl;
-    std::cout<<GridLogIRL <<"Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
-
-    //////////////////////////////////
+    
     // Restarting loop begins
-    //////////////////////////////////
     int iter;
     for(iter = 0; iter<MaxIter; ++iter){
       
-      OrthoTime=0.;
-
       std::cout<< GridLogMessage <<" **********************"<< std::endl;
       std::cout<< GridLogMessage <<" Restart iteration = "<< iter << std::endl;
       std::cout<< GridLogMessage <<" **********************"<< std::endl;
-
-      std::cout<<GridLogIRL <<" running "<<Nm-Nk <<" steps: "<<std::endl;
+      
       for(int k=Nk; k<Nm; ++k) step(eval,lme,evec,f,Nm,k);
+      
       f *= lme[Nm-1];
-
-      std::cout<<GridLogIRL <<" "<<Nm-Nk <<" steps done "<<std::endl;
-      std::cout<<GridLogIRL <<"Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
-	  
-      //////////////////////////////////
+      
       // getting eigenvalues
-      //////////////////////////////////
       for(int k=0; k<Nm; ++k){
 	eval2[k] = eval[k+k1-1];
 	lme2[k] = lme[k+k1-1];
       }
       Qt = Eigen::MatrixXd::Identity(Nm,Nm);
       diagonalize(eval2,lme2,Nm,Nm,Qt,grid);
-      std::cout<<GridLogIRL <<" diagonalized "<<std::endl;
 
-      //////////////////////////////////
       // sorting
-      //////////////////////////////////
-      eval2_copy = eval2;
-      std::partial_sort(eval2.begin(),eval2.begin()+Nm,eval2.end(),std::greater<RealD>());
-      std::cout<<GridLogIRL <<" evals sorted "<<std::endl;
-      const int chunk=8;
-      for(int io=0; io<k2;io+=chunk){
-	std::cout<<GridLogIRL << "eval "<< std::setw(3) << io ;
-	for(int ii=0;ii<chunk;ii++){
-	  if ( (io+ii)<k2 )
-	    std::cout<< " "<< std::setw(12)<< eval2[io+ii];
-	}
-	std::cout << std::endl;
-      }
-
-      //////////////////////////////////
+      _sort.push(eval2,Nm);
+      
       // Implicitly shifted QR transformations
-      //////////////////////////////////
       Qt = Eigen::MatrixXd::Identity(Nm,Nm);
       for(int ip=k2; ip<Nm; ++ip){ 
-	QR_decomp(eval,lme,Nm,Nm,Qt,eval2[ip],k1,Nm);
+	// Eigen replacement for qr_decomp ???
+	qr_decomp(eval,lme,Nm,Nm,Qt,eval2[ip],k1,Nm);
       }
-      std::cout<<GridLogIRL <<"QR decomposed "<<std::endl;
-
-      assert(k2<Nm);      assert(k2<Nm);      assert(k1>0);
-
-      basisRotate(evec,Qt,k1-1,k2+1,0,Nm,Nm); /// big constraint on the basis
-      std::cout<<GridLogIRL <<"basisRotated  by Qt"<<std::endl;
+    
+      for(int i=0; i<(Nk+1); ++i) B[i] = 0.0;
+	  
+      for(int j=k1-1; j<k2+1; ++j){
+	for(int k=0; k<Nm; ++k){
+	  B[j].checkerboard = evec[k].checkerboard;
+	  B[j] += Qt(j,k) * evec[k];
+	}
+      }
+      for(int j=k1-1; j<k2+1; ++j) evec[j] = B[j];
       
-      ////////////////////////////////////////////////////
       // Compressed vector f and beta(k2)
-      ////////////////////////////////////////////////////
       f *= Qt(k2-1,Nm-1);
       f += lme[k2-1] * evec[k2];
       beta_k = norm2(f);
       beta_k = sqrt(beta_k);
-      std::cout<<GridLogIRL<<" beta(k) = "<<beta_k<<std::endl;
-	  
+      std::cout<< GridLogMessage<<" beta(k) = "<<beta_k<<std::endl;
+      
       RealD betar = 1.0/beta_k;
       evec[k2] = betar * f;
       lme[k2-1] = beta_k;
-	  
-      ////////////////////////////////////////////////////
+      
       // Convergence test
-      ////////////////////////////////////////////////////
       for(int k=0; k<Nm; ++k){    
 	eval2[k] = eval[k];
 	lme2[k] = lme[k];
       }
       Qt = Eigen::MatrixXd::Identity(Nm,Nm);
       diagonalize(eval2,lme2,Nk,Nm,Qt,grid);
-      std::cout<<GridLogIRL <<" Diagonalized "<<std::endl;
-	  
-      Nconv = 0;
-      if (iter >= MinRestart) {
-
-	std::cout << GridLogIRL << "Test convergence: rotate subset of vectors to test convergence " << std::endl;
-
-	Field B(grid); B.checkerboard = evec[0].checkerboard;
-
-	//  power of two search pattern;  not every evalue in eval2 is assessed.
-	for(int jj = 1; jj<=Nstop; jj*=2){
-	  int j = Nstop-jj;
-	  RealD e = eval2_copy[j]; // Discard the evalue
-	  basisRotateJ(B,evec,Qt,j,0,Nk,Nm);	    
-	  if( _Tester.TestConvergence(j,eresid,B,e,evalMaxApprox) ) {
-	    if ( j > Nconv ) {
-	      Nconv=j+1;
-	      jj=Nstop; // Terminate the scan
-	    }
-	  }
-	}
-	// Do evec[0] for good measure
-	{ 
-	  int j=0;
-	  RealD e = eval2_copy[0]; 
-	  basisRotateJ(B,evec,Qt,j,0,Nk,Nm);	    
-	  _Tester.TestConvergence(j,eresid,B,e,evalMaxApprox);
-	}
-	// test if we converged, if so, terminate
-	std::cout<<GridLogIRL<<" #modes converged: >= "<<Nconv<<"/"<<Nstop<<std::endl;
-	//	if( Nconv>=Nstop || beta_k < betastp){
-	if( Nconv>=Nstop){
-	  goto converged;
-	}
-	  
-      } else {
-	std::cout << GridLogIRL << "iter < MinRestart: do not yet test for convergence\n";
-      } // end of iter loop
-    }
-
-    std::cout<<GridLogError<<"\n NOT converged.\n";
-    abort();
-	
-  converged:
-    {
-      Field B(grid); B.checkerboard = evec[0].checkerboard;
-      basisRotate(evec,Qt,0,Nk,0,Nk,Nm);	    
-      std::cout << GridLogIRL << " Rotated basis"<<std::endl;
-      Nconv=0;
-      //////////////////////////////////////////////////////////////////////
-      // Full final convergence test; unconditionally applied
-      //////////////////////////////////////////////////////////////////////
-      for(int j = 0; j<=Nk; j++){
-	B=evec[j];
-	if( _Tester.ReconstructEval(j,eresid,B,eval2[j],evalMaxApprox) ) {
-	  Nconv++;
+      
+      for(int k = 0; k<Nk; ++k) B[k]=0.0;
+      
+      for(int j = 0; j<Nk; ++j){
+	for(int k = 0; k<Nk; ++k){
+	  B[j].checkerboard = evec[k].checkerboard;
+	  B[j] += Qt(j,k) * evec[k];
 	}
       }
 
-      if ( Nconv < Nstop )
-	std::cout << GridLogIRL << "Nconv ("<<Nconv<<") < Nstop ("<<Nstop<<")"<<std::endl;
+      Nconv = 0;
+      for(int i=0; i<Nk; ++i){
+	
+	_Linop.HermOp(B[i],v);
+	    
+	RealD vnum = real(innerProduct(B[i],v)); // HermOp.
+	RealD vden = norm2(B[i]);
+	eval2[i] = vnum/vden;
+	v -= eval2[i]*B[i];
+	RealD vv = norm2(v);
+	
+	std::cout.precision(13);
+	std::cout << GridLogMessage << "[" << std::setw(3)<< std::setiosflags(std::ios_base::right) <<i<<"] ";
+	std::cout << "eval = "<<std::setw(25)<< std::setiosflags(std::ios_base::left)<< eval2[i];
+	std::cout << " |H B[i] - eval[i]B[i]|^2 "<< std::setw(25)<< std::setiosflags(std::ios_base::right)<< vv<< std::endl;
+	
+	// change the criteria as evals are supposed to be sorted, all evals smaller(larger) than Nstop should have converged
+	if((vv<eresid*eresid) && (i == Nconv) ){
+	  Iconv[Nconv] = i;
+	  ++Nconv;
+	}
+	
+      }  // i-loop end
+      
+      std::cout<< GridLogMessage <<" #modes converged: "<<Nconv<<std::endl;
 
-      eval=eval2;
-      
-      //Keep only converged
-      eval.resize(Nconv);// Nstop?
-      evec.resize(Nconv,grid);// Nstop?
-      basisSortInPlace(evec,eval,reverse);
-      
+      if( Nconv>=Nstop ){
+	goto converged;
+      }
+    } // end of iter loop
+    
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
+    std::cout<< GridLogError    <<" ImplicitlyRestartedLanczos::calc() NOT converged.";
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
+    abort();
+	
+  converged:
+    // Sorting
+    eval.resize(Nconv);
+    evec.resize(Nconv,grid);
+    for(int i=0; i<Nconv; ++i){
+      eval[i] = eval2[Iconv[i]];
+      evec[i] = B[Iconv[i]];
     }
-       
-    std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
-    std::cout << GridLogIRL << "ImplicitlyRestartedLanczos CONVERGED ; Summary :\n";
-    std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
-    std::cout << GridLogIRL << " -- Iterations  = "<< iter   << "\n";
-    std::cout << GridLogIRL << " -- beta(k)     = "<< beta_k << "\n";
-    std::cout << GridLogIRL << " -- Nconv       = "<< Nconv  << "\n";
-    std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
+    _sort.push(eval,evec,Nconv);
+    
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
+    std::cout << GridLogMessage << "ImplicitlyRestartedLanczos CONVERGED ; Summary :\n";
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
+    std::cout << GridLogMessage << " -- Iterations  = "<< iter   << "\n";
+    std::cout << GridLogMessage << " -- beta(k)     = "<< beta_k << "\n";
+    std::cout << GridLogMessage << " -- Nconv       = "<< Nconv  << "\n";
+    std::cout << GridLogMessage <<"**************************************************************************"<< std::endl;
   }
 
- private:
+private:
 /* Saad PP. 195
 1. Choose an initial vector v1 of 2-norm unity. Set β1 ≡ 0, v0 ≡ 0
 2. For k = 1,2,...,m Do:
@@ -578,38 +360,28 @@ until convergence
   {
     const RealD tiny = 1.0e-20;
     assert( k< Nm );
-
-    GridStopWatch gsw_op,gsw_o;
-
-    Field& evec_k = evec[k];
-
-    _PolyOp(evec_k,w);    std::cout<<GridLogIRL << "PolyOp" <<std::endl;
-
+    
+    _poly(_Linop,evec[k],w);      // 3. wk:=Avk−βkv_{k−1}
+    
     if(k>0) w -= lme[k-1] * evec[k-1];
-
-    ComplexD zalph = innerProduct(evec_k,w); // 4. αk:=(wk,vk)
+    
+    ComplexD zalph = innerProduct(evec[k],w); // 4. αk:=(wk,vk)
     RealD     alph = real(zalph);
-
-    w = w - alph * evec_k;// 5. wk:=wk−αkvk
-
+    
+    w = w - alph * evec[k];// 5. wk:=wk−αkvk
+    
     RealD beta = normalise(w); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
     // 7. vk+1 := wk/βk+1
-
+    
     lmd[k] = alph;
     lme[k] = beta;
-
-    if (k>0 && k % orth_period == 0) {
-      orthogonalize(w,evec,k); // orthonormalise
-      std::cout<<GridLogIRL << "Orthogonalised " <<std::endl;
-    }
-
-    if(k < Nm-1) evec[k+1] = w;
-
-    std::cout<<GridLogIRL << "alpha[" << k << "] = " << zalph << " beta[" << k << "] = "<<beta<<std::endl;
-    if ( beta < tiny ) 
-      std::cout<<GridLogIRL << " beta is tiny "<<beta<<std::endl;
+    
+    if ( k > 0 ) orthogonalize(w,evec,k); // orthonormalise
+    if ( k < Nm-1) evec[k+1] = w;
+    
+    if ( beta < tiny ) std::cout << GridLogMessage << " beta is tiny "<<beta<<std::endl;
   }
-
+      
   void diagonalize_Eigen(std::vector<RealD>& lmd, std::vector<RealD>& lme, 
 			 int Nk, int Nm,  
 			 Eigen::MatrixXd & Qt, // Nm x Nm
@@ -632,11 +404,11 @@ until convergence
       }
     }
   }
+  ///////////////////////////////////////////////////////////////////////////
+  // File could end here if settle on Eigen ???
+  ///////////////////////////////////////////////////////////////////////////
 
-  ///////////////////////////////////////////////////////////////////////////
-  // File could end here if settle on Eigen ??? !!!
-  ///////////////////////////////////////////////////////////////////////////
-  void QR_decomp(std::vector<RealD>& lmd,   // Nm 
+  void qr_decomp(std::vector<RealD>& lmd,   // Nm 
 		 std::vector<RealD>& lme,   // Nm 
 		 int Nk, int Nm,            // Nk, Nm
 		 Eigen::MatrixXd& Qt,       // Nm x Nm matrix
@@ -803,50 +575,51 @@ void diagonalize_lapack(std::vector<RealD>& lmd,
 #endif
 }
 
-void diagonalize_QR(std::vector<RealD>& lmd, std::vector<RealD>& lme, 
-		    int Nk, int Nm,   
-		    Eigen::MatrixXd & Qt,
-		    GridBase *grid)
-{
-  int QRiter = 100*Nm;
-  int kmin = 1;
-  int kmax = Nk;
-  
-  // (this should be more sophisticated)
-  for(int iter=0; iter<QRiter; ++iter){
-    
-    // determination of 2x2 leading submatrix
-    RealD dsub = lmd[kmax-1]-lmd[kmax-2];
-    RealD dd = sqrt(dsub*dsub + 4.0*lme[kmax-2]*lme[kmax-2]);
-    RealD Dsh = 0.5*(lmd[kmax-2]+lmd[kmax-1] +dd*(dsub/fabs(dsub)));
-    // (Dsh: shift)
-    
-    // transformation
-    QR_decomp(lmd,lme,Nk,Nm,Qt,Dsh,kmin,kmax); // Nk, Nm
-    
-    // Convergence criterion (redef of kmin and kamx)
-    for(int j=kmax-1; j>= kmin; --j){
-      RealD dds = fabs(lmd[j-1])+fabs(lmd[j]);
-      if(fabs(lme[j-1])+dds > dds){
-	kmax = j+1;
-	goto continued;
-      }
-    }
-    QRiter = iter;
-    return;
-    
-  continued:
-    for(int j=0; j<kmax-1; ++j){
-      RealD dds = fabs(lmd[j])+fabs(lmd[j+1]);
-      if(fabs(lme[j])+dds > dds){
-	kmin = j+1;
-	break;
+  void diagonalize_QR(std::vector<RealD>& lmd, std::vector<RealD>& lme, 
+		      int Nk, int Nm,   
+		      Eigen::MatrixXd & Qt,
+		      GridBase *grid)
+  {
+    int Niter = 100*Nm;
+    int kmin = 1;
+    int kmax = Nk;
+
+    // (this should be more sophisticated)
+    for(int iter=0; iter<Niter; ++iter){
+      
+      // determination of 2x2 leading submatrix
+      RealD dsub = lmd[kmax-1]-lmd[kmax-2];
+      RealD dd = sqrt(dsub*dsub + 4.0*lme[kmax-2]*lme[kmax-2]);
+      RealD Dsh = 0.5*(lmd[kmax-2]+lmd[kmax-1] +dd*(dsub/fabs(dsub)));
+      // (Dsh: shift)
+	
+      // transformation
+      qr_decomp(lmd,lme,Nk,Nm,Qt,Dsh,kmin,kmax); // Nk, Nm
+	
+      // Convergence criterion (redef of kmin and kamx)
+      for(int j=kmax-1; j>= kmin; --j){
+	RealD dds = fabs(lmd[j-1])+fabs(lmd[j]);
+	if(fabs(lme[j-1])+dds > dds){
+	  kmax = j+1;
+	  goto continued;
+	}
+      }
+      Niter = iter;
+      return;
+
+    continued:
+      for(int j=0; j<kmax-1; ++j){
+	RealD dds = fabs(lmd[j])+fabs(lmd[j+1]);
+	if(fabs(lme[j])+dds > dds){
+	  kmin = j+1;
+	  break;
+	}
       }
     }
+    std::cout << GridLogError << "[QL method] Error - Too many iteration: "<<Niter<<"\n";
+    abort();
   }
-  std::cout << GridLogError << "[QL method] Error - Too many iteration: "<<QRiter<<"\n";
-  abort();
-}
-};
+
+ };
 }
 #endif

lib/algorithms/iterative/LocalCoherenceLanczos.h

@@ -1,352 +0,0 @@
-    /*************************************************************************************
-
-    Grid physics library, www.github.com/paboyle/Grid 
-
-    Source file: ./lib/algorithms/iterative/LocalCoherenceLanczos.h
-
-    Copyright (C) 2015
-
-Author: Christoph Lehner <clehner@bnl.gov>
-Author: paboyle <paboyle@ph.ed.ac.uk>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    See the full license in the file "LICENSE" in the top level distribution directory
-    *************************************************************************************/
-    /*  END LEGAL */
-#ifndef GRID_LOCAL_COHERENCE_IRL_H
-#define GRID_LOCAL_COHERENCE_IRL_H
-namespace Grid { 
-struct LanczosParams : Serializable {
- public:
-  GRID_SERIALIZABLE_CLASS_MEMBERS(LanczosParams,
-				  ChebyParams, Cheby,/*Chebyshev*/
-				  int, Nstop,    /*Vecs in Lanczos must converge Nstop < Nk < Nm*/
-				  int, Nk,       /*Vecs in Lanczos seek converge*/
-				  int, Nm,       /*Total vecs in Lanczos include restart*/
-				  RealD, resid,  /*residual*/
- 				  int, MaxIt, 
-				  RealD, betastp,  /* ? */
-				  int, MinRes);    // Must restart
-};
-
-struct LocalCoherenceLanczosParams : Serializable {
- public:
-  GRID_SERIALIZABLE_CLASS_MEMBERS(LocalCoherenceLanczosParams,
-				  bool, doFine,
-				  bool, doFineRead,
-				  bool, doCoarse,
-	       			  bool, doCoarseRead,
-				  LanczosParams, FineParams,
-				  LanczosParams, CoarseParams,
-				  ChebyParams,   Smoother,
-				  RealD        , coarse_relax_tol,
-				  std::vector<int>, blockSize,
-				  std::string, config,
-				  std::vector < std::complex<double>  >, omega,
-				  RealD, mass,
-				  RealD, M5);
-};
-
-// Duplicate functionality; ProjectedFunctionHermOp could be used with the trivial function
-template<class Fobj,class CComplex,int nbasis>
-class ProjectedHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
-public:
-  typedef iVector<CComplex,nbasis >           CoarseSiteVector;
-  typedef Lattice<CoarseSiteVector>           CoarseField;
-  typedef Lattice<CComplex>   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<Fobj>          FineField;
-
-  LinearOperatorBase<FineField> &_Linop;
-  Aggregation<Fobj,CComplex,nbasis> &_Aggregate;
-
-  ProjectedHermOp(LinearOperatorBase<FineField>& linop,  Aggregation<Fobj,CComplex,nbasis> &aggregate) : 
-    _Linop(linop),
-    _Aggregate(aggregate)  {  };
-
-  void operator()(const CoarseField& in, CoarseField& out) {
-
-    GridBase *FineGrid = _Aggregate.FineGrid;
-    FineField fin(FineGrid);
-    FineField fout(FineGrid);
-
-    _Aggregate.PromoteFromSubspace(in,fin);    std::cout<<GridLogIRL<<"ProjectedHermop : Promote to fine"<<std::endl;
-    _Linop.HermOp(fin,fout);                   std::cout<<GridLogIRL<<"ProjectedHermop : HermOp (fine) "<<std::endl;
-    _Aggregate.ProjectToSubspace(out,fout);    std::cout<<GridLogIRL<<"ProjectedHermop : Project to coarse "<<std::endl;
-  }
-};
-
-template<class Fobj,class CComplex,int nbasis>
-class ProjectedFunctionHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
-public:
-  typedef iVector<CComplex,nbasis >           CoarseSiteVector;
-  typedef Lattice<CoarseSiteVector>           CoarseField;
-  typedef Lattice<CComplex>   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<Fobj>          FineField;
-
-
-  OperatorFunction<FineField>   & _poly;
-  LinearOperatorBase<FineField> &_Linop;
-  Aggregation<Fobj,CComplex,nbasis> &_Aggregate;
-
-  ProjectedFunctionHermOp(OperatorFunction<FineField> & poly,LinearOperatorBase<FineField>& linop, 
-			  Aggregation<Fobj,CComplex,nbasis> &aggregate) : 
-    _poly(poly),
-    _Linop(linop),
-    _Aggregate(aggregate)  {  };
-
-  void operator()(const CoarseField& in, CoarseField& out) {
-
-    GridBase *FineGrid = _Aggregate.FineGrid;
-
-    FineField fin(FineGrid) ;fin.checkerboard  =_Aggregate.checkerboard;
-    FineField fout(FineGrid);fout.checkerboard =_Aggregate.checkerboard;
-    
-    _Aggregate.PromoteFromSubspace(in,fin);    std::cout<<GridLogIRL<<"ProjectedFunctionHermop : Promote to fine"<<std::endl;
-    _poly(_Linop,fin,fout);                    std::cout<<GridLogIRL<<"ProjectedFunctionHermop : Poly "<<std::endl;
-    _Aggregate.ProjectToSubspace(out,fout);    std::cout<<GridLogIRL<<"ProjectedFunctionHermop : Project to coarse "<<std::endl;
-  }
-};
-
-template<class Fobj,class CComplex,int nbasis>
-class ImplicitlyRestartedLanczosSmoothedTester  : public ImplicitlyRestartedLanczosTester<Lattice<iVector<CComplex,nbasis > > >
-{
- public:
-  typedef iVector<CComplex,nbasis >           CoarseSiteVector;
-  typedef Lattice<CoarseSiteVector>           CoarseField;
-  typedef Lattice<CComplex>   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<Fobj>          FineField;
-
-  LinearFunction<CoarseField> & _Poly;
-  OperatorFunction<FineField>   & _smoother;
-  LinearOperatorBase<FineField> &_Linop;
-  Aggregation<Fobj,CComplex,nbasis> &_Aggregate;
-  RealD                             _coarse_relax_tol;
-  ImplicitlyRestartedLanczosSmoothedTester(LinearFunction<CoarseField>   &Poly,
-					   OperatorFunction<FineField>   &smoother,
-					   LinearOperatorBase<FineField> &Linop,
-					   Aggregation<Fobj,CComplex,nbasis> &Aggregate,
-					   RealD coarse_relax_tol=5.0e3) 
-    : _smoother(smoother), _Linop(Linop),_Aggregate(Aggregate), _Poly(Poly), _coarse_relax_tol(coarse_relax_tol)  {    };
-
-  int TestConvergence(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
-  {
-    CoarseField v(B);
-    RealD eval_poly = eval;
-    // Apply operator
-    _Poly(B,v);
-
-    RealD vnum = real(innerProduct(B,v)); // HermOp.
-    RealD vden = norm2(B);
-    RealD vv0  = norm2(v);
-    eval   = vnum/vden;
-    v -= eval*B;
-
-    RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
-
-    std::cout.precision(13);
-    std::cout<<GridLogIRL  << "[" << std::setw(3)<<j<<"] "
-	     <<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
-	     <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
-	     <<std::endl;
-
-    int conv=0;
-    if( (vv<eresid*eresid) ) conv = 1;
-    return conv;
-  }
-  int ReconstructEval(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
-  {
-    GridBase *FineGrid = _Aggregate.FineGrid;
-
-    int checkerboard   = _Aggregate.checkerboard;
-
-    FineField fB(FineGrid);fB.checkerboard =checkerboard;
-    FineField fv(FineGrid);fv.checkerboard =checkerboard;
-
-    _Aggregate.PromoteFromSubspace(B,fv);
-    _smoother(_Linop,fv,fB); 
-
-    RealD eval_poly = eval;
-    _Linop.HermOp(fB,fv);
-
-    RealD vnum = real(innerProduct(fB,fv)); // HermOp.
-    RealD vden = norm2(fB);
-    RealD vv0  = norm2(fv);
-    eval   = vnum/vden;
-    fv -= eval*fB;
-    RealD vv = norm2(fv) / ::pow(evalMaxApprox,2.0);
-
-    std::cout.precision(13);
-    std::cout<<GridLogIRL  << "[" << std::setw(3)<<j<<"] "
-	     <<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
-	     <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
-	     <<std::endl;
-    if ( j > nbasis ) eresid = eresid*_coarse_relax_tol;
-    if( (vv<eresid*eresid) ) return 1;
-    return 0;
-  }
-};
-
-////////////////////////////////////////////
-// Make serializable Lanczos params
-////////////////////////////////////////////
-template<class Fobj,class CComplex,int nbasis>
-class LocalCoherenceLanczos 
-{
-public:
-  typedef iVector<CComplex,nbasis >           CoarseSiteVector;
-  typedef Lattice<CComplex>                   CoarseScalar; // used for inner products on fine field
-  typedef Lattice<CoarseSiteVector>           CoarseField;
-  typedef Lattice<Fobj>                       FineField;
-
-protected:
-  GridBase *_CoarseGrid;
-  GridBase *_FineGrid;
-  int _checkerboard;
-  LinearOperatorBase<FineField>                 & _FineOp;
-  
-  // FIXME replace Aggregation with vector of fine; the code reuse is too small for
-  // the hassle and complexity of cross coupling.
-  Aggregation<Fobj,CComplex,nbasis>               _Aggregate;  
-  std::vector<RealD>                              evals_fine;
-  std::vector<RealD>                              evals_coarse; 
-  std::vector<CoarseField>                        evec_coarse;
-public:
-  LocalCoherenceLanczos(GridBase *FineGrid,
-		GridBase *CoarseGrid,
-		LinearOperatorBase<FineField> &FineOp,
-		int checkerboard) :
-    _CoarseGrid(CoarseGrid),
-    _FineGrid(FineGrid),
-    _Aggregate(CoarseGrid,FineGrid,checkerboard),
-    _FineOp(FineOp),
-    _checkerboard(checkerboard)
-  {
-    evals_fine.resize(0);
-    evals_coarse.resize(0);
-  };
-  void Orthogonalise(void ) { _Aggregate.Orthogonalise(); }
-
-  template<typename T>  static RealD normalise(T& v) 
-  {
-    RealD nn = norm2(v);
-    nn = ::sqrt(nn);
-    v = v * (1.0/nn);
-    return nn;
-  }
-
-  void fakeFine(void)
-  {
-    int Nk = nbasis;
-    _Aggregate.subspace.resize(Nk,_FineGrid);
-    _Aggregate.subspace[0]=1.0;
-    _Aggregate.subspace[0].checkerboard=_checkerboard;
-    normalise(_Aggregate.subspace[0]);
-    PlainHermOp<FineField>    Op(_FineOp);
-    for(int k=1;k<Nk;k++){
-      _Aggregate.subspace[k].checkerboard=_checkerboard;
-      Op(_Aggregate.subspace[k-1],_Aggregate.subspace[k]);
-      normalise(_Aggregate.subspace[k]);
-    }
-  }
-
-  void testFine(RealD resid) 
-  {
-    assert(evals_fine.size() == nbasis);
-    assert(_Aggregate.subspace.size() == nbasis);
-    PlainHermOp<FineField>    Op(_FineOp);
-    ImplicitlyRestartedLanczosHermOpTester<FineField> SimpleTester(Op);
-    for(int k=0;k<nbasis;k++){
-      assert(SimpleTester.ReconstructEval(k,resid,_Aggregate.subspace[k],evals_fine[k],1.0)==1);
-    }
-  }
-
-  void testCoarse(RealD resid,ChebyParams cheby_smooth,RealD relax) 
-  {
-    assert(evals_fine.size() == nbasis);
-    assert(_Aggregate.subspace.size() == nbasis);
-    //////////////////////////////////////////////////////////////////////////////////////////////////
-    // create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
-    //////////////////////////////////////////////////////////////////////////////////////////////////
-    Chebyshev<FineField>                          ChebySmooth(cheby_smooth);
-    ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (ChebySmooth,_FineOp,_Aggregate);
-    ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,_Aggregate,relax);
-
-    for(int k=0;k<evec_coarse.size();k++){
-      if ( k < nbasis ) { 
-	assert(ChebySmoothTester.ReconstructEval(k,resid,evec_coarse[k],evals_coarse[k],1.0)==1);
-      } else { 
-	assert(ChebySmoothTester.ReconstructEval(k,resid*relax,evec_coarse[k],evals_coarse[k],1.0)==1);
-      }
-    }
-  }
-
-  void calcFine(ChebyParams cheby_parms,int Nstop,int Nk,int Nm,RealD resid, 
-		RealD MaxIt, RealD betastp, int MinRes)
-  {
-    assert(nbasis<=Nm);
-    Chebyshev<FineField>      Cheby(cheby_parms);
-    FunctionHermOp<FineField> ChebyOp(Cheby,_FineOp);
-    PlainHermOp<FineField>    Op(_FineOp);
-
-    evals_fine.resize(Nm);
-    _Aggregate.subspace.resize(Nm,_FineGrid);
-
-    ImplicitlyRestartedLanczos<FineField> IRL(ChebyOp,Op,Nstop,Nk,Nm,resid,MaxIt,betastp,MinRes);
-
-    FineField src(_FineGrid); src=1.0; src.checkerboard = _checkerboard;
-
-    int Nconv;
-    IRL.calc(evals_fine,_Aggregate.subspace,src,Nconv,false);
-    
-    // Shrink down to number saved
-    assert(Nstop>=nbasis);
-    assert(Nconv>=nbasis);
-    evals_fine.resize(nbasis);
-    _Aggregate.subspace.resize(nbasis,_FineGrid);
-  }
-  void calcCoarse(ChebyParams cheby_op,ChebyParams cheby_smooth,RealD relax,
-		  int Nstop, int Nk, int Nm,RealD resid, 
-		  RealD MaxIt, RealD betastp, int MinRes)
-  {
-    Chebyshev<FineField>                          Cheby(cheby_op);
-    ProjectedHermOp<Fobj,CComplex,nbasis>         Op(_FineOp,_Aggregate);
-    ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (Cheby,_FineOp,_Aggregate);
-    //////////////////////////////////////////////////////////////////////////////////////////////////
-    // create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
-    //////////////////////////////////////////////////////////////////////////////////////////////////
-
-    Chebyshev<FineField>                                           ChebySmooth(cheby_smooth);
-    ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,_Aggregate,relax);
-
-    evals_coarse.resize(Nm);
-    evec_coarse.resize(Nm,_CoarseGrid);
-
-    CoarseField src(_CoarseGrid);     src=1.0; 
-
-    ImplicitlyRestartedLanczos<CoarseField> IRL(ChebyOp,ChebyOp,ChebySmoothTester,Nstop,Nk,Nm,resid,MaxIt,betastp,MinRes);
-    int Nconv=0;
-    IRL.calc(evals_coarse,evec_coarse,src,Nconv,false);
-    assert(Nconv>=Nstop);
-    evals_coarse.resize(Nstop);
-    evec_coarse.resize (Nstop,_CoarseGrid);
-    for (int i=0;i<Nstop;i++){
-      std::cout << i << " Coarse eval = " << evals_coarse[i]  << std::endl;
-    }
-  }
-};
-
-}
-#endif

lib/algorithms/iterative/SchurRedBlack.h

@@ -7,6 +7,7 @@
     Copyright (C) 2015
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Chulwoo Jung <chulwoo@bnl.gov>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -90,7 +91,7 @@ namespace Grid {
   // Take a matrix and form a Red Black solver calling a Herm solver
   // Use of RB info prevents making SchurRedBlackSolve conform to standard interface
   ///////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Now make the norm reflect extra factor of Mee
+
   template<class Field> class SchurRedBlackStaggeredSolve {
   private:
     OperatorFunction<Field> & _HermitianRBSolver;
@@ -123,14 +124,11 @@ namespace Grid {
       Field   tmp(grid);
       Field  Mtmp(grid);
       Field resid(fgrid);
-      
-      std::cout << GridLogMessage << " SchurRedBlackStaggeredSolve " <<std::endl;
+
       pickCheckerboard(Even,src_e,in);
       pickCheckerboard(Odd ,src_o,in);
       pickCheckerboard(Even,sol_e,out);
       pickCheckerboard(Odd ,sol_o,out);
-
-      std::cout << GridLogMessage << " SchurRedBlackStaggeredSolve checkerboards picked" <<std::endl;
     
       /////////////////////////////////////////////////////
       // src_o = (source_o - Moe MeeInv source_e)
@@ -139,15 +137,14 @@ namespace Grid {
       _Matrix.Meooe   (tmp,Mtmp);      assert( Mtmp.checkerboard ==Odd);     
       tmp=src_o-Mtmp;                  assert(  tmp.checkerboard ==Odd);     
 
-      //src_o = tmp;     assert(src_o.checkerboard ==Odd);
-      _Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source from dumb choice of matrix norm.
+      src_o = tmp;     assert(src_o.checkerboard ==Odd);
+      //  _Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source
 
       //////////////////////////////////////////////////////////////
       // Call the red-black solver
       //////////////////////////////////////////////////////////////
       std::cout<<GridLogMessage << "SchurRedBlackStaggeredSolver calling the Mpc solver" <<std::endl;
       _HermitianRBSolver(_HermOpEO,src_o,sol_o);  assert(sol_o.checkerboard==Odd);
-      std::cout<<GridLogMessage << "SchurRedBlackStaggeredSolver called  the Mpc solver" <<std::endl;
 
       ///////////////////////////////////////////////////
       // sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
@@ -156,20 +153,94 @@ namespace Grid {
       src_e = src_e-tmp;               assert(  src_e.checkerboard ==Even);
       _Matrix.MooeeInv(src_e,sol_e);   assert(  sol_e.checkerboard ==Even);
      
-      std::cout<<GridLogMessage << "SchurRedBlackStaggeredSolver reconstructed other CB" <<std::endl;
       setCheckerboard(out,sol_e); assert(  sol_e.checkerboard ==Even);
       setCheckerboard(out,sol_o); assert(  sol_o.checkerboard ==Odd );
-      std::cout<<GridLogMessage << "SchurRedBlackStaggeredSolver inserted solution" <<std::endl;
 
       // Verify the unprec residual
       _Matrix.M(out,resid); 
       resid = resid-in;
       RealD ns = norm2(in);
       RealD nr = norm2(resid);
+
       std::cout<<GridLogMessage << "SchurRedBlackStaggered solver true unprec resid "<< std::sqrt(nr/ns) <<" nr "<< nr <<" ns "<<ns << std::endl;
     }     
   };
-  template<class Field> using SchurRedBlackStagSolve = SchurRedBlackStaggeredSolve<Field>;
+//  template<class Field> using SchurRedBlackStagSolve = SchurRedBlackStaggeredSolve<Field>;
+  template<class Field> class SchurRedBlackStagSolve {
+  private:
+    OperatorFunction<Field> & _HermitianRBSolver;
+    int CBfactorise;
+  public:
+
+    /////////////////////////////////////////////////////
+    // Wrap the usual normal equations Schur trick
+    /////////////////////////////////////////////////////
+  SchurRedBlackStagSolve(OperatorFunction<Field> &HermitianRBSolver, int cb)  :
+     _HermitianRBSolver(HermitianRBSolver), CBfactorise(cb) {}
+
+    template<class Matrix>
+      void operator() (Matrix & _Matrix,const Field &in, Field &out){
+
+      // FIXME CGdiagonalMee not implemented virtual function
+      // FIXME use CBfactorise to control schur decomp
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      SchurStagOperator<Matrix,Field> _HermOpEO(_Matrix);
+      int Schur = CBfactorise;
+      int Other = 1 - CBfactorise;
+ 
+      Field src_e(grid);
+      Field src_o(grid);
+      Field sol_e(grid);
+      Field sol_o(grid);
+      Field   tmp(grid);
+      Field  Mtmp(grid);
+      Field resid(fgrid);
+
+      pickCheckerboard(Other,src_e,in);
+      pickCheckerboard(Schur ,src_o,in);
+      pickCheckerboard(Other,sol_e,out);
+      pickCheckerboard(Schur ,sol_o,out);
+    
+      /////////////////////////////////////////////////////
+      // src_o = Mdag * (source_o - Moe MeeInv source_e)
+      /////////////////////////////////////////////////////
+      _Matrix.MooeeInv(src_e,tmp);     assert(  tmp.checkerboard ==Other);
+      _Matrix.Meooe   (tmp,Mtmp);      assert( Mtmp.checkerboard ==Schur);     
+      tmp=src_o-Mtmp;                  assert(  tmp.checkerboard ==Schur);     
+
+#if 0
+      // get the right MpcDag
+//      _HermOpEO.MpcDag(tmp,src_o);     assert(src_o.checkerboard ==Schur);       
+#else
+      _Matrix.Mooee(tmp,src_o);     assert(src_o.checkerboard ==Schur);
+#endif
+      //////////////////////////////////////////////////////////////
+      // Call the red-black solver
+      //////////////////////////////////////////////////////////////
+      std::cout<<GridLogMessage << "SchurRedBlack solver calling the MpcDagMp solver" <<std::endl;
+      _HermitianRBSolver(_HermOpEO,src_o,sol_o);  assert(sol_o.checkerboard==Schur);
+
+      ///////////////////////////////////////////////////
+      // sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
+      ///////////////////////////////////////////////////
+      _Matrix.Meooe(sol_o,tmp);        assert(  tmp.checkerboard   ==Other);
+      src_e = src_e-tmp;               assert(  src_e.checkerboard ==Other);
+      _Matrix.MooeeInv(src_e,sol_e);   assert(  sol_e.checkerboard ==Other);
+     
+      setCheckerboard(out,sol_e); assert(  sol_e.checkerboard ==Other);
+      setCheckerboard(out,sol_o); assert(  sol_o.checkerboard ==Schur );
+
+      // Verify the unprec residual
+      _Matrix.M(out,resid); 
+      resid = resid-in;
+      RealD ns = norm2(in);
+      RealD nr = norm2(resid);
+
+      std::cout<<GridLogMessage << "SchurRedBlackStag solver true unprec resid "<< std::sqrt(nr/ns) <<" nr "<< nr <<" ns "<<ns << std::endl;
+    }     
+  };
 
   ///////////////////////////////////////////////////////////////////////////////////////////////////////
   // Take a matrix and form a Red Black solver calling a Herm solver
@@ -408,5 +479,77 @@ namespace Grid {
     }     
   };
 
+  template<class Field> class SchurRedBlackStagMixed {
+  private:
+    LinearFunction<Field> & _HermitianRBSolver;
+    int CBfactorise;
+  public:
+
+    /////////////////////////////////////////////////////
+    // Wrap the usual normal equations Schur trick
+    /////////////////////////////////////////////////////
+  SchurRedBlackStagMixed(LinearFunction<Field> &HermitianRBSolver, int cb)  :
+     _HermitianRBSolver(HermitianRBSolver), CBfactorise(cb) {}
+
+    template<class Matrix>
+      void operator() (Matrix & _Matrix,const Field &in, Field &out){
+
+      // FIXME CGdiagonalMee not implemented virtual function
+      // FIXME use CBfactorise to control schur decomp
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      SchurStagOperator<Matrix,Field> _HermOpEO(_Matrix);
+      int Schur = CBfactorise;
+      int Other = 1 - CBfactorise;
+ 
+      Field src_e(grid);
+      Field src_o(grid);
+      Field sol_e(grid);
+      Field sol_o(grid);
+      Field   tmp(grid);
+      Field  Mtmp(grid);
+      Field resid(fgrid);
+
+      pickCheckerboard(Other,src_e,in);
+      pickCheckerboard(Schur ,src_o,in);
+      pickCheckerboard(Other,sol_e,out);
+      pickCheckerboard(Schur ,sol_o,out);
+    
+      /////////////////////////////////////////////////////
+      // src_o = Mdag * (source_o - Moe MeeInv source_e)
+      /////////////////////////////////////////////////////
+      _Matrix.MooeeInv(src_e,tmp);     assert(  tmp.checkerboard ==Other);
+      _Matrix.Meooe   (tmp,Mtmp);      assert( Mtmp.checkerboard ==Schur);     
+      tmp=src_o-Mtmp;                  assert(  tmp.checkerboard ==Schur);     
+
+      _Matrix.Mooee(tmp,src_o);     assert(src_o.checkerboard ==Schur);
+      //////////////////////////////////////////////////////////////
+      // Call the red-black solver
+      //////////////////////////////////////////////////////////////
+      std::cout<<GridLogMessage << "SchurRedBlack solver calling the MpcDagMp solver" <<std::endl;
+//      _HermitianRBSolver(_HermOpEO,src_o,sol_o);  assert(sol_o.checkerboard==Schur);
+      _HermitianRBSolver(src_o,sol_o);  assert(sol_o.checkerboard==Other);
+
+      ///////////////////////////////////////////////////
+      // sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
+      ///////////////////////////////////////////////////
+      _Matrix.Meooe(sol_o,tmp);        assert(  tmp.checkerboard   ==Other);
+      src_e = src_e-tmp;               assert(  src_e.checkerboard ==Other);
+      _Matrix.MooeeInv(src_e,sol_e);   assert(  sol_e.checkerboard ==Other);
+     
+      setCheckerboard(out,sol_e); assert(  sol_e.checkerboard ==Other);
+      setCheckerboard(out,sol_o); assert(  sol_o.checkerboard ==Schur );
+
+      // Verify the unprec residual
+      _Matrix.M(out,resid); 
+      resid = resid-in;
+      RealD ns = norm2(in);
+      RealD nr = norm2(resid);
+
+      std::cout<<GridLogMessage << "SchurRedBlackStag solver true unprec resid "<< std::sqrt(nr/ns) <<" nr "<< nr <<" ns "<<ns << std::endl;
+    }     
+  };
+
 }
 #endif

lib/algorithms/iterative/SimpleLanczos.h

@@ -0,0 +1,933 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Chulwoo Jung <chulwoo@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef GRID_LANC_H
+#define GRID_LANC_H
+
+#include <string.h>		//memset
+
+#ifdef USE_LAPACK
+#ifdef USE_MKL
+#include<mkl_lapack.h>
+#else
+void LAPACK_dstegr (char *jobz, char *range, int *n, double *d, double *e,
+		    double *vl, double *vu, int *il, int *iu, double *abstol,
+		    int *m, double *w, double *z, int *ldz, int *isuppz,
+		    double *work, int *lwork, int *iwork, int *liwork,
+		    int *info);
+//#include <lapacke/lapacke.h>
+#endif
+#endif
+
+#include <Grid/algorithms/densematrix/DenseMatrix.h>
+//#include <Grid/algorithms/iterative/EigenSort.h>
+
+// eliminate temorary vector in calc()
+#define MEM_SAVE
+
+namespace Grid
+{
+
+  struct Bisection
+  {
+
+#if 0
+    static void get_eig2 (int row_num, std::vector < RealD > &ALPHA,
+			  std::vector < RealD > &BETA,
+			  std::vector < RealD > &eig)
+    {
+      int i, j;
+        std::vector < RealD > evec1 (row_num + 3);
+        std::vector < RealD > evec2 (row_num + 3);
+      RealD eps2;
+        ALPHA[1] = 0.;
+        BETHA[1] = 0.;
+      for (i = 0; i < row_num - 1; i++)
+	{
+	  ALPHA[i + 1] = A[i * (row_num + 1)].real ();
+	  BETHA[i + 2] = A[i * (row_num + 1) + 1].real ();
+	}
+      ALPHA[row_num] = A[(row_num - 1) * (row_num + 1)].real ();
+        bisec (ALPHA, BETHA, row_num, 1, row_num, 1e-10, 1e-10, evec1, eps2);
+        bisec (ALPHA, BETHA, row_num, 1, row_num, 1e-16, 1e-16, evec2, eps2);
+
+      // Do we really need to sort here?
+      int begin = 1;
+      int end = row_num;
+      int swapped = 1;
+      while (swapped)
+	{
+	  swapped = 0;
+	  for (i = begin; i < end; i++)
+	    {
+	      if (mag (evec2[i]) > mag (evec2[i + 1]))
+		{
+		  swap (evec2 + i, evec2 + i + 1);
+		  swapped = 1;
+		}
+	    }
+	  end--;
+	  for (i = end - 1; i >= begin; i--)
+	    {
+	      if (mag (evec2[i]) > mag (evec2[i + 1]))
+		{
+		  swap (evec2 + i, evec2 + i + 1);
+		  swapped = 1;
+		}
+	    }
+	  begin++;
+	}
+
+      for (i = 0; i < row_num; i++)
+	{
+	  for (j = 0; j < row_num; j++)
+	    {
+	      if (i == j)
+		H[i * row_num + j] = evec2[i + 1];
+	      else
+		H[i * row_num + j] = 0.;
+	    }
+	}
+    }
+#endif
+
+    static void bisec (std::vector < RealD > &c,
+		       std::vector < RealD > &b,
+		       int n,
+		       int m1,
+		       int m2,
+		       RealD eps1,
+		       RealD relfeh, std::vector < RealD > &x, RealD & eps2)
+    {
+      std::vector < RealD > wu (n + 2);
+
+      RealD h, q, x1, xu, x0, xmin, xmax;
+      int i, a, k;
+
+      b[1] = 0.0;
+      xmin = c[n] - fabs (b[n]);
+      xmax = c[n] + fabs (b[n]);
+      for (i = 1; i < n; i++)
+	{
+	  h = fabs (b[i]) + fabs (b[i + 1]);
+	  if (c[i] + h > xmax)
+	    xmax = c[i] + h;
+	  if (c[i] - h < xmin)
+	    xmin = c[i] - h;
+	}
+      xmax *= 2.;
+
+      eps2 = relfeh * ((xmin + xmax) > 0.0 ? xmax : -xmin);
+      if (eps1 <= 0.0)
+	eps1 = eps2;
+      eps2 = 0.5 * eps1 + 7.0 * (eps2);
+      x0 = xmax;
+      for (i = m1; i <= m2; i++)
+	{
+	  x[i] = xmax;
+	  wu[i] = xmin;
+	}
+
+      for (k = m2; k >= m1; k--)
+	{
+	  xu = xmin;
+	  i = k;
+	  do
+	    {
+	      if (xu < wu[i])
+		{
+		  xu = wu[i];
+		  i = m1 - 1;
+		}
+	      i--;
+	    }
+	  while (i >= m1);
+	  if (x0 > x[k])
+	    x0 = x[k];
+	  while ((x0 - xu) > 2 * relfeh * (fabs (xu) + fabs (x0)) + eps1)
+	    {
+	      x1 = (xu + x0) / 2;
+
+	      a = 0;
+	      q = 1.0;
+	      for (i = 1; i <= n; i++)
+		{
+		  q =
+		    c[i] - x1 -
+		    ((q != 0.0) ? b[i] * b[i] / q : fabs (b[i]) / relfeh);
+		  if (q < 0)
+		    a++;
+		}
+//      printf("x1=%0.14e a=%d\n",x1,a);
+	      if (a < k)
+		{
+		  if (a < m1)
+		    {
+		      xu = x1;
+		      wu[m1] = x1;
+		    }
+		  else
+		    {
+		      xu = x1;
+		      wu[a + 1] = x1;
+		      if (x[a] > x1)
+			x[a] = x1;
+		    }
+		}
+	      else
+		x0 = x1;
+	    }
+	  printf ("x0=%0.14e xu=%0.14e k=%d\n", x0, xu, k);
+	  x[k] = (x0 + xu) / 2;
+	}
+    }
+  };
+
+/////////////////////////////////////////////////////////////
+// Implicitly restarted lanczos
+/////////////////////////////////////////////////////////////
+
+
+  template < class Field > class SimpleLanczos
+  {
+
+    const RealD small = 1.0e-16;
+  public:
+    int lock;
+    int get;
+    int Niter;
+    int converged;
+
+    int Nstop;			// Number of evecs checked for convergence
+    int Nk;			// Number of converged sought
+    int Np;			// Np -- Number of spare vecs in kryloc space
+    int Nm;			// Nm -- total number of vectors
+
+
+    RealD OrthoTime;
+
+    RealD eresid;
+
+    SortEigen < Field > _sort;
+
+    LinearOperatorBase < Field > &_Linop;
+
+    OperatorFunction < Field > &_poly;
+
+    /////////////////////////
+    // Constructor
+    /////////////////////////
+    void init (void)
+    {
+    };
+    void Abort (int ff, DenseVector < RealD > &evals,
+		DenseVector < DenseVector < RealD > >&evecs);
+
+    SimpleLanczos (LinearOperatorBase < Field > &Linop,	// op
+		   OperatorFunction < Field > &poly,	// polynmial
+		   int _Nstop,	// sought vecs
+		   int _Nk,	// sought vecs
+		   int _Nm,	// spare vecs
+		   RealD _eresid,	// resid in lmdue deficit 
+		   int _Niter):	// Max iterations
+     
+      _Linop (Linop),
+      _poly (poly),
+      Nstop (_Nstop), Nk (_Nk), Nm (_Nm), eresid (_eresid), Niter (_Niter)
+    {
+      Np = Nm - Nk;
+      assert (Np > 0);
+    };
+
+    /////////////////////////
+    // Sanity checked this routine (step) against Saad.
+    /////////////////////////
+    void RitzMatrix (DenseVector < Field > &evec, int k)
+    {
+
+      if (1)
+	return;
+
+      GridBase *grid = evec[0]._grid;
+      Field w (grid);
+      std::cout << GridLogMessage << "RitzMatrix " << std::endl;
+      for (int i = 0; i < k; i++)
+	{
+	  _Linop.HermOp (evec[i], w);
+//      _poly(_Linop,evec[i],w);
+	  std::cout << GridLogMessage << "[" << i << "] ";
+	  for (int j = 0; j < k; j++)
+	    {
+	      ComplexD in = innerProduct (evec[j], w);
+	      if (fabs ((double) i - j) > 1)
+		{
+		  if (abs (in) > 1.0e-9)
+		    {
+		      std::cout << GridLogMessage << "oops" << std::endl;
+		      abort ();
+		    }
+		  else
+		    std::cout << GridLogMessage << " 0 ";
+		}
+	      else
+		{
+		  std::cout << GridLogMessage << " " << in << " ";
+		}
+	    }
+	  std::cout << GridLogMessage << std::endl;
+	}
+    }
+
+    void step (DenseVector < RealD > &lmd,
+	       DenseVector < RealD > &lme,
+	       Field & last, Field & current, Field & next, uint64_t k)
+    {
+      if (lmd.size () <= k)
+	lmd.resize (k + Nm);
+      if (lme.size () <= k)
+	lme.resize (k + Nm);
+
+
+//      _poly(_Linop,current,next );   // 3. wk:=Avk−βkv_{k−1}
+      _Linop.HermOp (current, next);	// 3. wk:=Avk−βkv_{k−1}
+      if (k > 0)
+	{
+	  next -= lme[k - 1] * last;
+	}
+//      std::cout<<GridLogMessage << "<last|next>" << innerProduct(last,next) <<std::endl;
+
+      ComplexD zalph = innerProduct (current, next);	// 4. αk:=(wk,vk)
+      RealD alph = real (zalph);
+
+      next = next - alph * current;	// 5. wk:=wk−αkvk
+//      std::cout<<GridLogMessage << "<current|next>" << innerProduct(current,next) <<std::endl;
+
+      RealD beta = normalise (next);	// 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
+      // 7. vk+1 := wk/βk+1
+//       norm=beta;
+
+      int interval = Nm / 100 + 1;
+      if ((k % interval) == 0)
+	std::
+	  cout << GridLogMessage << k << " : alpha = " << zalph << " beta " <<
+	  beta << std::endl;
+      const RealD tiny = 1.0e-20;
+      if (beta < tiny)
+	{
+	  std::cout << GridLogMessage << " beta is tiny " << beta << std::
+	    endl;
+	}
+      lmd[k] = alph;
+      lme[k] = beta;
+
+    }
+
+    void qr_decomp (DenseVector < RealD > &lmd,
+		    DenseVector < RealD > &lme,
+		    int Nk,
+		    int Nm,
+		    DenseVector < RealD > &Qt, RealD Dsh, int kmin, int kmax)
+    {
+      int k = kmin - 1;
+      RealD x;
+
+      RealD Fden = 1.0 / hypot (lmd[k] - Dsh, lme[k]);
+      RealD c = (lmd[k] - Dsh) * Fden;
+      RealD s = -lme[k] * Fden;
+
+      RealD tmpa1 = lmd[k];
+      RealD tmpa2 = lmd[k + 1];
+      RealD tmpb = lme[k];
+
+      lmd[k] = c * c * tmpa1 + s * s * tmpa2 - 2.0 * c * s * tmpb;
+      lmd[k + 1] = s * s * tmpa1 + c * c * tmpa2 + 2.0 * c * s * tmpb;
+      lme[k] = c * s * (tmpa1 - tmpa2) + (c * c - s * s) * tmpb;
+      x = -s * lme[k + 1];
+      lme[k + 1] = c * lme[k + 1];
+
+      for (int i = 0; i < Nk; ++i)
+	{
+	  RealD Qtmp1 = Qt[i + Nm * k];
+	  RealD Qtmp2 = Qt[i + Nm * (k + 1)];
+	  Qt[i + Nm * k] = c * Qtmp1 - s * Qtmp2;
+	  Qt[i + Nm * (k + 1)] = s * Qtmp1 + c * Qtmp2;
+	}
+
+      // Givens transformations
+      for (int k = kmin; k < kmax - 1; ++k)
+	{
+
+	  RealD Fden = 1.0 / hypot (x, lme[k - 1]);
+	  RealD c = lme[k - 1] * Fden;
+	  RealD s = -x * Fden;
+
+	  RealD tmpa1 = lmd[k];
+	  RealD tmpa2 = lmd[k + 1];
+	  RealD tmpb = lme[k];
+
+	  lmd[k] = c * c * tmpa1 + s * s * tmpa2 - 2.0 * c * s * tmpb;
+	  lmd[k + 1] = s * s * tmpa1 + c * c * tmpa2 + 2.0 * c * s * tmpb;
+	  lme[k] = c * s * (tmpa1 - tmpa2) + (c * c - s * s) * tmpb;
+	  lme[k - 1] = c * lme[k - 1] - s * x;
+
+	  if (k != kmax - 2)
+	    {
+	      x = -s * lme[k + 1];
+	      lme[k + 1] = c * lme[k + 1];
+	    }
+
+	  for (int i = 0; i < Nk; ++i)
+	    {
+	      RealD Qtmp1 = Qt[i + Nm * k];
+	      RealD Qtmp2 = Qt[i + Nm * (k + 1)];
+	      Qt[i + Nm * k] = c * Qtmp1 - s * Qtmp2;
+	      Qt[i + Nm * (k + 1)] = s * Qtmp1 + c * Qtmp2;
+	    }
+	}
+    }
+
+#ifdef USE_LAPACK
+#ifdef USE_MKL
+#define LAPACK_INT MKL_INT
+#else
+#define LAPACK_INT long long
+#endif
+    void diagonalize_lapack (DenseVector < RealD > &lmd, DenseVector < RealD > &lme, int N1,	// all
+			     int N2,	// get
+			     GridBase * grid)
+    {
+      const int size = Nm;
+      LAPACK_INT NN = N1;
+      double evals_tmp[NN];
+      double DD[NN];
+      double EE[NN];
+      for (int i = 0; i < NN; i++)
+	for (int j = i - 1; j <= i + 1; j++)
+	  if (j < NN && j >= 0)
+	    {
+	      if (i == j)
+		DD[i] = lmd[i];
+	      if (i == j)
+		evals_tmp[i] = lmd[i];
+	      if (j == (i - 1))
+		EE[j] = lme[j];
+	    }
+      LAPACK_INT evals_found;
+      LAPACK_INT lwork =
+	((18 * NN) >
+	 (1 + 4 * NN + NN * NN) ? (18 * NN) : (1 + 4 * NN + NN * NN));
+      LAPACK_INT liwork = 3 + NN * 10;
+      LAPACK_INT iwork[liwork];
+      double work[lwork];
+      LAPACK_INT isuppz[2 * NN];
+      char jobz = 'N';		// calculate evals only
+      char range = 'I';		// calculate il-th to iu-th evals
+      //    char range = 'A'; // calculate all evals
+      char uplo = 'U';		// refer to upper half of original matrix
+      char compz = 'I';		// Compute eigenvectors of tridiagonal matrix
+      int ifail[NN];
+      LAPACK_INT info;
+//  int total = QMP_get_number_of_nodes();
+//  int node = QMP_get_node_number();
+//  GridBase *grid = evec[0]._grid;
+      int total = grid->_Nprocessors;
+      int node = grid->_processor;
+      int interval = (NN / total) + 1;
+      double vl = 0.0, vu = 0.0;
+      LAPACK_INT il = interval * node + 1, iu = interval * (node + 1);
+      if (iu > NN)
+	iu = NN;
+      double tol = 0.0;
+      if (1)
+	{
+	  memset (evals_tmp, 0, sizeof (double) * NN);
+	  if (il <= NN)
+	    {
+	      printf ("total=%d node=%d il=%d iu=%d\n", total, node, il, iu);
+#ifdef USE_MKL
+	      dstegr (&jobz, &range, &NN,
+#else
+	      LAPACK_dstegr (&jobz, &range, &NN,
+#endif
+			     (double *) DD, (double *) EE, &vl, &vu, &il, &iu,	// these four are ignored if second parameteris 'A'
+			     &tol,	// tolerance
+			     &evals_found, evals_tmp, (double *) NULL, &NN,
+			     isuppz, work, &lwork, iwork, &liwork, &info);
+	      for (int i = iu - 1; i >= il - 1; i--)
+		{
+		  printf ("node=%d evals_found=%d evals_tmp[%d] = %g\n", node,
+			  evals_found, i - (il - 1), evals_tmp[i - (il - 1)]);
+		  evals_tmp[i] = evals_tmp[i - (il - 1)];
+		  if (il > 1)
+		    evals_tmp[i - (il - 1)] = 0.;
+		}
+	    }
+	  {
+	    grid->GlobalSumVector (evals_tmp, NN);
+	  }
+	}
+// cheating a bit. It is better to sort instead of just reversing it, but the document of the routine says evals are sorted in increasing order. qr gives evals in decreasing order.
+    }
+#undef LAPACK_INT
+#endif
+
+
+    void diagonalize (DenseVector < RealD > &lmd,
+		      DenseVector < RealD > &lme,
+		      int N2, int N1, GridBase * grid)
+    {
+
+#ifdef USE_LAPACK
+      const int check_lapack = 0;	// just use lapack if 0, check against lapack if 1
+
+      if (!check_lapack)
+	return diagonalize_lapack (lmd, lme, N2, N1, grid);
+
+//      diagonalize_lapack(lmd2,lme2,Nm2,Nm,Qt,grid);
+#endif
+    }
+
+#if 1
+    static RealD normalise (Field & v)
+    {
+      RealD nn = norm2 (v);
+      nn = sqrt (nn);
+      v = v * (1.0 / nn);
+      return nn;
+    }
+
+    void orthogonalize (Field & w, DenseVector < Field > &evec, int k)
+    {
+      double t0 = -usecond () / 1e6;
+      typedef typename Field::scalar_type MyComplex;
+      MyComplex ip;
+
+      if (0)
+	{
+	  for (int j = 0; j < k; ++j)
+	    {
+	      normalise (evec[j]);
+	      for (int i = 0; i < j; i++)
+		{
+		  ip = innerProduct (evec[i], evec[j]);	// are the evecs normalised? ; this assumes so.
+		  evec[j] = evec[j] - ip * evec[i];
+		}
+	    }
+	}
+
+      for (int j = 0; j < k; ++j)
+	{
+	  ip = innerProduct (evec[j], w);	// are the evecs normalised? ; this assumes so.
+	  w = w - ip * evec[j];
+	}
+      normalise (w);
+      t0 += usecond () / 1e6;
+      OrthoTime += t0;
+    }
+
+    void setUnit_Qt (int Nm, DenseVector < RealD > &Qt)
+    {
+      for (int i = 0; i < Qt.size (); ++i)
+	Qt[i] = 0.0;
+      for (int k = 0; k < Nm; ++k)
+	Qt[k + k * Nm] = 1.0;
+    }
+
+
+    void calc (DenseVector < RealD > &eval, const Field & src, int &Nconv)
+    {
+
+      GridBase *grid = src._grid;
+//      assert(grid == src._grid);
+
+      std::
+	cout << GridLogMessage << " -- Nk = " << Nk << " Np = " << Np << std::
+	endl;
+      std::cout << GridLogMessage << " -- Nm = " << Nm << std::endl;
+      std::cout << GridLogMessage << " -- size of eval   = " << eval.
+	size () << std::endl;
+
+//      assert(c.size() && Nm == eval.size());
+
+      DenseVector < RealD > lme (Nm);
+      DenseVector < RealD > lmd (Nm);
+
+
+      Field current (grid);
+      Field last (grid);
+      Field next (grid);
+
+      Nconv = 0;
+
+      RealD beta_k;
+
+      // Set initial vector
+      // (uniform vector) Why not src??
+      //      evec[0] = 1.0;
+      current = src;
+      std::cout << GridLogMessage << "norm2(src)= " << norm2 (src) << std::
+	endl;
+      normalise (current);
+      std::
+	cout << GridLogMessage << "norm2(evec[0])= " << norm2 (current) <<
+	std::endl;
+
+      // Initial Nk steps
+      OrthoTime = 0.;
+      double t0 = usecond () / 1e6;
+      RealD norm;		// sqrt norm of last vector
+
+      uint64_t iter = 0;
+
+      bool initted = false;
+      std::vector < RealD > low (Nstop * 10);
+      std::vector < RealD > high (Nstop * 10);
+      RealD cont = 0.;
+      while (1) {
+	  cont = 0.;
+	  std::vector < RealD > lme2 (Nm);
+	  std::vector < RealD > lmd2 (Nm);
+	  for (uint64_t k = 0; k < Nm; ++k, iter++) {
+	      step (lmd, lme, last, current, next, iter);
+	      last = current;
+	      current = next;
+	    }
+	  double t1 = usecond () / 1e6;
+	  std::cout << GridLogMessage << "IRL::Initial steps: " << t1 -
+	    t0 << "seconds" << std::endl;
+	  t0 = t1;
+	  std::
+	    cout << GridLogMessage << "IRL::Initial steps:OrthoTime " <<
+	    OrthoTime << "seconds" << std::endl;
+
+	  // getting eigenvalues
+	  lmd2.resize (iter + 2);
+	  lme2.resize (iter + 2);
+	  for (uint64_t k = 0; k < iter; ++k) {
+	      lmd2[k + 1] = lmd[k];
+	      lme2[k + 2] = lme[k];
+	    }
+	  t1 = usecond () / 1e6;
+	  std::cout << GridLogMessage << "IRL:: copy: " << t1 -
+	    t0 << "seconds" << std::endl;
+	  t0 = t1;
+	  {
+	    int total = grid->_Nprocessors;
+	    int node = grid->_processor;
+	    int interval = (Nstop / total) + 1;
+	    int iu = (iter + 1) - (interval * node + 1);
+	    int il = (iter + 1) - (interval * (node + 1));
+	    std::vector < RealD > eval2 (iter + 3);
+	    RealD eps2;
+	    Bisection::bisec (lmd2, lme2, iter, il, iu, 1e-16, 1e-10, eval2,
+			      eps2);
+//        diagonalize(eval2,lme2,iter,Nk,grid);
+	    RealD diff = 0.;
+	    for (int i = il; i <= iu; i++) {
+		if (initted)
+		  diff =
+		    fabs (eval2[i] - high[iu-i]) / (fabs (eval2[i]) +
+						      fabs (high[iu-i]));
+		if (initted && (diff > eresid))
+		  cont = 1.;
+		if (initted)
+		  printf ("eval[%d]=%0.14e %0.14e, %0.14e\n", i, eval2[i],
+			  high[iu-i], diff);
+		high[iu-i] = eval2[i];
+	      }
+	    il = (interval * node + 1);
+	    iu = (interval * (node + 1));
+	    Bisection::bisec (lmd2, lme2, iter, il, iu, 1e-16, 1e-10, eval2,
+			      eps2);
+	    for (int i = il; i <= iu; i++) {
+		if (initted)
+		  diff =
+		    fabs (eval2[i] - low[i]) / (fabs (eval2[i]) +
+						fabs (low[i]));
+		if (initted && (diff > eresid))
+		  cont = 1.;
+		if (initted)
+		  printf ("eval[%d]=%0.14e %0.14e, %0.14e\n", i, eval2[i],
+			  low[i], diff);
+		low[i] = eval2[i];
+	      }
+	    t1 = usecond () / 1e6;
+	    std::cout << GridLogMessage << "IRL:: diagonalize: " << t1 -
+	      t0 << "seconds" << std::endl;
+	    t0 = t1;
+	  }
+
+	  for (uint64_t k = 0; k < Nk; ++k) {
+//          eval[k] = eval2[k];
+	    }
+	  if (initted)
+	    {
+	      grid->GlobalSumVector (&cont, 1);
+	      if (cont < 1.) return;
+	    }
+	  initted = true;
+	}
+
+    }
+
+
+
+
+
+
+/**
+   There is some matrix Q such that for any vector y
+   Q.e_1 = y and Q is unitary.
+**/
+    template < class T >
+      static T orthQ (DenseMatrix < T > &Q, DenseVector < T > y)
+    {
+      int N = y.size ();	//Matrix Size
+      Fill (Q, 0.0);
+      T tau;
+      for (int i = 0; i < N; i++)
+	{
+	  Q[i][0] = y[i];
+	}
+      T sig = conj (y[0]) * y[0];
+      T tau0 = fabs (sqrt (sig));
+
+      for (int j = 1; j < N; j++)
+	{
+	  sig += conj (y[j]) * y[j];
+	  tau = abs (sqrt (sig));
+
+	  if (abs (tau0) > 0.0)
+	    {
+
+	      T gam = conj ((y[j] / tau) / tau0);
+	      for (int k = 0; k <= j - 1; k++)
+		{
+		  Q[k][j] = -gam * y[k];
+		}
+	      Q[j][j] = tau0 / tau;
+	    }
+	  else
+	    {
+	      Q[j - 1][j] = 1.0;
+	    }
+	  tau0 = tau;
+	}
+      return tau;
+    }
+
+/**
+	There is some matrix Q such that for any vector y
+	Q.e_k = y and Q is unitary.
+**/
+    template < class T >
+      static T orthU (DenseMatrix < T > &Q, DenseVector < T > y)
+    {
+      T tau = orthQ (Q, y);
+      SL (Q);
+      return tau;
+    }
+
+
+/**
+	Wind up with a matrix with the first con rows untouched
+
+say con = 2
+	Q is such that Qdag H Q has {x, x, val, 0, 0, 0, 0, ...} as 1st colum
+	and the matrix is upper hessenberg
+	and with f and Q appropriately modidied with Q is the arnoldi factorization
+
+**/
+
+    template < class T > static void Lock (DenseMatrix < T > &H,	///Hess mtx     
+					   DenseMatrix < T > &Q,	///Lock Transform
+					   T val,	///value to be locked
+					   int con,	///number already locked
+					   RealD small, int dfg, bool herm)
+    {
+      //ForceTridiagonal(H);
+
+      int M = H.dim;
+      DenseVector < T > vec;
+      Resize (vec, M - con);
+
+      DenseMatrix < T > AH;
+      Resize (AH, M - con, M - con);
+      AH = GetSubMtx (H, con, M, con, M);
+
+      DenseMatrix < T > QQ;
+      Resize (QQ, M - con, M - con);
+
+      Unity (Q);
+      Unity (QQ);
+
+      DenseVector < T > evals;
+      Resize (evals, M - con);
+      DenseMatrix < T > evecs;
+      Resize (evecs, M - con, M - con);
+
+      Wilkinson < T > (AH, evals, evecs, small);
+
+      int k = 0;
+      RealD cold = abs (val - evals[k]);
+      for (int i = 1; i < M - con; i++)
+	{
+	  RealD cnew = abs (val - evals[i]);
+	  if (cnew < cold)
+	    {
+	      k = i;
+	      cold = cnew;
+	    }
+	}
+      vec = evecs[k];
+
+      ComplexD tau;
+      orthQ (QQ, vec);
+      //orthQM(QQ,AH,vec);
+
+      AH = Hermitian (QQ) * AH;
+      AH = AH * QQ;
+
+      for (int i = con; i < M; i++)
+	{
+	  for (int j = con; j < M; j++)
+	    {
+	      Q[i][j] = QQ[i - con][j - con];
+	      H[i][j] = AH[i - con][j - con];
+	    }
+	}
+
+      for (int j = M - 1; j > con + 2; j--)
+	{
+
+	  DenseMatrix < T > U;
+	  Resize (U, j - 1 - con, j - 1 - con);
+	  DenseVector < T > z;
+	  Resize (z, j - 1 - con);
+	  T nm = norm (z);
+	  for (int k = con + 0; k < j - 1; k++)
+	    {
+	      z[k - con] = conj (H (j, k + 1));
+	    }
+	  normalise (z);
+
+	  RealD tmp = 0;
+	  for (int i = 0; i < z.size () - 1; i++)
+	    {
+	      tmp = tmp + abs (z[i]);
+	    }
+
+	  if (tmp < small / ((RealD) z.size () - 1.0))
+	    {
+	      continue;
+	    }
+
+	  tau = orthU (U, z);
+
+	  DenseMatrix < T > Hb;
+	  Resize (Hb, j - 1 - con, M);
+
+	  for (int a = 0; a < M; a++)
+	    {
+	      for (int b = 0; b < j - 1 - con; b++)
+		{
+		  T sum = 0;
+		  for (int c = 0; c < j - 1 - con; c++)
+		    {
+		      sum += H[a][con + 1 + c] * U[c][b];
+		    }		//sum += H(a,con+1+c)*U(c,b);}
+		  Hb[b][a] = sum;
+		}
+	    }
+
+	  for (int k = con + 1; k < j; k++)
+	    {
+	      for (int l = 0; l < M; l++)
+		{
+		  H[l][k] = Hb[k - 1 - con][l];
+		}
+	    }			//H(Hb[k-1-con][l] , l,k);}}
+
+	  DenseMatrix < T > Qb;
+	  Resize (Qb, M, M);
+
+	  for (int a = 0; a < M; a++)
+	    {
+	      for (int b = 0; b < j - 1 - con; b++)
+		{
+		  T sum = 0;
+		  for (int c = 0; c < j - 1 - con; c++)
+		    {
+		      sum += Q[a][con + 1 + c] * U[c][b];
+		    }		//sum += Q(a,con+1+c)*U(c,b);}
+		  Qb[b][a] = sum;
+		}
+	    }
+
+	  for (int k = con + 1; k < j; k++)
+	    {
+	      for (int l = 0; l < M; l++)
+		{
+		  Q[l][k] = Qb[k - 1 - con][l];
+		}
+	    }			//Q(Qb[k-1-con][l] , l,k);}}
+
+	  DenseMatrix < T > Hc;
+	  Resize (Hc, M, M);
+
+	  for (int a = 0; a < j - 1 - con; a++)
+	    {
+	      for (int b = 0; b < M; b++)
+		{
+		  T sum = 0;
+		  for (int c = 0; c < j - 1 - con; c++)
+		    {
+		      sum += conj (U[c][a]) * H[con + 1 + c][b];
+		    }		//sum += conj( U(c,a) )*H(con+1+c,b);}
+		  Hc[b][a] = sum;
+		}
+	    }
+
+	  for (int k = 0; k < M; k++)
+	    {
+	      for (int l = con + 1; l < j; l++)
+		{
+		  H[l][k] = Hc[k][l - 1 - con];
+		}
+	    }			//H(Hc[k][l-1-con] , l,k);}}
+
+	}
+    }
+#endif
+
+
+  };
+
+}
+#endif

lib/algorithms/iterative/bisec.c

@@ -0,0 +1,122 @@
+#include <math.h>
+#include <stdlib.h>
+#include <vector>
+
+struct Bisection {
+
+static void get_eig2(int row_num,std::vector<RealD> &ALPHA,std::vector<RealD> &BETA, std::vector<RealD> & eig)
+{
+  int i,j;
+  std::vector<RealD> evec1(row_num+3);
+  std::vector<RealD> evec2(row_num+3);
+  RealD eps2;
+  ALPHA[1]=0.;
+  BETHA[1]=0.;
+  for(i=0;i<row_num-1;i++) {
+    ALPHA[i+1] = A[i*(row_num+1)].real();
+    BETHA[i+2] = A[i*(row_num+1)+1].real();
+  }
+  ALPHA[row_num] = A[(row_num-1)*(row_num+1)].real();
+  bisec(ALPHA,BETHA,row_num,1,row_num,1e-10,1e-10,evec1,eps2);
+  bisec(ALPHA,BETHA,row_num,1,row_num,1e-16,1e-16,evec2,eps2);
+
+  // Do we really need to sort here?
+  int begin=1;
+  int end = row_num;
+  int swapped=1;
+  while(swapped) {
+    swapped=0;
+    for(i=begin;i<end;i++){
+      if(mag(evec2[i])>mag(evec2[i+1]))	{
+	swap(evec2+i,evec2+i+1);
+	swapped=1;
+      }
+    }
+    end--;
+    for(i=end-1;i>=begin;i--){
+      if(mag(evec2[i])>mag(evec2[i+1]))	{
+	swap(evec2+i,evec2+i+1);
+	swapped=1;
+      }
+    }
+    begin++;
+  }
+
+  for(i=0;i<row_num;i++){
+    for(j=0;j<row_num;j++) {
+      if(i==j) H[i*row_num+j]=evec2[i+1];
+      else H[i*row_num+j]=0.;
+    }
+  }
+}
+
+static void bisec(std::vector<RealD> &c,   
+		  std::vector<RealD> &b,
+		  int n,
+		  int m1,
+		  int m2,
+		  RealD eps1,
+		  RealD relfeh,
+		  std::vector<RealD> &x,
+		  RealD &eps2)
+{
+  std::vector<RealD> wu(n+2);
+
+  RealD h,q,x1,xu,x0,xmin,xmax; 
+  int i,a,k;
+
+  b[1]=0.0;
+  xmin=c[n]-fabs(b[n]);
+  xmax=c[n]+fabs(b[n]);
+  for(i=1;i<n;i++){
+    h=fabs(b[i])+fabs(b[i+1]);
+    if(c[i]+h>xmax) xmax= c[i]+h;
+    if(c[i]-h<xmin) xmin= c[i]-h;
+  }
+  xmax *=2.;
+
+  eps2=relfeh*((xmin+xmax)>0.0 ? xmax : -xmin);
+  if(eps1<=0.0) eps1=eps2;
+  eps2=0.5*eps1+7.0*(eps2);
+  x0=xmax;
+  for(i=m1;i<=m2;i++){
+    x[i]=xmax;
+    wu[i]=xmin;
+  }
+
+  for(k=m2;k>=m1;k--){
+    xu=xmin;
+    i=k;
+    do{
+      if(xu<wu[i]){
+	xu=wu[i];
+	i=m1-1;
+      }
+      i--;
+    }while(i>=m1);
+    if(x0>x[k]) x0=x[k];
+    while((x0-xu)>2*relfeh*(fabs(xu)+fabs(x0))+eps1){
+      x1=(xu+x0)/2;
+
+      a=0;
+      q=1.0;
+      for(i=1;i<=n;i++){
+	q=c[i]-x1-((q!=0.0)? b[i]*b[i]/q:fabs(b[i])/relfeh);
+	if(q<0) a++;
+      }
+      //			printf("x1=%e a=%d\n",x1,a);
+      if(a<k){
+	if(a<m1){
+	  xu=x1;
+	  wu[m1]=x1;
+	}else {
+	  xu=x1;
+	  wu[a+1]=x1;
+	  if(x[a]>x1) x[a]=x1;
+	}
+      }else x0=x1;
+    }
+    x[k]=(x0+xu)/2;
+  }
+}
+}

lib/allocator/AlignedAllocator.cc

@@ -3,12 +3,9 @@
 
 namespace Grid {
 
-MemoryStats *MemoryProfiler::stats = nullptr;
-bool         MemoryProfiler::debug = false;
-
 int PointerCache::victim;
 
-PointerCache::PointerCacheEntry PointerCache::Entries[PointerCache::Ncache];
+  PointerCache::PointerCacheEntry PointerCache::Entries[PointerCache::Ncache];
 
 void *PointerCache::Insert(void *ptr,size_t bytes) {
 
@@ -97,29 +94,4 @@ void check_huge_pages(void *Buf,uint64_t BYTES)
 #endif
 }
 
-std::string sizeString(const size_t bytes)
-{
-  constexpr unsigned int bufSize = 256;
-  const char             *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
-  char                   buf[256];
-  size_t                 s     = 0;
-  double                 count = bytes;
-  
-  while (count >= 1024 && s < 7)
-  {
-      s++;
-      count /= 1024;
-  }
-  if (count - floor(count) == 0.0)
-  {
-      snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
-  }
-  else
-  {
-      snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
-  }
-  
-  return std::string(buf);
-}
-
 }

lib/allocator/AlignedAllocator.h

@@ -63,64 +63,6 @@ namespace Grid {
     static void *Lookup(size_t bytes) ;
 
   };
-  
-  std::string sizeString(size_t bytes);
-
-  struct MemoryStats
-  {
-    size_t totalAllocated{0}, maxAllocated{0}, 
-           currentlyAllocated{0}, totalFreed{0};
-  };
-    
-  class MemoryProfiler
-  {
-  public:
-    static MemoryStats *stats;
-    static bool        debug;
-  };
-
-  #define memString(bytes) std::to_string(bytes) + " (" + sizeString(bytes) + ")"
-  #define profilerDebugPrint \
-  if (MemoryProfiler::stats)\
-  {\
-    auto s = MemoryProfiler::stats;\
-    std::cout << GridLogDebug << "[Memory debug] Stats " << MemoryProfiler::stats << std::endl;\
-    std::cout << GridLogDebug << "[Memory debug] total  : " << memString(s->totalAllocated) \
-              << std::endl;\
-    std::cout << GridLogDebug << "[Memory debug] max    : " << memString(s->maxAllocated) \
-              << std::endl;\
-    std::cout << GridLogDebug << "[Memory debug] current: " << memString(s->currentlyAllocated) \
-              << std::endl;\
-    std::cout << GridLogDebug << "[Memory debug] freed  : " << memString(s->totalFreed) \
-              << std::endl;\
-  }
-
-  #define profilerAllocate(bytes)\
-  if (MemoryProfiler::stats)\
-  {\
-    auto s = MemoryProfiler::stats;\
-    s->totalAllocated     += (bytes);\
-    s->currentlyAllocated += (bytes);\
-    s->maxAllocated        = std::max(s->maxAllocated, s->currentlyAllocated);\
-  }\
-  if (MemoryProfiler::debug)\
-  {\
-    std::cout << GridLogDebug << "[Memory debug] allocating " << memString(bytes) << std::endl;\
-    profilerDebugPrint;\
-  }
-
-  #define profilerFree(bytes)\
-  if (MemoryProfiler::stats)\
-  {\
-    auto s = MemoryProfiler::stats;\
-    s->totalFreed         += (bytes);\
-    s->currentlyAllocated -= (bytes);\
-  }\
-  if (MemoryProfiler::debug)\
-  {\
-    std::cout << GridLogDebug << "[Memory debug] freeing " << memString(bytes) << std::endl;\
-    profilerDebugPrint;\
-  }
 
   void check_huge_pages(void *Buf,uint64_t BYTES);
 
@@ -150,7 +92,6 @@ public:
   pointer allocate(size_type __n, const void* _p= 0)
   { 
     size_type bytes = __n*sizeof(_Tp);
-    profilerAllocate(bytes);
 
     _Tp *ptr = (_Tp *) PointerCache::Lookup(bytes);
     //    if ( ptr != NULL ) 
@@ -181,8 +122,6 @@ public:
   void deallocate(pointer __p, size_type __n) { 
     size_type bytes = __n * sizeof(_Tp);
 
-    profilerFree(bytes);
-
     pointer __freeme = (pointer)PointerCache::Insert((void *)__p,bytes);
 
 #ifdef HAVE_MM_MALLOC_H
@@ -233,13 +172,10 @@ public:
 #ifdef GRID_COMMS_SHMEM
   pointer allocate(size_type __n, const void* _p= 0)
   {
-    size_type bytes = __n*sizeof(_Tp);
-
-    profilerAllocate(bytes);
 #ifdef CRAY
-    _Tp *ptr = (_Tp *) shmem_align(bytes,64);
+    _Tp *ptr = (_Tp *) shmem_align(__n*sizeof(_Tp),64);
 #else
-    _Tp *ptr = (_Tp *) shmem_align(64,bytes);
+    _Tp *ptr = (_Tp *) shmem_align(64,__n*sizeof(_Tp));
 #endif
 #ifdef PARANOID_SYMMETRIC_HEAP
     static void * bcast;
@@ -257,23 +193,18 @@ public:
 #endif 
     return ptr;
   }
-  void deallocate(pointer __p, size_type __n) { 
-    size_type bytes = __n*sizeof(_Tp);
-
-    profilerFree(bytes);
+  void deallocate(pointer __p, size_type) { 
     shmem_free((void *)__p);
   }
 #else
   pointer allocate(size_type __n, const void* _p= 0) 
   {
-    size_type bytes = __n*sizeof(_Tp);
-    
-    profilerAllocate(bytes);
 #ifdef HAVE_MM_MALLOC_H
-    _Tp * ptr = (_Tp *) _mm_malloc(bytes, GRID_ALLOC_ALIGN);
+    _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),GRID_ALLOC_ALIGN);
 #else
-    _Tp * ptr = (_Tp *) memalign(GRID_ALLOC_ALIGN, bytes);
+    _Tp * ptr = (_Tp *) memalign(GRID_ALLOC_ALIGN,__n*sizeof(_Tp));
 #endif
+    size_type bytes = __n*sizeof(_Tp);
     uint8_t *cp = (uint8_t *)ptr;
     if ( ptr ) { 
     // One touch per 4k page, static OMP loop to catch same loop order
@@ -284,10 +215,7 @@ public:
     }
     return ptr;
   }
-  void deallocate(pointer __p, size_type __n) {
-    size_type bytes = __n*sizeof(_Tp);
-
-    profilerFree(bytes);
+  void deallocate(pointer __p, size_type) { 
 #ifdef HAVE_MM_MALLOC_H
     _mm_free((void *)__p); 
 #else

lib/cartesian/Cartesian_base.h

@@ -44,22 +44,16 @@ namespace Grid{
   class GridBase : public CartesianCommunicator , public GridThread {
 
 public:
-    int dummy;
+
     // Give Lattice access
     template<class object> friend class Lattice;
 
     GridBase(const std::vector<int> & processor_grid) : CartesianCommunicator(processor_grid) {};
     GridBase(const std::vector<int> & processor_grid,
-	     const CartesianCommunicator &parent,
-	     int &split_rank) 
-      : CartesianCommunicator(processor_grid,parent,split_rank) {};
-    GridBase(const std::vector<int> & processor_grid,
-	     const CartesianCommunicator &parent) 
-      : CartesianCommunicator(processor_grid,parent,dummy) {};
+	     const CartesianCommunicator &parent) : CartesianCommunicator(processor_grid,parent) {};
 
     virtual ~GridBase() = default;
 
-
     // Physics Grid information.
     std::vector<int> _simd_layout;// Which dimensions get relayed out over simd lanes.
     std::vector<int> _fdimensions;// (full) Global dimensions of array prior to cb removal
@@ -79,8 +73,6 @@ public:
     std::vector<int> _lstart;     // local start of array in gcoors _processor_coor[d]*_ldimensions[d]
     std::vector<int> _lend  ;     // local end of array in gcoors   _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1
 
-    bool _isCheckerBoarded; 
-
 public:
 
     ////////////////////////////////////////////////////////////////

lib/cartesian/Cartesian_full.h

@@ -38,7 +38,7 @@ namespace Grid{
 class GridCartesian: public GridBase {
 
 public:
-    int dummy;
+
     virtual int  CheckerBoardFromOindexTable (int Oindex) {
       return 0;
     }
@@ -67,14 +67,7 @@ public:
     GridCartesian(const std::vector<int> &dimensions,
 		  const std::vector<int> &simd_layout,
 		  const std::vector<int> &processor_grid,
-		  const GridCartesian &parent) : GridBase(processor_grid,parent,dummy)
-    {
-      Init(dimensions,simd_layout,processor_grid);
-    }
-    GridCartesian(const std::vector<int> &dimensions,
-		  const std::vector<int> &simd_layout,
-		  const std::vector<int> &processor_grid,
-		  const GridCartesian &parent,int &split_rank) : GridBase(processor_grid,parent,split_rank)
+		  const GridCartesian &parent) : GridBase(processor_grid,parent)
     {
       Init(dimensions,simd_layout,processor_grid);
     }
@@ -97,7 +90,6 @@ public:
       ///////////////////////
       // Grid information
       ///////////////////////
-      _isCheckerBoarded = false;
       _ndimension = dimensions.size();
 
       _fdimensions.resize(_ndimension);
@@ -123,7 +115,6 @@ public:
 
         // Use a reduced simd grid
         _ldimensions[d] = _gdimensions[d] / _processors[d]; //local dimensions
-        //std::cout << _ldimensions[d] << "  " << _gdimensions[d] << "  " << _processors[d] << std::endl;
         assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
 
         _rdimensions[d] = _ldimensions[d] / _simd_layout[d]; //overdecomposition
@@ -168,7 +159,6 @@ public:
         block = block * _rdimensions[d];
       }
     };
-
 };
 }
 #endif

lib/cartesian/Cartesian_red_black.h

@@ -171,8 +171,9 @@ public:
               const std::vector<int> &checker_dim_mask,
               int checker_dim)
     {
-
-      _isCheckerBoarded = true;
+      ///////////////////////
+      // Grid information
+      ///////////////////////
       _checker_dim = checker_dim;
       assert(checker_dim_mask[checker_dim] == 1);
       _ndimension = dimensions.size();
@@ -206,7 +207,6 @@ public:
         {
           assert((_gdimensions[d] & 0x1) == 0);
           _gdimensions[d] = _gdimensions[d] / 2; // Remove a checkerboard
-	  _gsites /= 2;
         }
         _ldimensions[d] = _gdimensions[d] / _processors[d];
         assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);

lib/communicator/Communicator.h

@@ -28,7 +28,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifndef GRID_COMMUNICATOR_H
 #define GRID_COMMUNICATOR_H
 
-#include <Grid/communicator/SharedMemory.h>
 #include <Grid/communicator/Communicator_base.h>
 
 #endif

lib/communicator/Communicator_base.cc

@@ -36,9 +36,33 @@ namespace Grid {
 ///////////////////////////////////////////////////////////////
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////
+void *              CartesianCommunicator::ShmCommBuf;
+uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 1024LL*1024LL*1024LL; 
 CartesianCommunicator::CommunicatorPolicy_t  
 CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent;
 int CartesianCommunicator::nCommThreads = -1;
+int CartesianCommunicator::Hugepages = 0;
+
+/////////////////////////////////
+// Alloc, free shmem region
+/////////////////////////////////
+void *CartesianCommunicator::ShmBufferMalloc(size_t bytes){
+  //  bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes
+  void *ptr = (void *)heap_top;
+  heap_top  += bytes;
+  heap_bytes+= bytes;
+  if (heap_bytes >= MAX_MPI_SHM_BYTES) {
+    std::cout<< " ShmBufferMalloc exceeded shared heap size -- try increasing with --shm <MB> flag" <<std::endl;
+    std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
+    std::cout<< " Current value is " << (MAX_MPI_SHM_BYTES/(1024*1024)) <<std::endl;
+    assert(heap_bytes<MAX_MPI_SHM_BYTES);
+  }
+  return ptr;
+}
+void CartesianCommunicator::ShmBufferFreeAll(void) { 
+  heap_top  =(size_t)ShmBufferSelf();
+  heap_bytes=0;
+}
 
 /////////////////////////////////
 // Grid information queries
@@ -71,6 +95,202 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
 {
   GlobalSumVector((double *)c,2*N);
 }
+
+
+#if defined( GRID_COMMS_MPI) || defined (GRID_COMMS_MPIT)
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
+{
+  _ndimension = processors.size();
+  
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // split the communicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  int Nparent;
+  MPI_Comm_size(parent.communicator,&Nparent);
+
+  int childsize=1;
+  for(int d=0;d<processors.size();d++) {
+    childsize *= processors[d];
+  }
+  int Nchild = Nparent/childsize;
+  assert (childsize * Nchild == Nparent);
+
+  std::vector<int> ccoor(_ndimension); // coor within subcommunicator
+  std::vector<int> scoor(_ndimension); // coor of split within parent
+  std::vector<int> ssize(_ndimension); // coor of split within parent
+
+  std::vector<int> pcoor(_ndimension,0); 
+  std::vector<int> pdims(_ndimension,1); 
+
+  if(parent._processors.size()==4 && _ndimension==5){
+      for(int i=0;i<4;i++) pcoor[i+1]=parent._processor_coor[i];
+      for(int i=0;i<4;i++) pdims[i+1]=parent._processors[i];
+  } else {
+      assert(_ndimension == parent._ndimension);
+      for(int i=0;i<_ndimension;i++) pcoor[i]=parent._processor_coor[i];
+      for(int i=0;i<_ndimension;i++) pdims[i]=parent._processors[i];
+  }
+
+  for(int d=0;d<_ndimension;d++){
+    ccoor[d] = pcoor[d] % processors[d];
+    scoor[d] = pcoor[d] / processors[d];
+    ssize[d] = pdims[d] / processors[d];
+  }
+  int crank,srank;  // rank within subcomm ; rank of subcomm within blocks of subcomms
+  Lexicographic::IndexFromCoor(ccoor,crank,processors);
+  Lexicographic::IndexFromCoor(scoor,srank,ssize);
+
+  MPI_Comm comm_split;
+  if ( Nchild > 1 ) { 
+
+    //    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
+    //    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
+    //    for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
+    //    std::cout<<std::endl;
+
+    //    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
+    //    for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
+    //    std::cout<<std::endl;
+
+    int ierr= MPI_Comm_split(parent.communicator,srank,crank,&comm_split);
+    assert(ierr==0);
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // Declare victory
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    //    std::cout << GridLogMessage<<"Divided communicator "<< parent._Nprocessors<<" into "
+    // 	      << Nchild <<" communicators with " << childsize << " ranks"<<std::endl;
+  } else {
+    comm_split=parent.communicator;
+  }
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Set up from the new split communicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  InitFromMPICommunicator(processors,comm_split);
+}
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+// Take an MPI_Comm and self assemble
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+void CartesianCommunicator::InitFromMPICommunicator(const std::vector<int> &processors, MPI_Comm communicator_base)
+{
+  //  if ( communicator_base != communicator_world ) {
+  //    std::cout << "Cartesian communicator created with a non-world communicator"<<std::endl;
+  //  }
+  _ndimension = processors.size();
+  _processor_coor.resize(_ndimension);
+
+  /////////////////////////////////
+  // Count the requested nodes
+  /////////////////////////////////
+  _Nprocessors=1;
+  _processors = processors;
+  for(int i=0;i<_ndimension;i++){
+    _Nprocessors*=_processors[i];
+  }
+
+  std::vector<int> periodic(_ndimension,1);
+  MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Comm_rank(communicator,&_processor);
+  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
+
+  int Size;
+  MPI_Comm_size(communicator,&Size);
+
+#ifdef GRID_COMMS_MPIT
+  communicator_halo.resize (2*_ndimension);
+  for(int i=0;i<_ndimension*2;i++){
+    MPI_Comm_dup(communicator,&communicator_halo[i]);
+  }
+#endif
+  
+  assert(Size==_Nprocessors);
+}
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors) 
+{
+  InitFromMPICommunicator(processors,communicator_world);
+}
+
+#endif
+
+#if !defined( GRID_COMMS_MPI3) 
+
+int                      CartesianCommunicator::NodeCount(void)    { return ProcessorCount();};
+int                      CartesianCommunicator::RankCount(void)    { return ProcessorCount();};
+#endif
+#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPIT)
+double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
+						     int xmit_to_rank,
+						     void *recv,
+						     int recv_from_rank,
+						     int bytes, int dir)
+{
+  std::vector<CommsRequest_t> list;
+  // Discard the "dir"
+  SendToRecvFromBegin   (list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
+  SendToRecvFromComplete(list);
+  return 2.0*bytes;
+}
+double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+							 void *xmit,
+							 int xmit_to_rank,
+							 void *recv,
+							 int recv_from_rank,
+							 int bytes, int dir)
+{
+  // Discard the "dir"
+  SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
+  return 2.0*bytes;
+}
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
+{
+  SendToRecvFromComplete(waitall);
+}
+#endif
+
+#if !defined( GRID_COMMS_MPI3) 
+
+void CartesianCommunicator::StencilBarrier(void){};
+
+commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
+
+void *CartesianCommunicator::ShmBufferSelf(void) { return ShmCommBuf; }
+
+void *CartesianCommunicator::ShmBuffer(int rank) {
+  return NULL;
+}
+void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { 
+  return NULL;
+}
+void CartesianCommunicator::ShmInitGeneric(void){
+#if 1
+  int mmap_flag =0;
+#ifdef MAP_ANONYMOUS
+  mmap_flag = mmap_flag| MAP_SHARED | MAP_ANONYMOUS;
+#endif
+#ifdef MAP_ANON
+  mmap_flag = mmap_flag| MAP_SHARED | MAP_ANON;
+#endif
+#ifdef MAP_HUGETLB
+  if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
+#endif
+  ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag, -1, 0); 
+  if (ShmCommBuf == (void *)MAP_FAILED) {
+    perror("mmap failed ");
+    exit(EXIT_FAILURE);  
+  }
+#ifdef MADV_HUGEPAGE
+  if (!Hugepages ) madvise(ShmCommBuf,MAX_MPI_SHM_BYTES,MADV_HUGEPAGE);
+#endif
+#else 
+  ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
+  ShmCommBuf=(void *)&ShmBufStorageVector[0];
+#endif
+  bzero(ShmCommBuf,MAX_MPI_SHM_BYTES);
+}
+
+#endif
   
 }