Hadrons: first full implementation of the scheduler

programs/Hadrons/Application.cc

@@ -83,6 +83,7 @@ void Application::parseParameterFile(void)
         {
             associatedModule_[n] = id.name;
         }
+        input_[id.name] = module_[id.name]->getInput();
     } while (reader.nextElement("module"));
     pop(reader);
     pop(reader);
@@ -113,19 +114,35 @@ void Application::schedule(void)
     }
     
     // topological sort
-    std::map<std::string, std::map<std::string, bool>> m;
+    unsigned int k = 0;
-    unsigned int                                       k = 0;
     
     std::vector<Graph<std::string>> con = moduleGraph.getConnectedComponents();
     LOG(Message) << "Program:" << std::endl;
     for (unsigned int i = 0; i < con.size(); ++i)
     {
         std::vector<std::vector<std::string>> t = con[i].allTopoSort();
+        int                                   memPeak, minMemPeak = -1;
+        unsigned int                          bestInd;
+        bool                                  msg;
 
-        m = makeDependencyMatrix(t);
+        env_.dryRun(true);
-        for (unsigned int j = 0; j < t[0].size(); ++j)
+        for (unsigned int p = 0; p < t.size(); ++p)
         {
-            program_.push_back(t[0][j]);
+            msg = HadronsLogMessage.isActive();
+            HadronsLogMessage.Active(false);
+            memPeak = execute(t[p]);
+            if ((memPeak < minMemPeak) or (minMemPeak < 0))
+            {
+                minMemPeak = memPeak;
+                bestInd = p;
+            }
+            HadronsLogMessage.Active(msg);
+            env_.freeAll();
+        }
+        env_.dryRun(false);
+        for (unsigned int j = 0; j < t[bestInd].size(); ++j)
+        {
+            program_.push_back(t[bestInd][j]);
             LOG(Message) << std::setw(4) << std::right << k << ": "
                          << program_[k] << std::endl;
             k++;
@@ -142,16 +159,42 @@ void Application::configLoop(void)
     {
         LOG(Message) << "Starting measurement for trajectory " << t
                      << std::endl;
-        execute();
+        execute(program_);
+        env_.freeAll();
     }
 }
 
-void Application::execute(void)
+unsigned int Application::execute(const std::vector<std::string> &program)
 {
-    for (unsigned int i = 0; i < program_.size(); ++i)
+    unsigned int memPeak = 0;
+    
+    for (unsigned int i = 0; i < program.size(); ++i)
     {
-        LOG(Message) << "Measurement step (" << i+1 << "/" << program_.size()
+        LOG(Message) << "Measurement step (" << i+1 << "/" << program.size()
                      << ")" << std::endl;
-        (*module_[program_[i]])(env_);
+        (*module_[program[i]])(env_);
+        LOG(Message) << "allocated propagators: " << env_.nProp() << std::endl;
+        if (env_.nProp() > memPeak)
+        {
+            memPeak = env_.nProp();
+        }
+        for (auto &n: associatedModule_)
+        {
+            bool canFree = true;
+            
+            for (unsigned int j = i + 1; j < program.size(); ++j)
+            {
+                auto &in = input_[program[j]];
+                auto it  = std::find(in.begin(), in.end(), n.first);
+                canFree  = canFree and (it == in.end());
+            }
+            if (canFree and env_.propExists(n.first))
+            {
+                LOG(Message) << "freeing '" << n.first << "'" << std::endl;
+                env_.free(n.first);
+            }
+        }
     }
+    
+    return memPeak;
 }

programs/Hadrons/Application.hpp

@@ -51,8 +51,7 @@ class GlobalPar: Serializable
 {
 public:
     GRID_SERIALIZABLE_CLASS_MEMBERS(GlobalPar,
-                                    std::vector<double>, latticeSize,
+                                    ConfigPar, configs);
-                                    ConfigPar,           configs);
 };
 
 /******************************************************************************
@@ -75,16 +74,17 @@ private:
     // schedule computation
     void schedule(void);
     // program execution
-    void configLoop(void);
+    void         configLoop(void);
-    void execute(void);
+    unsigned int execute(const std::vector<std::string> &program);
 private:
-    std::string                                    parameterFileName_;
+    std::string                                     parameterFileName_;
-    GlobalPar                                      par_;
+    GlobalPar                                       par_;
-    Environment                                    &env_;
+    Environment                                     &env_;
-    ModuleFactory                                  &modFactory_;
+    ModuleFactory                                   &modFactory_;
-    std::map<std::string, std::unique_ptr<Module>> module_;
+    std::map<std::string, std::unique_ptr<Module>>  module_;
-    std::map<std::string, std::string>             associatedModule_;
+    std::map<std::string, std::string>              associatedModule_;
-    std::vector<std::string>                       program_;
+    std::map<std::string, std::vector<std::string>> input_;
+    std::vector<std::string>                        program_;
 };
 
 END_HADRONS_NAMESPACE

programs/Hadrons/CMeson.cc

@@ -52,13 +52,11 @@ std::vector<std::string> CMeson::getOutput(void)
 }
 
 // memory footprint ////////////////////////////////////////////////////////////
-double CMeson::nCreatedProp(void)
+void CMeson::allocate(Environment &env)
-{
+{}
-    return 0.;
-}
 
 // execution ///////////////////////////////////////////////////////////////////
-void CMeson::operator()(Environment &env)
+void CMeson::execute(Environment &env)
 {
     LOG(Message) << "computing meson contraction '" << getName() << "'"
                  << std::endl;

programs/Hadrons/CMeson.hpp

@@ -50,10 +50,10 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-    // memory footprint
+    // allocation
-    virtual double nCreatedProp(void);
+    virtual void allocate(Environment &env);
     // execution
-    virtual void operator()(Environment &env);
+    virtual void execute(Environment &env);
 private:
     Par par_;
 };

programs/Hadrons/Environment.cc

@@ -20,6 +20,7 @@
 #include <Hadrons/Environment.hpp>
 
 using namespace Grid;
+using namespace QCD;
 using namespace Hadrons;
 
 /******************************************************************************
@@ -27,5 +28,132 @@ using namespace Hadrons;
  ******************************************************************************/
 // constructor /////////////////////////////////////////////////////////////////
 Environment::Environment(void)
-{}
+{
+    grid4d_.reset(SpaceTimeGrid::makeFourDimGrid(
+        GridDefaultLatt(), GridDefaultSimd(Nd, vComplex::Nsimd()),
+        GridDefaultMpi()));
+}
 
+// dry run /////////////////////////////////////////////////////////////////////
+void Environment::dryRun(const bool isDry)
+{
+    dryRun_ = isDry;
+}
+
+bool Environment::isDryRun(void)
+{
+    return dryRun_;
+}
+
+// quark propagators ///////////////////////////////////////////////////////////
+void Environment::addProp(const std::string name, const unsigned int Ls)
+{
+    if (propExists(name))
+    {
+        HADRON_ERROR("propagator '" + name + "' already exists");
+    }
+    if (Ls > 1)
+    {
+        GridCartesian *pt;
+        
+        try
+        {
+            pt = grid5d_.at(Ls).get();
+        }
+        catch(std::out_of_range &)
+        {
+            grid5d_[Ls].reset(SpaceTimeGrid::makeFiveDimGrid(Ls,
+                                                             grid4d_.get()));
+            pt = grid5d_[Ls].get();
+        }
+        if (!isDryRun())
+        {
+            prop_[name].reset(new LatticePropagator(pt));
+        }
+        else
+        {
+            prop_[name].reset(nullptr);
+        }
+        propSize_[name] = Ls;
+    }
+    else
+    {
+        if (!isDryRun())
+        {
+            prop_[name].reset(new LatticePropagator(grid4d_.get()));
+        }
+        else
+        {
+            prop_[name].reset(nullptr);
+        }
+        propSize_[name] = 1;
+    }
+}
+
+void Environment::freeProp(const std::string name)
+{
+    if (propExists(name))
+    {
+        prop_.erase(name);
+        propSize_.erase(name);
+    }
+    else
+    {
+        HADRON_ERROR("trying to free unknown propagator '" + name + "'");
+    }
+}
+
+LatticePropagator * Environment::getProp(const std::string name)
+{
+    if (propExists(name))
+    {
+        return prop_[name].get();
+    }
+    else
+    {
+        HADRON_ERROR("propagator '" + name + "' unknown");
+        
+        return nullptr;
+    }
+}
+
+bool Environment::propExists(const std::string name)
+{
+    auto it = prop_.find(name);
+    
+    if (it == prop_.end())
+    {
+        return false;
+    }
+    else
+    {
+        return true;
+    }
+}
+
+unsigned int Environment::nProp(void)
+{
+    unsigned int size = 0;
+    
+    for (auto &s: propSize_)
+    {
+        size += s.second;
+    }
+    
+    return size;
+}
+
+// general free ////////////////////////////////////////////////////////////////
+void Environment::free(const std::string name)
+{
+    if (propExists(name))
+    {
+        freeProp(name);
+    }
+}
+
+void Environment::freeAll(void)
+{
+    prop_.clear();
+    propSize_.clear();
+}

programs/Hadrons/Environment.hpp

@@ -30,6 +30,29 @@ BEGIN_HADRONS_NAMESPACE
 class Environment
 {
     SINGLETON(Environment);
+public:
+    typedef std::unique_ptr<GridCartesian>     GridPt;
+    typedef std::unique_ptr<LatticePropagator> PropPt;
+public:
+    // dry run
+    void                dryRun(const bool isDry);
+    bool                isDryRun(void);
+    // quark propagators
+    void                addProp(const std::string name,
+                                const unsigned int Ls = 1);
+    void                freeProp(const std::string name);
+    LatticePropagator * getProp(const std::string name);
+    bool                propExists(const std::string name);
+    unsigned int        nProp(void);
+    // general free
+    void                free(const std::string name);
+    void                freeAll(void);
+private:
+    bool                                dryRun_{false};
+    GridPt                              grid4d_;
+    std::map<unsigned int, GridPt>      grid5d_;
+    std::map<std::string, PropPt>       prop_;
+    std::map<std::string, unsigned int> propSize_;
 };
 
 END_HADRONS_NAMESPACE

programs/Hadrons/Global.hpp

@@ -26,7 +26,8 @@
 
 #define BEGIN_HADRONS_NAMESPACE \
 namespace Hadrons {\
-using namespace Grid;
+using namespace Grid;\
+using namespace QCD;
 #define END_HADRONS_NAMESPACE }
 
 BEGIN_HADRONS_NAMESPACE

programs/Hadrons/Graph.hpp

@@ -190,27 +190,27 @@ unsigned int Graph<T>::size(void) const
 template <typename T>
 bool Graph<T>::gotValue(const T &value) const
 {
-    try
+    auto it = isMarked_.find(value);
-    {
+    
-        isMarked_.at(value);
+    if (it == isMarked_.end())
-    }
-    catch (std::out_of_range &)
     {
         return false;
     }
-
+    else
-    return true;
+    {
+        return true;
+    }
 }
 
 // vertex marking //////////////////////////////////////////////////////////////
 template <typename T>
 void Graph<T>::mark(const T &value, const bool doMark)
 {
-    try
+    if (gotValue(value))
     {
-        isMarked_.at(value) = doMark;
+        isMarked_[value] = doMark;
     }
-    catch (std::out_of_range &)
+    else
     {
         HADRON_ERROR("vertex " << value << " does not exists");
     }
@@ -240,11 +240,11 @@ void Graph<T>::unmarkAll(void)
 template <typename T>
 bool Graph<T>::isMarked(const T &value) const
 {
-    try
+    if (gotValue(value))
     {
         return isMarked_.at(value);
     }
-    catch (std::out_of_range &)
+    else
     {
         HADRON_ERROR("vertex " << value << " does not exists");
         

programs/Hadrons/MQuark.cc

@@ -49,14 +49,20 @@ std::vector<std::string> MQuark::getOutput(void)
     return out;
 }
 
-// memory footprint
+// allocation //////////////////////////////////////////////////////////////////
-double MQuark::nCreatedProp(void)
+void MQuark::allocate(Environment &env)
 {
-    return static_cast<double>((par_.Ls > 1) ? par_.Ls + 1 : 1);
+    env.addProp(getName());
+    quark_ = env.getProp(getName());
+    if (par_.Ls > 1)
+    {
+        env.addProp(getName() + "_5d", par_.Ls);
+        quark5d_ = env.getProp(getName() + "_5d");
+    }
 }
 
 // execution
-void MQuark::operator()(Environment &env)
+void MQuark::execute(Environment &env)
 {
     LOG(Message) << "computing quark propagator '" << getName() << "'"
                  << std::endl;

programs/Hadrons/MQuark.hpp

@@ -47,12 +47,13 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void);
     virtual std::vector<std::string> getOutput(void);
-    // memory footprint
+    // allocation
-    virtual double nCreatedProp(void);
+    virtual void allocate(Environment &env);
     // execution
-    virtual void operator()(Environment &env);
+    virtual void execute(Environment &env);
 private:
     Par par_;
+    LatticePropagator *quark_{nullptr}, *quark5d_{nullptr};
 };
 
 MODULE_REGISTER(MQuark);

programs/Hadrons/Module.cc

@@ -35,3 +35,12 @@ std::string Module::getName(void) const
 {
     return name_;
 }
+
+void Module::operator()(Environment &env)
+{
+    allocate(env);
+    if (!env.isDryRun())
+    {
+        execute(env);
+    }
+}

programs/Hadrons/Module.hpp

@@ -21,6 +21,7 @@
 #define Hadrons_Module_hpp_
 
 #include <Hadrons/Global.hpp>
+#include <Hadrons/Environment.hpp>
 
 BEGIN_HADRONS_NAMESPACE
 
@@ -43,8 +44,6 @@ static mod##Registrar mod##RegistrarInstance;
 /******************************************************************************
  *                                 Module                                     *
  ******************************************************************************/
-class Environment;
-
 class Module
 {
 public:
@@ -59,10 +58,11 @@ public:
     // dependency relation
     virtual std::vector<std::string> getInput(void) = 0;
     virtual std::vector<std::string> getOutput(void) = 0;
-    // memory footprint
+    // allocation
-    virtual double nCreatedProp(void) = 0;
+    virtual void allocate(Environment &env) = 0;
     // execution
-    virtual void operator()(Environment &env) = 0;
+    void operator()(Environment &env);
+    virtual void execute(Environment &env) = 0;
 private:
     std::string name_;
 };