Hadrons: multi-file support for eigenpacks

extras/Hadrons/EigenPack.hpp

@@ -74,42 +74,47 @@ public:
         evec.resize(size, grid);
     }
 
-    virtual void read(const std::string fileStem, const int traj = -1)
+    virtual void read(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        std::string evecFilename, evalFilename;
-
-        makeFilenames(evecFilename, evalFilename, fileStem, traj);
-        XmlReader xmlReader(evalFilename);
-        LOG(Message) << "Reading " << eval.size() << " eigenvalues from '" 
-                     << evalFilename << "'" << std::endl;
-        Grid::read(xmlReader, "evals", eval);
-        basicRead(evec, evecFilename, evec.size());
+        if (multiFile)
+        {
+            for(int k = 0; k < evec.size(); ++k)
+            {
+                basicReadSingle(evec[k], eval[k], evecFilename(fileStem, k, traj), k);
+            }
+        }
+        else
+        {
+            basicRead(evec, eval, evecFilename(fileStem, -1, traj), evec.size());
+        }
     }
 
-    virtual void write(const std::string fileStem, const int traj = -1)
+    virtual void write(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        std::string evecFilename, evalFilename;
-
-        makeFilenames(evecFilename, evalFilename, fileStem, traj);
-        XmlWriter xmlWriter(evalFilename);
-        LOG(Message) << "Writing " << eval.size() << " eigenvalues to '" 
-                     << evalFilename << "'" << std::endl;
-        Grid::write(xmlWriter, "evals", eval);
-        basicWrite(evecFilename, evec, evec.size());
+        if (multiFile)
+        {
+            for(int k = 0; k < evec.size(); ++k)
+            {
+                basicWriteSingle(evecFilename(fileStem, k, traj), evec[k], eval[k], k);
+            }
+        }
+        else
+        {
+            basicWrite(evecFilename(fileStem, -1, traj), evec, eval, evec.size());
+        }
     }
 protected:
-    void makeFilenames(std::string &evecFilename, std::string &evalFilename,
-                       const std::string stem, const int traj = -1)
+    std::string evecFilename(const std::string stem, const int vec, const int traj)
     {
         std::string t = (traj < 0) ? "" : ("." + std::to_string(traj));
+        std::string v = (traj < 0) ? "" : (".v" + std::to_string(vec));
 
-        evecFilename = stem + "_evec" + t + ".bin";
-        evalFilename = stem + "_eval" + t + ".xml";
+        return stem + "_evec" + t + v + ".bin";
     }
 
     template <typename T>
-    void basicRead(std::vector<T> &evec, const std::string filename, 
-                   const unsigned int size)
+    void basicRead(std::vector<T> &evec, std::vector<double> &eval,
+                   const std::string filename, const unsigned int size)
     {
         ScidacReader    binReader;
 
@@ -124,16 +129,38 @@ protected:
             if (vecRecord.index != k)
             {
                 HADRONS_ERROR(Io, "Eigenvector " + std::to_string(k) + " has a"
-                             + " wrong index (expected " + std::to_string(vecRecord.index) 
-                             + ") in file '" + filename + "'");
+                              + " wrong index (expected " + std::to_string(vecRecord.index) 
+                              + ") in file '" + filename + "'");
             }
+            eval[k] = vecRecord.eval;
         }
         binReader.close();
     }
 
+    template <typename T>
+    void basicReadSingle(T &evec, double &eval, const std::string filename, 
+                         const unsigned int index)
+    {
+        ScidacReader binReader;
+        VecRecord    vecRecord;
+
+        binReader.open(filename);
+        binReader.skipPastObjectRecord(SCIDAC_FILE_XML);
+        LOG(Message) << "Reading eigenvector " << index << std::endl;
+        binReader.readScidacFieldRecord(evec, vecRecord);
+        if (vecRecord.index != index)
+        {
+            HADRONS_ERROR(Io, "Eigenvector " + std::to_string(index) + " has a"
+                          + " wrong index (expected " + std::to_string(vecRecord.index) 
+                          + ") in file '" + filename + "'");
+        }
+        eval = vecRecord.eval;
+        binReader.close();
+    }
+
     template <typename T>
     void basicWrite(const std::string filename, std::vector<T> &evec, 
-                    const unsigned int size)
+                    const std::vector<double> &eval, const unsigned int size)
     {
         ScidacWriter binWriter(evec[0]._grid->IsBoss());
         XmlWriter    xmlWriter("", "eigenPackPar");   
@@ -153,6 +180,25 @@ protected:
         }
         binWriter.close();
     }
+
+    template <typename T>
+    void basicWriteSingle(const std::string filename, T &evec, 
+                          const double eval, const unsigned int index)
+    {
+        ScidacWriter binWriter(evec._grid->IsBoss());
+        XmlWriter    xmlWriter("", "eigenPackPar");
+        VecRecord    vecRecord;
+
+        xmlWriter.pushXmlString(record.operatorXml);
+        xmlWriter.pushXmlString(record.solverXml);
+        binWriter.open(filename);
+        binWriter.writeLimeObject(1, 1, xmlWriter, "parameters", SCIDAC_FILE_XML);
+        vecRecord.index = index;
+        vecRecord.eval  = eval;
+        LOG(Message) << "Writing eigenvector " << index << std::endl;
+        binWriter.writeScidacFieldRecord(evec, vecRecord);
+        binWriter.close();
+    }
 };
 
 template <typename FineF, typename CoarseF>
@@ -181,77 +227,121 @@ public:
         evecCoarse.resize(sizeCoarse, gridCoarse);
     }
 
-    void readFine(const std::string fileStem, const int traj = -1)
+    void readFine(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        std::string evecFineFilename, evalFineFilename;
-        std::string evecCoarseFilename, evalCoarseFilename;
+        // std::string evecFineFilename, evalFineFilename;
+        // std::string evecCoarseFilename, evalCoarseFilename;
 
-        this->makeFilenames(evecFineFilename, evalFineFilename, 
-                            fileStem + "_fine", traj);
-        XmlReader xmlFineReader(evalFineFilename);
-        LOG(Message) << "Reading " << this->eval.size() << " fine eigenvalues from '" 
-                     << evalFineFilename << "'" << std::endl;
-        Grid::read(xmlFineReader, "evals", this->eval);
-        LOG(Message) << "Reading " << this->evec.size() << " fine eigenvectors from '" 
-                     << evecFineFilename << "'" << std::endl;
-        this->basicRead(this->evec, evecFineFilename, this->evec.size());
+        // this->makeFilenames(evecFineFilename, evalFineFilename, 
+        //                     fileStem + "_fine", traj);
+        // XmlReader xmlFineReader(evalFineFilename);
+        // LOG(Message) << "Reading " << this->eval.size() << " fine eigenvalues from '" 
+        //              << evalFineFilename << "'" << std::endl;
+        // Grid::read(xmlFineReader, "evals", this->eval);
+        // LOG(Message) << "Reading " << this->evec.size() << " fine eigenvectors from '" 
+        //              << evecFineFilename << "'" << std::endl;
+        // this->basicRead(this->evec, evecFineFilename, this->evec.size());
+        if (multiFile)
+        {
+            for(int k = 0; k < this->evec.size(); ++k)
+            {
+                this->basicReadSingle(this->evec[k], this->eval[k], this->evecFilename(fileStem + "_fine", k, traj), k);
+            }
+        }
+        else
+        {
+            this->basicRead(this->evec, this->eval, this->evecFilename(fileStem + "_fine", -1, traj), this->evec.size());
+        }
     }
 
-    void readCoarse(const std::string fileStem, const int traj = -1)
+    void readCoarse(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        std::string evecCoarseFilename, evalCoarseFilename;
+        // std::string evecCoarseFilename, evalCoarseFilename;
 
-        this->makeFilenames(evecCoarseFilename, evalCoarseFilename, 
-                            fileStem + "_coarse", traj);
-        XmlReader xmlCoarseReader(evalCoarseFilename);
-        LOG(Message) << "Reading " << evalCoarse.size() << " coarse eigenvalues from '" 
-                     << evalCoarseFilename << "'" << std::endl;
-        Grid::read(xmlCoarseReader, "evals", evalCoarse);
-        LOG(Message) << "Reading " << evecCoarse.size() << " coarse eigenvectors from '" 
-                     << evecCoarseFilename << "'" << std::endl;
-        this->basicRead(evecCoarse, evecCoarseFilename, evecCoarse.size());
+        // this->makeFilenames(evecCoarseFilename, evalCoarseFilename, 
+        //                     fileStem + "_coarse", traj);
+        // XmlReader xmlCoarseReader(evalCoarseFilename);
+        // LOG(Message) << "Reading " << evalCoarse.size() << " coarse eigenvalues from '" 
+        //              << evalCoarseFilename << "'" << std::endl;
+        // Grid::read(xmlCoarseReader, "evals", evalCoarse);
+        // LOG(Message) << "Reading " << evecCoarse.size() << " coarse eigenvectors from '" 
+        //              << evecCoarseFilename << "'" << std::endl;
+        // this->basicRead(evecCoarse, evecCoarseFilename, evecCoarse.size());
+        if (multiFile)
+        {
+            for(int k = 0; k < evecCoarse.size(); ++k)
+            {
+                this->basicReadSingle(evecCoarse[k], evalCoarse[k], this->evecFilename(fileStem + "_coarse", k, traj), k);
+            }
+        }
+        else
+        {
+            this->basicRead(evecCoarse, evalCoarse, this->evecFilename(fileStem + "_coarse", -1, traj), evecCoarse.size());
+        }
     }
 
-    virtual void read(const std::string fileStem, const int traj = -1)
+    virtual void read(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        readFine(fileStem, traj);
-        readCoarse(fileStem, traj);
+        readFine(fileStem, multiFile, traj);
+        readCoarse(fileStem, multiFile, traj);
     }
 
-    void writeFine(const std::string fileStem, const int traj = -1)
+    void writeFine(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        std::string evecFineFilename, evalFineFilename;
+        // std::string evecFineFilename, evalFineFilename;
 
-        this->makeFilenames(evecFineFilename, evalFineFilename, 
-                            fileStem + "_fine", traj);
-        XmlWriter xmlFineWriter(evalFineFilename);
-        LOG(Message) << "Writing " << this->eval.size() << " fine eigenvalues to '" 
-                     << evalFineFilename << "'" << std::endl;
-        Grid::write(xmlFineWriter, "evals", this->eval);
-        LOG(Message) << "Writing " << this->evec.size() << " fine eigenvectors to '" 
-                     << evecFineFilename << "'" << std::endl;
-        this->basicWrite(evecFineFilename, this->evec, this->evec.size());
+        // this->makeFilenames(evecFineFilename, evalFineFilename, 
+        //                     fileStem + "_fine", traj);
+        // XmlWriter xmlFineWriter(evalFineFilename);
+        // LOG(Message) << "Writing " << this->eval.size() << " fine eigenvalues to '" 
+        //              << evalFineFilename << "'" << std::endl;
+        // Grid::write(xmlFineWriter, "evals", this->eval);
+        // LOG(Message) << "Writing " << this->evec.size() << " fine eigenvectors to '" 
+        //              << evecFineFilename << "'" << std::endl;
+        // this->basicWrite(evecFineFilename, this->evec, this->evec.size());
+        if (multiFile)
+        {
+            for(int k = 0; k < this->evec.size(); ++k)
+            {
+                this->basicWriteSingle(this->evecFilename(fileStem + "_fine", k, traj), this->evec[k], this->eval[k], k);
+            }
+        }
+        else
+        {
+            this->basicWrite(this->evecFilename(fileStem + "_fine", -1, traj), this->evec, this->eval, this->evec.size());
+        }
     }
 
-    void writeCoarse(const std::string fileStem, const int traj = -1)
+    void writeCoarse(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        std::string evecCoarseFilename, evalCoarseFilename;
+        // std::string evecCoarseFilename, evalCoarseFilename;
 
-        this->makeFilenames(evecCoarseFilename, evalCoarseFilename,
-                            fileStem + "_coarse", traj);
-        XmlWriter xmlCoarseWriter(evalCoarseFilename);
-        LOG(Message) << "Writing " << evalCoarse.size() << " coarse eigenvalues to '" 
-                     << evalCoarseFilename << "'" << std::endl;
-        Grid::write(xmlCoarseWriter, "evals", evalCoarse);
-        LOG(Message) << "Writing " << evecCoarse.size() << " coarse eigenvectors to '" 
-                     << evecCoarseFilename << "'" << std::endl;
-        this->basicWrite(evecCoarseFilename, evecCoarse, evecCoarse.size());
+        // this->makeFilenames(evecCoarseFilename, evalCoarseFilename,
+        //                     fileStem + "_coarse", traj);
+        // XmlWriter xmlCoarseWriter(evalCoarseFilename);
+        // LOG(Message) << "Writing " << evalCoarse.size() << " coarse eigenvalues to '" 
+        //              << evalCoarseFilename << "'" << std::endl;
+        // Grid::write(xmlCoarseWriter, "evals", evalCoarse);
+        // LOG(Message) << "Writing " << evecCoarse.size() << " coarse eigenvectors to '" 
+        //              << evecCoarseFilename << "'" << std::endl;
+        // this->basicWrite(evecCoarseFilename, evecCoarse, evecCoarse.size());
+        if (multiFile)
+        {
+            for(int k = 0; k < evecCoarse.size(); ++k)
+            {
+                this->basicWriteSingle(this->evecFilename(fileStem + "_coarse", k, traj), evecCoarse[k], evalCoarse[k], k);
+            }
+        }
+        else
+        {
+            this->basicWrite(this->evecFilename(fileStem + "_coarse", -1, traj), evecCoarse, evalCoarse, evecCoarse.size());
+        }
     }
     
-    virtual void write(const std::string fileStem, const int traj = -1)
+    virtual void write(const std::string fileStem, const bool multiFile, const int traj = -1)
     {
-        writeFine(fileStem, traj);
-        writeCoarse(fileStem, traj);
+        writeFine(fileStem, multiFile, traj);
+        writeCoarse(fileStem, multiFile, traj);
     }
 };
 

extras/Hadrons/Modules/MIO/LoadCoarseEigenPack.hpp

@@ -45,6 +45,7 @@ class LoadCoarseEigenPackPar: Serializable
 public:
     GRID_SERIALIZABLE_CLASS_MEMBERS(LoadCoarseEigenPackPar,
                                     std::string, filestem,
+                                    bool,         multiFile,
                                     unsigned int, sizeFine,
                                     unsigned int, sizeCoarse,
                                     unsigned int, Ls,
@@ -120,7 +121,7 @@ void TLoadCoarseEigenPack<Pack>::execute(void)
     auto                 &epack = envGetDerived(BasePack, Pack, getName());
     Lattice<SiteComplex> dummy(cg);
 
-    epack.read(par().filestem, vm().getTrajectory());
+    epack.read(par().filestem, par().multiFile, vm().getTrajectory());
     LOG(Message) << "Block Gramm-Schmidt pass 1"<< std::endl;
     blockOrthogonalise(dummy, epack.evec);
     LOG(Message) << "Block Gramm-Schmidt pass 2"<< std::endl;

extras/Hadrons/Modules/MIO/LoadEigenPack.hpp

@@ -45,6 +45,7 @@ class LoadEigenPackPar: Serializable
 public:
     GRID_SERIALIZABLE_CLASS_MEMBERS(LoadEigenPackPar,
                                     std::string, filestem,
+                                    bool, multiFile,
                                     unsigned int, size,
                                     unsigned int, Ls);
 };
@@ -111,7 +112,7 @@ void TLoadEigenPack<Pack>::execute(void)
 {
     auto &epack = envGetDerived(BasePack, Pack, getName());
 
-    epack.read(par().filestem, vm().getTrajectory());
+    epack.read(par().filestem, par().multiFile, vm().getTrajectory());
     epack.eval.resize(par().size);
 }
 

extras/Hadrons/Modules/MSolver/LocalCoherenceLanczos.hpp

@@ -51,7 +51,8 @@ public:
                                     ChebyParams,   smoother,
                                     RealD,         coarseRelaxTol,
                                     std::string,   blockSize,
-                                    std::string,   output);
+                                    std::string,   output,
+                                    bool,          multiFile);
 };
 
 template <typename FImpl, int nBasis>
@@ -156,7 +157,7 @@ void TLocalCoherenceLanczos<FImpl, nBasis>::execute(void)
     solver.testFine(finePar.resid*100.0);
     if (!par().output.empty())
     {
-        epack.writeFine(par().output, vm().getTrajectory());
+        epack.writeFine(par().output, par().multiFile, vm().getTrajectory());
     }
     if (par().doCoarse)
     {
@@ -173,7 +174,7 @@ void TLocalCoherenceLanczos<FImpl, nBasis>::execute(void)
                         par().coarseRelaxTol);
         if (!par().output.empty())
         {
-            epack.writeCoarse(par().output, vm().getTrajectory());
+            epack.writeCoarse(par().output, par().multiFile, vm().getTrajectory());
         }
     }
 }