ILDG reader/writer works

Fill the xml header with the required information, todo.

lib/Grid.h

@@ -87,6 +87,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #include <Grid/qcd/hmc/NerscCheckpointer.h>
 #include <Grid/qcd/hmc/BinaryCheckpointer.h>
+#include <Grid/qcd/hmc/ILDGCheckpointer.h>
 #include <Grid/qcd/hmc/HmcRunner.h>
 #include <Grid/qcd/hmc/GenericHMCrunner.h>
 

lib/parallelIO/BinaryIO.h

@@ -449,10 +449,13 @@ class BinaryIO {
     return csum;
   }
 
-
+  template <class vobj, class fobj, class munger>
-  template<class vobj,class fobj,class munger>
+  static inline uint32_t readObjectParallel(Lattice<vobj> &Umu,
-  static inline uint32_t readObjectParallel(Lattice<vobj> &Umu,std::string file,munger munge,int offset,const std::string &format)
+                                            std::string file, 
-  {
+                                            munger munge,
+                                            int offset,
+                                            const std::string &format,
+                                            ILDGtype ILDG = ILDGtype()) {
     typedef typename vobj::scalar_object sobj;
 
     GridBase *grid = Umu._grid;
@@ -518,6 +521,7 @@ class BinaryIO {
     int myrank = grid->ThisRank();
     int iorank = grid->RankFromProcessorCoor(ioproc);
 
+    if (!ILDG.is_ILDG)
     	if ( IOnode ) { 
     		fin.open(file,std::ios::binary|std::ios::in);
     	}
@@ -562,8 +566,15 @@ class BinaryIO {
       ////////////////////////////////
       if (myrank == iorank) {
 
+      	if (ILDG.is_ILDG){
+      		// use C-LIME to populate the record
+          size_t sizeFO = sizeof(fileObj);
+          limeReaderSeek(ILDG.LR, g_idx*sizeFO, SEEK_SET);
+          int status = limeReaderReadData((void *)&fileObj, &sizeFO, ILDG.LR);
+        } else{
           fin.seekg(offset+g_idx*sizeof(fileObj));
           fin.read((char *)&fileObj,sizeof(fileObj));
+        }
         bytes+=sizeof(fileObj);
 
         if(ieee32big) be32toh_v((void *)&fileObj,sizeof(fileObj));
@@ -681,7 +692,7 @@ class BinaryIO {
     // Ideally one reader/writer per xy plane and read these contiguously
     // with comms from nominated I/O nodes.
     std::ofstream fout;
-    if (!ILDG.is_ILDG){
+    if (!ILDG.is_ILDG)
     	if (IOnode){
     		fout.open(file, std::ios::binary | std::ios::in | std::ios::out);
     		if (!fout.is_open()) {
@@ -690,7 +701,7 @@ class BinaryIO {
     			exit(0);
     		}
     	}
-    }
+
 
     //////////////////////////////////////////////////////////
     // Find the location of each site and send to primary node
@@ -753,10 +764,11 @@ class BinaryIO {
         if (ieee64big) htobe64_v((void *)&fileObj, sizeof(fileObj));
         if (ieee64) htole64_v((void *)&fileObj, sizeof(fileObj));
 
-        if (ILDG.is_ILDG){
+        if (ILDG.is_ILDG) {
           size_t sizeFO = sizeof(fileObj);
-        	int status = limeWriteRecordData((char*)&fileObj, &sizeFO, ILDG.LW);
+ 					limeWriterSeek(ILDG.LW, g_idx*sizeFO, SEEK_SET);
-        } else{
+          int status = limeWriteRecordData((void *)&fileObj, &sizeFO, ILDG.LW);
+        } else {
           fout.seekp(offset + g_idx * sizeof(fileObj));
           fout.write((char *)&fileObj, sizeof(fileObj));
         }

lib/parallelIO/IldgIO.h

@@ -87,7 +87,7 @@ struct ILDGMunger {
 };
 
 template <class fobj, class sobj>
-struct ILDGSimpleUnmunger {
+struct ILDGUnmunger {
   void operator()(sobj &in, fobj &out, uint32_t &csum) {
     for (int mu = 0; mu < 4; mu++) {
       for (int i = 0; i < 3; i++) {
@@ -103,20 +103,45 @@ struct ILDGSimpleUnmunger {
 ////////////////////////////////////////////////////////////////////////////////
 // Write and read from fstream; compute header offset for payload
 ////////////////////////////////////////////////////////////////////////////////
+enum ILDGstate {ILDGread, ILDGwrite};
+
 class ILDGIO : public BinaryIO {
-  FILE *outFile;
+  FILE *File;
   LimeWriter *LimeW;
   LimeRecordHeader *LimeHeader;
+  LimeReader *LimeR;
+  std::string filename;
+
 
  public:
-  ILDGIO(std::string file) {
+  ILDGIO(std::string file, ILDGstate RW) {
-    outFile = fopen(file.c_str(), "w");
+      filename = file;
+    if (RW == ILDGwrite){
+      File = fopen(file.c_str(), "w");
       // check if opened correctly
 
-    LimeW = limeCreateWriter(outFile);
+      LimeW = limeCreateWriter(File);
+    } else {
+      File = fopen(file.c_str(), "r");
+      // check if opened correctly
+
+      LimeR = limeCreateReader(File);
+    }
   }
 
-  ~ILDGIO() { fclose(outFile); }
+  ~ILDGIO() { fclose(File); }
+
+  int createHeader(std::string message, int MB, int ME, size_t PayloadSize, LimeWriter* L){
+    LimeRecordHeader *h;
+    h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
+    int status = limeWriteRecordHeader(h, L);
+    if (status < 0) {
+      std::cerr << "ILDG Header error\n";
+      return status;
+    }
+    limeDestroyHeader(h);
+    return LIME_SUCCESS;
+  }
 
   unsigned int writeHeader(ILDGField &header) {
     // write header in LIME
@@ -131,60 +156,86 @@ class ILDGIO : public BinaryIO {
     // save the xml header here
     // use the xml_writer to c++ streams in pugixml
     // and convert to char message
-    // limeWriteRecordData(message, &nbytes, LimeW);
+    limeWriteRecordData(message, &nbytes, LimeW);
     limeWriterCloseRecord(LimeW);
 
     return 0;
   }
 
-  unsigned int readHeader(std::string file, GridBase *grid, ILDGField &field) {
+  unsigned int readHeader(ILDGField &header) {
     return 0;
   }
 
   template <class vsimd>
-  int readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,
+  uint32_t readConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu) {
-                        ILDGField &header, std::string file) {
     typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
+    typedef LorentzColourMatrixD sobjd;
+    typedef LorentzColourMatrixF sobjf;
+    typedef iLorentzColourMatrix<vsimd> itype;
+    typedef LorentzColourMatrix sobj;
+    GridBase *grid = Umu._grid;
 
-    return 0;
+    ILDGField header;
+    readHeader(header);
+
+    // now just the conf, ignore the header
+    std::string format = std::string("IEEE64BIG");
+    do {limeReaderNextRecord(LimeR);}
+    while (strncmp(limeReaderType(LimeR), "ildg-binary-data",16));
+
+    n_uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration)
+
+
+    ILDGtype ILDGt(true, LimeR);
+    // this is special for double prec data, just for the moment
+    uint32_t csum = BinaryIO::readObjectParallel< itype, sobjd >(
+       Umu, filename, ILDGMunger<sobjd, sobj>(), 0, format, ILDGt);
+
+    // Check configuration 
+    // todo
+
+    return csum;
   }
 
   template <class vsimd>
-  int writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu,
+  uint32_t writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu, std::string format) {
-                         ILDGField &header, std::string file) {
     typedef Lattice<iLorentzColourMatrix<vsimd> > GaugeField;
     typedef iLorentzColourMatrix<vsimd> vobj;
     typedef typename vobj::scalar_object sobj;
     typedef LorentzColourMatrixD fobj;
 
-    ILDGSimpleUnmunger<fobj, sobj> munge;
+    ILDGField header;
+    // fill the header
+    header.floating_point = format;
+
+    ILDGUnmunger<fobj, sobj> munge;
     unsigned int offset = writeHeader(header);
 
     BinaryIO::Uint32Checksum<vobj, fobj>(Umu, munge, header.checksum);
 
-    // Write record header 
+    // Write data record header
-    LimeRecordHeader *h;
+    n_uint64_t PayloadSize = sizeof(fobj) * Umu._grid->_gsites;
-    std::cout << GridLogDebug << "ILDG Creating Header" << std::endl;
+    createHeader("ildg-binary-data", 0, 1, PayloadSize, LimeW);
-    char message[] = "ildg-binary-data";
-    h = limeCreateHeader(1, 1, message, strlen(message));
-
-    std::cout << GridLogDebug << "ILDG Writing Header" << std::endl;
-    int status = limeWriteRecordHeader(h, LimeW);
-
-    if (status < 0) {
-      std::cerr << "ILDG Header error\n";
-      return 1;
-    }
-
-    limeDestroyHeader(h);
 
     ILDGtype ILDGt(true, LimeW);
     uint32_t csum = BinaryIO::writeObjectParallel<vobj, fobj>(
-        Umu, file, munge, offset, header.floating_point, ILDGt);
+       Umu, filename, munge, 0, header.floating_point, ILDGt);
 
     limeWriterCloseRecord(LimeW);
 
-    return 0;
+    // Last record
+    // the logical file name LNF
+    // look into documentation on how to generate this string
+    std::string LNF = "empty"; 
+
+
+    PayloadSize = sizeof(LNF);
+    createHeader("ildg-binary-lfn", 1 , 1, PayloadSize, LimeW);
+    limeWriteRecordData(const_cast<char*>(LNF.c_str()), &PayloadSize, LimeW);
+
+    limeWriterCloseRecord(LimeW);
+
+    return csum;
   }
 
   // format for RNG?

lib/parallelIO/IldgIOtypes.h

@@ -36,9 +36,11 @@ namespace Grid {
   struct ILDGtype{
     bool is_ILDG;
     LimeWriter* LW;
+    LimeReader* LR;
     
-		ILDGtype(bool is, LimeWriter* L):is_ILDG(is),LW(L){}
+    ILDGtype(bool is, LimeWriter* L):is_ILDG(is),LW(L),LR(NULL){}
-    ILDGtype():is_ILDG(false),LW(NULL){}
+    ILDGtype(bool is, LimeReader* L):is_ILDG(is),LW(NULL),LR(L){}
+    ILDGtype():is_ILDG(false),LW(NULL),LR(NULL){}
   };
   
 

lib/parallelIO/NerscIO.h

@@ -424,8 +424,8 @@ static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu
     //int csum1=BinaryIO::writeObjectSerial<vobj,fobj2D>(Umu,file1,munge,offset,header.floating_point);
 
     
-    std::cout << GridLogMessage << " TESTING PARALLEL WRITE offsets " << offset1 << " "<< offset << std::endl;
+    //std::cout << GridLogMessage << " TESTING PARALLEL WRITE offsets " << offset1 << " "<< offset << std::endl;
-    std::cout << GridLogMessage << " TESTING PARALLEL WRITE csums   " << csum1 << " "<<std::hex<< csum << std::dec<< std::endl;
+    //std::cout << GridLogMessage << " TESTING PARALLEL WRITE csums   " << csum1 << " "<<std::hex<< csum << std::dec<< std::endl;
 
     assert(offset1==offset);  
     assert(csum1==csum);  
@@ -436,11 +436,10 @@ static inline void writeConfiguration(Lattice<iLorentzColourMatrix<vsimd> > &Umu
     NerscSimpleUnmunger<fobj3D,sobj> munge;
     BinaryIO::Uint32Checksum<vobj,fobj3D>(Umu, munge,header.checksum);
     offset = writeHeader(header,file);
-    //    csum=BinaryIO::writeObjectSerial<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point);
     csum=BinaryIO::writeObjectParallel<vobj,fobj3D>(Umu,file,munge,offset,header.floating_point);
   }
 
-  std::cout<<GridLogMessage <<"Written NERSC Configuration "<<file<< " checksum "<<std::hex<<csum<< std::dec<<" plaq "<< header.plaquette <<std::endl;
+  std::cout<<GridLogMessage <<"Written NERSC Configuration on "<< file << " checksum "<<std::hex<<csum<< std::dec<<" plaq "<< header.plaquette <<std::endl;
 
  }
 

lib/qcd/hmc/ILDGCheckpointer.h

@@ -0,0 +1,121 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/ILDGCheckpointer.h
+
+Copyright (C) 2015
+
+Author: Guido Cossu
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef ILDG_CHECKPOINTER
+#define ILDG_CHECKPOINTER
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+namespace Grid {
+namespace QCD {
+
+// Only for Gauge fields
+template <class Implementation>
+class ILDGHmcCheckpointer
+    : public HmcObservable<typename Implementation::GaugeField> {
+ private:
+  std::string configStem;
+  std::string rngStem;
+  int SaveInterval;
+  std::string format;
+
+ public:
+  INHERIT_GIMPL_TYPES(Implementation);  //
+
+  ILDGHmcCheckpointer(std::string cf, std::string rn, int savemodulo,
+                       std::string form = "IEEE64BIG") {
+    configStem = cf;
+    rngStem = rn;
+    SaveInterval = savemodulo;
+    format = form;
+
+    // check here that the format is valid
+    int ieee32big = (format == std::string("IEEE32BIG"));
+    int ieee32    = (format == std::string("IEEE32"));
+    int ieee64big = (format == std::string("IEEE64BIG"));
+    int ieee64    = (format == std::string("IEEE64"));
+
+    if (!(ieee64big ^ ieee32 ^ ieee32big ^ ieee64)) {
+      std::cout << GridLogMessage << "Invalid format: " << format << std::endl;
+      exit(0);
+    }
+  };
+
+  void TrajectoryComplete(int traj, GaugeField &U, GridSerialRNG &sRNG,
+                          GridParallelRNG &pRNG) {
+    if ((traj % SaveInterval) == 0) {
+      std::string rng;
+      {
+        std::ostringstream os;
+        os << rngStem << "." << traj;
+        rng = os.str();
+      }
+      std::string config;
+      {
+        std::ostringstream os;
+        os << configStem << "." << traj;
+        config = os.str();
+      }
+
+      ILDGIO IO(config, ILDGwrite);
+      BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0);
+      uint32_t csum  = IO.writeConfiguration(U, format);
+
+      std::cout << GridLogMessage << "Written ILDG Configuration on " << config
+                << " checksum " << std::hex << csum << std::dec << std::endl;
+    }
+  };
+
+  void CheckpointRestore(int traj, GaugeField &U, GridSerialRNG &sRNG,
+                         GridParallelRNG &pRNG) {
+    std::string rng;
+    {
+      std::ostringstream os;
+      os << rngStem << "." << traj;
+      rng = os.str();
+    }
+    std::string config;
+    {
+      std::ostringstream os;
+      os << configStem << "." << traj;
+      config = os.str();
+    }
+
+    ILDGIO IO(config, ILDGread);
+    BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0);
+    uint32_t csum = IO.readConfiguration(U);// format from the header
+
+    std::cout << GridLogMessage << "Read ILDG Configuration from " << config
+              << " checksum " << std::hex << csum << std::dec << std::endl;
+  };
+};
+}
+}
+#endif

tests/hmc/Test_hmc_WilsonGauge_Binary.cc

@@ -81,7 +81,7 @@ class HmcRunner : public BinaryHmcRunner {
     TheAction.push_back(Level1);
 
     // Add observables
-    BinaryHmcCheckpointer<BinaryHmcRunner::ImplPolicy> Checkpoint(
+    ILDGHmcCheckpointer<BinaryHmcRunner::ImplPolicy> Checkpoint(
         HMCPar.conf_prefix, HMCPar.rng_prefix, HMCPar.SaveInterval, HMCPar.format);
     // Can implement also a specific function in the hmcrunner
     // AddCheckpoint (...) that takes the same parameters + a string/tag