Merge branch 'develop' of https://github.com/paboyle/Grid into develop

benchmarks/Benchmark_comms.cc

@@ -450,7 +450,7 @@ int main (int argc, char ** argv)
   }    
 
 
-
+#ifdef GRID_OMP
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
@@ -537,7 +537,7 @@ int main (int argc, char ** argv)
  
     }
   }    
-
+#endif
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;

lib/parallelIO/BinaryIO.h

@@ -263,7 +263,7 @@ PARALLEL_CRITICAL
 			      GridBase *grid,
 			      std::vector<fobj> &iodata,
 			      std::string file,
-			      uint64_t offset,
+			      uint64_t& offset,
 			      const std::string &format, int control,
 			      uint32_t &nersc_csum,
 			      uint32_t &scidac_csuma,
@@ -495,6 +495,7 @@ PARALLEL_CRITICAL
 	  exit(1);
 #endif
 	}
+  offset  = fout.tellp();
 	fout.close();
       }
       timer.Stop();
@@ -699,7 +700,6 @@ PARALLEL_CRITICAL
 
     IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC,
 	     nersc_csum,scidac_csuma,scidac_csumb);
-
     iodata.resize(1);
     {
       std::vector<RngStateType> tmp(RngStateCount);

lib/qcd/hmc/HMCResourceManager.h

@@ -48,6 +48,22 @@ with this program; if not, write to the Free Software Foundation, Inc.,
     }                                                                    \
   }
 
+#define RegisterLoadCheckPointerMetadataFunction(NAME)                   \
+  template < class Metadata >                                            \
+  void Load##NAME##Checkpointer(const CheckpointerParameters& Params_, const Metadata& M_) { \
+    if (!have_CheckPointer) {                                            \
+      std::cout << GridLogDebug << "Loading Metadata Checkpointer " << #NAME      \
+                << std::endl;                                            \
+      CP = std::unique_ptr<CheckpointerBaseModule>(                      \
+        new NAME##CPModule<ImplementationPolicy, Metadata >(Params_, M_));   \
+      have_CheckPointer = true;                                          \
+    } else {                                                             \
+      std::cout << GridLogError << "Checkpointer already loaded "        \
+                << std::endl;                                            \
+      exit(1);                                                           \
+    }                                                                    \
+  }
+
 namespace Grid {
 namespace QCD {
 
@@ -77,7 +93,7 @@ class HMCResourceManager {
   bool have_CheckPointer;
 
   // NOTE: operator << is not overloaded for std::vector<string> 
-  // so thsi function is necessary
+  // so this function is necessary
   void output_vector_string(const std::vector<std::string> &vs){
     for (auto &i: vs)
       std::cout << i << " ";
@@ -254,6 +270,7 @@ class HMCResourceManager {
   RegisterLoadCheckPointerFunction(Nersc);
   #ifdef HAVE_LIME
   RegisterLoadCheckPointerFunction(ILDG);
+  RegisterLoadCheckPointerMetadataFunction(Scidac);
   #endif
 
   ////////////////////////////////////////////////////////

lib/qcd/hmc/checkpointers/BaseCheckpointer.h

@@ -76,6 +76,14 @@ class BaseHmcCheckpointer : public HmcObservable<typename Impl::Field> {
     }
  	} 
 
+  void check_filename(const std::string &filename){
+    std::ifstream f(filename.c_str());
+    if(!f.good()){
+      std::cout << GridLogError << "Filename " << filename << " not found. Aborting. " << std::endl;
+      abort();
+    };
+  }
+
   virtual void initialize(const CheckpointerParameters &Params) = 0;
 
   virtual void CheckpointRestore(int traj, typename Impl::Field &U,

lib/qcd/hmc/checkpointers/BinaryCheckpointer.h

@@ -93,6 +93,9 @@ class BinaryHmcCheckpointer : public BaseHmcCheckpointer<Impl> {
   void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) {
     std::string config, rng;
     this->build_filenames(traj, Params, config, rng);
+    this->check_filename(rng);
+    this->check_filename(config);
+
 
     BinarySimpleMunger<sobj_double, sobj> munge;
 

lib/qcd/hmc/checkpointers/CheckPointerModules.h

@@ -136,6 +136,20 @@ class ILDGCPModule: public CheckPointerModule< ImplementationPolicy> {
 
 };
 
+template<class ImplementationPolicy, class Metadata>
+class ScidacCPModule: public CheckPointerModule< ImplementationPolicy> {
+  typedef CheckPointerModule< ImplementationPolicy> CPBase;
+  Metadata M;
+
+  // acquire resource
+  virtual void initialize(){
+     this->CheckPointPtr.reset(new ScidacHmcCheckpointer<ImplementationPolicy, Metadata>(this->Par_, M));
+  }
+public:
+  ScidacCPModule(typename CPBase::APar Par, Metadata M_):M(M_), CPBase(Par) {}
+  template <class ReaderClass>
+  ScidacCPModule(Reader<ReaderClass>& Reader) : Parametrized<typename CPBase::APar>(Reader), M(Reader){};
+};
 #endif
 
 

lib/qcd/hmc/checkpointers/CheckPointers.h

@@ -34,6 +34,7 @@ directory
 #include <Grid/qcd/hmc/checkpointers/NerscCheckpointer.h>
 #include <Grid/qcd/hmc/checkpointers/BinaryCheckpointer.h>
 #include <Grid/qcd/hmc/checkpointers/ILDGCheckpointer.h>
+#include <Grid/qcd/hmc/checkpointers/ScidacCheckpointer.h>
 //#include <Grid/qcd/hmc/checkpointers/CheckPointerModules.h>
 
 

lib/qcd/hmc/checkpointers/ILDGCheckpointer.h

@@ -95,6 +95,10 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer<Implementation> {
                          GridParallelRNG &pRNG) {
     std::string config, rng;
     this->build_filenames(traj, Params, config, rng);
+    this->check_filename(rng);
+    this->check_filename(config);
+
+    
 
     uint32_t nersc_csum,scidac_csuma,scidac_csumb;
     BinaryIO::readRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb);

lib/qcd/hmc/checkpointers/NerscCheckpointer.h

@@ -69,6 +69,9 @@ class NerscHmcCheckpointer : public BaseHmcCheckpointer<Gimpl> {
                          GridParallelRNG &pRNG) {
     std::string config, rng;
     this->build_filenames(traj, Params, config, rng);
+    this->check_filename(rng);
+    this->check_filename(config);
+
 
     FieldMetaData header;
     NerscIO::readRNGState(sRNG, pRNG, header, rng);

lib/qcd/hmc/checkpointers/ScidacCheckpointer.h

@@ -0,0 +1,125 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/hmc/ScidacCheckpointer.h
+
+Copyright (C) 2018
+
+Author: Guido Cossu <guido.cossu@ed.ac.uk>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution
+directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef SCIDAC_CHECKPOINTER
+#define SCIDAC_CHECKPOINTER
+
+#ifdef HAVE_LIME
+
+#include <iostream>
+#include <sstream>
+#include <string>
+
+namespace Grid {
+namespace QCD {
+
+// For generic fields
+template <class Implementation, class Metadata>
+class ScidacHmcCheckpointer : public BaseHmcCheckpointer<Implementation> {
+ private:
+  CheckpointerParameters Params;
+  Metadata MData;
+
+  typedef typename Implementation::Field Field;
+
+ public:
+  //INHERIT_GIMPL_TYPES(Implementation);
+
+  ScidacHmcCheckpointer(const CheckpointerParameters &Params_) { initialize(Params_); }
+  ScidacHmcCheckpointer(const CheckpointerParameters &Params_, const Metadata& M_):MData(M_) { initialize(Params_); }
+
+  void initialize(const CheckpointerParameters &Params_) {
+    Params = Params_;
+
+    // check here that the format is valid
+    int ieee32big = (Params.format == std::string("IEEE32BIG"));
+    int ieee32    = (Params.format == std::string("IEEE32"));
+    int ieee64big = (Params.format == std::string("IEEE64BIG"));
+    int ieee64    = (Params.format == std::string("IEEE64"));
+
+    if (!(ieee64big || ieee32 || ieee32big || ieee64)) {
+      std::cout << GridLogError << "Unrecognized file format " << Params.format
+                << std::endl;
+      std::cout << GridLogError
+                << "Allowed: IEEE32BIG | IEEE32 | IEEE64BIG | IEEE64"
+                << std::endl;
+
+      exit(1);
+    }
+  }
+
+  void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG,
+                          GridParallelRNG &pRNG) {
+    if ((traj % Params.saveInterval) == 0) {
+      std::string config, rng;
+      this->build_filenames(traj, Params, config, rng);
+      GridBase *grid = U._grid;
+      uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+      BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb);
+      ScidacWriter _ScidacWriter(grid->IsBoss());
+      _ScidacWriter.open(config);
+      _ScidacWriter.writeScidacFieldRecord(U, MData);
+      _ScidacWriter.close();
+
+      std::cout << GridLogMessage << "Written Scidac Configuration on " << config
+                << " checksum " << std::hex << nersc_csum<<"/"
+		            << scidac_csuma<<"/" << scidac_csumb
+		            << std::dec << std::endl;
+    }
+  };
+
+  void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG,
+                         GridParallelRNG &pRNG) {
+    std::string config, rng;
+    this->build_filenames(traj, Params, config, rng);
+    this->check_filename(rng);
+    this->check_filename(config);
+
+
+    uint32_t nersc_csum,scidac_csuma,scidac_csumb;
+    BinaryIO::readRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb);
+
+    Metadata md_content;
+    ScidacReader _ScidacReader;
+    _ScidacReader.open(config);
+    _ScidacReader.readScidacFieldRecord(U,md_content);  // format from the header
+    _ScidacReader.close();
+
+    std::cout << GridLogMessage << "Read Scidac Configuration from " << config
+              << " checksum " << std::hex 
+	      << nersc_csum<<"/"
+	      << scidac_csuma<<"/"
+	      << scidac_csumb
+	      << std::dec << std::endl;
+  };
+};
+}
+}
+
+#endif  // HAVE_LIME
+#endif  // ILDG_CHECKPOINTER

tests/hmc/Test_hmc_ScalarActionNxN.cc

@@ -34,6 +34,8 @@ class ScalarActionParameters : Serializable {
     double, lambda,
     double, g);
 
+  ScalarActionParameters() = default;
+
     template <class ReaderClass >
   ScalarActionParameters(Reader<ReaderClass>& Reader){
     read(Reader, "ScalarAction", *this);
@@ -124,10 +126,13 @@ int main(int argc, char **argv) {
   ScalarGrid.set_rb(new GridRedBlackCartesian(ScalarGrid.get_full()));
   TheHMC.Resources.AddGrid("scalar", ScalarGrid);
   std::cout << "Lattice size : " << GridDefaultLatt() << std::endl;
+  
+  ScalarActionParameters SPar(Reader);
 
   // Checkpointer definition
   CheckpointerParameters CPparams(Reader);
-  TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  //TheHMC.Resources.LoadBinaryCheckpointer(CPparams);
+  TheHMC.Resources.LoadScidacCheckpointer(CPparams, SPar);
 
   RNGModuleParameters RNGpar(Reader);
   TheHMC.Resources.SetRNGSeeds(RNGpar);
@@ -140,7 +145,6 @@ int main(int argc, char **argv) {
   // Collect actions, here use more encapsulation
 
   // Scalar action in adjoint representation
-  ScalarActionParameters SPar(Reader);
   ScalarAction Saction(SPar.mass_squared, SPar.lambda, SPar.g);
 
   // Collect actions

tests/hmc/Test_hmc_WG_Production.cc

@@ -33,6 +33,7 @@ namespace Grid{
     GRID_SERIALIZABLE_CLASS_MEMBERS(ActionParameters,
 				    double, beta)
 
+    ActionParameters() = default;
 
     template <class ReaderClass >
     ActionParameters(Reader<ReaderClass>& Reader){
@@ -68,11 +69,15 @@ int main(int argc, char **argv) {
   }
   Serialiser Reader(TheHMC.ParameterFile);
 
-
+  // Read parameters from input file
+  ActionParameters WilsonPar(Reader);
 
   // Checkpointer definition
   CheckpointerParameters CPparams(Reader);
-  TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+  //TheHMC.Resources.LoadNerscCheckpointer(CPparams);
+
+  // Store metadata in the Scidac checkpointer
+  TheHMC.Resources.LoadScidacCheckpointer(CPparams, WilsonPar);
 
   RNGModuleParameters RNGpar(Reader);
   TheHMC.Resources.SetRNGSeeds(RNGpar);
@@ -91,8 +96,6 @@ int main(int argc, char **argv) {
   // need wrappers of the fermionic classes
   // that have a complex construction
   // standard
-  ActionParameters WilsonPar(Reader);
-  //RealD beta = 6.4 ;
   WilsonGaugeActionR Waction(WilsonPar.beta);
 
   ActionLevel<HMCWrapper::Field> Level1(1);