Merge branch 'feature/serialisation-hdf5' into feature/qed-fvol

configure.ac

@@ -99,6 +99,13 @@ case ${ac_MKL} in
         AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
 esac
 
+############### HDF5
+AC_ARG_WITH([hdf5],
+    [AS_HELP_STRING([--with-hdf5=prefix],
+    [try this for a non-standard install prefix of the HDF5 library])],
+    [AM_CXXFLAGS="-I$with_hdf5/include $AM_CXXFLAGS"]
+    [AM_LDFLAGS="-L$with_hdf5/lib $AM_LDFLAGS"])
+
 ############### first-touch
 AC_ARG_ENABLE([numa],
     [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
@@ -145,6 +152,12 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
                [have_fftw=true])
 
+AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
+               [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
+               [have_hdf5=true]
+               [LIBS="${LIBS} -lhdf5"], [], [-lhdf5])
+AM_CONDITIONAL(BUILD_HDF5, [ test "${have_hdf5}X" == "trueX" ])
+
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
@@ -411,6 +424,7 @@ RNG choice                  : ${ac_RNG}
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 LAPACK                      : ${ac_LAPACK}
 FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
 build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
 CXXFLAGS:

lib/Grid.h

@@ -59,13 +59,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 ///////////////////
 // Grid headers
 ///////////////////
-#include <Grid/serialisation/Serialisation.h>
 #include "Config.h"
 #include <Grid/Timer.h>
 #include <Grid/PerfCount.h>
 #include <Grid/Log.h>
 #include <Grid/AlignedAllocator.h>
 #include <Grid/Simd.h>
+#include <Grid/serialisation/Serialisation.h>
 #include <Grid/Threads.h>
 #include <Grid/Lexicographic.h>
 #include <Grid/Init.h>

lib/Makefile.am

@@ -1,4 +1,5 @@
 extra_sources=
+extra_headers=
 if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
   extra_sources+=communicator/Communicator_base.cc
@@ -24,6 +25,12 @@ if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_base.cc
 endif
 
+if BUILD_HDF5
+  extra_sources+=serialisation/Hdf5IO.cc 
+  extra_headers+=serialisation/Hdf5IO.h
+  extra_headers+=serialisation/Hdf5Type.h
+endif
+
 #
 # Libraries
 #
@@ -32,6 +39,9 @@ include Eigen.inc
 
 lib_LIBRARIES = libGrid.a
 
-libGrid_a_SOURCES              = $(CCFILES) $(extra_sources)
+CCFILES += $(extra_sources)
+HFILES  += $(extra_headers)
+
+libGrid_a_SOURCES              = $(CCFILES)
 libGrid_adir                   = $(pkgincludedir)
 nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h

lib/serialisation/BaseIO.h

@@ -32,6 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <type_traits>
 
 namespace Grid {
+  // Vector IO utilities ///////////////////////////////////////////////////////
   // helper function to read space-separated values
   template <typename T>
   std::vector<T> strToVec(const std::string s)
@@ -67,6 +68,77 @@ namespace Grid {
     return os;
   }
   
+  // Vector element trait //////////////////////////////////////////////////////  
+  template <typename T>
+  struct element
+  {
+    typedef T type;
+    static constexpr bool is_number = false;
+  };
+  
+  template <typename T>
+  struct element<std::vector<T>>
+  {
+    typedef typename element<T>::type type;
+    static constexpr bool is_number = std::is_arithmetic<T>::value
+                                      or is_complex<T>::value
+                                      or element<T>::is_number;
+  };
+  
+  // Vector flatening utility class ////////////////////////////////////////////
+  // Class to flatten a multidimensional std::vector
+  template <typename V>
+  class Flatten
+  {
+  public:
+    typedef typename element<V>::type Element;
+  public:
+    explicit                     Flatten(const V &vector);
+    const V &                    getVector(void);
+    const std::vector<Element> & getFlatVector(void);
+    const std::vector<size_t>  & getDim(void);
+  private:
+    void accumulate(const Element &e);
+    template <typename W>
+    void accumulate(const W &v);
+    void accumulateDim(const Element &e);
+    template <typename W>
+    void accumulateDim(const W &v);
+  private:
+    const V              &vector_;
+    std::vector<Element> flatVector_;
+    std::vector<size_t>  dim_;
+  };
+  
+  
+  // Class to reconstruct a multidimensional std::vector
+  template <typename V>
+  class Reconstruct
+  {
+  public:
+    typedef typename element<V>::type Element;
+  public:
+    Reconstruct(const std::vector<Element> &flatVector,
+                const std::vector<size_t> &dim);
+    const V &                    getVector(void);
+    const std::vector<Element> & getFlatVector(void);
+    const std::vector<size_t>  & getDim(void);
+  private:
+    void fill(std::vector<Element> &v);
+    template <typename W>
+    void fill(W &v);
+    void resize(std::vector<Element> &v, const unsigned int dim);
+    template <typename W>
+    void resize(W &v, const unsigned int dim);
+  private:
+    V                          vector_;
+    const std::vector<Element> &flatVector_;
+    std::vector<size_t>        dim_;
+    size_t                     ind_{0};
+    unsigned int               dimInd_{0};
+  };
+  
+  // Abstract writer/reader classes ////////////////////////////////////////////
   // static polymorphism implemented using CRTP idiom
   class Serializable;
   
@@ -132,7 +204,128 @@ namespace Grid {
     }
   };
   
-  // Generic writer interface
+  // Flatten class template implementation /////////////////////////////////////
+  template <typename V>
+  void Flatten<V>::accumulate(const Element &e)
+  {
+    flatVector_.push_back(e);
+  }
+  
+  template <typename V>
+  template <typename W>
+  void Flatten<V>::accumulate(const W &v)
+  {
+    for (auto &e: v)
+    {
+      accumulate(e);
+    }
+  }
+  
+  template <typename V>
+  void Flatten<V>::accumulateDim(const Element &e) {};
+  
+  template <typename V>
+  template <typename W>
+  void Flatten<V>::accumulateDim(const W &v)
+  {
+    dim_.push_back(v.size());
+    accumulateDim(v[0]);
+  }
+  
+  template <typename V>
+  Flatten<V>::Flatten(const V &vector)
+  : vector_(vector)
+  {
+    accumulate(vector_);
+    accumulateDim(vector_);
+  }
+  
+  template <typename V>
+  const V & Flatten<V>::getVector(void)
+  {
+    return vector_;
+  }
+  
+  template <typename V>
+  const std::vector<typename Flatten<V>::Element> &
+  Flatten<V>::getFlatVector(void)
+  {
+    return flatVector_;
+  }
+  
+  template <typename V>
+  const std::vector<size_t> & Flatten<V>::getDim(void)
+  {
+    return dim_;
+  }
+  
+  // Reconstruct class template implementation /////////////////////////////////
+  template <typename V>
+  void Reconstruct<V>::fill(std::vector<Element> &v)
+  {
+    for (auto &e: v)
+    {
+      e = flatVector_[ind_++];
+    }
+  }
+  
+  template <typename V>
+  template <typename W>
+  void Reconstruct<V>::fill(W &v)
+  {
+    for (auto &e: v)
+    {
+      fill(e);
+    }
+  }
+  
+  template <typename V>
+  void Reconstruct<V>::resize(std::vector<Element> &v, const unsigned int dim)
+  {
+    v.resize(dim_[dim]);
+  }
+  
+  template <typename V>
+  template <typename W>
+  void Reconstruct<V>::resize(W &v, const unsigned int dim)
+  {
+    v.resize(dim_[dim]);
+    for (auto &e: v)
+    {
+      resize(e, dim + 1);
+    }
+  }
+  
+  template <typename V>
+  Reconstruct<V>::Reconstruct(const std::vector<Element> &flatVector,
+                              const std::vector<size_t> &dim)
+  : flatVector_(flatVector)
+  , dim_(dim)
+  {
+    resize(vector_, 0);
+    fill(vector_);
+  }
+  
+  template <typename V>
+  const V & Reconstruct<V>::getVector(void)
+  {
+    return vector_;
+  }
+  
+  template <typename V>
+  const std::vector<typename Reconstruct<V>::Element> &
+  Reconstruct<V>::getFlatVector(void)
+  {
+    return flatVector_;
+  }
+  
+  template <typename V>
+  const std::vector<size_t> & Reconstruct<V>::getDim(void)
+  {
+    return dim_;
+  }
+  
+  // Generic writer interface //////////////////////////////////////////////////
   template <typename T>
   inline void push(Writer<T> &w, const std::string &s)
   {
@@ -217,7 +410,7 @@ namespace Grid {
     upcast->writeDefault(s, output);
   }
   
-  // Reader template implementation ////////////////////////////////////////////
+  // Reader template implementation
   template <typename T>
   Reader<T>::Reader(void)
   {

lib/serialisation/Hdf5IO.cc

@@ -0,0 +1,103 @@
+#include <Grid.h>
+
+using namespace Grid;
+#ifndef H5_NO_NAMESPACE
+using namespace H5NS;
+#endif
+
+// Writer implementation ///////////////////////////////////////////////////////
+Hdf5Writer::Hdf5Writer(const std::string &fileName)
+: fileName_(fileName)
+, file_(fileName.c_str(), H5F_ACC_TRUNC)
+{
+  group_ = file_.openGroup("/");
+  writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
+                       Hdf5Type<unsigned int>::type());
+}
+
+void Hdf5Writer::push(const std::string &s)
+{
+  group_ = group_.createGroup(s);
+  path_.push_back(s);
+}
+
+void Hdf5Writer::pop(void)
+{
+  path_.pop_back();
+  if (path_.empty())
+  {
+    group_ = file_.openGroup("/");
+  }
+  else
+  {
+    auto binOp = [](const std::string &a, const std::string &b)->std::string
+    {
+      return a + "/" + b;
+    };
+    
+    group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(),
+                                              std::string(""), binOp));
+  }
+}
+
+template <>
+void Hdf5Writer::writeDefault(const std::string &s, const std::string &x)
+{
+  StrType     strType(PredType::C_S1, x.size());
+  
+  writeSingleAttribute(*(x.data()), s, strType);
+}
+
+void Hdf5Writer::writeDefault(const std::string &s, const char *x)
+{
+  std::string sx(x);
+  
+  writeDefault(s, sx);
+}
+
+// Reader implementation ///////////////////////////////////////////////////////
+Hdf5Reader::Hdf5Reader(const std::string &fileName)
+: fileName_(fileName)
+, file_(fileName.c_str(), H5F_ACC_RDONLY)
+{
+  group_ = file_.openGroup("/");
+  readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
+                      Hdf5Type<unsigned int>::type());
+}
+
+void Hdf5Reader::push(const std::string &s)
+{
+  group_ = group_.openGroup(s);
+  path_.push_back(s);
+}
+
+void Hdf5Reader::pop(void)
+{
+  path_.pop_back();
+  if (path_.empty())
+  {
+    group_ = file_.openGroup("/");
+  }
+  else
+  {
+    auto binOp = [](const std::string &a, const std::string &b)->std::string
+    {
+      return a + "/" + b;
+    };
+    
+    group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(),
+                                              std::string(""), binOp));
+  }
+}
+
+template <>
+void Hdf5Reader::readDefault(const std::string &s, std::string &x)
+{
+  Attribute attribute;
+  
+  attribute       = group_.openAttribute(s);
+  StrType strType = attribute.getStrType();
+  
+  x.resize(strType.getSize());
+  attribute.read(strType, &(x[0]));
+}

lib/serialisation/Hdf5IO.h

@@ -0,0 +1,242 @@
+#ifndef GRID_SERIALISATION_HDF5_H
+#define GRID_SERIALISATION_HDF5_H
+
+#include <stack>
+#include <string>
+#include <vector>
+#include <H5Cpp.h>
+#include "Hdf5Type.h"
+
+#ifndef H5_NO_NAMESPACE
+#define H5NS H5
+#endif
+
+// default thresold above which datasets are used instead of attributes
+#ifndef HDF5_DEF_DATASET_THRES
+#define HDF5_DEF_DATASET_THRES 6u
+#endif
+
+// name guard for Grid metadata
+#define HDF5_GRID_GUARD "_Grid_"
+
+namespace Grid
+{
+  class Hdf5Writer: public Writer<Hdf5Writer>
+  {
+  public:
+    Hdf5Writer(const std::string &fileName);
+    virtual ~Hdf5Writer(void) = default;
+    void push(const std::string &s);
+    void pop(void);
+    void writeDefault(const std::string &s, const char *x);
+    template <typename U>
+    void writeDefault(const std::string &s, const U &x);
+    template <typename U>
+    typename std::enable_if<element<std::vector<U>>::is_number, void>::type
+    writeDefault(const std::string &s, const std::vector<U> &x);
+    template <typename U>
+    typename std::enable_if<!element<std::vector<U>>::is_number, void>::type
+    writeDefault(const std::string &s, const std::vector<U> &x);
+  private:
+    template <typename U>
+    void writeSingleAttribute(const U &x, const std::string &name,
+                              const H5NS::DataType &type);
+  private:
+    std::string              fileName_;
+    std::vector<std::string> path_;
+    H5NS::H5File             file_;
+    H5NS::Group              group_;
+    unsigned int             dataSetThres_{HDF5_DEF_DATASET_THRES};
+  };
+  
+  class Hdf5Reader: public Reader<Hdf5Reader>
+  {
+  public:
+    Hdf5Reader(const std::string &fileName);
+    virtual ~Hdf5Reader(void) = default;
+    void push(const std::string &s);
+    void pop(void);
+    template <typename U>
+    void readDefault(const std::string &s, U &output);
+    template <typename U>
+    typename std::enable_if<element<std::vector<U>>::is_number, void>::type
+    readDefault(const std::string &s, std::vector<U> &x);
+    template <typename U>
+    typename std::enable_if<!element<std::vector<U>>::is_number, void>::type
+    readDefault(const std::string &s, std::vector<U> &x);
+  private:
+    template <typename U>
+    void readSingleAttribute(U &x, const std::string &name,
+                             const H5NS::DataType &type);
+  private:
+    std::string              fileName_;
+    std::vector<std::string> path_;
+    H5NS::H5File             file_;
+    H5NS::Group              group_;
+    unsigned int             dataSetThres_;
+  };
+  
+  // Writer template implementation ////////////////////////////////////////////
+  template <typename U>
+  void Hdf5Writer::writeSingleAttribute(const U &x, const std::string &name,
+                                        const H5NS::DataType &type)
+  {
+    H5NS::Attribute attribute;
+    hsize_t         attrDim = 1;
+    H5NS::DataSpace attrSpace(1, &attrDim);
+    
+    attribute = group_.createAttribute(name, type, attrSpace);
+    attribute.write(type, &x);
+  }
+  
+  template <typename U>
+  void Hdf5Writer::writeDefault(const std::string &s, const U &x)
+  {
+    writeSingleAttribute(x, s, Hdf5Type<U>::type());
+  }
+  
+  template <>
+  void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
+  
+  template <typename U>
+  typename std::enable_if<element<std::vector<U>>::is_number, void>::type
+  Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
+  {
+    // alias to element type
+    typedef typename element<std::vector<U>>::type Element;
+    
+    // flatten the vector and getting dimensions
+    Flatten<std::vector<U>> flat(x);
+    std::vector<hsize_t> dim;
+    const auto           &flatx = flat.getFlatVector();
+    
+    for (auto &d: flat.getDim())
+    {
+      dim.push_back(d);
+    }
+    
+    // write to file
+    H5NS::DataSpace dataSpace(dim.size(), dim.data());
+    
+    if (flatx.size() > dataSetThres_)
+    {
+      H5NS::DataSet dataSet;
+      
+      dataSet = group_.createDataSet(s, Hdf5Type<Element>::type(), dataSpace);
+      dataSet.write(flatx.data(), Hdf5Type<Element>::type());
+    }
+    else
+    {
+      H5NS::Attribute attribute;
+      
+      attribute = group_.createAttribute(s, Hdf5Type<Element>::type(), dataSpace);
+      attribute.write(Hdf5Type<Element>::type(), flatx.data());
+    }
+  }
+  
+  template <typename U>
+  typename std::enable_if<!element<std::vector<U>>::is_number, void>::type
+  Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
+  {
+    push(s);
+    writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size",
+                         Hdf5Type<uint64_t>::type());
+    for (hsize_t i = 0; i < x.size(); ++i)
+    {
+      write(s + "_" + std::to_string(i), x[i]);
+    }
+    pop();
+  }
+  
+  // Reader template implementation ////////////////////////////////////////////
+  template <typename U>
+  void Hdf5Reader::readSingleAttribute(U &x, const std::string &name,
+                                       const H5NS::DataType &type)
+  {
+    H5NS::Attribute attribute;
+    
+    attribute = group_.openAttribute(name);
+    attribute.read(type, &x);
+  }
+  
+  template <typename U>
+  void Hdf5Reader::readDefault(const std::string &s, U &output)
+  {
+    readSingleAttribute(output, s, Hdf5Type<U>::type());
+  }
+  
+  template <>
+  void Hdf5Reader::readDefault(const std::string &s, std::string &x);
+  
+  template <typename U>
+  typename std::enable_if<element<std::vector<U>>::is_number, void>::type
+  Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
+  {
+    // alias to element type
+    typedef typename element<std::vector<U>>::type Element;
+    
+    // read the dimensions
+    H5NS::DataSpace       dataSpace;
+    std::vector<hsize_t>  hdim;
+    std::vector<size_t>   dim;
+    hsize_t               size = 1;
+    
+    if (group_.attrExists(s))
+    {
+      dataSpace = group_.openAttribute(s).getSpace();
+    }
+    else
+    {
+      dataSpace = group_.openDataSet(s).getSpace();
+    }
+    hdim.resize(dataSpace.getSimpleExtentNdims());
+    dataSpace.getSimpleExtentDims(hdim.data());
+    for (auto &d: hdim)
+    {
+      dim.push_back(d);
+      size *= d;
+    }
+    
+    // read the flat vector
+    std::vector<Element> buf(size);
+
+    if (size > dataSetThres_)
+    {
+      H5NS::DataSet dataSet;
+      
+      dataSet = group_.openDataSet(s);
+      dataSet.read(buf.data(), Hdf5Type<Element>::type());
+    }
+    else
+    {
+      H5NS::Attribute attribute;
+      
+      attribute = group_.openAttribute(s);
+      attribute.read(Hdf5Type<Element>::type(), buf.data());
+    }
+    
+    // reconstruct the multidimensional vector
+    Reconstruct<std::vector<U>> r(buf, dim);
+    
+    x = r.getVector();
+  }
+  
+  template <typename U>
+  typename std::enable_if<!element<std::vector<U>>::is_number, void>::type
+  Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
+  {
+    uint64_t size;
+    
+    push(s);
+    readSingleAttribute(size, HDF5_GRID_GUARD "vector_size",
+                        Hdf5Type<uint64_t>::type());
+    x.resize(size);
+    for (hsize_t i = 0; i < x.size(); ++i)
+    {
+      read(s + "_" + std::to_string(i), x[i]);
+    }
+    pop();
+  }
+}
+
+#endif

lib/serialisation/Hdf5Type.h

@@ -0,0 +1,77 @@
+#ifndef GRID_SERIALISATION_HDF5_TYPE_H
+#define GRID_SERIALISATION_HDF5_TYPE_H
+
+#include <H5Cpp.h>
+#include <complex>
+#include <memory>
+
+#ifndef H5_NO_NAMESPACE
+#define H5NS H5
+#endif
+
+#define HDF5_NATIVE_TYPE(predType, cType)\
+template <>\
+class Hdf5Type<cType>\
+{\
+public:\
+  static inline const H5NS::DataType & type(void)\
+  {\
+    return H5NS::PredType::predType;\
+  }\
+  static constexpr bool isNative = true;\
+};
+
+#define DEFINE_HDF5_NATIVE_TYPES \
+HDF5_NATIVE_TYPE(NATIVE_B8,      bool);\
+HDF5_NATIVE_TYPE(NATIVE_CHAR,    char);\
+HDF5_NATIVE_TYPE(NATIVE_SCHAR,   signed char);\
+HDF5_NATIVE_TYPE(NATIVE_UCHAR,   unsigned char);\
+HDF5_NATIVE_TYPE(NATIVE_SHORT,   short);\
+HDF5_NATIVE_TYPE(NATIVE_USHORT,  unsigned short);\
+HDF5_NATIVE_TYPE(NATIVE_INT,     int);\
+HDF5_NATIVE_TYPE(NATIVE_UINT,    unsigned int);\
+HDF5_NATIVE_TYPE(NATIVE_LONG,    long);\
+HDF5_NATIVE_TYPE(NATIVE_ULONG,   unsigned long);\
+HDF5_NATIVE_TYPE(NATIVE_LLONG,   long long);\
+HDF5_NATIVE_TYPE(NATIVE_ULLONG,  unsigned long long);\
+HDF5_NATIVE_TYPE(NATIVE_FLOAT,   float);\
+HDF5_NATIVE_TYPE(NATIVE_DOUBLE,  double);\
+HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double);
+
+namespace Grid
+{
+  template <typename T> class Hdf5Type
+  {
+  public:
+    static constexpr bool isNative = false;
+  };
+  
+  DEFINE_HDF5_NATIVE_TYPES;
+  
+  template <typename R>
+  class Hdf5Type<std::complex<R>>
+  {
+  public:
+    static inline const H5NS::DataType & type(void)
+    {
+      if (typePtr_ == nullptr)
+      {
+        typePtr_.reset(new H5NS::CompType(sizeof(std::complex<R>)));
+        typePtr_->insertMember("re", 0,         Hdf5Type<R>::type());
+        typePtr_->insertMember("im", sizeof(R), Hdf5Type<R>::type());
+      }
+
+      return *typePtr_;
+    }
+    static constexpr bool isNative = false;
+  private:
+    static std::unique_ptr<H5NS::CompType> typePtr_;
+  };
+  
+  template <typename R>
+  std::unique_ptr<H5NS::CompType> Hdf5Type<std::complex<R>>::typePtr_ = nullptr;
+}
+
+#undef HDF5_NATIVE_TYPE
+
+#endif /* GRID_SERIALISATION_HDF5_TYPE_H */

lib/serialisation/MacroMagic.h

@@ -109,38 +109,36 @@ THE SOFTWARE.
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 #define GRID_MACRO_MEMBER(A,B)        A B;
+#define GRID_MACRO_COMP_MEMBER(A,B) result = (result and (lhs. B == rhs. B));
 #define GRID_MACRO_OS_WRITE_MEMBER(A,B) os<< #A <<" "#B <<" = "<< obj. B <<" ; " <<std::endl;
 #define GRID_MACRO_READ_MEMBER(A,B) Grid::read(RD,#B,obj. B);
 #define GRID_MACRO_WRITE_MEMBER(A,B) Grid::write(WR,#B,obj. B);
 
-#define GRID_SERIALIZABLE_CLASS_MEMBERS(cname,...)		\
+#define GRID_SERIALIZABLE_CLASS_MEMBERS(cname,...)\
-\
+GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_MEMBER,__VA_ARGS__))\
-\
-GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_MEMBER,__VA_ARGS__))		\
-\
-\
 template <typename T>\
 static inline void write(Writer<T> &WR,const std::string &s, const cname &obj){ \
   push(WR,s);\
   GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_WRITE_MEMBER,__VA_ARGS__))	\
   pop(WR);\
-} \
+}\
-\
-\
 template <typename T>\
 static inline void read(Reader<T> &RD,const std::string &s, cname &obj){	\
   push(RD,s);\
   GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_READ_MEMBER,__VA_ARGS__))	\
   pop(RD);\
-} \
+}\
-\
-\
 friend inline std::ostream & operator << (std::ostream &os, const cname &obj ) { \
   os<<"class "<<#cname<<" {"<<std::endl;\
   GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_OS_WRITE_MEMBER,__VA_ARGS__))	\
     os<<"}";								\
   return os;\
-};  
+}\
+friend inline bool operator==(const cname &lhs, const cname &rhs) {\
+  bool result = true;\
+  GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_COMP_MEMBER,__VA_ARGS__))\
+  return result;\
+}
 
 #define GRID_ENUM_TYPE(obj) std::remove_reference<decltype(obj)>::type
 #define GRID_MACRO_ENUMVAL(A,B) A = B,
@@ -149,7 +147,7 @@ friend inline std::ostream & operator << (std::ostream &os, const cname &obj ) {
 #define GRID_MACRO_ENUMCASEIO(A,B) case GRID_ENUM_TYPE(obj)::A: os << #A; break;
 
 #define GRID_SERIALIZABLE_ENUM(name,undefname,...)\
-class name: public Serializable\
+class name: public Grid::Serializable\
 {\
 public:\
   enum EnumType\
@@ -161,7 +159,7 @@ public:\
   name(void): value_(undefname) {};\
   name(EnumType value): value_(value) {};\
   template <typename T>\
-  static inline void write(Writer<T> &WR,const std::string &s, const name &obj)\
+  static inline void write(Grid::Writer<T> &WR,const std::string &s, const name &obj)\
   {\
     switch (obj.value_)\
     {\
@@ -171,7 +169,7 @@ public:\
   }\
   \
   template <typename T>\
-  static inline void read(Reader<T> &RD,const std::string &s, name &obj)\
+  static inline void read(Grid::Reader<T> &RD,const std::string &s, name &obj)\
   {\
     std::string buf;\
     Grid::read(RD, s, buf);\

lib/serialisation/Serialisation.h

@@ -36,6 +36,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include "BinaryIO.h"
 #include "TextIO.h"
 #include "XmlIO.h"
+#ifdef HAVE_HDF5
+#include "Hdf5IO.h"
+#endif
 //////////////////////////////////////////
 // Todo:
 //////////////////////////////////////////

scripts/filelist

@@ -4,9 +4,8 @@ home=`pwd`
 
 # library Make.inc
 cd $home/lib
-HFILES=`find . -type f -name '*.h' -not -path '*/Old/*' -not -path '*/Eigen/*'`
+HFILES=`find . -type f -name '*.h' -not -name '*Hdf5*' -not -path '*/Old/*' -not -path '*/Eigen/*'`
-HFILES="$HFILES"
+CCFILES=`find . -type f -name '*.cc' -not -name '*Communicator*.cc' -not -name '*Hdf5*'`
-CCFILES=`find . -type f -name '*.cc' -not  -name '*ommunicator*.cc'`
 echo HFILES=$HFILES > Make.inc
 echo >> Make.inc
 echo CCFILES=$CCFILES >> Make.inc

tests/IO/Test_serialisation.cc

@@ -28,130 +28,152 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include <Grid/Grid.h>
 
-namespace Grid {
-  
-  GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3);
-    
-  class myclass: Serializable {
-  public:
-    
-    GRID_SERIALIZABLE_CLASS_MEMBERS(myclass,
-                            myenum, e,
-                            std::vector<myenum>, ve,
-                            std::string, name,
-                            int, x,
-                            double, y,
-                            bool , b,
-                            std::vector<double>, array,
-                            std::vector<std::vector<double>>, twodimarray,
-                            );
-    
-    myclass() {}
-    myclass(int i)
-    : array(4,5.1), twodimarray(3,std::vector<double>(2,1.23456)), ve(2, myenum::blue)
-    {
-      e=myenum::red;
-      x=i;
-      y=2*i;
-      b=true;
-      name="bother said pooh";
-    }
-  };
-  
-}
-
 using namespace Grid;
 
-int16_t i16 = 1;
+GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3);
+  
+class myclass: Serializable {
+public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(myclass,
+                          myenum, e,
+                          std::vector<myenum>, ve,
+                          std::string, name,
+                          int, x,
+                          double, y,
+                          bool , b,
+                          std::vector<double>, array,
+                          std::vector<std::vector<double>>, twodimarray,
+                          std::vector<std::vector<std::vector<Complex>>>, cmplx3darray
+                          );
+  myclass() {}
+  myclass(int i)
+  : array(4,5.1)
+  , twodimarray(3,std::vector<double>(5, 1.23456))
+  , cmplx3darray(3,std::vector<std::vector<Complex>>(5, std::vector<Complex>(7, Complex(1.2, 3.4))))
+  , ve(2, myenum::blue)
+  {
+    e=myenum::red;
+    x=i;
+    y=2*i;
+    b=true;
+    name="bother said pooh";
+  }
+};
+
+int16_t  i16 = 1;
 uint16_t u16 = 2;
-int32_t i32 = 3;
+int32_t  i32 = 3;
 uint32_t u32 = 4;
-int64_t i64 = 5;
+int64_t  i64 = 5;
 uint64_t u64 = 6;
-float    f = M_PI;
+float    f   = M_PI;
-double   d = 2*M_PI;
+double   d   = 2*M_PI;
-bool     b = false;
+bool     b   = false;
+
+template <typename W, typename R, typename O>
+void ioTest(const std::string &filename, const O &object, const std::string &name)
+{
+  // writer needs to be destroyed so that writing physically happens
+  {
+    W writer(filename);
+    
+    write(writer, "testobject", object);
+  }
+  
+  R    reader(filename);
+  O    buf;
+  bool good;
+  
+  read(reader, "testobject", buf);
+  good = (object == buf);
+  std::cout << name << " IO test: " << (good ? "success" : "failure");
+  std::cout << std::endl;
+  if (!good) exit(EXIT_FAILURE);
+}
 
 int main(int argc,char **argv)
 {
-  {
+  std::cout << "==== basic IO" << std::endl;
-    XmlWriter WR("bother.xml");
+  XmlWriter WR("bother.xml");
   
-    // test basic type writing
+  // test basic type writing
-    push(WR,"BasicTypes");
+  std::cout << "-- basic writing to 'bother.xml'..." << std::endl;
-    write(WR,std::string("i16"),i16);
+  push(WR,"BasicTypes");
-    write(WR,"u16",u16);
+  write(WR,std::string("i16"),i16);
-    write(WR,"i32",i32);
+  write(WR,"u16",u16);
-    write(WR,"u32",u32);
+  write(WR,"i32",i32);
-    write(WR,"i64",i64);
+  write(WR,"u32",u32);
-    write(WR,"u64",u64);
+  write(WR,"i64",i64);
-    write(WR,"f",f);
+  write(WR,"u64",u64);
-    write(WR,"d",d);
+  write(WR,"f",f);
-    write(WR,"b",b);
+  write(WR,"d",d);
-    pop(WR);
+  write(WR,"b",b);
+  pop(WR);
   
-    // test serializable class writing
+  // test serializable class writing
-    myclass obj(1234); // non-trivial constructor
+  myclass              obj(1234); // non-trivial constructor
-    write(WR,"obj",obj);
+  std::vector<myclass> vec;
-    WR.write("obj2", obj);
-    std::cout << obj << std::endl;
   
-    std::vector<myclass> vec;
+  std::cout << "-- serialisable class writing to 'bother.xml'..." << std::endl;
-    vec.push_back(myclass(1234));
+  write(WR,"obj",obj);
-    vec.push_back(myclass(5678));
+  WR.write("obj2", obj);
-    vec.push_back(myclass(3838));
+  vec.push_back(myclass(1234));
-    write(WR, "objvec", vec);
+  vec.push_back(myclass(5678));
-  };
+  vec.push_back(myclass(3838));
+  write(WR, "objvec", vec);
+  std::cout << "-- serialisable class writing to std::cout:" << std::endl;
+  std::cout << obj << std::endl;
+  std::cout << "-- serialisable class comparison:" << std::endl;
+  std::cout << "vec[0] == obj: " << ((vec[0] == obj) ? "true" : "false") << std::endl;
+  std::cout << "vec[1] == obj: " << ((vec[1] == obj) ? "true" : "false") << std::endl;
   
   // read tests
-  myclass copy1, copy2, copy3;
+  std::cout << "\n==== IO self-consistency tests" << std::endl;
-  std::vector<myclass> veccopy1, veccopy2, veccopy3;
   //// XML
-  {
+  ioTest<XmlWriter, XmlReader>("iotest.xml", obj, "XML    (object)           ");
-    XmlReader RD("bother.xml");
+  ioTest<XmlWriter, XmlReader>("iotest.xml", vec, "XML    (vector of objects)");
-    read(RD,"obj",copy1);
-    read(RD,"objvec", veccopy1);
-    std::cout << "Loaded (XML) -----------------" << std::endl;
-    std::cout << copy1 << std::endl << veccopy1 << std::endl;
-  }
   //// binary
-  {
+  ioTest<BinaryWriter, BinaryReader>("iotest.bin", obj, "binary (object)           ");
-    BinaryWriter BWR("bother.bin");
+  ioTest<BinaryWriter, BinaryReader>("iotest.bin", vec, "binary (vector of objects)");
-    write(BWR,"discard",copy1 );
-    write(BWR,"discard",veccopy1 );
-  }
-  {
-    BinaryReader BRD("bother.bin");
-    read (BRD,"discard",copy2 );
-    read (BRD,"discard",veccopy2 );
-    std::cout << "Loaded (bin) -----------------" << std::endl;
-    std::cout << copy2 << std::endl << veccopy2 << std::endl;
-  }
   //// text
-  {
+  ioTest<TextWriter, TextReader>("iotest.dat", obj, "text   (object)           ");
-    TextWriter TWR("bother.txt");
+  ioTest<TextWriter, TextReader>("iotest.dat", vec, "text   (vector of objects)");
-    write(TWR,"discard",copy1 );
+  //// HDF5
-    write(TWR,"discard",veccopy1 );
+#ifdef HAVE_HDF5
-  }
+  ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", obj, "HDF5   (object)           ");
-  {
+  ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", vec, "HDF5   (vector of objects)");
-    TextReader TRD("bother.txt");
+#endif
-    read (TRD,"discard",copy3 );
-    read (TRD,"discard",veccopy3 );
-    std::cout << "Loaded (txt) -----------------" << std::endl;
-    std::cout << copy3 << std::endl << veccopy3 << std::endl;
-  }
   
-  std::vector<int> iv = strToVec<int>("1 2 2 4");
+  std::cout << "\n==== vector flattening/reconstruction" << std::endl;
-  std::vector<std::string> sv = strToVec<std::string>("bli bla blu");
+  typedef std::vector<std::vector<std::vector<double>>> vec3d;
   
-  for (auto &e: iv)
+  vec3d dv, buf;
+  double d = 0.;
+  
+  dv.resize(4);
+  for (auto &v1: dv)
   {
-    std::cout << e << " ";
+    v1.resize(3);
+    for (auto &v2: v1)
+    {
+      v2.resize(5);
+      for (auto &x: v2)
+      {
+        x = d++;
+      }
+    }
   }
-  std::cout << std::endl;
+  std::cout << "original 3D vector:" << std::endl;
-  for (auto &e: sv)
+  std::cout << dv << std::endl;
-  {
+  
-    std::cout << e << " ";
+  Flatten<vec3d> flatdv(dv);
-  }
+  
-  std::cout << std::endl;
+  std::cout << "\ndimensions:" << std::endl;
+  std::cout << flatdv.getDim() << std::endl;
+  std::cout << "\nflattened vector:" << std::endl;
+  std::cout << flatdv.getFlatVector() << std::endl;
+  
+  Reconstruct<vec3d> rec(flatdv.getFlatVector(), flatdv.getDim());
+  std::cout << "\nreconstructed vector:" << std::endl;
+  std::cout << flatdv.getVector() << std::endl;
 }