First implementation of HDF5 serial IO writer, reader is still empty

configure.ac

@@ -99,6 +99,13 @@ case ${ac_MKL} in
         AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);;
 esac
 
+############### HDF5
+AC_ARG_WITH([hdf5],
+    [AS_HELP_STRING([--with-hdf5=prefix],
+    [try this for a non-standard install prefix of the HDF5 library])],
+    [AM_CXXFLAGS="-I$with_hdf5/include $AM_CXXFLAGS"]
+    [AM_LDFLAGS="-L$with_hdf5/lib $AM_LDFLAGS"])
+
 ############### first-touch
 AC_ARG_ENABLE([numa],
     [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], 
@@ -145,6 +152,12 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3],
                [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])]
                [have_fftw=true])
 
+AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp],
+               [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])]
+               [have_hdf5=true]
+               [LIBS="${LIBS} -lhdf5"], [], [-lhdf5])
+AM_CONDITIONAL(BUILD_HDF5, [ test "${have_hdf5}X" == "trueX" ])
+
 CXXFLAGS=$CXXFLAGS_CPY
 LDFLAGS=$LDFLAGS_CPY
 
@@ -410,6 +423,7 @@ RNG choice                  : ${ac_RNG}
 GMP                         : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi`
 LAPACK                      : ${ac_LAPACK}
 FFTW                        : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi`
+HDF5                        : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi`
 build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi`
 ----- BUILD FLAGS -------------------------------------
 CXXFLAGS:

lib/Grid.h

@@ -59,8 +59,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 ///////////////////
 // Grid headers
 ///////////////////
-#include <Grid/serialisation/Serialisation.h>
 #include "Config.h"
+#include <Grid/serialisation/Serialisation.h>
 #include <Grid/Timer.h>
 #include <Grid/PerfCount.h>
 #include <Grid/Log.h>

lib/Makefile.am

@@ -1,4 +1,5 @@
 extra_sources=
+extra_headers=
 if BUILD_COMMS_MPI
   extra_sources+=communicator/Communicator_mpi.cc
   extra_sources+=communicator/Communicator_base.cc
@@ -24,6 +25,13 @@ if BUILD_COMMS_NONE
   extra_sources+=communicator/Communicator_base.cc
 endif
 
+if BUILD_HDF5
+  extra_sources+=serialisation/Hdf5IO.cc 
+  extra_sources+=serialisation/Hdf5Type.cc
+  extra_headers+=serialisation/Hdf5IO.h
+  extra_headers+=serialisation/Hdf5Type.h
+endif
+
 #
 # Libraries
 #
@@ -32,6 +40,9 @@ include Eigen.inc
 
 lib_LIBRARIES = libGrid.a
 
-libGrid_a_SOURCES              = $(CCFILES) $(extra_sources)
+CCFILES += $(extra_sources)
+HFILES  += $(extra_headers)
+
+libGrid_a_SOURCES              = $(CCFILES)
 libGrid_adir                   = $(pkgincludedir)
 nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h

lib/serialisation/Hdf5IO.cc

@@ -0,0 +1,84 @@
+#include <Grid.h>
+
+using namespace Grid;
+#ifndef H5_NO_NAMESPACE
+using namespace H5NS;
+#endif
+
+// Writer implementation ///////////////////////////////////////////////////////
+Hdf5Writer::Hdf5Writer(const std::string &fileName)
+: fileName_(fileName)
+, file_(fileName.c_str(), H5F_ACC_TRUNC)
+{
+  group_ = file_.openGroup("/");
+}
+
+Hdf5Writer::~Hdf5Writer(void)
+{
+  file_.close();
+}
+
+void Hdf5Writer::push(const std::string &s)
+{
+  group_ = group_.createGroup(s);
+  path_.push_back(s);
+}
+
+void Hdf5Writer::pop(void)
+{
+  path_.pop_back();
+  if (path_.empty())
+  {
+    group_ = file_.openGroup("/");
+  }
+  else
+  {
+    auto binOp = [](const std::string &a, const std::string &b)->std::string
+    {
+      return a + "/" + b;
+    };
+    
+    group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(),
+                                              std::string(""), binOp));
+  }
+}
+
+template <>
+void Hdf5Writer::writeDefault(const std::string &s, const std::string &x)
+{
+  StrType     strType(PredType::C_S1, x.size());
+  Attribute   attribute;
+  hsize_t     attrDim = 1;
+  DataSpace   attrSpace(1, &attrDim);
+  
+  attribute = group_.createAttribute(s, strType, attrSpace);
+  attribute.write(strType, x.data());
+}
+
+void Hdf5Writer::writeDefault(const std::string &s, const char *x)
+{
+  std::string sx(x);
+  
+  writeDefault(s, sx);
+}
+
+// Reader implementation ///////////////////////////////////////////////////////
+Hdf5Reader::Hdf5Reader(const std::string &fileName)
+{
+    
+}
+
+Hdf5Reader::~Hdf5Reader(void)
+{
+    
+}
+
+void Hdf5Reader::push(const std::string &s)
+{
+    
+}
+
+void Hdf5Reader::pop(void)
+{
+    
+}

lib/serialisation/Hdf5IO.h

@@ -0,0 +1,169 @@
+#ifndef GRID_SERIALISATION_HDF5_H
+#define GRID_SERIALISATION_HDF5_H
+
+#include <stack>
+#include <string>
+#include <vector>
+#include <H5Cpp.h>
+#include "Hdf5Type.h"
+
+#ifndef H5_NO_NAMESPACE
+#define H5NS H5
+#endif
+
+// default thresold above which datasets are used instead of attributes
+#ifndef H5_DEF_DATASET_THRES
+#define H5_DEF_DATASET_THRES 6u
+#endif
+
+namespace Grid
+{
+  template <typename T>
+  struct is_arithmetic_vector
+  {
+    static constexpr bool value = false;
+  };
+  
+  template <typename T>
+  struct is_arithmetic_vector<std::vector<T>>
+  {
+    static constexpr bool value = std::is_arithmetic<T>::value
+                                  or is_arithmetic_vector<T>::value;
+  };
+  
+  class Hdf5Writer: public Writer<Hdf5Writer>
+  {
+  public:
+    Hdf5Writer(const std::string &fileName);
+    virtual ~Hdf5Writer(void);
+    void push(const std::string &s);
+    void pop(void);
+    void writeDefault(const std::string &s, const char *x);
+    template <typename U>
+    void writeDefault(const std::string &s, const U &x);
+    template <typename U>
+    typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+                            and std::is_arithmetic<U>::value, void>::type
+    writeDefault(const std::string &s, const std::vector<U> &x);
+    template <typename U>
+    typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+                            and !std::is_arithmetic<U>::value, void>::type
+    writeDefault(const std::string &s, const std::vector<U> &x);
+    template <typename U>
+    typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
+    writeDefault(const std::string &s, const std::vector<U> &x);
+  private:
+    std::string              fileName_;
+    std::vector<std::string> path_;
+    std::vector<hsize_t>     dim_;
+    bool                     multiDim_{true};
+    H5NS::H5File             file_;
+    H5NS::Group              group_;
+    unsigned int             datasetThres_{H5_DEF_DATASET_THRES};
+  };
+  
+  class Hdf5Reader: public Reader<Hdf5Reader>
+  {
+  public:
+    Hdf5Reader(const std::string &fileName);
+    virtual ~Hdf5Reader(void);
+    void push(const std::string &s);
+    void pop(void);
+    template <typename U>
+    void readDefault(const std::string &s, U &output);
+    template <typename U>
+    void readDefault(const std::string &s, std::vector<U> &output);
+  private:
+  };
+  
+  // Writer template implementation ////////////////////////////////////////////
+  template <typename U>
+  void Hdf5Writer::writeDefault(const std::string &s, const U &x)
+  {
+    H5NS::Attribute attribute;
+    hsize_t         attrDim = 1;
+    H5NS::DataSpace attrSpace(1, &attrDim);
+    
+    attribute = group_.createAttribute(s, *Hdf5Type<U>::type, attrSpace);
+    attribute.write(*Hdf5Type<U>::type, &x);
+  }
+  
+  template <>
+  void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
+  
+  template <typename U>
+  typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+                          and std::is_arithmetic<U>::value, void>::type
+  Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
+  {
+    hsize_t size = 1;
+    
+    dim_.push_back(x.size());
+    for (auto d: dim_)
+    {
+      size *= d;
+    }
+    
+    H5NS::DataSpace dataspace(dim_.size(), dim_.data());
+    
+    if (size > datasetThres_)
+    {
+      H5NS::DataSet dataset;
+      
+      dataset = group_.createDataSet(s, *Hdf5Type<U>::type, dataspace);
+      dataset.write(x.data(), *Hdf5Type<U>::type);
+    }
+    else
+    {
+      H5NS::Attribute attribute;
+      
+      attribute = group_.createAttribute(s, *Hdf5Type<U>::type, dataspace);
+      attribute.write(*Hdf5Type<U>::type, x.data());
+    }
+    dim_.clear();
+    multiDim_ = true;
+  }
+  
+  template <typename U>
+  typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+                          and !std::is_arithmetic<U>::value, void>::type
+  Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
+  {
+    hsize_t firstSize = x[0].size();
+    
+    for (auto &v: x)
+    {
+      multiDim_ = (multiDim_ and (v.size() == firstSize));
+    }
+    assert(multiDim_);
+    dim_.push_back(x.size());
+    writeDefault(s, x[0]);
+  }
+  
+  template <typename U>
+  typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
+  Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
+  {
+    push(s);
+    for (hsize_t i = 0; i < x.size(); ++i)
+    {
+      write(s + "_" + std::to_string(i), x[i]);
+    }
+    pop();
+  }
+  
+  // Reader template implementation ////////////////////////////////////////////
+  template <typename U>
+  void Hdf5Reader::readDefault(const std::string &s, U &output)
+  {
+    
+  }
+  
+  template <typename U>
+  void Hdf5Reader::readDefault(const std::string &s, std::vector<U> &output)
+  {
+    
+  }
+}
+
+#endif

lib/serialisation/Hdf5Type.cc

@@ -0,0 +1,8 @@
+#include "Hdf5Type.h"
+
+using namespace Grid;
+
+#define HDF5_NATIVE_TYPE(predType, cType)\
+const H5NS::PredType * Hdf5Type<cType>::type = &H5NS::PredType::predType;
+
+DEFINE_HDF5_NATIVE_TYPES;

lib/serialisation/Hdf5Type.h

@@ -0,0 +1,48 @@
+#ifndef GRID_SERIALISATION_HDF5_TYPE_H
+#define GRID_SERIALISATION_HDF5_TYPE_H
+
+#include <H5Cpp.h>
+#include <vector>
+
+#ifndef H5_NO_NAMESPACE
+#define H5NS H5
+#endif
+
+#define HDF5_NATIVE_TYPE(predType, cType)\
+template <>\
+struct Hdf5Type<cType>\
+{\
+static const H5NS::PredType *type;\
+static constexpr bool       isNative = true;\
+};
+
+#define DEFINE_HDF5_NATIVE_TYPES \
+HDF5_NATIVE_TYPE(NATIVE_B8,      bool);\
+HDF5_NATIVE_TYPE(NATIVE_CHAR,    char);\
+HDF5_NATIVE_TYPE(NATIVE_SCHAR,   signed char);\
+HDF5_NATIVE_TYPE(NATIVE_UCHAR,   unsigned char);\
+HDF5_NATIVE_TYPE(NATIVE_SHORT,   short);\
+HDF5_NATIVE_TYPE(NATIVE_USHORT,  unsigned short);\
+HDF5_NATIVE_TYPE(NATIVE_INT,     int);\
+HDF5_NATIVE_TYPE(NATIVE_UINT,    unsigned int);\
+HDF5_NATIVE_TYPE(NATIVE_LONG,    long);\
+HDF5_NATIVE_TYPE(NATIVE_ULONG,   unsigned long);\
+HDF5_NATIVE_TYPE(NATIVE_LLONG,   long long);\
+HDF5_NATIVE_TYPE(NATIVE_ULLONG,  unsigned long long);\
+HDF5_NATIVE_TYPE(NATIVE_FLOAT,   float);\
+HDF5_NATIVE_TYPE(NATIVE_DOUBLE,  double);\
+HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double);
+
+namespace Grid
+{
+  template <typename T> struct Hdf5Type
+  {
+    static constexpr bool isNative = false;
+  };
+  
+  DEFINE_HDF5_NATIVE_TYPES;
+}
+
+#undef HDF5_NATIVE_TYPE
+
+#endif /* GRID_SERIALISATION_HDF5_TYPE_H */

lib/serialisation/Serialisation.h

@@ -36,6 +36,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include "BinaryIO.h"
 #include "TextIO.h"
 #include "XmlIO.h"
+#ifdef HAVE_HDF5
+#include "Hdf5IO.h"
+#endif
 //////////////////////////////////////////
 // Todo:
 //////////////////////////////////////////

scripts/filelist

@@ -4,9 +4,8 @@ home=`pwd`
 
 # library Make.inc
 cd $home/lib
-HFILES=`find . -type f -name '*.h' -not -path '*/Old/*' -not -path '*/Eigen/*'`
+HFILES=`find . -type f -name '*.h' -not -name '*Hdf5*' -not -path '*/Old/*' -not -path '*/Eigen/*'`
-HFILES="$HFILES"
+CCFILES=`find . -type f -name '*.cc' -not -name '*Communicator*.cc' -not -name '*Hdf5*'`
-CCFILES=`find . -type f -name '*.cc' -not  -name '*ommunicator*.cc'`
 echo HFILES=$HFILES > Make.inc
 echo >> Make.inc
 echo CCFILES=$CCFILES >> Make.inc

tests/IO/Test_serialisation.cc

@@ -140,6 +140,22 @@ int main(int argc,char **argv)
     std::cout << "Loaded (txt) -----------------" << std::endl;
     std::cout << copy3 << std::endl << veccopy3 << std::endl;
   }
+#ifdef HAVE_HDF5
+  //// HDF5
+  //// HDF5 does not accept elements with the duplicated names, hence "discard2"
+  {
+    Hdf5Writer TWR("bother.h5");
+    write(TWR,"discard",copy1 );
+    write(TWR,"discard2",veccopy1 );
+  }
+  {
+    Hdf5Reader TRD("bother.h5");
+    read (TRD,"discard",copy3 );
+    read (TRD,"discard2",veccopy3 );
+    std::cout << "Loaded (h5) -----------------" << std::endl;
+    std::cout << copy3 << std::endl << veccopy3 << std::endl;
+  }
+#endif
   
   std::vector<int> iv = strToVec<int>("1 2 2 4");
   std::vector<std::string> sv = strToVec<std::string>("bli bla blu");