HDF5 serial IO implemented and tested

lib/serialisation/BaseIO.h

@@ -32,6 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <type_traits>
 
 namespace Grid {
+  // Vector IO utilities ///////////////////////////////////////////////////////
   // helper function to read space-separated values
   template <typename T>
   std::vector<T> strToVec(const std::string s)
@@ -67,6 +68,76 @@ namespace Grid {
     return os;
   }
   
+  // Vector element trait //////////////////////////////////////////////////////
+  template <typename T>
+  struct element
+  {
+    typedef T type;
+    static constexpr bool is_arithmetic = false;
+  };
+  
+  template <typename T>
+  struct element<std::vector<T>>
+  {
+    typedef typename element<T>::type type;
+    static constexpr bool is_arithmetic = std::is_arithmetic<T>::value
+                                          or element<T>::is_arithmetic;
+  };
+  
+  // Vector flatening utility class ////////////////////////////////////////////
+  // Class to flatten a multidimensional std::vector
+  template <typename V>
+  class Flatten
+  {
+  public:
+    typedef typename element<V>::type Element;
+  public:
+    explicit                     Flatten(const V &vector);
+    const V &                    getVector(void);
+    const std::vector<Element> & getFlatVector(void);
+    const std::vector<size_t>  & getDim(void);
+  private:
+    void accumulate(const Element &e);
+    template <typename W>
+    void accumulate(const W &v);
+    void accumulateDim(const Element &e);
+    template <typename W>
+    void accumulateDim(const W &v);
+  private:
+    const V              &vector_;
+    std::vector<Element> flatVector_;
+    std::vector<size_t>  dim_;
+  };
+  
+  
+  // Class to reconstruct a multidimensional std::vector
+  template <typename V>
+  class Reconstruct
+  {
+  public:
+    typedef typename element<V>::type Element;
+  public:
+    Reconstruct(const std::vector<Element> &flatVector,
+                const std::vector<size_t> &dim);
+    const V &                    getVector(void);
+    const std::vector<Element> & getFlatVector(void);
+    const std::vector<size_t>  & getDim(void);
+  private:
+    void fill(std::vector<Element> &v);
+    template <typename W>
+    void fill(W &v);
+    void resize(std::vector<Element> &v, const unsigned int dim);
+    template <typename W>
+    void resize(W &v, const unsigned int dim);
+  private:
+    V                          vector_;
+    const std::vector<Element> &flatVector_;
+    std::vector<size_t>        dim_;
+    size_t                     ind_{0};
+    unsigned int               dimInd_{0};
+  };
+  
+  // Abstract writer/reader classes ////////////////////////////////////////////
   // static polymorphism implemented using CRTP idiom
   class Serializable;
   
@@ -132,7 +203,128 @@ namespace Grid {
     }
   };
   
-  // Generic writer interface
+  // Flatten class template implementation /////////////////////////////////////
+  template <typename V>
+  void Flatten<V>::accumulate(const Element &e)
+  {
+    flatVector_.push_back(e);
+  }
+  
+  template <typename V>
+  template <typename W>
+  void Flatten<V>::accumulate(const W &v)
+  {
+    for (auto &e: v)
+    {
+      accumulate(e);
+    }
+  }
+  
+  template <typename V>
+  void Flatten<V>::accumulateDim(const Element &e) {};
+  
+  template <typename V>
+  template <typename W>
+  void Flatten<V>::accumulateDim(const W &v)
+  {
+    dim_.push_back(v.size());
+    accumulateDim(v[0]);
+  }
+  
+  template <typename V>
+  Flatten<V>::Flatten(const V &vector)
+  : vector_(vector)
+  {
+    accumulate(vector_);
+    accumulateDim(vector_);
+  }
+  
+  template <typename V>
+  const V & Flatten<V>::getVector(void)
+  {
+    return vector_;
+  }
+  
+  template <typename V>
+  const std::vector<typename Flatten<V>::Element> &
+  Flatten<V>::getFlatVector(void)
+  {
+    return flatVector_;
+  }
+  
+  template <typename V>
+  const std::vector<size_t> & Flatten<V>::getDim(void)
+  {
+    return dim_;
+  }
+  
+  // Reconstruct class template implementation /////////////////////////////////
+  template <typename V>
+  void Reconstruct<V>::fill(std::vector<Element> &v)
+  {
+    for (auto &e: v)
+    {
+      e = flatVector_[ind_++];
+    }
+  }
+  
+  template <typename V>
+  template <typename W>
+  void Reconstruct<V>::fill(W &v)
+  {
+    for (auto &e: v)
+    {
+      fill(e);
+    }
+  }
+  
+  template <typename V>
+  void Reconstruct<V>::resize(std::vector<Element> &v, const unsigned int dim)
+  {
+    v.resize(dim_[dim]);
+  }
+  
+  template <typename V>
+  template <typename W>
+  void Reconstruct<V>::resize(W &v, const unsigned int dim)
+  {
+    v.resize(dim_[dim]);
+    for (auto &e: v)
+    {
+      resize(e, dim + 1);
+    }
+  }
+  
+  template <typename V>
+  Reconstruct<V>::Reconstruct(const std::vector<Element> &flatVector,
+                              const std::vector<size_t> &dim)
+  : flatVector_(flatVector)
+  , dim_(dim)
+  {
+    resize(vector_, 0);
+    fill(vector_);
+  }
+  
+  template <typename V>
+  const V & Reconstruct<V>::Reconstruct<V>::getVector(void)
+  {
+    return vector_;
+  }
+  
+  template <typename V>
+  const std::vector<typename Reconstruct<V>::Element> &
+  Reconstruct<V>::getFlatVector(void)
+  {
+    return flatVector_;
+  }
+  
+  template <typename V>
+  const std::vector<size_t> & Reconstruct<V>::getDim(void)
+  {
+    return dim_;
+  }
+  
+  // Generic writer interface //////////////////////////////////////////////////
   template <typename T>
   inline void push(Writer<T> &w, const std::string &s)
   {
@@ -217,7 +409,7 @@ namespace Grid {
     upcast->writeDefault(s, output);
   }
   
-  // Reader template implementation ////////////////////////////////////////////
+  // Reader template implementation
   template <typename T>
   Reader<T>::Reader(void)
   {

lib/serialisation/Hdf5IO.cc

@@ -11,11 +11,8 @@ Hdf5Writer::Hdf5Writer(const std::string &fileName)
 , file_(fileName.c_str(), H5F_ACC_TRUNC)
 {
   group_ = file_.openGroup("/");
-}
+  writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
-
+                       *Hdf5Type<unsigned int>::type);
-Hdf5Writer::~Hdf5Writer(void)
-{
-  file_.close();
 }
 
 void Hdf5Writer::push(const std::string &s)
@@ -47,12 +44,8 @@ template <>
 void Hdf5Writer::writeDefault(const std::string &s, const std::string &x)
 {
   StrType     strType(PredType::C_S1, x.size());
-  Attribute   attribute;
-  hsize_t     attrDim = 1;
-  DataSpace   attrSpace(1, &attrDim);
   
-  attribute = group_.createAttribute(s, strType, attrSpace);
+  writeSingleAttribute(*(x.data()), s, strType);
-  attribute.write(strType, x.data());
 }
 
 void Hdf5Writer::writeDefault(const std::string &s, const char *x)
@@ -64,21 +57,55 @@ void Hdf5Writer::writeDefault(const std::string &s, const char *x)
 
 // Reader implementation ///////////////////////////////////////////////////////
 Hdf5Reader::Hdf5Reader(const std::string &fileName)
+: fileName_(fileName)
+, file_(fileName.c_str(), H5F_ACC_RDONLY)
 {
-    
+  group_ = file_.openGroup("/");
-}
+  readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
-
+                      *Hdf5Type<unsigned int>::type);
-Hdf5Reader::~Hdf5Reader(void)
-{
-    
 }
 
 void Hdf5Reader::push(const std::string &s)
 {
-    
+  group_ = group_.openGroup(s);
+  path_.push_back(s);
 }
 
 void Hdf5Reader::pop(void)
 {
+  path_.pop_back();
+  if (path_.empty())
+  {
+    group_ = file_.openGroup("/");
+  }
+  else
+  {
+    auto binOp = [](const std::string &a, const std::string &b)->std::string
+    {
+      return a + "/" + b;
+    };
     
+    group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(),
+                                              std::string(""), binOp));
+  }
+}
+
+template <>
+void Hdf5Reader::readDefault(const std::string &s, std::string &x)
+{
+  Attribute attribute;
+  
+  attribute       = group_.openAttribute(s);
+  StrType strType = attribute.getStrType();
+  
+  x.resize(strType.getSize());
+  attribute.read(strType, &(x[0]));
+  
+  std::cout << "length: " << strType.getSize() << std::endl;
+  std::cout << "string: |";
+  for (auto &c: x)
+  {
+    std::cout << "'" << c << "'|";
+  }
+  std::cout << std::endl;
 }

lib/serialisation/Hdf5IO.h

@@ -12,139 +12,135 @@
 #endif
 
 // default thresold above which datasets are used instead of attributes
-#ifndef H5_DEF_DATASET_THRES
+#ifndef HDF5_DEF_DATASET_THRES
-#define H5_DEF_DATASET_THRES 6u
+#define HDF5_DEF_DATASET_THRES 6u
 #endif
 
+// name guard for Grid metadata
+#define HDF5_GRID_GUARD "_Grid_"
+
 namespace Grid
 {
-  template <typename T>
-  struct is_arithmetic_vector
-  {
-    static constexpr bool value = false;
-  };
-  
-  template <typename T>
-  struct is_arithmetic_vector<std::vector<T>>
-  {
-    static constexpr bool value = std::is_arithmetic<T>::value
-                                  or is_arithmetic_vector<T>::value;
-  };
-  
   class Hdf5Writer: public Writer<Hdf5Writer>
   {
   public:
     Hdf5Writer(const std::string &fileName);
-    virtual ~Hdf5Writer(void);
+    virtual ~Hdf5Writer(void) = default;
     void push(const std::string &s);
     void pop(void);
     void writeDefault(const std::string &s, const char *x);
     template <typename U>
     void writeDefault(const std::string &s, const U &x);
     template <typename U>
-    typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+    typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
-                            and std::is_arithmetic<U>::value, void>::type
     writeDefault(const std::string &s, const std::vector<U> &x);
     template <typename U>
-    typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+    typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
-                            and !std::is_arithmetic<U>::value, void>::type
     writeDefault(const std::string &s, const std::vector<U> &x);
+  private:
     template <typename U>
-    typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
+    void writeSingleAttribute(const U &x, const std::string &name,
-    writeDefault(const std::string &s, const std::vector<U> &x);
+                              const H5NS::DataType &type);
   private:
     std::string              fileName_;
     std::vector<std::string> path_;
-    std::vector<hsize_t>     dim_;
-    bool                     multiDim_{true};
     H5NS::H5File             file_;
     H5NS::Group              group_;
-    unsigned int             datasetThres_{H5_DEF_DATASET_THRES};
+    unsigned int             dataSetThres_{HDF5_DEF_DATASET_THRES};
   };
   
   class Hdf5Reader: public Reader<Hdf5Reader>
   {
   public:
     Hdf5Reader(const std::string &fileName);
-    virtual ~Hdf5Reader(void);
+    virtual ~Hdf5Reader(void) = default;
     void push(const std::string &s);
     void pop(void);
     template <typename U>
     void readDefault(const std::string &s, U &output);
     template <typename U>
-    void readDefault(const std::string &s, std::vector<U> &output);
+    typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
+    readDefault(const std::string &s, std::vector<U> &x);
+    template <typename U>
+    typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
+    readDefault(const std::string &s, std::vector<U> &x);
   private:
+    template <typename U>
+    void readSingleAttribute(U &x, const std::string &name,
+                             const H5NS::DataType &type);
+  private:
+    std::string              fileName_;
+    std::vector<std::string> path_;
+    H5NS::H5File             file_;
+    H5NS::Group              group_;
+    unsigned int             dataSetThres_;
   };
   
   // Writer template implementation ////////////////////////////////////////////
   template <typename U>
-  void Hdf5Writer::writeDefault(const std::string &s, const U &x)
+  void Hdf5Writer::writeSingleAttribute(const U &x, const std::string &name,
+                                        const H5NS::DataType &type)
   {
     H5NS::Attribute attribute;
     hsize_t         attrDim = 1;
     H5NS::DataSpace attrSpace(1, &attrDim);
     
-    attribute = group_.createAttribute(s, *Hdf5Type<U>::type, attrSpace);
+    attribute = group_.createAttribute(name, type, attrSpace);
-    attribute.write(*Hdf5Type<U>::type, &x);
+    attribute.write(type, &x);
+  }
+  
+  template <typename U>
+  void Hdf5Writer::writeDefault(const std::string &s, const U &x)
+  {
+    writeSingleAttribute(x, s, *Hdf5Type<U>::type);
   }
   
   template <>
   void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
   
   template <typename U>
-  typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+  typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
-                          and std::is_arithmetic<U>::value, void>::type
   Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
   {
-    hsize_t size = 1;
+    // alias to element type
+    typedef typename element<std::vector<U>>::type Element;
     
-    dim_.push_back(x.size());
+    // flatten the vector and getting dimensions
-    for (auto d: dim_)
+    Flatten<std::vector<U>> flat(x);
+    std::vector<hsize_t> dim;
+    const auto           &flatx = flat.getFlatVector();
+    
+    for (auto &d: flat.getDim())
     {
-      size *= d;
+      dim.push_back(d);
     }
     
-    H5NS::DataSpace dataspace(dim_.size(), dim_.data());
+    // write to file
+    H5NS::DataSpace dataSpace(dim.size(), dim.data());
     
-    if (size > datasetThres_)
+    if (flatx.size() > dataSetThres_)
     {
-      H5NS::DataSet dataset;
+      H5NS::DataSet dataSet;
       
-      dataset = group_.createDataSet(s, *Hdf5Type<U>::type, dataspace);
+      dataSet = group_.createDataSet(s, *Hdf5Type<Element>::type, dataSpace);
-      dataset.write(x.data(), *Hdf5Type<U>::type);
+      dataSet.write(flatx.data(), *Hdf5Type<Element>::type);
     }
     else
     {
       H5NS::Attribute attribute;
       
-      attribute = group_.createAttribute(s, *Hdf5Type<U>::type, dataspace);
+      attribute = group_.createAttribute(s, *Hdf5Type<Element>::type, dataSpace);
-      attribute.write(*Hdf5Type<U>::type, x.data());
+      attribute.write(*Hdf5Type<Element>::type, flatx.data());
     }
-    dim_.clear();
-    multiDim_ = true;
   }
   
   template <typename U>
-  typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
+  typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
-                          and !std::is_arithmetic<U>::value, void>::type
-  Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
-  {
-    hsize_t firstSize = x[0].size();
-    
-    for (auto &v: x)
-    {
-      multiDim_ = (multiDim_ and (v.size() == firstSize));
-    }
-    assert(multiDim_);
-    dim_.push_back(x.size());
-    writeDefault(s, x[0]);
-  }
-  
-  template <typename U>
-  typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
   Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
   {
     push(s);
+    writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size",
+                         *Hdf5Type<uint64_t>::type);
     for (hsize_t i = 0; i < x.size(); ++i)
     {
       write(s + "_" + std::to_string(i), x[i]);
@@ -154,15 +150,97 @@ namespace Grid
   
   // Reader template implementation ////////////////////////////////////////////
   template <typename U>
-  void Hdf5Reader::readDefault(const std::string &s, U &output)
+  void Hdf5Reader::readSingleAttribute(U &x, const std::string &name,
+                                       const H5NS::DataType &type)
   {
+    H5NS::Attribute attribute;
     
+    attribute = group_.openAttribute(name);
+    attribute.read(type, &x);
   }
   
   template <typename U>
-  void Hdf5Reader::readDefault(const std::string &s, std::vector<U> &output)
+  void Hdf5Reader::readDefault(const std::string &s, U &output)
   {
+    readSingleAttribute(output, s, *Hdf5Type<U>::type);
+  }
+  
+  template <>
+  void Hdf5Reader::readDefault(const std::string &s, std::string &x);
+  
+  template <typename U>
+  typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
+  Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
+  {
+    // alias to element type
+    typedef typename element<std::vector<U>>::type Element;
     
+    // read the dimensions
+    H5NS::DataSpace       dataSpace;
+    H5E_auto2_t           func;
+    void *                client_data;
+    std::vector<hsize_t>  hdim;
+    std::vector<size_t>   dim;
+    hsize_t               size = 1;
+    
+    H5NS::Exception::getAutoPrint(func, &client_data);
+    try
+    {
+      H5NS::Exception::dontPrint();
+      dataSpace = group_.openDataSet(s).getSpace();
+    }
+    catch (H5NS::Exception &e)
+    {
+      H5NS::Exception::setAutoPrint(func, client_data);
+      dataSpace = group_.openAttribute(s).getSpace();
+    }
+    hdim.resize(dataSpace.getSimpleExtentNdims());
+    dataSpace.getSimpleExtentDims(hdim.data());
+    for (auto &d: hdim)
+    {
+      dim.push_back(d);
+      size *= d;
+    }
+    
+    // read the flat vector
+    std::vector<Element> buf(size);
+
+    if (size > dataSetThres_)
+    {
+      H5NS::DataSet dataSet;
+      
+      dataSet = group_.openDataSet(s);
+      dataSet.read(buf.data(), *Hdf5Type<Element>::type);
+    }
+    else
+    {
+      H5NS::Attribute attribute;
+      
+      attribute = group_.openAttribute(s);
+      attribute.read(*Hdf5Type<Element>::type, buf.data());
+    }
+    
+    // reconstruct the multidimensional vector
+    Reconstruct<std::vector<U>> r(buf, dim);
+    
+    x = r.getVector();
+  }
+  
+  template <typename U>
+  typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
+  Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
+  {
+    uint64_t size;
+    
+    push(s);
+    readSingleAttribute(size, HDF5_GRID_GUARD "vector_size",
+                        *Hdf5Type<uint64_t>::type);
+    x.resize(size);
+    for (hsize_t i = 0; i < x.size(); ++i)
+    {
+      read(s + "_" + std::to_string(i), x[i]);
+    }
+    pop();
   }
 }
 

tests/IO/Test_serialisation.cc

@@ -104,8 +104,8 @@ int main(int argc,char **argv)
   };
   
   // read tests
-  myclass copy1, copy2, copy3;
+  myclass copy1, copy2, copy3, copy4;
-  std::vector<myclass> veccopy1, veccopy2, veccopy3;
+  std::vector<myclass> veccopy1, veccopy2, veccopy3, veccopy4;
   //// XML
   {
     XmlReader RD("bother.xml");
@@ -150,24 +150,41 @@ int main(int argc,char **argv)
   }
   {
     Hdf5Reader TRD("bother.h5");
-    read (TRD,"discard",copy3 );
+    std::cout << "read single" << std::endl;
-    read (TRD,"discard2",veccopy3 );
+    read (TRD,"discard",copy4 );
+    std::cout << "read vec" << std::endl;
+    read (TRD,"discard2",veccopy4 );
     std::cout << "Loaded (h5) -----------------" << std::endl;
-    std::cout << copy3 << std::endl << veccopy3 << std::endl;
+    std::cout << copy3 << std::endl << veccopy4 << std::endl;
   }
 #endif
   
-  std::vector<int> iv = strToVec<int>("1 2 2 4");
+  typedef std::vector<std::vector<std::vector<double>>> vec3d;
-  std::vector<std::string> sv = strToVec<std::string>("bli bla blu");
   
-  for (auto &e: iv)
+  vec3d dv, buf;
+  double d = 0.;
+  
+  dv.resize(4);
+  for (auto &v1: dv)
   {
-    std::cout << e << " ";
+    v1.resize(3);
+    for (auto &v2: v1)
+    {
+      v2.resize(5);
+      for (auto &x: v2)
+      {
+        x = d++;
+      }
+    }
   }
-  std::cout << std::endl;
+  std::cout << dv << std::endl;
-  for (auto &e: sv)
+  
-  {
+  Flatten<vec3d> flatdv(dv);
-    std::cout << e << " ";
+  
-  }
+  std::cout << flatdv.getDim() << std::endl;
-  std::cout << std::endl;
+  std::cout << flatdv.getFlatVector() << std::endl;
+  
+  Reconstruct<vec3d> rec(flatdv.getFlatVector(), flatdv.getDim());
+  
+  std::cout << flatdv.getVector() << std::endl;
 }