From 5803933aea55df67c733cf1e4201f354d8a8965b Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Tue, 17 Jan 2017 16:21:18 -0800 Subject: [PATCH 01/14] First implementation of HDF5 serial IO writer, reader is still empty --- configure.ac | 14 +++ lib/Grid.h | 2 +- lib/Makefile.am | 13 ++- lib/serialisation/Hdf5IO.cc | 84 +++++++++++++++ lib/serialisation/Hdf5IO.h | 169 ++++++++++++++++++++++++++++++ lib/serialisation/Hdf5Type.cc | 8 ++ lib/serialisation/Hdf5Type.h | 48 +++++++++ lib/serialisation/Serialisation.h | 3 + scripts/filelist | 5 +- tests/IO/Test_serialisation.cc | 16 +++ 10 files changed, 357 insertions(+), 5 deletions(-) create mode 100644 lib/serialisation/Hdf5IO.cc create mode 100644 lib/serialisation/Hdf5IO.h create mode 100644 lib/serialisation/Hdf5Type.cc create mode 100644 lib/serialisation/Hdf5Type.h diff --git a/configure.ac b/configure.ac index f413cde8..f848bd23 100644 --- a/configure.ac +++ b/configure.ac @@ -99,6 +99,13 @@ case ${ac_MKL} in AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);; esac +############### HDF5 +AC_ARG_WITH([hdf5], + [AS_HELP_STRING([--with-hdf5=prefix], + [try this for a non-standard install prefix of the HDF5 library])], + [AM_CXXFLAGS="-I$with_hdf5/include $AM_CXXFLAGS"] + [AM_LDFLAGS="-L$with_hdf5/lib $AM_LDFLAGS"]) + ############### first-touch AC_ARG_ENABLE([numa], [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], @@ -145,6 +152,12 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3], [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])] [have_fftw=true]) +AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp], + [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])] + [have_hdf5=true] + [LIBS="${LIBS} -lhdf5"], [], [-lhdf5]) +AM_CONDITIONAL(BUILD_HDF5, [ test "${have_hdf5}X" == "trueX" ]) + CXXFLAGS=$CXXFLAGS_CPY LDFLAGS=$LDFLAGS_CPY @@ -410,6 +423,7 @@ RNG choice : ${ac_RNG} GMP : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi` LAPACK : ${ac_LAPACK} FFTW : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi` +HDF5 : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi` build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi` ----- BUILD FLAGS ------------------------------------- CXXFLAGS: diff --git a/lib/Grid.h b/lib/Grid.h index 0c5983f3..0f57c8a6 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -59,8 +59,8 @@ Author: paboyle /////////////////// // Grid headers /////////////////// -#include #include "Config.h" +#include #include #include #include diff --git a/lib/Makefile.am b/lib/Makefile.am index a779135f..9aa6af92 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,4 +1,5 @@ extra_sources= +extra_headers= if BUILD_COMMS_MPI extra_sources+=communicator/Communicator_mpi.cc extra_sources+=communicator/Communicator_base.cc @@ -24,6 +25,13 @@ if BUILD_COMMS_NONE extra_sources+=communicator/Communicator_base.cc endif +if BUILD_HDF5 + extra_sources+=serialisation/Hdf5IO.cc + extra_sources+=serialisation/Hdf5Type.cc + extra_headers+=serialisation/Hdf5IO.h + extra_headers+=serialisation/Hdf5Type.h +endif + # # Libraries # @@ -32,6 +40,9 @@ include Eigen.inc lib_LIBRARIES = libGrid.a -libGrid_a_SOURCES = $(CCFILES) $(extra_sources) +CCFILES += $(extra_sources) +HFILES += $(extra_headers) + +libGrid_a_SOURCES = $(CCFILES) libGrid_adir = $(pkgincludedir) nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc new file mode 100644 index 00000000..0d62fdd8 --- /dev/null +++ b/lib/serialisation/Hdf5IO.cc @@ -0,0 +1,84 @@ +#include + +using namespace Grid; +#ifndef H5_NO_NAMESPACE +using namespace H5NS; +#endif + +// Writer implementation /////////////////////////////////////////////////////// +Hdf5Writer::Hdf5Writer(const std::string &fileName) +: fileName_(fileName) +, file_(fileName.c_str(), H5F_ACC_TRUNC) +{ + group_ = file_.openGroup("/"); +} + +Hdf5Writer::~Hdf5Writer(void) +{ + file_.close(); +} + +void Hdf5Writer::push(const std::string &s) +{ + group_ = group_.createGroup(s); + path_.push_back(s); +} + +void Hdf5Writer::pop(void) +{ + path_.pop_back(); + if (path_.empty()) + { + group_ = file_.openGroup("/"); + } + else + { + auto binOp = [](const std::string &a, const std::string &b)->std::string + { + return a + "/" + b; + }; + + group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(), + std::string(""), binOp)); + } +} + +template <> +void Hdf5Writer::writeDefault(const std::string &s, const std::string &x) +{ + StrType strType(PredType::C_S1, x.size()); + Attribute attribute; + hsize_t attrDim = 1; + DataSpace attrSpace(1, &attrDim); + + attribute = group_.createAttribute(s, strType, attrSpace); + attribute.write(strType, x.data()); +} + +void Hdf5Writer::writeDefault(const std::string &s, const char *x) +{ + std::string sx(x); + + writeDefault(s, sx); +} + +// Reader implementation /////////////////////////////////////////////////////// +Hdf5Reader::Hdf5Reader(const std::string &fileName) +{ + +} + +Hdf5Reader::~Hdf5Reader(void) +{ + +} + +void Hdf5Reader::push(const std::string &s) +{ + +} + +void Hdf5Reader::pop(void) +{ + +} diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h new file mode 100644 index 00000000..481fa1cf --- /dev/null +++ b/lib/serialisation/Hdf5IO.h @@ -0,0 +1,169 @@ +#ifndef GRID_SERIALISATION_HDF5_H +#define GRID_SERIALISATION_HDF5_H + +#include +#include +#include +#include +#include "Hdf5Type.h" + +#ifndef H5_NO_NAMESPACE +#define H5NS H5 +#endif + +// default thresold above which datasets are used instead of attributes +#ifndef H5_DEF_DATASET_THRES +#define H5_DEF_DATASET_THRES 6u +#endif + +namespace Grid +{ + template + struct is_arithmetic_vector + { + static constexpr bool value = false; + }; + + template + struct is_arithmetic_vector> + { + static constexpr bool value = std::is_arithmetic::value + or is_arithmetic_vector::value; + }; + + class Hdf5Writer: public Writer + { + public: + Hdf5Writer(const std::string &fileName); + virtual ~Hdf5Writer(void); + void push(const std::string &s); + void pop(void); + void writeDefault(const std::string &s, const char *x); + template + void writeDefault(const std::string &s, const U &x); + template + typename std::enable_if>::value + and std::is_arithmetic::value, void>::type + writeDefault(const std::string &s, const std::vector &x); + template + typename std::enable_if>::value + and !std::is_arithmetic::value, void>::type + writeDefault(const std::string &s, const std::vector &x); + template + typename std::enable_if>::value, void>::type + writeDefault(const std::string &s, const std::vector &x); + private: + std::string fileName_; + std::vector path_; + std::vector dim_; + bool multiDim_{true}; + H5NS::H5File file_; + H5NS::Group group_; + unsigned int datasetThres_{H5_DEF_DATASET_THRES}; + }; + + class Hdf5Reader: public Reader + { + public: + Hdf5Reader(const std::string &fileName); + virtual ~Hdf5Reader(void); + void push(const std::string &s); + void pop(void); + template + void readDefault(const std::string &s, U &output); + template + void readDefault(const std::string &s, std::vector &output); + private: + }; + + // Writer template implementation //////////////////////////////////////////// + template + void Hdf5Writer::writeDefault(const std::string &s, const U &x) + { + H5NS::Attribute attribute; + hsize_t attrDim = 1; + H5NS::DataSpace attrSpace(1, &attrDim); + + attribute = group_.createAttribute(s, *Hdf5Type::type, attrSpace); + attribute.write(*Hdf5Type::type, &x); + } + + template <> + void Hdf5Writer::writeDefault(const std::string &s, const std::string &x); + + template + typename std::enable_if>::value + and std::is_arithmetic::value, void>::type + Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) + { + hsize_t size = 1; + + dim_.push_back(x.size()); + for (auto d: dim_) + { + size *= d; + } + + H5NS::DataSpace dataspace(dim_.size(), dim_.data()); + + if (size > datasetThres_) + { + H5NS::DataSet dataset; + + dataset = group_.createDataSet(s, *Hdf5Type::type, dataspace); + dataset.write(x.data(), *Hdf5Type::type); + } + else + { + H5NS::Attribute attribute; + + attribute = group_.createAttribute(s, *Hdf5Type::type, dataspace); + attribute.write(*Hdf5Type::type, x.data()); + } + dim_.clear(); + multiDim_ = true; + } + + template + typename std::enable_if>::value + and !std::is_arithmetic::value, void>::type + Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) + { + hsize_t firstSize = x[0].size(); + + for (auto &v: x) + { + multiDim_ = (multiDim_ and (v.size() == firstSize)); + } + assert(multiDim_); + dim_.push_back(x.size()); + writeDefault(s, x[0]); + } + + template + typename std::enable_if>::value, void>::type + Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) + { + push(s); + for (hsize_t i = 0; i < x.size(); ++i) + { + write(s + "_" + std::to_string(i), x[i]); + } + pop(); + } + + // Reader template implementation //////////////////////////////////////////// + template + void Hdf5Reader::readDefault(const std::string &s, U &output) + { + + } + + template + void Hdf5Reader::readDefault(const std::string &s, std::vector &output) + { + + } +} + +#endif diff --git a/lib/serialisation/Hdf5Type.cc b/lib/serialisation/Hdf5Type.cc new file mode 100644 index 00000000..75c7692e --- /dev/null +++ b/lib/serialisation/Hdf5Type.cc @@ -0,0 +1,8 @@ +#include "Hdf5Type.h" + +using namespace Grid; + +#define HDF5_NATIVE_TYPE(predType, cType)\ +const H5NS::PredType * Hdf5Type::type = &H5NS::PredType::predType; + +DEFINE_HDF5_NATIVE_TYPES; diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h new file mode 100644 index 00000000..beb509c2 --- /dev/null +++ b/lib/serialisation/Hdf5Type.h @@ -0,0 +1,48 @@ +#ifndef GRID_SERIALISATION_HDF5_TYPE_H +#define GRID_SERIALISATION_HDF5_TYPE_H + +#include +#include + +#ifndef H5_NO_NAMESPACE +#define H5NS H5 +#endif + +#define HDF5_NATIVE_TYPE(predType, cType)\ +template <>\ +struct Hdf5Type\ +{\ +static const H5NS::PredType *type;\ +static constexpr bool isNative = true;\ +}; + +#define DEFINE_HDF5_NATIVE_TYPES \ +HDF5_NATIVE_TYPE(NATIVE_B8, bool);\ +HDF5_NATIVE_TYPE(NATIVE_CHAR, char);\ +HDF5_NATIVE_TYPE(NATIVE_SCHAR, signed char);\ +HDF5_NATIVE_TYPE(NATIVE_UCHAR, unsigned char);\ +HDF5_NATIVE_TYPE(NATIVE_SHORT, short);\ +HDF5_NATIVE_TYPE(NATIVE_USHORT, unsigned short);\ +HDF5_NATIVE_TYPE(NATIVE_INT, int);\ +HDF5_NATIVE_TYPE(NATIVE_UINT, unsigned int);\ +HDF5_NATIVE_TYPE(NATIVE_LONG, long);\ +HDF5_NATIVE_TYPE(NATIVE_ULONG, unsigned long);\ +HDF5_NATIVE_TYPE(NATIVE_LLONG, long long);\ +HDF5_NATIVE_TYPE(NATIVE_ULLONG, unsigned long long);\ +HDF5_NATIVE_TYPE(NATIVE_FLOAT, float);\ +HDF5_NATIVE_TYPE(NATIVE_DOUBLE, double);\ +HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double); + +namespace Grid +{ + template struct Hdf5Type + { + static constexpr bool isNative = false; + }; + + DEFINE_HDF5_NATIVE_TYPES; +} + +#undef HDF5_NATIVE_TYPE + +#endif /* GRID_SERIALISATION_HDF5_TYPE_H */ diff --git a/lib/serialisation/Serialisation.h b/lib/serialisation/Serialisation.h index 8f405d73..aa84e989 100644 --- a/lib/serialisation/Serialisation.h +++ b/lib/serialisation/Serialisation.h @@ -36,6 +36,9 @@ Author: Peter Boyle #include "BinaryIO.h" #include "TextIO.h" #include "XmlIO.h" +#ifdef HAVE_HDF5 +#include "Hdf5IO.h" +#endif ////////////////////////////////////////// // Todo: ////////////////////////////////////////// diff --git a/scripts/filelist b/scripts/filelist index 1ab95c7c..bf2fbc41 100755 --- a/scripts/filelist +++ b/scripts/filelist @@ -4,9 +4,8 @@ home=`pwd` # library Make.inc cd $home/lib -HFILES=`find . -type f -name '*.h' -not -path '*/Old/*' -not -path '*/Eigen/*'` -HFILES="$HFILES" -CCFILES=`find . -type f -name '*.cc' -not -name '*ommunicator*.cc'` +HFILES=`find . -type f -name '*.h' -not -name '*Hdf5*' -not -path '*/Old/*' -not -path '*/Eigen/*'` +CCFILES=`find . -type f -name '*.cc' -not -name '*Communicator*.cc' -not -name '*Hdf5*'` echo HFILES=$HFILES > Make.inc echo >> Make.inc echo CCFILES=$CCFILES >> Make.inc diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index e23aa1a3..7250d618 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -140,6 +140,22 @@ int main(int argc,char **argv) std::cout << "Loaded (txt) -----------------" << std::endl; std::cout << copy3 << std::endl << veccopy3 << std::endl; } +#ifdef HAVE_HDF5 + //// HDF5 + //// HDF5 does not accept elements with the duplicated names, hence "discard2" + { + Hdf5Writer TWR("bother.h5"); + write(TWR,"discard",copy1 ); + write(TWR,"discard2",veccopy1 ); + } + { + Hdf5Reader TRD("bother.h5"); + read (TRD,"discard",copy3 ); + read (TRD,"discard2",veccopy3 ); + std::cout << "Loaded (h5) -----------------" << std::endl; + std::cout << copy3 << std::endl << veccopy3 << std::endl; + } +#endif std::vector iv = strToVec("1 2 2 4"); std::vector sv = strToVec("bli bla blu"); From f599cb5b177ce38a2f4acc5a97bbf556cc6d0784 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Wed, 18 Jan 2017 16:50:21 -0800 Subject: [PATCH 02/14] HDF5 serial IO implemented and tested --- lib/serialisation/BaseIO.h | 196 ++++++++++++++++++++++++++++++- lib/serialisation/Hdf5IO.cc | 61 +++++++--- lib/serialisation/Hdf5IO.h | 206 +++++++++++++++++++++++---------- tests/IO/Test_serialisation.cc | 47 +++++--- 4 files changed, 412 insertions(+), 98 deletions(-) diff --git a/lib/serialisation/BaseIO.h b/lib/serialisation/BaseIO.h index 1095baf1..5b5ef427 100644 --- a/lib/serialisation/BaseIO.h +++ b/lib/serialisation/BaseIO.h @@ -32,6 +32,7 @@ Author: Peter Boyle #include namespace Grid { + // Vector IO utilities /////////////////////////////////////////////////////// // helper function to read space-separated values template std::vector strToVec(const std::string s) @@ -67,6 +68,76 @@ namespace Grid { return os; } + // Vector element trait ////////////////////////////////////////////////////// + template + struct element + { + typedef T type; + static constexpr bool is_arithmetic = false; + }; + + template + struct element> + { + typedef typename element::type type; + static constexpr bool is_arithmetic = std::is_arithmetic::value + or element::is_arithmetic; + }; + + // Vector flatening utility class //////////////////////////////////////////// + // Class to flatten a multidimensional std::vector + template + class Flatten + { + public: + typedef typename element::type Element; + public: + explicit Flatten(const V &vector); + const V & getVector(void); + const std::vector & getFlatVector(void); + const std::vector & getDim(void); + private: + void accumulate(const Element &e); + template + void accumulate(const W &v); + void accumulateDim(const Element &e); + template + void accumulateDim(const W &v); + private: + const V &vector_; + std::vector flatVector_; + std::vector dim_; + }; + + + // Class to reconstruct a multidimensional std::vector + template + class Reconstruct + { + public: + typedef typename element::type Element; + public: + Reconstruct(const std::vector &flatVector, + const std::vector &dim); + const V & getVector(void); + const std::vector & getFlatVector(void); + const std::vector & getDim(void); + private: + void fill(std::vector &v); + template + void fill(W &v); + void resize(std::vector &v, const unsigned int dim); + template + void resize(W &v, const unsigned int dim); + private: + V vector_; + const std::vector &flatVector_; + std::vector dim_; + size_t ind_{0}; + unsigned int dimInd_{0}; + }; + + // Abstract writer/reader classes //////////////////////////////////////////// // static polymorphism implemented using CRTP idiom class Serializable; @@ -132,7 +203,128 @@ namespace Grid { } }; - // Generic writer interface + // Flatten class template implementation ///////////////////////////////////// + template + void Flatten::accumulate(const Element &e) + { + flatVector_.push_back(e); + } + + template + template + void Flatten::accumulate(const W &v) + { + for (auto &e: v) + { + accumulate(e); + } + } + + template + void Flatten::accumulateDim(const Element &e) {}; + + template + template + void Flatten::accumulateDim(const W &v) + { + dim_.push_back(v.size()); + accumulateDim(v[0]); + } + + template + Flatten::Flatten(const V &vector) + : vector_(vector) + { + accumulate(vector_); + accumulateDim(vector_); + } + + template + const V & Flatten::getVector(void) + { + return vector_; + } + + template + const std::vector::Element> & + Flatten::getFlatVector(void) + { + return flatVector_; + } + + template + const std::vector & Flatten::getDim(void) + { + return dim_; + } + + // Reconstruct class template implementation ///////////////////////////////// + template + void Reconstruct::fill(std::vector &v) + { + for (auto &e: v) + { + e = flatVector_[ind_++]; + } + } + + template + template + void Reconstruct::fill(W &v) + { + for (auto &e: v) + { + fill(e); + } + } + + template + void Reconstruct::resize(std::vector &v, const unsigned int dim) + { + v.resize(dim_[dim]); + } + + template + template + void Reconstruct::resize(W &v, const unsigned int dim) + { + v.resize(dim_[dim]); + for (auto &e: v) + { + resize(e, dim + 1); + } + } + + template + Reconstruct::Reconstruct(const std::vector &flatVector, + const std::vector &dim) + : flatVector_(flatVector) + , dim_(dim) + { + resize(vector_, 0); + fill(vector_); + } + + template + const V & Reconstruct::Reconstruct::getVector(void) + { + return vector_; + } + + template + const std::vector::Element> & + Reconstruct::getFlatVector(void) + { + return flatVector_; + } + + template + const std::vector & Reconstruct::getDim(void) + { + return dim_; + } + + // Generic writer interface ////////////////////////////////////////////////// template inline void push(Writer &w, const std::string &s) { @@ -217,7 +409,7 @@ namespace Grid { upcast->writeDefault(s, output); } - // Reader template implementation //////////////////////////////////////////// + // Reader template implementation template Reader::Reader(void) { diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc index 0d62fdd8..02356220 100644 --- a/lib/serialisation/Hdf5IO.cc +++ b/lib/serialisation/Hdf5IO.cc @@ -11,11 +11,8 @@ Hdf5Writer::Hdf5Writer(const std::string &fileName) , file_(fileName.c_str(), H5F_ACC_TRUNC) { group_ = file_.openGroup("/"); -} - -Hdf5Writer::~Hdf5Writer(void) -{ - file_.close(); + writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold", + *Hdf5Type::type); } void Hdf5Writer::push(const std::string &s) @@ -47,12 +44,8 @@ template <> void Hdf5Writer::writeDefault(const std::string &s, const std::string &x) { StrType strType(PredType::C_S1, x.size()); - Attribute attribute; - hsize_t attrDim = 1; - DataSpace attrSpace(1, &attrDim); - attribute = group_.createAttribute(s, strType, attrSpace); - attribute.write(strType, x.data()); + writeSingleAttribute(*(x.data()), s, strType); } void Hdf5Writer::writeDefault(const std::string &s, const char *x) @@ -64,21 +57,55 @@ void Hdf5Writer::writeDefault(const std::string &s, const char *x) // Reader implementation /////////////////////////////////////////////////////// Hdf5Reader::Hdf5Reader(const std::string &fileName) +: fileName_(fileName) +, file_(fileName.c_str(), H5F_ACC_RDONLY) { - -} - -Hdf5Reader::~Hdf5Reader(void) -{ - + group_ = file_.openGroup("/"); + readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold", + *Hdf5Type::type); } void Hdf5Reader::push(const std::string &s) { - + group_ = group_.openGroup(s); + path_.push_back(s); } void Hdf5Reader::pop(void) { + path_.pop_back(); + if (path_.empty()) + { + group_ = file_.openGroup("/"); + } + else + { + auto binOp = [](const std::string &a, const std::string &b)->std::string + { + return a + "/" + b; + }; + group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(), + std::string(""), binOp)); + } +} + +template <> +void Hdf5Reader::readDefault(const std::string &s, std::string &x) +{ + Attribute attribute; + + attribute = group_.openAttribute(s); + StrType strType = attribute.getStrType(); + + x.resize(strType.getSize()); + attribute.read(strType, &(x[0])); + + std::cout << "length: " << strType.getSize() << std::endl; + std::cout << "string: |"; + for (auto &c: x) + { + std::cout << "'" << c << "'|"; + } + std::cout << std::endl; } diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index 481fa1cf..b58c86ed 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -12,139 +12,135 @@ #endif // default thresold above which datasets are used instead of attributes -#ifndef H5_DEF_DATASET_THRES -#define H5_DEF_DATASET_THRES 6u +#ifndef HDF5_DEF_DATASET_THRES +#define HDF5_DEF_DATASET_THRES 6u #endif +// name guard for Grid metadata +#define HDF5_GRID_GUARD "_Grid_" + namespace Grid { - template - struct is_arithmetic_vector - { - static constexpr bool value = false; - }; - - template - struct is_arithmetic_vector> - { - static constexpr bool value = std::is_arithmetic::value - or is_arithmetic_vector::value; - }; - class Hdf5Writer: public Writer { public: Hdf5Writer(const std::string &fileName); - virtual ~Hdf5Writer(void); + virtual ~Hdf5Writer(void) = default; void push(const std::string &s); void pop(void); void writeDefault(const std::string &s, const char *x); template void writeDefault(const std::string &s, const U &x); template - typename std::enable_if>::value - and std::is_arithmetic::value, void>::type + typename std::enable_if>::is_arithmetic, void>::type writeDefault(const std::string &s, const std::vector &x); template - typename std::enable_if>::value - and !std::is_arithmetic::value, void>::type + typename std::enable_if>::is_arithmetic, void>::type writeDefault(const std::string &s, const std::vector &x); + private: template - typename std::enable_if>::value, void>::type - writeDefault(const std::string &s, const std::vector &x); + void writeSingleAttribute(const U &x, const std::string &name, + const H5NS::DataType &type); private: std::string fileName_; std::vector path_; - std::vector dim_; - bool multiDim_{true}; H5NS::H5File file_; H5NS::Group group_; - unsigned int datasetThres_{H5_DEF_DATASET_THRES}; + unsigned int dataSetThres_{HDF5_DEF_DATASET_THRES}; }; class Hdf5Reader: public Reader { public: Hdf5Reader(const std::string &fileName); - virtual ~Hdf5Reader(void); + virtual ~Hdf5Reader(void) = default; void push(const std::string &s); void pop(void); template void readDefault(const std::string &s, U &output); template - void readDefault(const std::string &s, std::vector &output); + typename std::enable_if>::is_arithmetic, void>::type + readDefault(const std::string &s, std::vector &x); + template + typename std::enable_if>::is_arithmetic, void>::type + readDefault(const std::string &s, std::vector &x); private: + template + void readSingleAttribute(U &x, const std::string &name, + const H5NS::DataType &type); + private: + std::string fileName_; + std::vector path_; + H5NS::H5File file_; + H5NS::Group group_; + unsigned int dataSetThres_; }; // Writer template implementation //////////////////////////////////////////// template - void Hdf5Writer::writeDefault(const std::string &s, const U &x) + void Hdf5Writer::writeSingleAttribute(const U &x, const std::string &name, + const H5NS::DataType &type) { H5NS::Attribute attribute; hsize_t attrDim = 1; H5NS::DataSpace attrSpace(1, &attrDim); - attribute = group_.createAttribute(s, *Hdf5Type::type, attrSpace); - attribute.write(*Hdf5Type::type, &x); + attribute = group_.createAttribute(name, type, attrSpace); + attribute.write(type, &x); + } + + template + void Hdf5Writer::writeDefault(const std::string &s, const U &x) + { + writeSingleAttribute(x, s, *Hdf5Type::type); } template <> void Hdf5Writer::writeDefault(const std::string &s, const std::string &x); template - typename std::enable_if>::value - and std::is_arithmetic::value, void>::type + typename std::enable_if>::is_arithmetic, void>::type Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) { - hsize_t size = 1; + // alias to element type + typedef typename element>::type Element; - dim_.push_back(x.size()); - for (auto d: dim_) + // flatten the vector and getting dimensions + Flatten> flat(x); + std::vector dim; + const auto &flatx = flat.getFlatVector(); + + for (auto &d: flat.getDim()) { - size *= d; + dim.push_back(d); } - H5NS::DataSpace dataspace(dim_.size(), dim_.data()); + // write to file + H5NS::DataSpace dataSpace(dim.size(), dim.data()); - if (size > datasetThres_) + if (flatx.size() > dataSetThres_) { - H5NS::DataSet dataset; + H5NS::DataSet dataSet; - dataset = group_.createDataSet(s, *Hdf5Type::type, dataspace); - dataset.write(x.data(), *Hdf5Type::type); + dataSet = group_.createDataSet(s, *Hdf5Type::type, dataSpace); + dataSet.write(flatx.data(), *Hdf5Type::type); } else { H5NS::Attribute attribute; - attribute = group_.createAttribute(s, *Hdf5Type::type, dataspace); - attribute.write(*Hdf5Type::type, x.data()); + attribute = group_.createAttribute(s, *Hdf5Type::type, dataSpace); + attribute.write(*Hdf5Type::type, flatx.data()); } - dim_.clear(); - multiDim_ = true; } template - typename std::enable_if>::value - and !std::is_arithmetic::value, void>::type - Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) - { - hsize_t firstSize = x[0].size(); - - for (auto &v: x) - { - multiDim_ = (multiDim_ and (v.size() == firstSize)); - } - assert(multiDim_); - dim_.push_back(x.size()); - writeDefault(s, x[0]); - } - - template - typename std::enable_if>::value, void>::type + typename std::enable_if>::is_arithmetic, void>::type Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) { push(s); + writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size", + *Hdf5Type::type); for (hsize_t i = 0; i < x.size(); ++i) { write(s + "_" + std::to_string(i), x[i]); @@ -154,15 +150,97 @@ namespace Grid // Reader template implementation //////////////////////////////////////////// template - void Hdf5Reader::readDefault(const std::string &s, U &output) + void Hdf5Reader::readSingleAttribute(U &x, const std::string &name, + const H5NS::DataType &type) { + H5NS::Attribute attribute; + attribute = group_.openAttribute(name); + attribute.read(type, &x); } template - void Hdf5Reader::readDefault(const std::string &s, std::vector &output) + void Hdf5Reader::readDefault(const std::string &s, U &output) { + readSingleAttribute(output, s, *Hdf5Type::type); + } + + template <> + void Hdf5Reader::readDefault(const std::string &s, std::string &x); + + template + typename std::enable_if>::is_arithmetic, void>::type + Hdf5Reader::readDefault(const std::string &s, std::vector &x) + { + // alias to element type + typedef typename element>::type Element; + // read the dimensions + H5NS::DataSpace dataSpace; + H5E_auto2_t func; + void * client_data; + std::vector hdim; + std::vector dim; + hsize_t size = 1; + + H5NS::Exception::getAutoPrint(func, &client_data); + try + { + H5NS::Exception::dontPrint(); + dataSpace = group_.openDataSet(s).getSpace(); + } + catch (H5NS::Exception &e) + { + H5NS::Exception::setAutoPrint(func, client_data); + dataSpace = group_.openAttribute(s).getSpace(); + } + hdim.resize(dataSpace.getSimpleExtentNdims()); + dataSpace.getSimpleExtentDims(hdim.data()); + for (auto &d: hdim) + { + dim.push_back(d); + size *= d; + } + + // read the flat vector + std::vector buf(size); + + if (size > dataSetThres_) + { + H5NS::DataSet dataSet; + + dataSet = group_.openDataSet(s); + dataSet.read(buf.data(), *Hdf5Type::type); + } + else + { + H5NS::Attribute attribute; + + attribute = group_.openAttribute(s); + attribute.read(*Hdf5Type::type, buf.data()); + } + + // reconstruct the multidimensional vector + Reconstruct> r(buf, dim); + + x = r.getVector(); + } + + template + typename std::enable_if>::is_arithmetic, void>::type + Hdf5Reader::readDefault(const std::string &s, std::vector &x) + { + uint64_t size; + + push(s); + readSingleAttribute(size, HDF5_GRID_GUARD "vector_size", + *Hdf5Type::type); + x.resize(size); + for (hsize_t i = 0; i < x.size(); ++i) + { + read(s + "_" + std::to_string(i), x[i]); + } + pop(); } } diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index 7250d618..d3bbabe4 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -104,8 +104,8 @@ int main(int argc,char **argv) }; // read tests - myclass copy1, copy2, copy3; - std::vector veccopy1, veccopy2, veccopy3; + myclass copy1, copy2, copy3, copy4; + std::vector veccopy1, veccopy2, veccopy3, veccopy4; //// XML { XmlReader RD("bother.xml"); @@ -150,24 +150,41 @@ int main(int argc,char **argv) } { Hdf5Reader TRD("bother.h5"); - read (TRD,"discard",copy3 ); - read (TRD,"discard2",veccopy3 ); + std::cout << "read single" << std::endl; + read (TRD,"discard",copy4 ); + std::cout << "read vec" << std::endl; + read (TRD,"discard2",veccopy4 ); std::cout << "Loaded (h5) -----------------" << std::endl; - std::cout << copy3 << std::endl << veccopy3 << std::endl; + std::cout << copy3 << std::endl << veccopy4 << std::endl; } #endif - std::vector iv = strToVec("1 2 2 4"); - std::vector sv = strToVec("bli bla blu"); + typedef std::vector>> vec3d; - for (auto &e: iv) + vec3d dv, buf; + double d = 0.; + + dv.resize(4); + for (auto &v1: dv) { - std::cout << e << " "; + v1.resize(3); + for (auto &v2: v1) + { + v2.resize(5); + for (auto &x: v2) + { + x = d++; + } + } } - std::cout << std::endl; - for (auto &e: sv) - { - std::cout << e << " "; - } - std::cout << std::endl; + std::cout << dv << std::endl; + + Flatten flatdv(dv); + + std::cout << flatdv.getDim() << std::endl; + std::cout << flatdv.getFlatVector() << std::endl; + + Reconstruct rec(flatdv.getFlatVector(), flatdv.getDim()); + + std::cout << flatdv.getVector() << std::endl; } From 4be08ebccc3caa704dbb7552b353c92ecd43b782 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Wed, 18 Jan 2017 17:39:59 -0800 Subject: [PATCH 03/14] debug code cleaning --- lib/serialisation/Hdf5IO.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc index 02356220..8b6581ea 100644 --- a/lib/serialisation/Hdf5IO.cc +++ b/lib/serialisation/Hdf5IO.cc @@ -100,12 +100,4 @@ void Hdf5Reader::readDefault(const std::string &s, std::string &x) x.resize(strType.getSize()); attribute.read(strType, &(x[0])); - - std::cout << "length: " << strType.getSize() << std::endl; - std::cout << "string: |"; - for (auto &c: x) - { - std::cout << "'" << c << "'|"; - } - std::cout << std::endl; } From 654e0b0fd0b23ad468c405ae053dd3f44748d28e Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Wed, 18 Jan 2017 17:40:32 -0800 Subject: [PATCH 04/14] Serialisable object are now comparable with == --- lib/serialisation/MacroMagic.h | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/lib/serialisation/MacroMagic.h b/lib/serialisation/MacroMagic.h index c9137dfe..8b027f30 100644 --- a/lib/serialisation/MacroMagic.h +++ b/lib/serialisation/MacroMagic.h @@ -109,38 +109,36 @@ THE SOFTWARE. ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #define GRID_MACRO_MEMBER(A,B) A B; +#define GRID_MACRO_COMP_MEMBER(A,B) result = (result and (lhs. B == rhs. B)); #define GRID_MACRO_OS_WRITE_MEMBER(A,B) os<< #A <<" "#B <<" = "<< obj. B <<" ; " <\ static inline void write(Writer &WR,const std::string &s, const cname &obj){ \ push(WR,s);\ GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_WRITE_MEMBER,__VA_ARGS__)) \ pop(WR);\ -} \ -\ -\ +}\ template \ static inline void read(Reader &RD,const std::string &s, cname &obj){ \ push(RD,s);\ GRID_MACRO_EVAL(GRID_MACRO_MAP(GRID_MACRO_READ_MEMBER,__VA_ARGS__)) \ pop(RD);\ -} \ -\ -\ +}\ friend inline std::ostream & operator << (std::ostream &os, const cname &obj ) { \ os<<"class "<<#cname<<" {"<::type #define GRID_MACRO_ENUMVAL(A,B) A = B, @@ -149,7 +147,7 @@ friend inline std::ostream & operator << (std::ostream &os, const cname &obj ) { #define GRID_MACRO_ENUMCASEIO(A,B) case GRID_ENUM_TYPE(obj)::A: os << #A; break; #define GRID_SERIALIZABLE_ENUM(name,undefname,...)\ -class name: public Serializable\ +class name: public Grid::Serializable\ {\ public:\ enum EnumType\ @@ -161,7 +159,7 @@ public:\ name(void): value_(undefname) {};\ name(EnumType value): value_(value) {};\ template \ - static inline void write(Writer &WR,const std::string &s, const name &obj)\ + static inline void write(Grid::Writer &WR,const std::string &s, const name &obj)\ {\ switch (obj.value_)\ {\ @@ -171,7 +169,7 @@ public:\ }\ \ template \ - static inline void read(Reader &RD,const std::string &s, name &obj)\ + static inline void read(Grid::Reader &RD,const std::string &s, name &obj)\ {\ std::string buf;\ Grid::read(RD, s, buf);\ From f3f0b6fef99b25cbc6857f4d8b87731a2aa1d2e7 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Wed, 18 Jan 2017 17:41:05 -0800 Subject: [PATCH 05/14] serious rewriting of Test_serialisation, now crashes if IO inconsistent --- tests/IO/Test_serialisation.cc | 217 +++++++++++++++------------------ 1 file changed, 101 insertions(+), 116 deletions(-) diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index d3bbabe4..b7158b2b 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -28,137 +28,119 @@ Author: Peter Boyle /* END LEGAL */ #include -namespace Grid { - - GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3); - - class myclass: Serializable { - public: - - GRID_SERIALIZABLE_CLASS_MEMBERS(myclass, - myenum, e, - std::vector, ve, - std::string, name, - int, x, - double, y, - bool , b, - std::vector, array, - std::vector>, twodimarray, - ); - - myclass() {} - myclass(int i) - : array(4,5.1), twodimarray(3,std::vector(2,1.23456)), ve(2, myenum::blue) - { - e=myenum::red; - x=i; - y=2*i; - b=true; - name="bother said pooh"; - } - }; - -} - using namespace Grid; -int16_t i16 = 1; +GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3); + +class myclass: Serializable { +public: + GRID_SERIALIZABLE_CLASS_MEMBERS(myclass, + myenum, e, + std::vector, ve, + std::string, name, + int, x, + double, y, + bool , b, + std::vector, array, + std::vector>, twodimarray, + ); + myclass() {} + myclass(int i) + : array(4,5.1), twodimarray(3,std::vector(2,1.23456)), ve(2, myenum::blue) + { + e=myenum::red; + x=i; + y=2*i; + b=true; + name="bother said pooh"; + } +}; + +int16_t i16 = 1; uint16_t u16 = 2; -int32_t i32 = 3; +int32_t i32 = 3; uint32_t u32 = 4; -int64_t i64 = 5; +int64_t i64 = 5; uint64_t u64 = 6; -float f = M_PI; -double d = 2*M_PI; -bool b = false; +float f = M_PI; +double d = 2*M_PI; +bool b = false; + +template +void ioTest(const std::string &filename, const O &object, const std::string &name) +{ + // writer needs to be destroyed so that writing physically happens + { + W writer(filename); + + write(writer, "testobject", object); + } + + R reader(filename); + O buf; + bool good; + + read(reader, "testobject", buf); + good = (object == buf); + std::cout << name << " IO test: " << (good ? "success" : "failure"); + std::cout << std::endl; + if (!good) exit(EXIT_FAILURE); +} int main(int argc,char **argv) { - { - XmlWriter WR("bother.xml"); - - // test basic type writing - push(WR,"BasicTypes"); - write(WR,std::string("i16"),i16); - write(WR,"u16",u16); - write(WR,"i32",i32); - write(WR,"u32",u32); - write(WR,"i64",i64); - write(WR,"u64",u64); - write(WR,"f",f); - write(WR,"d",d); - write(WR,"b",b); - pop(WR); - - // test serializable class writing - myclass obj(1234); // non-trivial constructor - write(WR,"obj",obj); - WR.write("obj2", obj); - std::cout << obj << std::endl; - - std::vector vec; - vec.push_back(myclass(1234)); - vec.push_back(myclass(5678)); - vec.push_back(myclass(3838)); - write(WR, "objvec", vec); - }; + std::cout << "==== basic IO" << std::endl; + XmlWriter WR("bother.xml"); + + // test basic type writing + std::cout << "-- basic writing to 'bother.xml'..." << std::endl; + push(WR,"BasicTypes"); + write(WR,std::string("i16"),i16); + write(WR,"u16",u16); + write(WR,"i32",i32); + write(WR,"u32",u32); + write(WR,"i64",i64); + write(WR,"u64",u64); + write(WR,"f",f); + write(WR,"d",d); + write(WR,"b",b); + pop(WR); + + // test serializable class writing + myclass obj(1234); // non-trivial constructor + std::vector vec; + + std::cout << "-- serialisable class writing to 'bother.xml'..." << std::endl; + write(WR,"obj",obj); + WR.write("obj2", obj); + vec.push_back(myclass(1234)); + vec.push_back(myclass(5678)); + vec.push_back(myclass(3838)); + write(WR, "objvec", vec); + std::cout << "-- serialisable class writing to std::cout:" << std::endl; + std::cout << obj << std::endl; + std::cout << "-- serialisable class comparison:" << std::endl; + std::cout << "vec[0] == obj: " << ((vec[0] == obj) ? "true" : "false") << std::endl; + std::cout << "vec[1] == obj: " << ((vec[1] == obj) ? "true" : "false") << std::endl; // read tests - myclass copy1, copy2, copy3, copy4; - std::vector veccopy1, veccopy2, veccopy3, veccopy4; + std::cout << "\n==== IO self-consistency tests" << std::endl; //// XML - { - XmlReader RD("bother.xml"); - read(RD,"obj",copy1); - read(RD,"objvec", veccopy1); - std::cout << "Loaded (XML) -----------------" << std::endl; - std::cout << copy1 << std::endl << veccopy1 << std::endl; - } + ioTest("iotest.xml", obj, "XML (object) "); + ioTest("iotest.xml", vec, "XML (vector of objects)"); //// binary - { - BinaryWriter BWR("bother.bin"); - write(BWR,"discard",copy1 ); - write(BWR,"discard",veccopy1 ); - } - { - BinaryReader BRD("bother.bin"); - read (BRD,"discard",copy2 ); - read (BRD,"discard",veccopy2 ); - std::cout << "Loaded (bin) -----------------" << std::endl; - std::cout << copy2 << std::endl << veccopy2 << std::endl; - } + ioTest("iotest.bin", obj, "binary (object) "); + ioTest("iotest.bin", vec, "binary (vector of objects)"); //// text - { - TextWriter TWR("bother.txt"); - write(TWR,"discard",copy1 ); - write(TWR,"discard",veccopy1 ); - } - { - TextReader TRD("bother.txt"); - read (TRD,"discard",copy3 ); - read (TRD,"discard",veccopy3 ); - std::cout << "Loaded (txt) -----------------" << std::endl; - std::cout << copy3 << std::endl << veccopy3 << std::endl; - } -#ifdef HAVE_HDF5 + ioTest("iotest.dat", obj, "text (object) "); + ioTest("iotest.dat", vec, "text (vector of objects)"); //// HDF5 - //// HDF5 does not accept elements with the duplicated names, hence "discard2" - { - Hdf5Writer TWR("bother.h5"); - write(TWR,"discard",copy1 ); - write(TWR,"discard2",veccopy1 ); - } - { - Hdf5Reader TRD("bother.h5"); - std::cout << "read single" << std::endl; - read (TRD,"discard",copy4 ); - std::cout << "read vec" << std::endl; - read (TRD,"discard2",veccopy4 ); - std::cout << "Loaded (h5) -----------------" << std::endl; - std::cout << copy3 << std::endl << veccopy4 << std::endl; - } +#ifdef HAVE_HDF5 + ioTest("iotest.h5", obj, "HDF5 (object) "); + ioTest("iotest.h5", vec, "HDF5 (vector of objects)"); #endif + std::cout << "\n==== vector flattening/reconstruction" << std::endl; typedef std::vector>> vec3d; vec3d dv, buf; @@ -177,14 +159,17 @@ int main(int argc,char **argv) } } } + std::cout << "original 3D vector:" << std::endl; std::cout << dv << std::endl; Flatten flatdv(dv); + std::cout << "\ndimensions:" << std::endl; std::cout << flatdv.getDim() << std::endl; + std::cout << "\nflattened vector:" << std::endl; std::cout << flatdv.getFlatVector() << std::endl; Reconstruct rec(flatdv.getFlatVector(), flatdv.getDim()); - + std::cout << "\nreconstructed vector:" << std::endl; std::cout << flatdv.getVector() << std::endl; } From 5405526424a43ef7c78a186c831b1eecc2de985f Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Wed, 18 Jan 2017 22:42:19 -0800 Subject: [PATCH 06/14] Code typo --- lib/serialisation/BaseIO.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/serialisation/BaseIO.h b/lib/serialisation/BaseIO.h index 5b5ef427..eca5cff2 100644 --- a/lib/serialisation/BaseIO.h +++ b/lib/serialisation/BaseIO.h @@ -306,7 +306,7 @@ namespace Grid { } template - const V & Reconstruct::Reconstruct::getVector(void) + const V & Reconstruct::getVector(void) { return vector_; } From 2c673666dacec2087306309b4f5824073aa13f0e Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Thu, 19 Jan 2017 17:19:12 -0800 Subject: [PATCH 07/14] Standardisation of HDF5 types --- lib/serialisation/Hdf5Type.h | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h index beb509c2..52c6ed24 100644 --- a/lib/serialisation/Hdf5Type.h +++ b/lib/serialisation/Hdf5Type.h @@ -17,21 +17,20 @@ static constexpr bool isNative = true;\ }; #define DEFINE_HDF5_NATIVE_TYPES \ -HDF5_NATIVE_TYPE(NATIVE_B8, bool);\ -HDF5_NATIVE_TYPE(NATIVE_CHAR, char);\ -HDF5_NATIVE_TYPE(NATIVE_SCHAR, signed char);\ -HDF5_NATIVE_TYPE(NATIVE_UCHAR, unsigned char);\ -HDF5_NATIVE_TYPE(NATIVE_SHORT, short);\ -HDF5_NATIVE_TYPE(NATIVE_USHORT, unsigned short);\ -HDF5_NATIVE_TYPE(NATIVE_INT, int);\ -HDF5_NATIVE_TYPE(NATIVE_UINT, unsigned int);\ -HDF5_NATIVE_TYPE(NATIVE_LONG, long);\ -HDF5_NATIVE_TYPE(NATIVE_ULONG, unsigned long);\ -HDF5_NATIVE_TYPE(NATIVE_LLONG, long long);\ -HDF5_NATIVE_TYPE(NATIVE_ULLONG, unsigned long long);\ -HDF5_NATIVE_TYPE(NATIVE_FLOAT, float);\ -HDF5_NATIVE_TYPE(NATIVE_DOUBLE, double);\ -HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double); +HDF5_NATIVE_TYPE(STD_B8LE, bool);\ +HDF5_NATIVE_TYPE(STD_I8LE, char);\ +HDF5_NATIVE_TYPE(STD_U8LE, unsigned char);\ +HDF5_NATIVE_TYPE(STD_I16LE, short);\ +HDF5_NATIVE_TYPE(STD_U16LE, unsigned short);\ +HDF5_NATIVE_TYPE(STD_I32LE, int);\ +HDF5_NATIVE_TYPE(STD_U32LE, unsigned int);\ +HDF5_NATIVE_TYPE(STD_I64LE, long);\ +HDF5_NATIVE_TYPE(STD_U64LE, unsigned long);\ +HDF5_NATIVE_TYPE(STD_I64LE, long long);\ +HDF5_NATIVE_TYPE(STD_U64LE, unsigned long long);\ +HDF5_NATIVE_TYPE(IEEE_F32LE, float);\ +HDF5_NATIVE_TYPE(IEEE_F64LE, double);\ +HDF5_NATIVE_TYPE(IEEE_F64LE, long double); namespace Grid { From 6eea9e4da71227d205a971adfb968721fac09bc5 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Thu, 19 Jan 2017 18:02:53 -0800 Subject: [PATCH 08/14] HDF5 types static initialisation is mysteriously buggy on BG/Q, changing strategy --- lib/Makefile.am | 1 - lib/serialisation/Hdf5IO.cc | 4 ++-- lib/serialisation/Hdf5IO.h | 20 ++++++++++---------- lib/serialisation/Hdf5Type.cc | 8 -------- lib/serialisation/Hdf5Type.h | 5 ++++- 5 files changed, 16 insertions(+), 22 deletions(-) delete mode 100644 lib/serialisation/Hdf5Type.cc diff --git a/lib/Makefile.am b/lib/Makefile.am index 9aa6af92..fac622ca 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -27,7 +27,6 @@ endif if BUILD_HDF5 extra_sources+=serialisation/Hdf5IO.cc - extra_sources+=serialisation/Hdf5Type.cc extra_headers+=serialisation/Hdf5IO.h extra_headers+=serialisation/Hdf5Type.h endif diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc index 8b6581ea..4d5a2df5 100644 --- a/lib/serialisation/Hdf5IO.cc +++ b/lib/serialisation/Hdf5IO.cc @@ -12,7 +12,7 @@ Hdf5Writer::Hdf5Writer(const std::string &fileName) { group_ = file_.openGroup("/"); writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold", - *Hdf5Type::type); + *Hdf5Type::type()); } void Hdf5Writer::push(const std::string &s) @@ -62,7 +62,7 @@ Hdf5Reader::Hdf5Reader(const std::string &fileName) { group_ = file_.openGroup("/"); readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold", - *Hdf5Type::type); + *Hdf5Type::type()); } void Hdf5Reader::push(const std::string &s) diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index b58c86ed..9281e5a0 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -92,7 +92,7 @@ namespace Grid template void Hdf5Writer::writeDefault(const std::string &s, const U &x) { - writeSingleAttribute(x, s, *Hdf5Type::type); + writeSingleAttribute(x, s, *Hdf5Type::type()); } template <> @@ -122,15 +122,15 @@ namespace Grid { H5NS::DataSet dataSet; - dataSet = group_.createDataSet(s, *Hdf5Type::type, dataSpace); - dataSet.write(flatx.data(), *Hdf5Type::type); + dataSet = group_.createDataSet(s, *Hdf5Type::type(), dataSpace); + dataSet.write(flatx.data(), *Hdf5Type::type()); } else { H5NS::Attribute attribute; - attribute = group_.createAttribute(s, *Hdf5Type::type, dataSpace); - attribute.write(*Hdf5Type::type, flatx.data()); + attribute = group_.createAttribute(s, *Hdf5Type::type(), dataSpace); + attribute.write(*Hdf5Type::type(), flatx.data()); } } @@ -140,7 +140,7 @@ namespace Grid { push(s); writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size", - *Hdf5Type::type); + *Hdf5Type::type()); for (hsize_t i = 0; i < x.size(); ++i) { write(s + "_" + std::to_string(i), x[i]); @@ -162,7 +162,7 @@ namespace Grid template void Hdf5Reader::readDefault(const std::string &s, U &output) { - readSingleAttribute(output, s, *Hdf5Type::type); + readSingleAttribute(output, s, *Hdf5Type::type()); } template <> @@ -210,14 +210,14 @@ namespace Grid H5NS::DataSet dataSet; dataSet = group_.openDataSet(s); - dataSet.read(buf.data(), *Hdf5Type::type); + dataSet.read(buf.data(), *Hdf5Type::type()); } else { H5NS::Attribute attribute; attribute = group_.openAttribute(s); - attribute.read(*Hdf5Type::type, buf.data()); + attribute.read(*Hdf5Type::type(), buf.data()); } // reconstruct the multidimensional vector @@ -234,7 +234,7 @@ namespace Grid push(s); readSingleAttribute(size, HDF5_GRID_GUARD "vector_size", - *Hdf5Type::type); + *Hdf5Type::type()); x.resize(size); for (hsize_t i = 0; i < x.size(); ++i) { diff --git a/lib/serialisation/Hdf5Type.cc b/lib/serialisation/Hdf5Type.cc deleted file mode 100644 index 75c7692e..00000000 --- a/lib/serialisation/Hdf5Type.cc +++ /dev/null @@ -1,8 +0,0 @@ -#include "Hdf5Type.h" - -using namespace Grid; - -#define HDF5_NATIVE_TYPE(predType, cType)\ -const H5NS::PredType * Hdf5Type::type = &H5NS::PredType::predType; - -DEFINE_HDF5_NATIVE_TYPES; diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h index 52c6ed24..8b56c406 100644 --- a/lib/serialisation/Hdf5Type.h +++ b/lib/serialisation/Hdf5Type.h @@ -12,7 +12,10 @@ template <>\ struct Hdf5Type\ {\ -static const H5NS::PredType *type;\ +static inline const H5NS::PredType *type(void)\ +{\ + return &H5NS::PredType::predType;\ +}\ static constexpr bool isNative = true;\ }; From ade1058e5f249531217c7f6874f3596ad85479fe Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Thu, 19 Jan 2017 18:23:55 -0800 Subject: [PATCH 09/14] Hdf5Type does not need to be a pointer anymore --- lib/serialisation/Hdf5IO.cc | 4 ++-- lib/serialisation/Hdf5IO.h | 14 +++++++------- lib/serialisation/Hdf5Type.h | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc index 4d5a2df5..c5313495 100644 --- a/lib/serialisation/Hdf5IO.cc +++ b/lib/serialisation/Hdf5IO.cc @@ -12,7 +12,7 @@ Hdf5Writer::Hdf5Writer(const std::string &fileName) { group_ = file_.openGroup("/"); writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold", - *Hdf5Type::type()); + Hdf5Type::type()); } void Hdf5Writer::push(const std::string &s) @@ -62,7 +62,7 @@ Hdf5Reader::Hdf5Reader(const std::string &fileName) { group_ = file_.openGroup("/"); readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold", - *Hdf5Type::type()); + Hdf5Type::type()); } void Hdf5Reader::push(const std::string &s) diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index 9281e5a0..1c73bea3 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -92,7 +92,7 @@ namespace Grid template void Hdf5Writer::writeDefault(const std::string &s, const U &x) { - writeSingleAttribute(x, s, *Hdf5Type::type()); + writeSingleAttribute(x, s, Hdf5Type::type()); } template <> @@ -122,7 +122,7 @@ namespace Grid { H5NS::DataSet dataSet; - dataSet = group_.createDataSet(s, *Hdf5Type::type(), dataSpace); + dataSet = group_.createDataSet(s, Hdf5Type::type(), dataSpace); dataSet.write(flatx.data(), *Hdf5Type::type()); } else @@ -140,7 +140,7 @@ namespace Grid { push(s); writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size", - *Hdf5Type::type()); + Hdf5Type::type()); for (hsize_t i = 0; i < x.size(); ++i) { write(s + "_" + std::to_string(i), x[i]); @@ -162,7 +162,7 @@ namespace Grid template void Hdf5Reader::readDefault(const std::string &s, U &output) { - readSingleAttribute(output, s, *Hdf5Type::type()); + readSingleAttribute(output, s, Hdf5Type::type()); } template <> @@ -210,14 +210,14 @@ namespace Grid H5NS::DataSet dataSet; dataSet = group_.openDataSet(s); - dataSet.read(buf.data(), *Hdf5Type::type()); + dataSet.read(buf.data(), Hdf5Type::type()); } else { H5NS::Attribute attribute; attribute = group_.openAttribute(s); - attribute.read(*Hdf5Type::type(), buf.data()); + attribute.read(Hdf5Type::type(), buf.data()); } // reconstruct the multidimensional vector @@ -234,7 +234,7 @@ namespace Grid push(s); readSingleAttribute(size, HDF5_GRID_GUARD "vector_size", - *Hdf5Type::type()); + Hdf5Type::type()); x.resize(size); for (hsize_t i = 0; i < x.size(); ++i) { diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h index 8b56c406..75575bf9 100644 --- a/lib/serialisation/Hdf5Type.h +++ b/lib/serialisation/Hdf5Type.h @@ -12,9 +12,9 @@ template <>\ struct Hdf5Type\ {\ -static inline const H5NS::PredType *type(void)\ +static inline const H5NS::PredType & type(void)\ {\ - return &H5NS::PredType::predType;\ + return H5NS::PredType::predType;\ }\ static constexpr bool isNative = true;\ }; From 81e66d6631ab07ee52522a50478ff3bb304a09d6 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Thu, 19 Jan 2017 18:24:53 -0800 Subject: [PATCH 10/14] HDF5: revert back to native types --- lib/serialisation/Hdf5Type.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h index 75575bf9..2e02128e 100644 --- a/lib/serialisation/Hdf5Type.h +++ b/lib/serialisation/Hdf5Type.h @@ -20,20 +20,21 @@ static constexpr bool isNative = true;\ }; #define DEFINE_HDF5_NATIVE_TYPES \ -HDF5_NATIVE_TYPE(STD_B8LE, bool);\ -HDF5_NATIVE_TYPE(STD_I8LE, char);\ -HDF5_NATIVE_TYPE(STD_U8LE, unsigned char);\ -HDF5_NATIVE_TYPE(STD_I16LE, short);\ -HDF5_NATIVE_TYPE(STD_U16LE, unsigned short);\ -HDF5_NATIVE_TYPE(STD_I32LE, int);\ -HDF5_NATIVE_TYPE(STD_U32LE, unsigned int);\ -HDF5_NATIVE_TYPE(STD_I64LE, long);\ -HDF5_NATIVE_TYPE(STD_U64LE, unsigned long);\ -HDF5_NATIVE_TYPE(STD_I64LE, long long);\ -HDF5_NATIVE_TYPE(STD_U64LE, unsigned long long);\ -HDF5_NATIVE_TYPE(IEEE_F32LE, float);\ -HDF5_NATIVE_TYPE(IEEE_F64LE, double);\ -HDF5_NATIVE_TYPE(IEEE_F64LE, long double); +HDF5_NATIVE_TYPE(NATIVE_B8, bool);\ +HDF5_NATIVE_TYPE(NATIVE_CHAR, char);\ +HDF5_NATIVE_TYPE(NATIVE_SCHAR, signed char);\ +HDF5_NATIVE_TYPE(NATIVE_UCHAR, unsigned char);\ +HDF5_NATIVE_TYPE(NATIVE_SHORT, short);\ +HDF5_NATIVE_TYPE(NATIVE_USHORT, unsigned short);\ +HDF5_NATIVE_TYPE(NATIVE_INT, int);\ +HDF5_NATIVE_TYPE(NATIVE_UINT, unsigned int);\ +HDF5_NATIVE_TYPE(NATIVE_LONG, long);\ +HDF5_NATIVE_TYPE(NATIVE_ULONG, unsigned long);\ +HDF5_NATIVE_TYPE(NATIVE_LLONG, long long);\ +HDF5_NATIVE_TYPE(NATIVE_ULLONG, unsigned long long);\ +HDF5_NATIVE_TYPE(NATIVE_FLOAT, float);\ +HDF5_NATIVE_TYPE(NATIVE_DOUBLE, double);\ +HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double); namespace Grid { From 7423a352c5b2f97e800667000533c7df9ab06c07 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Thu, 19 Jan 2017 18:33:04 -0800 Subject: [PATCH 11/14] HDF5: typos --- lib/serialisation/Hdf5IO.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index 1c73bea3..0fb277b0 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -123,14 +123,14 @@ namespace Grid H5NS::DataSet dataSet; dataSet = group_.createDataSet(s, Hdf5Type::type(), dataSpace); - dataSet.write(flatx.data(), *Hdf5Type::type()); + dataSet.write(flatx.data(), Hdf5Type::type()); } else { H5NS::Attribute attribute; - attribute = group_.createAttribute(s, *Hdf5Type::type(), dataSpace); - attribute.write(*Hdf5Type::type(), flatx.data()); + attribute = group_.createAttribute(s, Hdf5Type::type(), dataSpace); + attribute.write(Hdf5Type::type(), flatx.data()); } } From 6b5259cc104d79b714ec93ffbb95e31b039a95b0 Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Fri, 20 Jan 2017 11:03:19 -0800 Subject: [PATCH 12/14] HDF5 detects if a name is a dataset or not without using exception catching --- lib/serialisation/Hdf5IO.h | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index 0fb277b0..3edb7d10 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -177,23 +177,18 @@ namespace Grid // read the dimensions H5NS::DataSpace dataSpace; - H5E_auto2_t func; - void * client_data; std::vector hdim; std::vector dim; hsize_t size = 1; - H5NS::Exception::getAutoPrint(func, &client_data); - try + if (group_.attrExists(s)) { - H5NS::Exception::dontPrint(); - dataSpace = group_.openDataSet(s).getSpace(); - } - catch (H5NS::Exception &e) - { - H5NS::Exception::setAutoPrint(func, client_data); dataSpace = group_.openAttribute(s).getSpace(); } + else + { + dataSpace = group_.openDataSet(s).getSpace(); + } hdim.resize(dataSpace.getSimpleExtentNdims()); dataSpace.getSimpleExtentDims(hdim.data()); for (auto &d: hdim) From afa095d33d1665dbf7648dddef95f78901e7e6cd Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Fri, 20 Jan 2017 12:10:41 -0800 Subject: [PATCH 13/14] HDF5: better complex number support --- lib/Grid.h | 2 +- lib/serialisation/BaseIO.h | 9 ++++---- lib/serialisation/Hdf5IO.h | 16 +++++++------- lib/serialisation/Hdf5Type.h | 39 ++++++++++++++++++++++++++++------ tests/IO/Test_serialisation.cc | 6 +++++- 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/lib/Grid.h b/lib/Grid.h index 0f57c8a6..cb55d0c8 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -60,12 +60,12 @@ Author: paboyle // Grid headers /////////////////// #include "Config.h" -#include #include #include #include #include #include +#include #include #include #include diff --git a/lib/serialisation/BaseIO.h b/lib/serialisation/BaseIO.h index eca5cff2..0357915d 100644 --- a/lib/serialisation/BaseIO.h +++ b/lib/serialisation/BaseIO.h @@ -68,20 +68,21 @@ namespace Grid { return os; } - // Vector element trait ////////////////////////////////////////////////////// + // Vector element trait ////////////////////////////////////////////////////// template struct element { typedef T type; - static constexpr bool is_arithmetic = false; + static constexpr bool is_number = false; }; template struct element> { typedef typename element::type type; - static constexpr bool is_arithmetic = std::is_arithmetic::value - or element::is_arithmetic; + static constexpr bool is_number = std::is_arithmetic::value + or is_complex::value + or element::is_number; }; // Vector flatening utility class //////////////////////////////////////////// diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h index 3edb7d10..2f891cd4 100644 --- a/lib/serialisation/Hdf5IO.h +++ b/lib/serialisation/Hdf5IO.h @@ -32,10 +32,10 @@ namespace Grid template void writeDefault(const std::string &s, const U &x); template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type writeDefault(const std::string &s, const std::vector &x); template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type writeDefault(const std::string &s, const std::vector &x); private: template @@ -59,10 +59,10 @@ namespace Grid template void readDefault(const std::string &s, U &output); template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type readDefault(const std::string &s, std::vector &x); template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type readDefault(const std::string &s, std::vector &x); private: template @@ -99,7 +99,7 @@ namespace Grid void Hdf5Writer::writeDefault(const std::string &s, const std::string &x); template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) { // alias to element type @@ -135,7 +135,7 @@ namespace Grid } template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) { push(s); @@ -169,7 +169,7 @@ namespace Grid void Hdf5Reader::readDefault(const std::string &s, std::string &x); template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type Hdf5Reader::readDefault(const std::string &s, std::vector &x) { // alias to element type @@ -222,7 +222,7 @@ namespace Grid } template - typename std::enable_if>::is_arithmetic, void>::type + typename std::enable_if>::is_number, void>::type Hdf5Reader::readDefault(const std::string &s, std::vector &x) { uint64_t size; diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h index 2e02128e..cf682138 100644 --- a/lib/serialisation/Hdf5Type.h +++ b/lib/serialisation/Hdf5Type.h @@ -10,13 +10,14 @@ #define HDF5_NATIVE_TYPE(predType, cType)\ template <>\ -struct Hdf5Type\ +class Hdf5Type\ {\ -static inline const H5NS::PredType & type(void)\ -{\ - return H5NS::PredType::predType;\ -}\ -static constexpr bool isNative = true;\ +public:\ + static inline const H5NS::DataType & type(void)\ + {\ + return H5NS::PredType::predType;\ + }\ + static constexpr bool isNative = true;\ }; #define DEFINE_HDF5_NATIVE_TYPES \ @@ -38,12 +39,36 @@ HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double); namespace Grid { - template struct Hdf5Type + template class Hdf5Type { + public: static constexpr bool isNative = false; }; DEFINE_HDF5_NATIVE_TYPES; + + template + class Hdf5Type> + { + public: + static inline const H5NS::DataType & type(void) + { + if (typePtr_ == nullptr) + { + typePtr_.reset(new H5NS::CompType(sizeof(std::complex))); + typePtr_->insertMember("re", 0, Hdf5Type::type()); + typePtr_->insertMember("im", sizeof(R), Hdf5Type::type()); + } + + return *typePtr_; + } + static constexpr bool isNative = false; + private: + static std::unique_ptr typePtr_; + }; + + template + std::unique_ptr Hdf5Type>::typePtr_ = nullptr; } #undef HDF5_NATIVE_TYPE diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index b7158b2b..8204b05b 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -43,10 +43,14 @@ public: bool , b, std::vector, array, std::vector>, twodimarray, + std::vector>>, cmplx3darray ); myclass() {} myclass(int i) - : array(4,5.1), twodimarray(3,std::vector(2,1.23456)), ve(2, myenum::blue) + : array(4,5.1) + , twodimarray(3,std::vector(5, 1.23456)) + , cmplx3darray(3,std::vector>(5, std::vector(7, Complex(1.2, 3.4)))) + , ve(2, myenum::blue) { e=myenum::red; x=i; From 4c75095c616c6c246b268c775fee13cbe7ae84da Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Fri, 20 Jan 2017 12:14:01 -0800 Subject: [PATCH 14/14] HDF5: header fix --- lib/serialisation/Hdf5Type.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h index cf682138..8634f35b 100644 --- a/lib/serialisation/Hdf5Type.h +++ b/lib/serialisation/Hdf5Type.h @@ -2,7 +2,8 @@ #define GRID_SERIALISATION_HDF5_TYPE_H #include -#include +#include +#include #ifndef H5_NO_NAMESPACE #define H5NS H5