From 5803933aea55df67c733cf1e4201f354d8a8965b Mon Sep 17 00:00:00 2001 From: Antonin Portelli Date: Tue, 17 Jan 2017 16:21:18 -0800 Subject: [PATCH] First implementation of HDF5 serial IO writer, reader is still empty --- configure.ac | 14 +++ lib/Grid.h | 2 +- lib/Makefile.am | 13 ++- lib/serialisation/Hdf5IO.cc | 84 +++++++++++++++ lib/serialisation/Hdf5IO.h | 169 ++++++++++++++++++++++++++++++ lib/serialisation/Hdf5Type.cc | 8 ++ lib/serialisation/Hdf5Type.h | 48 +++++++++ lib/serialisation/Serialisation.h | 3 + scripts/filelist | 5 +- tests/IO/Test_serialisation.cc | 16 +++ 10 files changed, 357 insertions(+), 5 deletions(-) create mode 100644 lib/serialisation/Hdf5IO.cc create mode 100644 lib/serialisation/Hdf5IO.h create mode 100644 lib/serialisation/Hdf5Type.cc create mode 100644 lib/serialisation/Hdf5Type.h diff --git a/configure.ac b/configure.ac index f413cde8..f848bd23 100644 --- a/configure.ac +++ b/configure.ac @@ -99,6 +99,13 @@ case ${ac_MKL} in AC_DEFINE([USE_MKL], [1], [Define to 1 if you use the Intel MKL]);; esac +############### HDF5 +AC_ARG_WITH([hdf5], + [AS_HELP_STRING([--with-hdf5=prefix], + [try this for a non-standard install prefix of the HDF5 library])], + [AM_CXXFLAGS="-I$with_hdf5/include $AM_CXXFLAGS"] + [AM_LDFLAGS="-L$with_hdf5/lib $AM_LDFLAGS"]) + ############### first-touch AC_ARG_ENABLE([numa], [AC_HELP_STRING([--enable-numa=yes|no|prefix], [enable first touch numa opt])], @@ -145,6 +152,12 @@ AC_SEARCH_LIBS([fftw_execute], [fftw3], [AC_DEFINE([HAVE_FFTW], [1], [Define to 1 if you have the `FFTW' library])] [have_fftw=true]) +AC_SEARCH_LIBS([H5Fopen], [hdf5_cpp], + [AC_DEFINE([HAVE_HDF5], [1], [Define to 1 if you have the `HDF5' library])] + [have_hdf5=true] + [LIBS="${LIBS} -lhdf5"], [], [-lhdf5]) +AM_CONDITIONAL(BUILD_HDF5, [ test "${have_hdf5}X" == "trueX" ]) + CXXFLAGS=$CXXFLAGS_CPY LDFLAGS=$LDFLAGS_CPY @@ -410,6 +423,7 @@ RNG choice : ${ac_RNG} GMP : `if test "x$have_gmp" = xtrue; then echo yes; else echo no; fi` LAPACK : ${ac_LAPACK} FFTW : `if test "x$have_fftw" = xtrue; then echo yes; else echo no; fi` +HDF5 : `if test "x$have_hdf5" = xtrue; then echo yes; else echo no; fi` build DOXYGEN documentation : `if test "$DX_FLAG_doc" = '1'; then echo yes; else echo no; fi` ----- BUILD FLAGS ------------------------------------- CXXFLAGS: diff --git a/lib/Grid.h b/lib/Grid.h index 0c5983f3..0f57c8a6 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -59,8 +59,8 @@ Author: paboyle /////////////////// // Grid headers /////////////////// -#include #include "Config.h" +#include #include #include #include diff --git a/lib/Makefile.am b/lib/Makefile.am index a779135f..9aa6af92 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,4 +1,5 @@ extra_sources= +extra_headers= if BUILD_COMMS_MPI extra_sources+=communicator/Communicator_mpi.cc extra_sources+=communicator/Communicator_base.cc @@ -24,6 +25,13 @@ if BUILD_COMMS_NONE extra_sources+=communicator/Communicator_base.cc endif +if BUILD_HDF5 + extra_sources+=serialisation/Hdf5IO.cc + extra_sources+=serialisation/Hdf5Type.cc + extra_headers+=serialisation/Hdf5IO.h + extra_headers+=serialisation/Hdf5Type.h +endif + # # Libraries # @@ -32,6 +40,9 @@ include Eigen.inc lib_LIBRARIES = libGrid.a -libGrid_a_SOURCES = $(CCFILES) $(extra_sources) +CCFILES += $(extra_sources) +HFILES += $(extra_headers) + +libGrid_a_SOURCES = $(CCFILES) libGrid_adir = $(pkgincludedir) nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) Config.h diff --git a/lib/serialisation/Hdf5IO.cc b/lib/serialisation/Hdf5IO.cc new file mode 100644 index 00000000..0d62fdd8 --- /dev/null +++ b/lib/serialisation/Hdf5IO.cc @@ -0,0 +1,84 @@ +#include + +using namespace Grid; +#ifndef H5_NO_NAMESPACE +using namespace H5NS; +#endif + +// Writer implementation /////////////////////////////////////////////////////// +Hdf5Writer::Hdf5Writer(const std::string &fileName) +: fileName_(fileName) +, file_(fileName.c_str(), H5F_ACC_TRUNC) +{ + group_ = file_.openGroup("/"); +} + +Hdf5Writer::~Hdf5Writer(void) +{ + file_.close(); +} + +void Hdf5Writer::push(const std::string &s) +{ + group_ = group_.createGroup(s); + path_.push_back(s); +} + +void Hdf5Writer::pop(void) +{ + path_.pop_back(); + if (path_.empty()) + { + group_ = file_.openGroup("/"); + } + else + { + auto binOp = [](const std::string &a, const std::string &b)->std::string + { + return a + "/" + b; + }; + + group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(), + std::string(""), binOp)); + } +} + +template <> +void Hdf5Writer::writeDefault(const std::string &s, const std::string &x) +{ + StrType strType(PredType::C_S1, x.size()); + Attribute attribute; + hsize_t attrDim = 1; + DataSpace attrSpace(1, &attrDim); + + attribute = group_.createAttribute(s, strType, attrSpace); + attribute.write(strType, x.data()); +} + +void Hdf5Writer::writeDefault(const std::string &s, const char *x) +{ + std::string sx(x); + + writeDefault(s, sx); +} + +// Reader implementation /////////////////////////////////////////////////////// +Hdf5Reader::Hdf5Reader(const std::string &fileName) +{ + +} + +Hdf5Reader::~Hdf5Reader(void) +{ + +} + +void Hdf5Reader::push(const std::string &s) +{ + +} + +void Hdf5Reader::pop(void) +{ + +} diff --git a/lib/serialisation/Hdf5IO.h b/lib/serialisation/Hdf5IO.h new file mode 100644 index 00000000..481fa1cf --- /dev/null +++ b/lib/serialisation/Hdf5IO.h @@ -0,0 +1,169 @@ +#ifndef GRID_SERIALISATION_HDF5_H +#define GRID_SERIALISATION_HDF5_H + +#include +#include +#include +#include +#include "Hdf5Type.h" + +#ifndef H5_NO_NAMESPACE +#define H5NS H5 +#endif + +// default thresold above which datasets are used instead of attributes +#ifndef H5_DEF_DATASET_THRES +#define H5_DEF_DATASET_THRES 6u +#endif + +namespace Grid +{ + template + struct is_arithmetic_vector + { + static constexpr bool value = false; + }; + + template + struct is_arithmetic_vector> + { + static constexpr bool value = std::is_arithmetic::value + or is_arithmetic_vector::value; + }; + + class Hdf5Writer: public Writer + { + public: + Hdf5Writer(const std::string &fileName); + virtual ~Hdf5Writer(void); + void push(const std::string &s); + void pop(void); + void writeDefault(const std::string &s, const char *x); + template + void writeDefault(const std::string &s, const U &x); + template + typename std::enable_if>::value + and std::is_arithmetic::value, void>::type + writeDefault(const std::string &s, const std::vector &x); + template + typename std::enable_if>::value + and !std::is_arithmetic::value, void>::type + writeDefault(const std::string &s, const std::vector &x); + template + typename std::enable_if>::value, void>::type + writeDefault(const std::string &s, const std::vector &x); + private: + std::string fileName_; + std::vector path_; + std::vector dim_; + bool multiDim_{true}; + H5NS::H5File file_; + H5NS::Group group_; + unsigned int datasetThres_{H5_DEF_DATASET_THRES}; + }; + + class Hdf5Reader: public Reader + { + public: + Hdf5Reader(const std::string &fileName); + virtual ~Hdf5Reader(void); + void push(const std::string &s); + void pop(void); + template + void readDefault(const std::string &s, U &output); + template + void readDefault(const std::string &s, std::vector &output); + private: + }; + + // Writer template implementation //////////////////////////////////////////// + template + void Hdf5Writer::writeDefault(const std::string &s, const U &x) + { + H5NS::Attribute attribute; + hsize_t attrDim = 1; + H5NS::DataSpace attrSpace(1, &attrDim); + + attribute = group_.createAttribute(s, *Hdf5Type::type, attrSpace); + attribute.write(*Hdf5Type::type, &x); + } + + template <> + void Hdf5Writer::writeDefault(const std::string &s, const std::string &x); + + template + typename std::enable_if>::value + and std::is_arithmetic::value, void>::type + Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) + { + hsize_t size = 1; + + dim_.push_back(x.size()); + for (auto d: dim_) + { + size *= d; + } + + H5NS::DataSpace dataspace(dim_.size(), dim_.data()); + + if (size > datasetThres_) + { + H5NS::DataSet dataset; + + dataset = group_.createDataSet(s, *Hdf5Type::type, dataspace); + dataset.write(x.data(), *Hdf5Type::type); + } + else + { + H5NS::Attribute attribute; + + attribute = group_.createAttribute(s, *Hdf5Type::type, dataspace); + attribute.write(*Hdf5Type::type, x.data()); + } + dim_.clear(); + multiDim_ = true; + } + + template + typename std::enable_if>::value + and !std::is_arithmetic::value, void>::type + Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) + { + hsize_t firstSize = x[0].size(); + + for (auto &v: x) + { + multiDim_ = (multiDim_ and (v.size() == firstSize)); + } + assert(multiDim_); + dim_.push_back(x.size()); + writeDefault(s, x[0]); + } + + template + typename std::enable_if>::value, void>::type + Hdf5Writer::writeDefault(const std::string &s, const std::vector &x) + { + push(s); + for (hsize_t i = 0; i < x.size(); ++i) + { + write(s + "_" + std::to_string(i), x[i]); + } + pop(); + } + + // Reader template implementation //////////////////////////////////////////// + template + void Hdf5Reader::readDefault(const std::string &s, U &output) + { + + } + + template + void Hdf5Reader::readDefault(const std::string &s, std::vector &output) + { + + } +} + +#endif diff --git a/lib/serialisation/Hdf5Type.cc b/lib/serialisation/Hdf5Type.cc new file mode 100644 index 00000000..75c7692e --- /dev/null +++ b/lib/serialisation/Hdf5Type.cc @@ -0,0 +1,8 @@ +#include "Hdf5Type.h" + +using namespace Grid; + +#define HDF5_NATIVE_TYPE(predType, cType)\ +const H5NS::PredType * Hdf5Type::type = &H5NS::PredType::predType; + +DEFINE_HDF5_NATIVE_TYPES; diff --git a/lib/serialisation/Hdf5Type.h b/lib/serialisation/Hdf5Type.h new file mode 100644 index 00000000..beb509c2 --- /dev/null +++ b/lib/serialisation/Hdf5Type.h @@ -0,0 +1,48 @@ +#ifndef GRID_SERIALISATION_HDF5_TYPE_H +#define GRID_SERIALISATION_HDF5_TYPE_H + +#include +#include + +#ifndef H5_NO_NAMESPACE +#define H5NS H5 +#endif + +#define HDF5_NATIVE_TYPE(predType, cType)\ +template <>\ +struct Hdf5Type\ +{\ +static const H5NS::PredType *type;\ +static constexpr bool isNative = true;\ +}; + +#define DEFINE_HDF5_NATIVE_TYPES \ +HDF5_NATIVE_TYPE(NATIVE_B8, bool);\ +HDF5_NATIVE_TYPE(NATIVE_CHAR, char);\ +HDF5_NATIVE_TYPE(NATIVE_SCHAR, signed char);\ +HDF5_NATIVE_TYPE(NATIVE_UCHAR, unsigned char);\ +HDF5_NATIVE_TYPE(NATIVE_SHORT, short);\ +HDF5_NATIVE_TYPE(NATIVE_USHORT, unsigned short);\ +HDF5_NATIVE_TYPE(NATIVE_INT, int);\ +HDF5_NATIVE_TYPE(NATIVE_UINT, unsigned int);\ +HDF5_NATIVE_TYPE(NATIVE_LONG, long);\ +HDF5_NATIVE_TYPE(NATIVE_ULONG, unsigned long);\ +HDF5_NATIVE_TYPE(NATIVE_LLONG, long long);\ +HDF5_NATIVE_TYPE(NATIVE_ULLONG, unsigned long long);\ +HDF5_NATIVE_TYPE(NATIVE_FLOAT, float);\ +HDF5_NATIVE_TYPE(NATIVE_DOUBLE, double);\ +HDF5_NATIVE_TYPE(NATIVE_LDOUBLE, long double); + +namespace Grid +{ + template struct Hdf5Type + { + static constexpr bool isNative = false; + }; + + DEFINE_HDF5_NATIVE_TYPES; +} + +#undef HDF5_NATIVE_TYPE + +#endif /* GRID_SERIALISATION_HDF5_TYPE_H */ diff --git a/lib/serialisation/Serialisation.h b/lib/serialisation/Serialisation.h index 8f405d73..aa84e989 100644 --- a/lib/serialisation/Serialisation.h +++ b/lib/serialisation/Serialisation.h @@ -36,6 +36,9 @@ Author: Peter Boyle #include "BinaryIO.h" #include "TextIO.h" #include "XmlIO.h" +#ifdef HAVE_HDF5 +#include "Hdf5IO.h" +#endif ////////////////////////////////////////// // Todo: ////////////////////////////////////////// diff --git a/scripts/filelist b/scripts/filelist index 1ab95c7c..bf2fbc41 100755 --- a/scripts/filelist +++ b/scripts/filelist @@ -4,9 +4,8 @@ home=`pwd` # library Make.inc cd $home/lib -HFILES=`find . -type f -name '*.h' -not -path '*/Old/*' -not -path '*/Eigen/*'` -HFILES="$HFILES" -CCFILES=`find . -type f -name '*.cc' -not -name '*ommunicator*.cc'` +HFILES=`find . -type f -name '*.h' -not -name '*Hdf5*' -not -path '*/Old/*' -not -path '*/Eigen/*'` +CCFILES=`find . -type f -name '*.cc' -not -name '*Communicator*.cc' -not -name '*Hdf5*'` echo HFILES=$HFILES > Make.inc echo >> Make.inc echo CCFILES=$CCFILES >> Make.inc diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index e23aa1a3..7250d618 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -140,6 +140,22 @@ int main(int argc,char **argv) std::cout << "Loaded (txt) -----------------" << std::endl; std::cout << copy3 << std::endl << veccopy3 << std::endl; } +#ifdef HAVE_HDF5 + //// HDF5 + //// HDF5 does not accept elements with the duplicated names, hence "discard2" + { + Hdf5Writer TWR("bother.h5"); + write(TWR,"discard",copy1 ); + write(TWR,"discard2",veccopy1 ); + } + { + Hdf5Reader TRD("bother.h5"); + read (TRD,"discard",copy3 ); + read (TRD,"discard2",veccopy3 ); + std::cout << "Loaded (h5) -----------------" << std::endl; + std::cout << copy3 << std::endl << veccopy3 << std::endl; + } +#endif std::vector iv = strToVec("1 2 2 4"); std::vector sv = strToVec("bli bla blu");