1
0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-10 07:55:35 +00:00

HDF5 serial IO implemented and tested

This commit is contained in:
Antonin Portelli 2017-01-18 16:50:21 -08:00
parent 5803933aea
commit f599cb5b17
4 changed files with 412 additions and 98 deletions

View File

@ -32,6 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
#include <type_traits>
namespace Grid {
// Vector IO utilities ///////////////////////////////////////////////////////
// helper function to read space-separated values
template <typename T>
std::vector<T> strToVec(const std::string s)
@ -67,6 +68,76 @@ namespace Grid {
return os;
}
// Vector element trait //////////////////////////////////////////////////////
template <typename T>
struct element
{
typedef T type;
static constexpr bool is_arithmetic = false;
};
template <typename T>
struct element<std::vector<T>>
{
typedef typename element<T>::type type;
static constexpr bool is_arithmetic = std::is_arithmetic<T>::value
or element<T>::is_arithmetic;
};
// Vector flatening utility class ////////////////////////////////////////////
// Class to flatten a multidimensional std::vector
template <typename V>
class Flatten
{
public:
typedef typename element<V>::type Element;
public:
explicit Flatten(const V &vector);
const V & getVector(void);
const std::vector<Element> & getFlatVector(void);
const std::vector<size_t> & getDim(void);
private:
void accumulate(const Element &e);
template <typename W>
void accumulate(const W &v);
void accumulateDim(const Element &e);
template <typename W>
void accumulateDim(const W &v);
private:
const V &vector_;
std::vector<Element> flatVector_;
std::vector<size_t> dim_;
};
// Class to reconstruct a multidimensional std::vector
template <typename V>
class Reconstruct
{
public:
typedef typename element<V>::type Element;
public:
Reconstruct(const std::vector<Element> &flatVector,
const std::vector<size_t> &dim);
const V & getVector(void);
const std::vector<Element> & getFlatVector(void);
const std::vector<size_t> & getDim(void);
private:
void fill(std::vector<Element> &v);
template <typename W>
void fill(W &v);
void resize(std::vector<Element> &v, const unsigned int dim);
template <typename W>
void resize(W &v, const unsigned int dim);
private:
V vector_;
const std::vector<Element> &flatVector_;
std::vector<size_t> dim_;
size_t ind_{0};
unsigned int dimInd_{0};
};
// Abstract writer/reader classes ////////////////////////////////////////////
// static polymorphism implemented using CRTP idiom
class Serializable;
@ -132,7 +203,128 @@ namespace Grid {
}
};
// Generic writer interface
// Flatten class template implementation /////////////////////////////////////
template <typename V>
void Flatten<V>::accumulate(const Element &e)
{
flatVector_.push_back(e);
}
template <typename V>
template <typename W>
void Flatten<V>::accumulate(const W &v)
{
for (auto &e: v)
{
accumulate(e);
}
}
template <typename V>
void Flatten<V>::accumulateDim(const Element &e) {};
template <typename V>
template <typename W>
void Flatten<V>::accumulateDim(const W &v)
{
dim_.push_back(v.size());
accumulateDim(v[0]);
}
template <typename V>
Flatten<V>::Flatten(const V &vector)
: vector_(vector)
{
accumulate(vector_);
accumulateDim(vector_);
}
template <typename V>
const V & Flatten<V>::getVector(void)
{
return vector_;
}
template <typename V>
const std::vector<typename Flatten<V>::Element> &
Flatten<V>::getFlatVector(void)
{
return flatVector_;
}
template <typename V>
const std::vector<size_t> & Flatten<V>::getDim(void)
{
return dim_;
}
// Reconstruct class template implementation /////////////////////////////////
template <typename V>
void Reconstruct<V>::fill(std::vector<Element> &v)
{
for (auto &e: v)
{
e = flatVector_[ind_++];
}
}
template <typename V>
template <typename W>
void Reconstruct<V>::fill(W &v)
{
for (auto &e: v)
{
fill(e);
}
}
template <typename V>
void Reconstruct<V>::resize(std::vector<Element> &v, const unsigned int dim)
{
v.resize(dim_[dim]);
}
template <typename V>
template <typename W>
void Reconstruct<V>::resize(W &v, const unsigned int dim)
{
v.resize(dim_[dim]);
for (auto &e: v)
{
resize(e, dim + 1);
}
}
template <typename V>
Reconstruct<V>::Reconstruct(const std::vector<Element> &flatVector,
const std::vector<size_t> &dim)
: flatVector_(flatVector)
, dim_(dim)
{
resize(vector_, 0);
fill(vector_);
}
template <typename V>
const V & Reconstruct<V>::Reconstruct<V>::getVector(void)
{
return vector_;
}
template <typename V>
const std::vector<typename Reconstruct<V>::Element> &
Reconstruct<V>::getFlatVector(void)
{
return flatVector_;
}
template <typename V>
const std::vector<size_t> & Reconstruct<V>::getDim(void)
{
return dim_;
}
// Generic writer interface //////////////////////////////////////////////////
template <typename T>
inline void push(Writer<T> &w, const std::string &s)
{
@ -217,7 +409,7 @@ namespace Grid {
upcast->writeDefault(s, output);
}
// Reader template implementation ////////////////////////////////////////////
// Reader template implementation
template <typename T>
Reader<T>::Reader(void)
{

View File

@ -11,11 +11,8 @@ Hdf5Writer::Hdf5Writer(const std::string &fileName)
, file_(fileName.c_str(), H5F_ACC_TRUNC)
{
group_ = file_.openGroup("/");
}
Hdf5Writer::~Hdf5Writer(void)
{
file_.close();
writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
*Hdf5Type<unsigned int>::type);
}
void Hdf5Writer::push(const std::string &s)
@ -47,12 +44,8 @@ template <>
void Hdf5Writer::writeDefault(const std::string &s, const std::string &x)
{
StrType strType(PredType::C_S1, x.size());
Attribute attribute;
hsize_t attrDim = 1;
DataSpace attrSpace(1, &attrDim);
attribute = group_.createAttribute(s, strType, attrSpace);
attribute.write(strType, x.data());
writeSingleAttribute(*(x.data()), s, strType);
}
void Hdf5Writer::writeDefault(const std::string &s, const char *x)
@ -64,21 +57,55 @@ void Hdf5Writer::writeDefault(const std::string &s, const char *x)
// Reader implementation ///////////////////////////////////////////////////////
Hdf5Reader::Hdf5Reader(const std::string &fileName)
: fileName_(fileName)
, file_(fileName.c_str(), H5F_ACC_RDONLY)
{
}
Hdf5Reader::~Hdf5Reader(void)
{
group_ = file_.openGroup("/");
readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
*Hdf5Type<unsigned int>::type);
}
void Hdf5Reader::push(const std::string &s)
{
group_ = group_.openGroup(s);
path_.push_back(s);
}
void Hdf5Reader::pop(void)
{
path_.pop_back();
if (path_.empty())
{
group_ = file_.openGroup("/");
}
else
{
auto binOp = [](const std::string &a, const std::string &b)->std::string
{
return a + "/" + b;
};
group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(),
std::string(""), binOp));
}
}
template <>
void Hdf5Reader::readDefault(const std::string &s, std::string &x)
{
Attribute attribute;
attribute = group_.openAttribute(s);
StrType strType = attribute.getStrType();
x.resize(strType.getSize());
attribute.read(strType, &(x[0]));
std::cout << "length: " << strType.getSize() << std::endl;
std::cout << "string: |";
for (auto &c: x)
{
std::cout << "'" << c << "'|";
}
std::cout << std::endl;
}

View File

@ -12,139 +12,135 @@
#endif
// default thresold above which datasets are used instead of attributes
#ifndef H5_DEF_DATASET_THRES
#define H5_DEF_DATASET_THRES 6u
#ifndef HDF5_DEF_DATASET_THRES
#define HDF5_DEF_DATASET_THRES 6u
#endif
// name guard for Grid metadata
#define HDF5_GRID_GUARD "_Grid_"
namespace Grid
{
template <typename T>
struct is_arithmetic_vector
{
static constexpr bool value = false;
};
template <typename T>
struct is_arithmetic_vector<std::vector<T>>
{
static constexpr bool value = std::is_arithmetic<T>::value
or is_arithmetic_vector<T>::value;
};
class Hdf5Writer: public Writer<Hdf5Writer>
{
public:
Hdf5Writer(const std::string &fileName);
virtual ~Hdf5Writer(void);
virtual ~Hdf5Writer(void) = default;
void push(const std::string &s);
void pop(void);
void writeDefault(const std::string &s, const char *x);
template <typename U>
void writeDefault(const std::string &s, const U &x);
template <typename U>
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
and std::is_arithmetic<U>::value, void>::type
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
writeDefault(const std::string &s, const std::vector<U> &x);
template <typename U>
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
and !std::is_arithmetic<U>::value, void>::type
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
writeDefault(const std::string &s, const std::vector<U> &x);
private:
template <typename U>
typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
writeDefault(const std::string &s, const std::vector<U> &x);
void writeSingleAttribute(const U &x, const std::string &name,
const H5NS::DataType &type);
private:
std::string fileName_;
std::vector<std::string> path_;
std::vector<hsize_t> dim_;
bool multiDim_{true};
H5NS::H5File file_;
H5NS::Group group_;
unsigned int datasetThres_{H5_DEF_DATASET_THRES};
unsigned int dataSetThres_{HDF5_DEF_DATASET_THRES};
};
class Hdf5Reader: public Reader<Hdf5Reader>
{
public:
Hdf5Reader(const std::string &fileName);
virtual ~Hdf5Reader(void);
virtual ~Hdf5Reader(void) = default;
void push(const std::string &s);
void pop(void);
template <typename U>
void readDefault(const std::string &s, U &output);
template <typename U>
void readDefault(const std::string &s, std::vector<U> &output);
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
readDefault(const std::string &s, std::vector<U> &x);
template <typename U>
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
readDefault(const std::string &s, std::vector<U> &x);
private:
template <typename U>
void readSingleAttribute(U &x, const std::string &name,
const H5NS::DataType &type);
private:
std::string fileName_;
std::vector<std::string> path_;
H5NS::H5File file_;
H5NS::Group group_;
unsigned int dataSetThres_;
};
// Writer template implementation ////////////////////////////////////////////
template <typename U>
void Hdf5Writer::writeDefault(const std::string &s, const U &x)
void Hdf5Writer::writeSingleAttribute(const U &x, const std::string &name,
const H5NS::DataType &type)
{
H5NS::Attribute attribute;
hsize_t attrDim = 1;
H5NS::DataSpace attrSpace(1, &attrDim);
attribute = group_.createAttribute(s, *Hdf5Type<U>::type, attrSpace);
attribute.write(*Hdf5Type<U>::type, &x);
attribute = group_.createAttribute(name, type, attrSpace);
attribute.write(type, &x);
}
template <typename U>
void Hdf5Writer::writeDefault(const std::string &s, const U &x)
{
writeSingleAttribute(x, s, *Hdf5Type<U>::type);
}
template <>
void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
template <typename U>
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
and std::is_arithmetic<U>::value, void>::type
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
{
hsize_t size = 1;
// alias to element type
typedef typename element<std::vector<U>>::type Element;
dim_.push_back(x.size());
for (auto d: dim_)
// flatten the vector and getting dimensions
Flatten<std::vector<U>> flat(x);
std::vector<hsize_t> dim;
const auto &flatx = flat.getFlatVector();
for (auto &d: flat.getDim())
{
size *= d;
dim.push_back(d);
}
H5NS::DataSpace dataspace(dim_.size(), dim_.data());
// write to file
H5NS::DataSpace dataSpace(dim.size(), dim.data());
if (size > datasetThres_)
if (flatx.size() > dataSetThres_)
{
H5NS::DataSet dataset;
H5NS::DataSet dataSet;
dataset = group_.createDataSet(s, *Hdf5Type<U>::type, dataspace);
dataset.write(x.data(), *Hdf5Type<U>::type);
dataSet = group_.createDataSet(s, *Hdf5Type<Element>::type, dataSpace);
dataSet.write(flatx.data(), *Hdf5Type<Element>::type);
}
else
{
H5NS::Attribute attribute;
attribute = group_.createAttribute(s, *Hdf5Type<U>::type, dataspace);
attribute.write(*Hdf5Type<U>::type, x.data());
attribute = group_.createAttribute(s, *Hdf5Type<Element>::type, dataSpace);
attribute.write(*Hdf5Type<Element>::type, flatx.data());
}
dim_.clear();
multiDim_ = true;
}
template <typename U>
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
and !std::is_arithmetic<U>::value, void>::type
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
{
hsize_t firstSize = x[0].size();
for (auto &v: x)
{
multiDim_ = (multiDim_ and (v.size() == firstSize));
}
assert(multiDim_);
dim_.push_back(x.size());
writeDefault(s, x[0]);
}
template <typename U>
typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
{
push(s);
writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size",
*Hdf5Type<uint64_t>::type);
for (hsize_t i = 0; i < x.size(); ++i)
{
write(s + "_" + std::to_string(i), x[i]);
@ -154,15 +150,97 @@ namespace Grid
// Reader template implementation ////////////////////////////////////////////
template <typename U>
void Hdf5Reader::readDefault(const std::string &s, U &output)
void Hdf5Reader::readSingleAttribute(U &x, const std::string &name,
const H5NS::DataType &type)
{
H5NS::Attribute attribute;
attribute = group_.openAttribute(name);
attribute.read(type, &x);
}
template <typename U>
void Hdf5Reader::readDefault(const std::string &s, std::vector<U> &output)
void Hdf5Reader::readDefault(const std::string &s, U &output)
{
readSingleAttribute(output, s, *Hdf5Type<U>::type);
}
template <>
void Hdf5Reader::readDefault(const std::string &s, std::string &x);
template <typename U>
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
{
// alias to element type
typedef typename element<std::vector<U>>::type Element;
// read the dimensions
H5NS::DataSpace dataSpace;
H5E_auto2_t func;
void * client_data;
std::vector<hsize_t> hdim;
std::vector<size_t> dim;
hsize_t size = 1;
H5NS::Exception::getAutoPrint(func, &client_data);
try
{
H5NS::Exception::dontPrint();
dataSpace = group_.openDataSet(s).getSpace();
}
catch (H5NS::Exception &e)
{
H5NS::Exception::setAutoPrint(func, client_data);
dataSpace = group_.openAttribute(s).getSpace();
}
hdim.resize(dataSpace.getSimpleExtentNdims());
dataSpace.getSimpleExtentDims(hdim.data());
for (auto &d: hdim)
{
dim.push_back(d);
size *= d;
}
// read the flat vector
std::vector<Element> buf(size);
if (size > dataSetThres_)
{
H5NS::DataSet dataSet;
dataSet = group_.openDataSet(s);
dataSet.read(buf.data(), *Hdf5Type<Element>::type);
}
else
{
H5NS::Attribute attribute;
attribute = group_.openAttribute(s);
attribute.read(*Hdf5Type<Element>::type, buf.data());
}
// reconstruct the multidimensional vector
Reconstruct<std::vector<U>> r(buf, dim);
x = r.getVector();
}
template <typename U>
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
{
uint64_t size;
push(s);
readSingleAttribute(size, HDF5_GRID_GUARD "vector_size",
*Hdf5Type<uint64_t>::type);
x.resize(size);
for (hsize_t i = 0; i < x.size(); ++i)
{
read(s + "_" + std::to_string(i), x[i]);
}
pop();
}
}

View File

@ -104,8 +104,8 @@ int main(int argc,char **argv)
};
// read tests
myclass copy1, copy2, copy3;
std::vector<myclass> veccopy1, veccopy2, veccopy3;
myclass copy1, copy2, copy3, copy4;
std::vector<myclass> veccopy1, veccopy2, veccopy3, veccopy4;
//// XML
{
XmlReader RD("bother.xml");
@ -150,24 +150,41 @@ int main(int argc,char **argv)
}
{
Hdf5Reader TRD("bother.h5");
read (TRD,"discard",copy3 );
read (TRD,"discard2",veccopy3 );
std::cout << "read single" << std::endl;
read (TRD,"discard",copy4 );
std::cout << "read vec" << std::endl;
read (TRD,"discard2",veccopy4 );
std::cout << "Loaded (h5) -----------------" << std::endl;
std::cout << copy3 << std::endl << veccopy3 << std::endl;
std::cout << copy3 << std::endl << veccopy4 << std::endl;
}
#endif
std::vector<int> iv = strToVec<int>("1 2 2 4");
std::vector<std::string> sv = strToVec<std::string>("bli bla blu");
typedef std::vector<std::vector<std::vector<double>>> vec3d;
for (auto &e: iv)
vec3d dv, buf;
double d = 0.;
dv.resize(4);
for (auto &v1: dv)
{
std::cout << e << " ";
v1.resize(3);
for (auto &v2: v1)
{
v2.resize(5);
for (auto &x: v2)
{
x = d++;
}
}
}
std::cout << std::endl;
for (auto &e: sv)
{
std::cout << e << " ";
}
std::cout << std::endl;
std::cout << dv << std::endl;
Flatten<vec3d> flatdv(dv);
std::cout << flatdv.getDim() << std::endl;
std::cout << flatdv.getFlatVector() << std::endl;
Reconstruct<vec3d> rec(flatdv.getFlatVector(), flatdv.getDim());
std::cout << flatdv.getVector() << std::endl;
}