mirror of
https://github.com/paboyle/Grid.git
synced 2025-11-04 05:54:32 +00:00
HDF5 serial IO implemented and tested
This commit is contained in:
@@ -32,6 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
#include <type_traits>
|
||||
|
||||
namespace Grid {
|
||||
// Vector IO utilities ///////////////////////////////////////////////////////
|
||||
// helper function to read space-separated values
|
||||
template <typename T>
|
||||
std::vector<T> strToVec(const std::string s)
|
||||
@@ -67,6 +68,76 @@ namespace Grid {
|
||||
return os;
|
||||
}
|
||||
|
||||
// Vector element trait //////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
struct element
|
||||
{
|
||||
typedef T type;
|
||||
static constexpr bool is_arithmetic = false;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct element<std::vector<T>>
|
||||
{
|
||||
typedef typename element<T>::type type;
|
||||
static constexpr bool is_arithmetic = std::is_arithmetic<T>::value
|
||||
or element<T>::is_arithmetic;
|
||||
};
|
||||
|
||||
// Vector flatening utility class ////////////////////////////////////////////
|
||||
// Class to flatten a multidimensional std::vector
|
||||
template <typename V>
|
||||
class Flatten
|
||||
{
|
||||
public:
|
||||
typedef typename element<V>::type Element;
|
||||
public:
|
||||
explicit Flatten(const V &vector);
|
||||
const V & getVector(void);
|
||||
const std::vector<Element> & getFlatVector(void);
|
||||
const std::vector<size_t> & getDim(void);
|
||||
private:
|
||||
void accumulate(const Element &e);
|
||||
template <typename W>
|
||||
void accumulate(const W &v);
|
||||
void accumulateDim(const Element &e);
|
||||
template <typename W>
|
||||
void accumulateDim(const W &v);
|
||||
private:
|
||||
const V &vector_;
|
||||
std::vector<Element> flatVector_;
|
||||
std::vector<size_t> dim_;
|
||||
};
|
||||
|
||||
|
||||
// Class to reconstruct a multidimensional std::vector
|
||||
template <typename V>
|
||||
class Reconstruct
|
||||
{
|
||||
public:
|
||||
typedef typename element<V>::type Element;
|
||||
public:
|
||||
Reconstruct(const std::vector<Element> &flatVector,
|
||||
const std::vector<size_t> &dim);
|
||||
const V & getVector(void);
|
||||
const std::vector<Element> & getFlatVector(void);
|
||||
const std::vector<size_t> & getDim(void);
|
||||
private:
|
||||
void fill(std::vector<Element> &v);
|
||||
template <typename W>
|
||||
void fill(W &v);
|
||||
void resize(std::vector<Element> &v, const unsigned int dim);
|
||||
template <typename W>
|
||||
void resize(W &v, const unsigned int dim);
|
||||
private:
|
||||
V vector_;
|
||||
const std::vector<Element> &flatVector_;
|
||||
std::vector<size_t> dim_;
|
||||
size_t ind_{0};
|
||||
unsigned int dimInd_{0};
|
||||
};
|
||||
|
||||
// Abstract writer/reader classes ////////////////////////////////////////////
|
||||
// static polymorphism implemented using CRTP idiom
|
||||
class Serializable;
|
||||
|
||||
@@ -132,7 +203,128 @@ namespace Grid {
|
||||
}
|
||||
};
|
||||
|
||||
// Generic writer interface
|
||||
// Flatten class template implementation /////////////////////////////////////
|
||||
template <typename V>
|
||||
void Flatten<V>::accumulate(const Element &e)
|
||||
{
|
||||
flatVector_.push_back(e);
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
template <typename W>
|
||||
void Flatten<V>::accumulate(const W &v)
|
||||
{
|
||||
for (auto &e: v)
|
||||
{
|
||||
accumulate(e);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
void Flatten<V>::accumulateDim(const Element &e) {};
|
||||
|
||||
template <typename V>
|
||||
template <typename W>
|
||||
void Flatten<V>::accumulateDim(const W &v)
|
||||
{
|
||||
dim_.push_back(v.size());
|
||||
accumulateDim(v[0]);
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
Flatten<V>::Flatten(const V &vector)
|
||||
: vector_(vector)
|
||||
{
|
||||
accumulate(vector_);
|
||||
accumulateDim(vector_);
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
const V & Flatten<V>::getVector(void)
|
||||
{
|
||||
return vector_;
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
const std::vector<typename Flatten<V>::Element> &
|
||||
Flatten<V>::getFlatVector(void)
|
||||
{
|
||||
return flatVector_;
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
const std::vector<size_t> & Flatten<V>::getDim(void)
|
||||
{
|
||||
return dim_;
|
||||
}
|
||||
|
||||
// Reconstruct class template implementation /////////////////////////////////
|
||||
template <typename V>
|
||||
void Reconstruct<V>::fill(std::vector<Element> &v)
|
||||
{
|
||||
for (auto &e: v)
|
||||
{
|
||||
e = flatVector_[ind_++];
|
||||
}
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
template <typename W>
|
||||
void Reconstruct<V>::fill(W &v)
|
||||
{
|
||||
for (auto &e: v)
|
||||
{
|
||||
fill(e);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
void Reconstruct<V>::resize(std::vector<Element> &v, const unsigned int dim)
|
||||
{
|
||||
v.resize(dim_[dim]);
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
template <typename W>
|
||||
void Reconstruct<V>::resize(W &v, const unsigned int dim)
|
||||
{
|
||||
v.resize(dim_[dim]);
|
||||
for (auto &e: v)
|
||||
{
|
||||
resize(e, dim + 1);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
Reconstruct<V>::Reconstruct(const std::vector<Element> &flatVector,
|
||||
const std::vector<size_t> &dim)
|
||||
: flatVector_(flatVector)
|
||||
, dim_(dim)
|
||||
{
|
||||
resize(vector_, 0);
|
||||
fill(vector_);
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
const V & Reconstruct<V>::Reconstruct<V>::getVector(void)
|
||||
{
|
||||
return vector_;
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
const std::vector<typename Reconstruct<V>::Element> &
|
||||
Reconstruct<V>::getFlatVector(void)
|
||||
{
|
||||
return flatVector_;
|
||||
}
|
||||
|
||||
template <typename V>
|
||||
const std::vector<size_t> & Reconstruct<V>::getDim(void)
|
||||
{
|
||||
return dim_;
|
||||
}
|
||||
|
||||
// Generic writer interface //////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
inline void push(Writer<T> &w, const std::string &s)
|
||||
{
|
||||
@@ -217,7 +409,7 @@ namespace Grid {
|
||||
upcast->writeDefault(s, output);
|
||||
}
|
||||
|
||||
// Reader template implementation ////////////////////////////////////////////
|
||||
// Reader template implementation
|
||||
template <typename T>
|
||||
Reader<T>::Reader(void)
|
||||
{
|
||||
|
||||
@@ -11,11 +11,8 @@ Hdf5Writer::Hdf5Writer(const std::string &fileName)
|
||||
, file_(fileName.c_str(), H5F_ACC_TRUNC)
|
||||
{
|
||||
group_ = file_.openGroup("/");
|
||||
}
|
||||
|
||||
Hdf5Writer::~Hdf5Writer(void)
|
||||
{
|
||||
file_.close();
|
||||
writeSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
|
||||
*Hdf5Type<unsigned int>::type);
|
||||
}
|
||||
|
||||
void Hdf5Writer::push(const std::string &s)
|
||||
@@ -47,12 +44,8 @@ template <>
|
||||
void Hdf5Writer::writeDefault(const std::string &s, const std::string &x)
|
||||
{
|
||||
StrType strType(PredType::C_S1, x.size());
|
||||
Attribute attribute;
|
||||
hsize_t attrDim = 1;
|
||||
DataSpace attrSpace(1, &attrDim);
|
||||
|
||||
attribute = group_.createAttribute(s, strType, attrSpace);
|
||||
attribute.write(strType, x.data());
|
||||
writeSingleAttribute(*(x.data()), s, strType);
|
||||
}
|
||||
|
||||
void Hdf5Writer::writeDefault(const std::string &s, const char *x)
|
||||
@@ -64,21 +57,55 @@ void Hdf5Writer::writeDefault(const std::string &s, const char *x)
|
||||
|
||||
// Reader implementation ///////////////////////////////////////////////////////
|
||||
Hdf5Reader::Hdf5Reader(const std::string &fileName)
|
||||
: fileName_(fileName)
|
||||
, file_(fileName.c_str(), H5F_ACC_RDONLY)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
Hdf5Reader::~Hdf5Reader(void)
|
||||
{
|
||||
|
||||
group_ = file_.openGroup("/");
|
||||
readSingleAttribute(dataSetThres_, HDF5_GRID_GUARD "dataset_threshold",
|
||||
*Hdf5Type<unsigned int>::type);
|
||||
}
|
||||
|
||||
void Hdf5Reader::push(const std::string &s)
|
||||
{
|
||||
|
||||
group_ = group_.openGroup(s);
|
||||
path_.push_back(s);
|
||||
}
|
||||
|
||||
void Hdf5Reader::pop(void)
|
||||
{
|
||||
path_.pop_back();
|
||||
if (path_.empty())
|
||||
{
|
||||
group_ = file_.openGroup("/");
|
||||
}
|
||||
else
|
||||
{
|
||||
auto binOp = [](const std::string &a, const std::string &b)->std::string
|
||||
{
|
||||
return a + "/" + b;
|
||||
};
|
||||
|
||||
group_ = group_.openGroup(std::accumulate(path_.begin(), path_.end(),
|
||||
std::string(""), binOp));
|
||||
}
|
||||
}
|
||||
|
||||
template <>
|
||||
void Hdf5Reader::readDefault(const std::string &s, std::string &x)
|
||||
{
|
||||
Attribute attribute;
|
||||
|
||||
attribute = group_.openAttribute(s);
|
||||
StrType strType = attribute.getStrType();
|
||||
|
||||
x.resize(strType.getSize());
|
||||
attribute.read(strType, &(x[0]));
|
||||
|
||||
std::cout << "length: " << strType.getSize() << std::endl;
|
||||
std::cout << "string: |";
|
||||
for (auto &c: x)
|
||||
{
|
||||
std::cout << "'" << c << "'|";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
@@ -12,139 +12,135 @@
|
||||
#endif
|
||||
|
||||
// default thresold above which datasets are used instead of attributes
|
||||
#ifndef H5_DEF_DATASET_THRES
|
||||
#define H5_DEF_DATASET_THRES 6u
|
||||
#ifndef HDF5_DEF_DATASET_THRES
|
||||
#define HDF5_DEF_DATASET_THRES 6u
|
||||
#endif
|
||||
|
||||
// name guard for Grid metadata
|
||||
#define HDF5_GRID_GUARD "_Grid_"
|
||||
|
||||
namespace Grid
|
||||
{
|
||||
template <typename T>
|
||||
struct is_arithmetic_vector
|
||||
{
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct is_arithmetic_vector<std::vector<T>>
|
||||
{
|
||||
static constexpr bool value = std::is_arithmetic<T>::value
|
||||
or is_arithmetic_vector<T>::value;
|
||||
};
|
||||
|
||||
class Hdf5Writer: public Writer<Hdf5Writer>
|
||||
{
|
||||
public:
|
||||
Hdf5Writer(const std::string &fileName);
|
||||
virtual ~Hdf5Writer(void);
|
||||
virtual ~Hdf5Writer(void) = default;
|
||||
void push(const std::string &s);
|
||||
void pop(void);
|
||||
void writeDefault(const std::string &s, const char *x);
|
||||
template <typename U>
|
||||
void writeDefault(const std::string &s, const U &x);
|
||||
template <typename U>
|
||||
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
|
||||
and std::is_arithmetic<U>::value, void>::type
|
||||
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
|
||||
writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
template <typename U>
|
||||
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
|
||||
and !std::is_arithmetic<U>::value, void>::type
|
||||
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
|
||||
writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
private:
|
||||
template <typename U>
|
||||
typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
|
||||
writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
void writeSingleAttribute(const U &x, const std::string &name,
|
||||
const H5NS::DataType &type);
|
||||
private:
|
||||
std::string fileName_;
|
||||
std::vector<std::string> path_;
|
||||
std::vector<hsize_t> dim_;
|
||||
bool multiDim_{true};
|
||||
H5NS::H5File file_;
|
||||
H5NS::Group group_;
|
||||
unsigned int datasetThres_{H5_DEF_DATASET_THRES};
|
||||
unsigned int dataSetThres_{HDF5_DEF_DATASET_THRES};
|
||||
};
|
||||
|
||||
class Hdf5Reader: public Reader<Hdf5Reader>
|
||||
{
|
||||
public:
|
||||
Hdf5Reader(const std::string &fileName);
|
||||
virtual ~Hdf5Reader(void);
|
||||
virtual ~Hdf5Reader(void) = default;
|
||||
void push(const std::string &s);
|
||||
void pop(void);
|
||||
template <typename U>
|
||||
void readDefault(const std::string &s, U &output);
|
||||
template <typename U>
|
||||
void readDefault(const std::string &s, std::vector<U> &output);
|
||||
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
|
||||
readDefault(const std::string &s, std::vector<U> &x);
|
||||
template <typename U>
|
||||
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
|
||||
readDefault(const std::string &s, std::vector<U> &x);
|
||||
private:
|
||||
template <typename U>
|
||||
void readSingleAttribute(U &x, const std::string &name,
|
||||
const H5NS::DataType &type);
|
||||
private:
|
||||
std::string fileName_;
|
||||
std::vector<std::string> path_;
|
||||
H5NS::H5File file_;
|
||||
H5NS::Group group_;
|
||||
unsigned int dataSetThres_;
|
||||
};
|
||||
|
||||
// Writer template implementation ////////////////////////////////////////////
|
||||
template <typename U>
|
||||
void Hdf5Writer::writeDefault(const std::string &s, const U &x)
|
||||
void Hdf5Writer::writeSingleAttribute(const U &x, const std::string &name,
|
||||
const H5NS::DataType &type)
|
||||
{
|
||||
H5NS::Attribute attribute;
|
||||
hsize_t attrDim = 1;
|
||||
H5NS::DataSpace attrSpace(1, &attrDim);
|
||||
|
||||
attribute = group_.createAttribute(s, *Hdf5Type<U>::type, attrSpace);
|
||||
attribute.write(*Hdf5Type<U>::type, &x);
|
||||
attribute = group_.createAttribute(name, type, attrSpace);
|
||||
attribute.write(type, &x);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void Hdf5Writer::writeDefault(const std::string &s, const U &x)
|
||||
{
|
||||
writeSingleAttribute(x, s, *Hdf5Type<U>::type);
|
||||
}
|
||||
|
||||
template <>
|
||||
void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
|
||||
and std::is_arithmetic<U>::value, void>::type
|
||||
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
|
||||
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
|
||||
{
|
||||
hsize_t size = 1;
|
||||
// alias to element type
|
||||
typedef typename element<std::vector<U>>::type Element;
|
||||
|
||||
dim_.push_back(x.size());
|
||||
for (auto d: dim_)
|
||||
// flatten the vector and getting dimensions
|
||||
Flatten<std::vector<U>> flat(x);
|
||||
std::vector<hsize_t> dim;
|
||||
const auto &flatx = flat.getFlatVector();
|
||||
|
||||
for (auto &d: flat.getDim())
|
||||
{
|
||||
size *= d;
|
||||
dim.push_back(d);
|
||||
}
|
||||
|
||||
H5NS::DataSpace dataspace(dim_.size(), dim_.data());
|
||||
// write to file
|
||||
H5NS::DataSpace dataSpace(dim.size(), dim.data());
|
||||
|
||||
if (size > datasetThres_)
|
||||
if (flatx.size() > dataSetThres_)
|
||||
{
|
||||
H5NS::DataSet dataset;
|
||||
H5NS::DataSet dataSet;
|
||||
|
||||
dataset = group_.createDataSet(s, *Hdf5Type<U>::type, dataspace);
|
||||
dataset.write(x.data(), *Hdf5Type<U>::type);
|
||||
dataSet = group_.createDataSet(s, *Hdf5Type<Element>::type, dataSpace);
|
||||
dataSet.write(flatx.data(), *Hdf5Type<Element>::type);
|
||||
}
|
||||
else
|
||||
{
|
||||
H5NS::Attribute attribute;
|
||||
|
||||
attribute = group_.createAttribute(s, *Hdf5Type<U>::type, dataspace);
|
||||
attribute.write(*Hdf5Type<U>::type, x.data());
|
||||
attribute = group_.createAttribute(s, *Hdf5Type<Element>::type, dataSpace);
|
||||
attribute.write(*Hdf5Type<Element>::type, flatx.data());
|
||||
}
|
||||
dim_.clear();
|
||||
multiDim_ = true;
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<is_arithmetic_vector<std::vector<U>>::value
|
||||
and !std::is_arithmetic<U>::value, void>::type
|
||||
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
|
||||
{
|
||||
hsize_t firstSize = x[0].size();
|
||||
|
||||
for (auto &v: x)
|
||||
{
|
||||
multiDim_ = (multiDim_ and (v.size() == firstSize));
|
||||
}
|
||||
assert(multiDim_);
|
||||
dim_.push_back(x.size());
|
||||
writeDefault(s, x[0]);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<!is_arithmetic_vector<std::vector<U>>::value, void>::type
|
||||
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
|
||||
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
|
||||
{
|
||||
push(s);
|
||||
writeSingleAttribute(x.size(), HDF5_GRID_GUARD "vector_size",
|
||||
*Hdf5Type<uint64_t>::type);
|
||||
for (hsize_t i = 0; i < x.size(); ++i)
|
||||
{
|
||||
write(s + "_" + std::to_string(i), x[i]);
|
||||
@@ -154,15 +150,97 @@ namespace Grid
|
||||
|
||||
// Reader template implementation ////////////////////////////////////////////
|
||||
template <typename U>
|
||||
void Hdf5Reader::readDefault(const std::string &s, U &output)
|
||||
void Hdf5Reader::readSingleAttribute(U &x, const std::string &name,
|
||||
const H5NS::DataType &type)
|
||||
{
|
||||
H5NS::Attribute attribute;
|
||||
|
||||
attribute = group_.openAttribute(name);
|
||||
attribute.read(type, &x);
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void Hdf5Reader::readDefault(const std::string &s, std::vector<U> &output)
|
||||
void Hdf5Reader::readDefault(const std::string &s, U &output)
|
||||
{
|
||||
readSingleAttribute(output, s, *Hdf5Type<U>::type);
|
||||
}
|
||||
|
||||
template <>
|
||||
void Hdf5Reader::readDefault(const std::string &s, std::string &x);
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<element<std::vector<U>>::is_arithmetic, void>::type
|
||||
Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
|
||||
{
|
||||
// alias to element type
|
||||
typedef typename element<std::vector<U>>::type Element;
|
||||
|
||||
// read the dimensions
|
||||
H5NS::DataSpace dataSpace;
|
||||
H5E_auto2_t func;
|
||||
void * client_data;
|
||||
std::vector<hsize_t> hdim;
|
||||
std::vector<size_t> dim;
|
||||
hsize_t size = 1;
|
||||
|
||||
H5NS::Exception::getAutoPrint(func, &client_data);
|
||||
try
|
||||
{
|
||||
H5NS::Exception::dontPrint();
|
||||
dataSpace = group_.openDataSet(s).getSpace();
|
||||
}
|
||||
catch (H5NS::Exception &e)
|
||||
{
|
||||
H5NS::Exception::setAutoPrint(func, client_data);
|
||||
dataSpace = group_.openAttribute(s).getSpace();
|
||||
}
|
||||
hdim.resize(dataSpace.getSimpleExtentNdims());
|
||||
dataSpace.getSimpleExtentDims(hdim.data());
|
||||
for (auto &d: hdim)
|
||||
{
|
||||
dim.push_back(d);
|
||||
size *= d;
|
||||
}
|
||||
|
||||
// read the flat vector
|
||||
std::vector<Element> buf(size);
|
||||
|
||||
if (size > dataSetThres_)
|
||||
{
|
||||
H5NS::DataSet dataSet;
|
||||
|
||||
dataSet = group_.openDataSet(s);
|
||||
dataSet.read(buf.data(), *Hdf5Type<Element>::type);
|
||||
}
|
||||
else
|
||||
{
|
||||
H5NS::Attribute attribute;
|
||||
|
||||
attribute = group_.openAttribute(s);
|
||||
attribute.read(*Hdf5Type<Element>::type, buf.data());
|
||||
}
|
||||
|
||||
// reconstruct the multidimensional vector
|
||||
Reconstruct<std::vector<U>> r(buf, dim);
|
||||
|
||||
x = r.getVector();
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<!element<std::vector<U>>::is_arithmetic, void>::type
|
||||
Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
|
||||
{
|
||||
uint64_t size;
|
||||
|
||||
push(s);
|
||||
readSingleAttribute(size, HDF5_GRID_GUARD "vector_size",
|
||||
*Hdf5Type<uint64_t>::type);
|
||||
x.resize(size);
|
||||
for (hsize_t i = 0; i < x.size(); ++i)
|
||||
{
|
||||
read(s + "_" + std::to_string(i), x[i]);
|
||||
}
|
||||
pop();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user