mirror of
https://github.com/paboyle/Grid.git
synced 2025-06-18 15:57:05 +01:00
Eigen::Tensor serialisation. Tested on single and double precision builds
This commit is contained in:
Michael Marshall
@ -33,12 +33,66 @@ Author: Guido Cossu <guido.cossu@ed.ac.uk>
|
||||
#include <type_traits>
|
||||
#include <Grid/tensors/Tensors.h>
|
||||
#include <Grid/serialisation/VectorUtils.h>
|
||||
#include <Grid/Eigen/unsupported/CXX11/Tensor>
|
||||
|
||||
namespace Grid {
|
||||
namespace EigenIO {
|
||||
// EigenIO works for scalars that are not just Grid supported scalars
|
||||
template<typename T, typename V = void> struct is_complex : public std::false_type {};
|
||||
// Support all complex types (not just Grid complex types) - even if the definitions overlap (!)
|
||||
template<typename T> struct is_complex< T , typename
|
||||
std::enable_if< ::Grid::is_complex< T >::value>::type> : public std::true_type {};
|
||||
template<typename T> struct is_complex<std::complex<T>, typename
|
||||
std::enable_if<!::Grid::is_complex<std::complex<T>>::value>::type> : public std::true_type {};
|
||||
|
||||
// Helpers to support I/O for Eigen tensors of arithmetic scalars, complex types, or Grid tensors
|
||||
template<typename T, typename V = void> struct is_scalar : public std::false_type {};
|
||||
template<typename T> struct is_scalar<T, typename std::enable_if<std::is_arithmetic<T>::value || is_complex<T>::value>::type> : public std::true_type {};
|
||||
|
||||
// Is this an Eigen tensor
|
||||
template<typename T> struct is_tensor : std::integral_constant<bool,
|
||||
std::is_base_of<Eigen::TensorBase<T, Eigen::ReadOnlyAccessors>, T>::value> {};
|
||||
|
||||
// Is this an Eigen tensor of a supported scalar
|
||||
template<typename T, typename V = void> struct is_tensor_of_scalar : public std::false_type {};
|
||||
template<typename T> struct is_tensor_of_scalar<T, typename std::enable_if<is_tensor<T>::value && is_scalar<typename T::Scalar>::value>::type> : public std::true_type {};
|
||||
|
||||
// Is this an Eigen tensor of a supported container
|
||||
template<typename T, typename V = void> struct is_tensor_of_container : public std::false_type {};
|
||||
template<typename T> struct is_tensor_of_container<T, typename std::enable_if<is_tensor<T>::value && isGridTensor<typename T::Scalar>::value>::type> : public std::true_type {};
|
||||
|
||||
// Traits are the default for scalars, or come from GridTypeMapper for GridTensors
|
||||
template<typename T, typename V = void> struct Traits {};
|
||||
template<typename T> struct Traits<T, typename std::enable_if<is_tensor_of_scalar<T>::value>::type> : public GridTypeMapper_Base {
|
||||
using scalar_type = typename T::Scalar;
|
||||
static constexpr bool is_complex = ::Grid::EigenIO::is_complex<scalar_type>::value;
|
||||
};
|
||||
template<typename T> struct Traits<T, typename std::enable_if<is_tensor_of_container<T>::value>::type> : public GridTypeMapper<typename T::Scalar> {
|
||||
using scalar_type = typename GridTypeMapper<typename T::Scalar>::scalar_type;
|
||||
static constexpr bool is_complex = ::Grid::EigenIO::is_complex<scalar_type>::value;
|
||||
};
|
||||
|
||||
// Is this a fixed-size Eigen tensor
|
||||
template<typename T> struct is_tensor_fixed : public std::false_type {};
|
||||
template<typename Scalar_, typename Dimensions_, int Options_, typename IndexType>
|
||||
struct is_tensor_fixed<Eigen::TensorFixedSize<Scalar_, Dimensions_, Options_, IndexType>>
|
||||
: public std::true_type {};
|
||||
template<typename Scalar_, typename Dimensions_, int Options_, typename IndexType,
|
||||
int MapOptions_, template <class> class MapPointer_>
|
||||
struct is_tensor_fixed<Eigen::TensorMap<Eigen::TensorFixedSize<Scalar_, Dimensions_,
|
||||
Options_, IndexType>, MapOptions_, MapPointer_>>
|
||||
: public std::true_type {};
|
||||
|
||||
// Is this a variable-size Eigen tensor
|
||||
template<typename T, typename V = void> struct is_tensor_variable : public std::false_type {};
|
||||
template<typename T> struct is_tensor_variable<T, typename std::enable_if<is_tensor<T>::value
|
||||
&& !is_tensor_fixed<T>::value>::type> : public std::true_type {};
|
||||
}
|
||||
|
||||
// Abstract writer/reader classes ////////////////////////////////////////////
|
||||
// static polymorphism implemented using CRTP idiom
|
||||
class Serializable;
|
||||
|
||||
|
||||
// Static abstract writer
|
||||
template <typename T>
|
||||
class Writer
|
||||
@ -49,10 +103,10 @@ namespace Grid {
|
||||
void push(const std::string &s);
|
||||
void pop(void);
|
||||
template <typename U>
|
||||
typename std::enable_if<std::is_base_of<Serializable, U>::value, void>::type
|
||||
typename std::enable_if<std::is_base_of<Serializable, U>::value>::type
|
||||
write(const std::string& s, const U &output);
|
||||
template <typename U>
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value, void>::type
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value && !EigenIO::is_tensor<U>::value>::type
|
||||
write(const std::string& s, const U &output);
|
||||
template <typename U>
|
||||
void write(const std::string &s, const iScalar<U> &output);
|
||||
@ -60,6 +114,42 @@ namespace Grid {
|
||||
void write(const std::string &s, const iVector<U, N> &output);
|
||||
template <typename U, int N>
|
||||
void write(const std::string &s, const iMatrix<U, N> &output);
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor<ETensor>::value>::type
|
||||
write(const std::string &s, const ETensor &output);
|
||||
|
||||
// Helper functions for Scalar vs Container specialisations
|
||||
template <typename ETensor>
|
||||
inline typename std::enable_if<EigenIO::is_tensor_of_scalar<ETensor>::value,
|
||||
const typename ETensor::Scalar *>::type
|
||||
getFirstScalar(const ETensor &output)
|
||||
{
|
||||
return output.data();
|
||||
}
|
||||
|
||||
template <typename ETensor>
|
||||
inline typename std::enable_if<EigenIO::is_tensor_of_container<ETensor>::value,
|
||||
const typename EigenIO::Traits<ETensor>::scalar_type *>::type
|
||||
getFirstScalar(const ETensor &output)
|
||||
{
|
||||
return output.data()->begin();
|
||||
}
|
||||
|
||||
template <typename S>
|
||||
inline typename std::enable_if<EigenIO::is_scalar<S>::value, void>::type
|
||||
copyScalars(S * &pCopy, const S &Source)
|
||||
{
|
||||
* pCopy ++ = Source;
|
||||
}
|
||||
|
||||
template <typename S>
|
||||
inline typename std::enable_if<isGridTensor<S>::value, void>::type
|
||||
copyScalars(typename GridTypeMapper<S>::scalar_type * &pCopy, const S &Source)
|
||||
{
|
||||
for( const typename GridTypeMapper<S>::scalar_type &item : Source )
|
||||
* pCopy ++ = item;
|
||||
}
|
||||
|
||||
void scientificFormat(const bool set);
|
||||
bool isScientific(void);
|
||||
void setPrecision(const unsigned int prec);
|
||||
@ -83,7 +173,8 @@ namespace Grid {
|
||||
typename std::enable_if<std::is_base_of<Serializable, U>::value, void>::type
|
||||
read(const std::string& s, U &output);
|
||||
template <typename U>
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value, void>::type
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value
|
||||
&& !EigenIO::is_tensor<U>::value, void>::type
|
||||
read(const std::string& s, U &output);
|
||||
template <typename U>
|
||||
void read(const std::string &s, iScalar<U> &output);
|
||||
@ -91,6 +182,32 @@ namespace Grid {
|
||||
void read(const std::string &s, iVector<U, N> &output);
|
||||
template <typename U, int N>
|
||||
void read(const std::string &s, iMatrix<U, N> &output);
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor<ETensor>::value, void>::type
|
||||
read(const std::string &s, ETensor &output);
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor_fixed<ETensor>::value, void>::type
|
||||
Reshape(ETensor &t, const std::array<typename ETensor::Index, ETensor::NumDimensions> &dims );
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor_variable<ETensor>::value, void>::type
|
||||
Reshape(ETensor &t, const std::array<typename ETensor::Index, ETensor::NumDimensions> &dims );
|
||||
|
||||
// Helper functions for Scalar vs Container specialisations
|
||||
template <typename S>
|
||||
inline typename std::enable_if<EigenIO::is_scalar<S>::value, void>::type
|
||||
copyScalars(S &Dest, const S * &pSource)
|
||||
{
|
||||
Dest = * pSource ++;
|
||||
}
|
||||
|
||||
template <typename S>
|
||||
inline typename std::enable_if<isGridTensor<S>::value, void>::type
|
||||
copyScalars(S &Dest, const typename GridTypeMapper<S>::scalar_type * &pSource)
|
||||
{
|
||||
for( typename GridTypeMapper<S>::scalar_type &item : Dest )
|
||||
item = * pSource ++;
|
||||
}
|
||||
|
||||
protected:
|
||||
template <typename U>
|
||||
void fromString(U &output, const std::string &s);
|
||||
@ -135,12 +252,14 @@ namespace Grid {
|
||||
|
||||
template <typename T>
|
||||
template <typename U>
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value, void>::type
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value
|
||||
&& !EigenIO::is_tensor<U>::value, void>::type
|
||||
Writer<T>::write(const std::string &s, const U &output)
|
||||
{
|
||||
upcast->writeDefault(s, output);
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
template <typename U>
|
||||
void Writer<T>::write(const std::string &s, const iScalar<U> &output)
|
||||
@ -161,6 +280,58 @@ namespace Grid {
|
||||
{
|
||||
upcast->writeDefault(s, tensorToVec(output));
|
||||
}
|
||||
|
||||
// Eigen::Tensors of Grid tensors (iScalar, iVector, iMatrix)
|
||||
template <typename T>
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor<ETensor>::value, void>::type
|
||||
Writer<T>::write(const std::string &s, const ETensor &output)
|
||||
{
|
||||
using Index = typename ETensor::Index;
|
||||
using Container = typename ETensor::Scalar; // NB: could be same as scalar
|
||||
using Traits = EigenIO::Traits<ETensor>;
|
||||
using Scalar = typename Traits::scalar_type; // type of the underlying scalar
|
||||
constexpr unsigned int TensorRank{ETensor::NumIndices};
|
||||
constexpr unsigned int ContainerRank{Traits::Rank}; // Only non-zero for containers
|
||||
constexpr unsigned int TotalRank{TensorRank + ContainerRank};
|
||||
const Index NumElements{output.size()};
|
||||
assert( NumElements > 0 );
|
||||
|
||||
// Get the dimensionality of the tensor
|
||||
std::vector<std::size_t> TotalDims(TotalRank);
|
||||
for(auto i = 0; i < TensorRank; i++ ) {
|
||||
auto dim = output.dimension(i);
|
||||
TotalDims[i] = static_cast<size_t>(dim);
|
||||
assert( TotalDims[i] == dim ); // check we didn't lose anything in the conversion
|
||||
}
|
||||
for(auto i = 0; i < ContainerRank; i++ )
|
||||
TotalDims[TensorRank + i] = Traits::Dimension(i);
|
||||
|
||||
// If the Tensor isn't in Row-Major order, then we'll need to copy it's data
|
||||
const bool CopyData{NumElements > 1 && ETensor::Layout != Eigen::StorageOptions::RowMajor};
|
||||
const Scalar * pWriteBuffer;
|
||||
Scalar * pCopyBuffer = nullptr;
|
||||
const Index TotalNumElements = NumElements * Traits::count;
|
||||
if( !CopyData ) {
|
||||
pWriteBuffer = getFirstScalar( output );
|
||||
} else {
|
||||
// Regardless of the Eigen::Tensor storage order, the copy will be Row Major
|
||||
pCopyBuffer = new Scalar[TotalNumElements];
|
||||
pWriteBuffer = pCopyBuffer;
|
||||
Scalar * pCopy = pCopyBuffer;
|
||||
std::array<Index, TensorRank> MyIndex;
|
||||
for( auto &idx : MyIndex ) idx = 0;
|
||||
for( auto n = 0; n < NumElements; n++ ) {
|
||||
const Container & c = output( MyIndex );
|
||||
copyScalars( pCopy, c );
|
||||
// Now increment the index
|
||||
for( int i = output.NumDimensions - 1; i >= 0 && ++MyIndex[i] == output.dimension(i); i-- )
|
||||
MyIndex[i] = 0;
|
||||
}
|
||||
}
|
||||
upcast->template writeMultiDim<Scalar>(s, TotalDims, pWriteBuffer, TotalNumElements);
|
||||
if( pCopyBuffer ) delete [] pCopyBuffer;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void Writer<T>::scientificFormat(const bool set)
|
||||
@ -215,7 +386,8 @@ namespace Grid {
|
||||
|
||||
template <typename T>
|
||||
template <typename U>
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value, void>::type
|
||||
typename std::enable_if<!std::is_base_of<Serializable, U>::value
|
||||
&& !EigenIO::is_tensor<U>::value, void>::type
|
||||
Reader<T>::read(const std::string &s, U &output)
|
||||
{
|
||||
upcast->readDefault(s, output);
|
||||
@ -251,6 +423,79 @@ namespace Grid {
|
||||
vecToTensor(output, v);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor<ETensor>::value, void>::type
|
||||
Reader<T>::read(const std::string &s, ETensor &output)
|
||||
{
|
||||
using Index = typename ETensor::Index;
|
||||
using Container = typename ETensor::Scalar; // NB: could be same as scalar
|
||||
using Traits = EigenIO::Traits<ETensor>;
|
||||
using Scalar = typename Traits::scalar_type; // type of the underlying scalar
|
||||
constexpr unsigned int TensorRank{ETensor::NumIndices};
|
||||
constexpr unsigned int ContainerRank{Traits::Rank}; // Only non-zero for containers
|
||||
constexpr unsigned int TotalRank{TensorRank + ContainerRank};
|
||||
using ETDims = std::array<Index, TensorRank>; // Dimensions of the tensor
|
||||
|
||||
// read the (flat) data and dimensionality
|
||||
std::vector<std::size_t> dimData;
|
||||
std::vector<Scalar> buf;
|
||||
upcast->readMultiDim( s, buf, dimData );
|
||||
assert(dimData.size() == TotalRank && "EigenIO: Tensor rank mismatch" );
|
||||
// Make sure that the number of elements read matches dimensions read
|
||||
std::size_t NumContainers = 1;
|
||||
for( auto i = 0 ; i < TensorRank ; i++ )
|
||||
NumContainers *= dimData[i];
|
||||
// If our scalar object is a Container, make sure it's dimensions match what we read back
|
||||
std::size_t ElementsPerContainer = 1;
|
||||
for( auto i = 0 ; i < ContainerRank ; i++ ) {
|
||||
assert( dimData[TensorRank+i] == Traits::Dimension(i) && "Tensor Container dimensions don't match data" );
|
||||
ElementsPerContainer *= dimData[TensorRank+i];
|
||||
}
|
||||
assert( NumContainers * ElementsPerContainer == buf.size() && "EigenIO: Number of elements != product of dimensions" );
|
||||
// Now see whether the tensor is the right shape, or can be made to be
|
||||
const auto & dims{output.dimensions()};
|
||||
bool bShapeOK = (output.data() != nullptr);
|
||||
for( auto i = 0; bShapeOK && i < TensorRank ; i++ )
|
||||
if( dims[i] != dimData[i] )
|
||||
bShapeOK = false;
|
||||
// Make the tensor the same size as the data read
|
||||
ETDims MyIndex;
|
||||
if( !bShapeOK ) {
|
||||
for( auto i = 0 ; i < TensorRank ; i++ )
|
||||
MyIndex[i] = dimData[i];
|
||||
Reshape(output, MyIndex);
|
||||
}
|
||||
// Copy the data into the tensor
|
||||
for( auto &d : MyIndex ) d = 0;
|
||||
const Scalar * pSource = &buf[0];
|
||||
for( std::size_t n = 0 ; n < NumContainers ; n++ ) {
|
||||
Container & c = output( MyIndex );
|
||||
copyScalars( c, pSource );
|
||||
// Now increment the index
|
||||
for( int i = TensorRank - 1; i != -1 && ++MyIndex[i] == dims[i]; i-- )
|
||||
MyIndex[i] = 0;
|
||||
}
|
||||
assert( pSource == &buf[NumContainers * ElementsPerContainer] );
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor_fixed<ETensor>::value, void>::type
|
||||
Reader<T>::Reshape(ETensor &t, const std::array<typename ETensor::Index, ETensor::NumDimensions> &dims )
|
||||
{
|
||||
assert( 0 && "EigenIO: Fixed tensor dimensions can't be changed" );
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename ETensor>
|
||||
typename std::enable_if<EigenIO::is_tensor_variable<ETensor>::value, void>::type
|
||||
Reader<T>::Reshape(ETensor &t, const std::array<typename ETensor::Index, ETensor::NumDimensions> &dims )
|
||||
{
|
||||
//t.reshape( dims );
|
||||
t.resize( dims );
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename U>
|
||||
void Reader<T>::fromString(U &output, const std::string &s)
|
||||
@ -289,8 +534,50 @@ namespace Grid {
|
||||
{
|
||||
return os;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2>
|
||||
static inline typename std::enable_if<!EigenIO::is_tensor<T1>::value || !EigenIO::is_tensor<T2>::value, bool>::type
|
||||
CompareMember(const T1 &lhs, const T2 &rhs) {
|
||||
return lhs == rhs;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2>
|
||||
static inline typename std::enable_if<EigenIO::is_tensor<T1>::value && EigenIO::is_tensor<T2>::value, bool>::type
|
||||
CompareMember(const T1 &lhs, const T2 &rhs) {
|
||||
// First check whether dimensions match (Eigen tensor library will assert if they don't match)
|
||||
bool bReturnValue = (T1::NumIndices == T2::NumIndices);
|
||||
for( auto i = 0 ; bReturnValue && i < T1::NumIndices ; i++ )
|
||||
bReturnValue = ( lhs.dimension(i) == rhs.dimension(i) );
|
||||
if( bReturnValue ) {
|
||||
Eigen::Tensor<bool, 0, T1::Options> bResult = (lhs == rhs).all();
|
||||
bReturnValue = bResult(0);
|
||||
}
|
||||
return bReturnValue;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<EigenIO::is_tensor<T>::value, bool>::type
|
||||
CompareMember(const std::vector<T> &lhs, const std::vector<T> &rhs) {
|
||||
const auto NumElements{lhs.size()};
|
||||
bool bResult = ( NumElements == rhs.size() );
|
||||
for( auto i = 0 ; i < NumElements && bResult ; i++ )
|
||||
bResult = CompareMember(lhs[i], rhs[i]);
|
||||
return bResult;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<!EigenIO::is_tensor<T>::value, void>::type
|
||||
WriteMember(std::ostream &os, const T &object) {
|
||||
os << object;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static inline typename std::enable_if<EigenIO::is_tensor<T>::value, void>::type
|
||||
WriteMember(std::ostream &os, const T &object) {
|
||||
os << "Eigen::Tensor";
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Generic writer interface //////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
inline void push(Writer<T> &w, const std::string &s) {
|
||||
|
||||
@ -51,6 +51,8 @@ namespace Grid {
|
||||
template <typename U>
|
||||
void writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
void writeDefault(const std::string &s, const char *x);
|
||||
template <typename U>
|
||||
void writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements);
|
||||
private:
|
||||
std::ofstream file_;
|
||||
};
|
||||
@ -66,6 +68,8 @@ namespace Grid {
|
||||
void readDefault(const std::string &s, U &output);
|
||||
template <typename U>
|
||||
void readDefault(const std::string &s, std::vector<U> &output);
|
||||
template <typename U>
|
||||
void readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim);
|
||||
private:
|
||||
std::ifstream file_;
|
||||
};
|
||||
@ -92,6 +96,27 @@ namespace Grid {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void BinaryWriter::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
|
||||
{
|
||||
uint64_t rank = static_cast<uint64_t>( Dimensions.size() );
|
||||
uint64_t tmp = 1;
|
||||
for( auto i = 0 ; i < rank ; i++ )
|
||||
tmp *= Dimensions[i];
|
||||
assert( tmp == NumElements && "Dimensions don't match size of data being written" );
|
||||
// Total number of elements
|
||||
write("", tmp);
|
||||
// Number of dimensions
|
||||
write("", rank);
|
||||
// Followed by each dimension
|
||||
for( auto i = 0 ; i < rank ; i++ ) {
|
||||
tmp = Dimensions[i];
|
||||
write("", tmp);
|
||||
}
|
||||
for( auto i = 0; i < NumElements; ++i)
|
||||
write("", pDataRowMajor[i]);
|
||||
}
|
||||
|
||||
// Reader template implementation ////////////////////////////////////////////
|
||||
template <typename U>
|
||||
void BinaryReader::readDefault(const std::string &s, U &output)
|
||||
@ -114,6 +139,30 @@ namespace Grid {
|
||||
read("", output[i]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void BinaryReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
|
||||
{
|
||||
// Number of elements
|
||||
uint64_t NumElements;
|
||||
read("", NumElements);
|
||||
// Number of dimensions
|
||||
uint64_t rank;
|
||||
read("", rank);
|
||||
// Followed by each dimension
|
||||
uint64_t count = 1;
|
||||
dim.resize(rank);
|
||||
uint64_t tmp;
|
||||
for( auto i = 0 ; i < rank ; i++ ) {
|
||||
read("", tmp);
|
||||
dim[i] = tmp;
|
||||
count *= tmp;
|
||||
}
|
||||
assert( count == NumElements && "Dimensions don't match size of data being read" );
|
||||
buf.resize(count);
|
||||
for( auto i = 0; i < count; ++i)
|
||||
read("", buf[i]);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
|
||||
#include <stack>
|
||||
#include <string>
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include <H5Cpp.h>
|
||||
#include <Grid/tensors/Tensors.h>
|
||||
@ -38,6 +39,8 @@ namespace Grid
|
||||
template <typename U>
|
||||
typename std::enable_if<!element<std::vector<U>>::is_number, void>::type
|
||||
writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
template <typename U>
|
||||
void writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements);
|
||||
H5NS::Group & getGroup(void);
|
||||
private:
|
||||
template <typename U>
|
||||
@ -66,6 +69,8 @@ namespace Grid
|
||||
template <typename U>
|
||||
typename std::enable_if<!element<std::vector<U>>::is_number, void>::type
|
||||
readDefault(const std::string &s, std::vector<U> &x);
|
||||
template <typename U>
|
||||
void readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim);
|
||||
H5NS::Group & getGroup(void);
|
||||
private:
|
||||
template <typename U>
|
||||
@ -101,6 +106,90 @@ namespace Grid
|
||||
template <>
|
||||
void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
|
||||
|
||||
template <typename U>
|
||||
void Hdf5Writer::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
|
||||
{
|
||||
// Hdf5 needs the dimensions as hsize_t
|
||||
const int rank = static_cast<int>(Dimensions.size());
|
||||
std::vector<hsize_t> dim(rank);
|
||||
for(int i = 0; i < rank; i++)
|
||||
dim[i] = Dimensions[i];
|
||||
// write the entire dataset to file
|
||||
H5NS::DataSpace dataSpace(rank, dim.data());
|
||||
|
||||
size_t DataSize = NumElements * sizeof(U);
|
||||
if (DataSize > dataSetThres_)
|
||||
{
|
||||
// First few prime numbers from https://oeis.org/A000040
|
||||
static const unsigned short Primes[] = { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31,
|
||||
37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109,
|
||||
113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193,
|
||||
197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271 };
|
||||
constexpr int NumPrimes = sizeof( Primes ) / sizeof( Primes[0] );
|
||||
// Make sure 1) each dimension; and 2) chunk size is < 4GB
|
||||
const hsize_t MaxElements = ( sizeof( U ) == 1 ) ? 0xffffffff : 0x100000000 / sizeof( U );
|
||||
hsize_t ElementsPerChunk = 1;
|
||||
bool bTooBig = false;
|
||||
for( int i = rank - 1 ; i != -1 ; i-- ) {
|
||||
auto &d = dim[i];
|
||||
if( bTooBig )
|
||||
d = 1; // Chunk size is already as big as can be - remaining dimensions = 1
|
||||
else {
|
||||
// If individual dimension too big, reduce by prime factors if possible
|
||||
for( int PrimeIdx = 0; d > MaxElements && PrimeIdx < NumPrimes; ) {
|
||||
if( d % Primes[PrimeIdx] )
|
||||
PrimeIdx++;
|
||||
else
|
||||
d /= Primes[PrimeIdx];
|
||||
}
|
||||
const char ErrorMsg[] = " dimension > 4GB without small prime factors. "
|
||||
"Hdf5IO chunk size will be inefficient. NB Serialisation is not intended for large datasets - please consider alternatives.";
|
||||
if( d > MaxElements ) {
|
||||
std::cout << GridLogWarning << "Individual" << ErrorMsg << std::endl;
|
||||
hsize_t quotient = d / MaxElements;
|
||||
if( d % MaxElements )
|
||||
quotient++;
|
||||
d /= quotient;
|
||||
}
|
||||
// Now make sure overall size is not too big
|
||||
hsize_t OverflowCheck = ElementsPerChunk;
|
||||
ElementsPerChunk *= d;
|
||||
assert( OverflowCheck == ElementsPerChunk / d && "Product of dimensions overflowed hsize_t" );
|
||||
// If product of dimensions too big, reduce by prime factors
|
||||
for( int PrimeIdx = 0; ElementsPerChunk > MaxElements && PrimeIdx < NumPrimes; ) {
|
||||
bTooBig = true;
|
||||
if( d % Primes[PrimeIdx] )
|
||||
PrimeIdx++;
|
||||
else {
|
||||
d /= Primes[PrimeIdx];
|
||||
ElementsPerChunk /= Primes[PrimeIdx];
|
||||
}
|
||||
}
|
||||
if( ElementsPerChunk > MaxElements ) {
|
||||
std::cout << GridLogMessage << "Product of" << ErrorMsg << std::endl;
|
||||
hsize_t quotient = ElementsPerChunk / MaxElements;
|
||||
if( ElementsPerChunk % MaxElements )
|
||||
quotient++;
|
||||
d /= quotient;
|
||||
ElementsPerChunk /= quotient;
|
||||
}
|
||||
}
|
||||
}
|
||||
H5NS::DataSet dataSet;
|
||||
H5NS::DSetCreatPropList plist;
|
||||
plist.setChunk(rank, dim.data());
|
||||
plist.setFletcher32();
|
||||
dataSet = group_.createDataSet(s, Hdf5Type<U>::type(), dataSpace, plist);
|
||||
dataSet.write(pDataRowMajor, Hdf5Type<U>::type());
|
||||
}
|
||||
else
|
||||
{
|
||||
H5NS::Attribute attribute;
|
||||
attribute = group_.createAttribute(s, Hdf5Type<U>::type(), dataSpace);
|
||||
attribute.write(Hdf5Type<U>::type(), pDataRowMajor);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<element<std::vector<U>>::is_number, void>::type
|
||||
Hdf5Writer::writeDefault(const std::string &s, const std::vector<U> &x)
|
||||
@ -110,34 +199,11 @@ namespace Grid
|
||||
|
||||
// flatten the vector and getting dimensions
|
||||
Flatten<std::vector<U>> flat(x);
|
||||
std::vector<hsize_t> dim;
|
||||
std::vector<size_t> dim;
|
||||
const auto &flatx = flat.getFlatVector();
|
||||
|
||||
for (auto &d: flat.getDim())
|
||||
{
|
||||
dim.push_back(d);
|
||||
}
|
||||
|
||||
// write to file
|
||||
H5NS::DataSpace dataSpace(dim.size(), dim.data());
|
||||
|
||||
if (flatx.size() > dataSetThres_)
|
||||
{
|
||||
H5NS::DataSet dataSet;
|
||||
H5NS::DSetCreatPropList plist;
|
||||
|
||||
plist.setChunk(dim.size(), dim.data());
|
||||
plist.setFletcher32();
|
||||
dataSet = group_.createDataSet(s, Hdf5Type<Element>::type(), dataSpace, plist);
|
||||
dataSet.write(flatx.data(), Hdf5Type<Element>::type());
|
||||
}
|
||||
else
|
||||
{
|
||||
H5NS::Attribute attribute;
|
||||
|
||||
attribute = group_.createAttribute(s, Hdf5Type<Element>::type(), dataSpace);
|
||||
attribute.write(Hdf5Type<Element>::type(), flatx.data());
|
||||
}
|
||||
writeMultiDim<Element>(s, dim, &flatx[0], flatx.size());
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
@ -173,10 +239,9 @@ namespace Grid
|
||||
|
||||
template <>
|
||||
void Hdf5Reader::readDefault(const std::string &s, std::string &x);
|
||||
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<element<std::vector<U>>::is_number, void>::type
|
||||
Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
|
||||
void Hdf5Reader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
|
||||
{
|
||||
// alias to element type
|
||||
typedef typename element<std::vector<U>>::type Element;
|
||||
@ -184,7 +249,6 @@ namespace Grid
|
||||
// read the dimensions
|
||||
H5NS::DataSpace dataSpace;
|
||||
std::vector<hsize_t> hdim;
|
||||
std::vector<size_t> dim;
|
||||
hsize_t size = 1;
|
||||
|
||||
if (group_.attrExists(s))
|
||||
@ -204,9 +268,9 @@ namespace Grid
|
||||
}
|
||||
|
||||
// read the flat vector
|
||||
std::vector<Element> buf(size);
|
||||
|
||||
if (size > dataSetThres_)
|
||||
buf.resize(size);
|
||||
|
||||
if (size * sizeof(Element) > dataSetThres_)
|
||||
{
|
||||
H5NS::DataSet dataSet;
|
||||
|
||||
@ -220,7 +284,19 @@ namespace Grid
|
||||
attribute = group_.openAttribute(s);
|
||||
attribute.read(Hdf5Type<Element>::type(), buf.data());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
typename std::enable_if<element<std::vector<U>>::is_number, void>::type
|
||||
Hdf5Reader::readDefault(const std::string &s, std::vector<U> &x)
|
||||
{
|
||||
// alias to element type
|
||||
typedef typename element<std::vector<U>>::type Element;
|
||||
|
||||
std::vector<size_t> dim;
|
||||
std::vector<Element> buf;
|
||||
readMultiDim( s, buf, dim );
|
||||
|
||||
// reconstruct the multidimensional vector
|
||||
Reconstruct<std::vector<U>> r(buf, dim);
|
||||
|
||||
|
||||
@ -109,8 +109,8 @@ THE SOFTWARE.
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#define GRID_MACRO_MEMBER(A,B) A B;
|
||||
#define GRID_MACRO_COMP_MEMBER(A,B) result = (result and (lhs. B == rhs. B));
|
||||
#define GRID_MACRO_OS_WRITE_MEMBER(A,B) os<< #A <<" " #B << " = " << obj. B << " ; " <<std::endl;
|
||||
#define GRID_MACRO_COMP_MEMBER(A,B) result = (result and CompareMember(lhs. B, rhs. B));
|
||||
#define GRID_MACRO_OS_WRITE_MEMBER(A,B) os<< #A <<" " #B << " = "; WriteMember( os, obj. B ); os << " ; " <<std::endl;
|
||||
#define GRID_MACRO_READ_MEMBER(A,B) Grid::read(RD,#B,obj. B);
|
||||
#define GRID_MACRO_WRITE_MEMBER(A,B) Grid::write(WR,#B,obj. B);
|
||||
|
||||
|
||||
@ -51,6 +51,8 @@ namespace Grid
|
||||
void writeDefault(const std::string &s, const U &x);
|
||||
template <typename U>
|
||||
void writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
template <typename U>
|
||||
void writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements);
|
||||
private:
|
||||
void indent(void);
|
||||
private:
|
||||
@ -69,6 +71,8 @@ namespace Grid
|
||||
void readDefault(const std::string &s, U &output);
|
||||
template <typename U>
|
||||
void readDefault(const std::string &s, std::vector<U> &output);
|
||||
template <typename U>
|
||||
void readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim);
|
||||
private:
|
||||
void checkIndent(void);
|
||||
private:
|
||||
@ -95,7 +99,18 @@ namespace Grid
|
||||
write(s, x[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template <typename U>
|
||||
void TextWriter::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
|
||||
{
|
||||
uint64_t Rank = Dimensions.size();
|
||||
write(s, Rank);
|
||||
for( uint64_t d : Dimensions )
|
||||
write(s, d);
|
||||
while( NumElements-- )
|
||||
write(s, *pDataRowMajor++);
|
||||
}
|
||||
|
||||
// Reader template implementation ////////////////////////////////////////////
|
||||
template <typename U>
|
||||
void TextReader::readDefault(const std::string &s, U &output)
|
||||
@ -121,6 +136,23 @@ namespace Grid
|
||||
read("", output[i]);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void TextReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
|
||||
{
|
||||
const char sz[] = "";
|
||||
uint64_t Rank;
|
||||
read(sz, Rank);
|
||||
dim.resize( Rank );
|
||||
size_t NumElements = 1;
|
||||
for( auto &d : dim ) {
|
||||
read(sz, d);
|
||||
NumElements *= d;
|
||||
}
|
||||
buf.resize( NumElements );
|
||||
for( auto &x : buf )
|
||||
read(s, x);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -53,6 +53,17 @@ namespace Grid {
|
||||
return os;
|
||||
}
|
||||
|
||||
// std::vector<std:vector<...>> nested to specified Rank //////////////////////////////////
|
||||
template<typename T, unsigned int Rank>
|
||||
struct NestedStdVector {
|
||||
typedef typename std::vector<typename NestedStdVector<T, Rank - 1>::type> type;
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct NestedStdVector<T,0> {
|
||||
typedef T type;
|
||||
};
|
||||
|
||||
// Grid scalar tensors to nested std::vectors //////////////////////////////////
|
||||
template <typename T>
|
||||
struct TensorToVec
|
||||
@ -436,4 +447,4 @@ std::string vecToStr(const std::vector<T> &v)
|
||||
return sstr.str();
|
||||
}
|
||||
|
||||
#endif
|
||||
#endif
|
||||
|
||||
@ -57,6 +57,8 @@ namespace Grid
|
||||
void writeDefault(const std::string &s, const U &x);
|
||||
template <typename U>
|
||||
void writeDefault(const std::string &s, const std::vector<U> &x);
|
||||
template <typename U>
|
||||
void writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements);
|
||||
std::string docString(void);
|
||||
std::string string(void);
|
||||
private:
|
||||
@ -79,6 +81,8 @@ namespace Grid
|
||||
void readDefault(const std::string &s, U &output);
|
||||
template <typename U>
|
||||
void readDefault(const std::string &s, std::vector<U> &output);
|
||||
template <typename U>
|
||||
void readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim);
|
||||
void readCurrentSubtree(std::string &s);
|
||||
private:
|
||||
void checkParse(const pugi::xml_parse_result &result, const std::string name);
|
||||
@ -122,13 +126,45 @@ namespace Grid
|
||||
void XmlWriter::writeDefault(const std::string &s, const std::vector<U> &x)
|
||||
{
|
||||
push(s);
|
||||
for (auto &x_i: x)
|
||||
for( auto &u : x )
|
||||
{
|
||||
write("elem", x_i);
|
||||
write("elem", u);
|
||||
}
|
||||
pop();
|
||||
}
|
||||
|
||||
|
||||
template <typename U>
|
||||
void XmlWriter::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
|
||||
{
|
||||
push(s);
|
||||
size_t count = 1;
|
||||
const int Rank = static_cast<int>( Dimensions.size() );
|
||||
write("rank", Rank );
|
||||
std::vector<size_t> MyIndex( Rank );
|
||||
for( auto d : Dimensions ) {
|
||||
write("dim", d);
|
||||
count *= d;
|
||||
}
|
||||
assert( count == NumElements && "XmlIO : element count doesn't match dimensions" );
|
||||
static const char sName[] = "tensor";
|
||||
for( int i = 0 ; i < Rank ; i++ ) {
|
||||
MyIndex[i] = 0;
|
||||
push(sName);
|
||||
}
|
||||
while (NumElements--) {
|
||||
write("elem", *pDataRowMajor++);
|
||||
int i;
|
||||
for( i = Rank - 1 ; i != -1 && ++MyIndex[i] == Dimensions[i] ; i-- )
|
||||
MyIndex[i] = 0;
|
||||
int Rollover = Rank - 1 - i;
|
||||
for( i = 0 ; i < Rollover ; i++ )
|
||||
pop();
|
||||
for( i = 0 ; NumElements && i < Rollover ; i++ )
|
||||
push(sName);
|
||||
}
|
||||
pop();
|
||||
}
|
||||
|
||||
// Reader template implementation ////////////////////////////////////////////
|
||||
template <typename U>
|
||||
void XmlReader::readDefault(const std::string &s, U &output)
|
||||
@ -145,25 +181,66 @@ namespace Grid
|
||||
template <typename U>
|
||||
void XmlReader::readDefault(const std::string &s, std::vector<U> &output)
|
||||
{
|
||||
std::string buf;
|
||||
unsigned int i = 0;
|
||||
|
||||
if (!push(s))
|
||||
{
|
||||
std::cout << GridLogWarning << "XML: cannot open node '" << s << "'";
|
||||
std::cout << std::endl;
|
||||
|
||||
return;
|
||||
} else {
|
||||
for(unsigned int i = 0; node_.child("elem"); )
|
||||
{
|
||||
output.resize(i + 1);
|
||||
read("elem", output[i++]);
|
||||
node_.child("elem").set_name("elem-done");
|
||||
}
|
||||
pop();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename U>
|
||||
void XmlReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
|
||||
{
|
||||
if (!push(s))
|
||||
{
|
||||
std::cout << GridLogWarning << "XML: cannot open node '" << s << "'";
|
||||
std::cout << std::endl;
|
||||
} else {
|
||||
static const char sName[] = "tensor";
|
||||
static const char sNameDone[] = "tensor-done";
|
||||
int Rank;
|
||||
read("rank", Rank);
|
||||
dim.resize( Rank );
|
||||
size_t NumElements = 1;
|
||||
for( auto &d : dim )
|
||||
{
|
||||
read("dim", d);
|
||||
node_.child("dim").set_name("dim-done");
|
||||
NumElements *= d;
|
||||
}
|
||||
buf.resize( NumElements );
|
||||
std::vector<size_t> MyIndex( Rank );
|
||||
for( int i = 0 ; i < Rank ; i++ ) {
|
||||
MyIndex[i] = 0;
|
||||
push(sName);
|
||||
}
|
||||
|
||||
for( auto &x : buf )
|
||||
{
|
||||
NumElements--;
|
||||
read("elem", x);
|
||||
node_.child("elem").set_name("elem-done");
|
||||
int i;
|
||||
for( i = Rank - 1 ; i != -1 && ++MyIndex[i] == dim[i] ; i-- )
|
||||
MyIndex[i] = 0;
|
||||
int Rollover = Rank - 1 - i;
|
||||
for( i = 0 ; i < Rollover ; i++ ) {
|
||||
node_.set_name(sNameDone);
|
||||
pop();
|
||||
}
|
||||
for( i = 0 ; NumElements && i < Rollover ; i++ )
|
||||
push(sName);
|
||||
}
|
||||
pop();
|
||||
}
|
||||
while (node_.child("elem"))
|
||||
{
|
||||
output.resize(i + 1);
|
||||
read("elem", output[i]);
|
||||
node_.child("elem").set_name("elem-done");
|
||||
i++;
|
||||
}
|
||||
pop();
|
||||
}
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user