portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-10 07:55:35 +00:00
Code Releases Activity

Ensured Hdf5 chunk size always less than 4GB

Browse Source
This commit is contained in:
Michael Marshall 2019-02-22 15:14:32 +00:00
parent 292ff33f7f
commit 44a2d4854a
2 changed files with 38 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Grid/serialisation/Hdf5IO.h
View File

@ -105,11 +105,23 @@ namespace Grid
template <> template <>
void Hdf5Writer::writeDefault(const std::string &s, const std::string &x); void Hdf5Writer::writeDefault(const std::string &s, const std::string &x);
static hsize_t alignup(hsize_t n)
{
n--; // 1000 0011 --> 1000 0010
n |= n >> 1; // 1000 0010 | 0100 0001 = 1100 0011
n |= n >> 2; // 1100 0011 | 0011 0000 = 1111 0011
n |= n >> 4; // 1111 0011 | 0000 1111 = 1111 1111
n |= n >> 8; // ... (At this point all bits are 1, so further bitwise-or
n |= n >> 16; // operations produce no effect.)
n++; // 1111 1111 --> 1 0000 0000
return n;
};
template <typename U> template <typename U>
void Hdf5Writer::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements) void Hdf5Writer::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
{ {
// Hdf5 needs the dimensions as hsize_t // Hdf5 needs the dimensions as hsize_t
int rank = static_cast<int>(Dimensions.size()); const int rank = static_cast<int>(Dimensions.size());
std::vector<hsize_t> dim(rank); std::vector<hsize_t> dim(rank);
for(int i = 0; i < rank; i++) for(int i = 0; i < rank; i++)
dim[i] = Dimensions[i]; dim[i] = Dimensions[i];
@ -119,9 +131,29 @@ namespace Grid
size_t DataSize = NumElements * sizeof(U); size_t DataSize = NumElements * sizeof(U);
if (DataSize > dataSetThres_) if (DataSize > dataSetThres_)
{ {
// Make sure the chunk size is < 4GB
const hsize_t MaxElements = ( sizeof( U ) == 1 ) ? 0xffffffff : 0x100000000 / sizeof( U );
hsize_t ElementsPerChunk = 1;
bool bTooBig = false;
for( unsigned int i = rank - 1; i != -1; i-- ) {
if( bTooBig )
// Chunk size is already as big as can be - remaining dimensions = 1
dim[i] = 1;
else {
// Now make sure overall size is not too big
ElementsPerChunk *= dim[i];
if( ElementsPerChunk >= MaxElements ) {
bTooBig = true;
hsize_t dividend = ElementsPerChunk / MaxElements;
hsize_t remainder = ElementsPerChunk % MaxElements;
if( remainder )
dividend++;
dim[i] = dim[i] / dividend;
}
}
}
H5NS::DataSet dataSet; H5NS::DataSet dataSet;
H5NS::DSetCreatPropList plist; H5NS::DSetCreatPropList plist;
plist.setChunk(rank, dim.data()); plist.setChunk(rank, dim.data());
plist.setFletcher32(); plist.setFletcher32();
dataSet = group_.createDataSet(s, Hdf5Type<U>::type(), dataSpace, plist); dataSet = group_.createDataSet(s, Hdf5Type<U>::type(), dataSpace, plist);

5
tests/hadrons/Test_hadrons_distil.cc
View File

@ -655,7 +655,10 @@ int main(int argc, char *argv[])
<< ", sizeof(size_t) = " << sizeof(size_t) << ", sizeof(size_t) = " << sizeof(size_t)
<< ", sizeof(std::size_t) = " << sizeof(std::size_t) << ", sizeof(std::size_t) = " << sizeof(std::size_t)
<< ", sizeof(std::streamsize) = " << sizeof(std::streamsize) << ", sizeof(std::streamsize) = " << sizeof(std::streamsize)
<< ", sizeof(Eigen::Index) = " << sizeof(Eigen::Index) << std::endl; << ", sizeof(Eigen::Index) = " << sizeof(Eigen::Index)
<< ", sizeof(hsize_t) = " << sizeof(hsize_t)
<< ", sizeof(unsigned long long) = " << sizeof(unsigned long long)
<< std::endl;
if( DebugEigenTest() ) return 0; if( DebugEigenTest() ) return 0;
if(DebugGridTensorTest()) return 0; if(DebugGridTensorTest()) return 0;
#endif #endif