portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-09 23:45:36 +00:00
Code Releases Activity

Tweaked format and memory use on Xml format. Still crashes (out of memory) on large read on my laptop

Browse Source
This commit is contained in:
Michael Marshall 2019-02-25 22:03:21 +00:00
parent 81b3f3d2ca
commit 578eb177e7
3 changed files with 50 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Grid/serialisation/TextIO.h
View File

@ -150,8 +150,8 @@ namespace Grid
NumElements *= d;
}
buf.resize( NumElements );
for( auto i = 0; i < NumElements; i++ )
read(s, buf[i]);
for( auto &x : buf )
read(s, x);
}
}

76
Grid/serialisation/XmlIO.h
View File

@ -125,24 +125,28 @@ namespace Grid
template <typename U>
void XmlWriter::writeDefault(const std::string &s, const std::vector<U> &x)
{
std::vector<size_t> dims(1);
dims[0] = x.size();
writeMultiDim(s, dims, &x[0], dims[0]);
push(s);
for( auto &u : x )
{
write("elem", u);
}
pop();
}
template <typename U>
void XmlWriter::writeMultiDim(const std::string &s, const std::vector<size_t> & Dimensions, const U * pDataRowMajor, size_t NumElements)
{
push(s);
if( Dimensions.size() > 1 )
{
for( auto d : Dimensions )
write("dim", d);
size_t count = 1;
const size_t Rank = Dimensions.size();
write("rank", Rank );
for( auto d : Dimensions ) {
write("dim", d);
count *= d;
}
assert( count == NumElements && "XmlIO : element count doesn't match dimensions" );
while (NumElements--)
{
write("elem", *pDataRowMajor++);
}
pop();
}
@ -162,38 +166,46 @@ namespace Grid
template <typename U>
void XmlReader::readDefault(const std::string &s, std::vector<U> &output)
{
std::vector<size_t> dims;
readMultiDim(s, output, dims);
assert( dims.size() == 1 && dims[0] == output.size() && "XmlIO: Expected 1D vector" );
}
template <typename U>
void XmlReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
{
unsigned int i = 0;
unsigned int Rank = 0;
if (!push(s))
{
std::cout << GridLogWarning << "XML: cannot open node '" << s << "'";
std::cout << std::endl;
} else {
while (node_.child("dim"))
for(unsigned int i = 0; node_.child("elem"); )
{
dim.resize(Rank + 1);
read("dim", dim[Rank]);
node_.child("dim").set_name("dim-done");
Rank++;
}
while (node_.child("elem"))
{
buf.resize(i + 1);
read("elem", buf[i]);
output.resize(i + 1);
read("elem", output[i++]);
node_.child("elem").set_name("elem-done");
}
pop();
}
}
template <typename U>
void XmlReader::readMultiDim(const std::string &s, std::vector<U> &buf, std::vector<size_t> &dim)
{
if (!push(s))
{
std::cout << GridLogWarning << "XML: cannot open node '" << s << "'";
std::cout << std::endl;
} else {
size_t Rank;
read("rank", Rank);
dim.resize( Rank );
size_t NumElements = 1;
for( auto &d : dim )
{
read("dim", d);
node_.child("dim").set_name("dim-done");
NumElements *= d;
}
buf.resize( NumElements );
for( auto &x : buf )
{
read("elem", x);
node_.child("elem").set_name("elem-done");
i++;
}
pop();
if( Rank == 0 )
dim.push_back(i);
}
}
}

8
tests/IO/Test_serialisation.cc
View File

@ -113,7 +113,7 @@ typedef Eigen::Tensor<TestScalar, 3, Eigen::StorageOptions::RowMajor> TensorRank
typedef Eigen::TensorFixedSize<TestScalar, Eigen::Sizes<9,4,2>, Eigen::StorageOptions::RowMajor> Tensor_9_4_2;
typedef std::vector<Tensor_9_4_2> aTensor_9_4_2;
typedef Eigen::TensorFixedSize<SpinColourVector, Eigen::Sizes<6,5>> LSCTensor;
#ifdef DEBUG
#ifndef DEBUG
typedef Eigen::TensorFixedSize<iMatrix<iVector<iMatrix<iVector<LorentzColourMatrix,5>,2>,7>,3>, Eigen::Sizes<2,4,11,10,9>, Eigen::StorageOptions::RowMajor> LCMTensor;
#endif
@ -193,7 +193,7 @@ void EigenHdf5IOTest(const char * pszExtension)
}
TensorWriteRead ( LSCTensor )
TensorWriteReadLarge( PerambIOTestClass )
#ifdef DEBUG
#ifndef DEBUG
std::cout << "sizeof( LCMTensor ) = " << sizeof( LCMTensor ) / 1024 / 1024 << " MB" << std::endl;
TensorWriteReadLarge ( LCMTensor )
// Also write > 4GB of complex numbers (I suspect this will fail inside Hdf5)
@ -295,12 +295,12 @@ int main(int argc,char **argv)
std::cout << "\n==== detailed Hdf5 tensor tests (Grid::EigenIO)" << std::endl;
EigenHdf5IOTest<Hdf5Writer, Hdf5Reader>(".h5");
#endif
std::cout << "\n==== detailed xml tensor tests (Grid::EigenIO)" << std::endl;
EigenHdf5IOTest<XmlWriter, XmlReader>(".xml");
std::cout << "\n==== detailed binary tensor tests (Grid::EigenIO)" << std::endl;
EigenHdf5IOTest<BinaryWriter, BinaryReader>(".bin");
std::cout << "\n==== detailed text tensor tests (Grid::EigenIO)" << std::endl;
EigenHdf5IOTest<TextWriter, TextReader>(".dat");
std::cout << "\n==== detailed xml tensor tests (Grid::EigenIO)" << std::endl;
EigenHdf5IOTest<XmlWriter, XmlReader>(".xml");
std::cout << "\n==== vector flattening/reconstruction" << std::endl;
typedef std::vector<std::vector<std::vector<double>>> vec3d;