From b3d4ba86578f8b2599bbd664fa91508dfc41d426 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Sat, 2 Mar 2019 00:24:37 +0000 Subject: [PATCH] Fixed issues with Eigen Tensor serialisation. Fixed issues with precision to text streams --- Grid/serialisation/BaseIO.h | 18 ++++--- tests/IO/Test_serialisation.cc | 86 +++++++++++++++++----------------- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/Grid/serialisation/BaseIO.h b/Grid/serialisation/BaseIO.h index 289e7902..30d7a42d 100644 --- a/Grid/serialisation/BaseIO.h +++ b/Grid/serialisation/BaseIO.h @@ -282,7 +282,7 @@ namespace Grid { im *= n; if( Precision ) { std::stringstream s; - s << std::scientific << std::setprecision(Precision) << re; + s << std::setprecision(Precision) << re; s >> re; s.clear(); s << im; @@ -741,13 +741,16 @@ namespace Grid { upcast->readMultiDim( s, buf, dimData ); assert(dimData.size() == TotalRank && "EigenIO: Tensor rank mismatch" ); // Make sure that the number of elements read matches dimensions read - std::size_t NumElements = 1; - for( auto d : dimData ) - NumElements *= d; - assert( NumElements == buf.size() && "EigenIO: Number of elements != product of dimensions" ); + std::size_t NumContainers = 1; + for( auto i = 0 ; i < TensorRank ; i++ ) + NumContainers *= dimData[i]; // If our scalar object is a Container, make sure it's dimensions match what we read back - for( auto i = 0 ; i < ContainerRank ; i++ ) + std::size_t ElementsPerContainer = 1; + for( auto i = 0 ; i < ContainerRank ; i++ ) { assert( dimData[TensorRank+i] == Traits::Dimension(i) && "Tensor Container dimensions don't match data" ); + ElementsPerContainer *= dimData[TensorRank+i]; + } + assert( NumContainers * ElementsPerContainer == buf.size() && "EigenIO: Number of elements != product of dimensions" ); // Now see whether the tensor is the right shape, or can be made to be const auto & dims{output.dimensions()}; bool bShapeOK = (output.data() != nullptr); @@ -764,13 +767,14 @@ namespace Grid { // Copy the data into the tensor for( auto &d : MyIndex ) d = 0; const Scalar * pSource = &buf[0]; - for( auto n = 0 ; n < NumElements ; n++ ) { + for( std::size_t n = 0 ; n < NumContainers ; n++ ) { Container & c = output( MyIndex ); copyScalars( c, pSource ); // Now increment the index for( int i = TensorRank - 1; i != -1 && ++MyIndex[i] == dims[i]; i-- ) MyIndex[i] = 0; } + assert( pSource == &buf[NumContainers * ElementsPerContainer] ); } template diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index c8d7906f..f3df9bec 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -4,11 +4,12 @@ Source file: ./tests/Test_serialisation.cc - Copyright (C) 2015-2016 + Copyright (C) 2015-2019 Author: Guido Cossu Author: Antonin Portelli Author: Peter Boyle +Author: Michael Marshall This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -107,7 +108,6 @@ void ioTest(const std::string &filename, const O &object, const std::string &nam std::cout << " done." << std::endl; } -#ifdef HAVE_HDF5 typedef std::complex TestScalar; typedef Eigen::TensorFixedSize> TensorRank5UShort; typedef Eigen::TensorFixedSize, Eigen::StorageOptions::RowMajor> TensorRank5UShortAlt; @@ -115,9 +115,6 @@ typedef Eigen::Tensor TensorRank typedef Eigen::TensorFixedSize, Eigen::StorageOptions::RowMajor> Tensor_9_4_2; typedef std::vector aTensor_9_4_2; typedef Eigen::TensorFixedSize> LSCTensor; -#ifndef NO_STRESS_TESTS -typedef Eigen::TensorFixedSize,2>,7>,3>, Eigen::Sizes<2,4,11,10,9>, Eigen::StorageOptions::RowMajor> LCMTensor; -#endif class PerambIOTestClass: Serializable { Grid_complex Flag; @@ -156,35 +153,41 @@ public: } }; -#define TensorWriteReadInnerNoInit( T ) \ - filename = "iotest_" + std::to_string(++TestNum) + "_" #T + pszExtension; \ - ioTest(filename, t, #T, #T); -#define TensorWriteReadInner( T ) SequentialInit( t, Flag, Precision ); TensorWriteReadInnerNoInit( T ) -#define TensorWriteRead( T ) { T t ; TensorWriteReadInner( T ) } -#define TensorWriteReadV(T, ... ) { T t( __VA_ARGS__ ); TensorWriteReadInner( T ) } -#define TensorWriteReadLarge( T ) { std::unique_ptr p{new T}; T &t{*p}; TensorWriteReadInnerNoInit(T) } +#define TEST_PARAMS( T ) #T, Flag, Precision, filename, pszExtension, TestNum + +template +void EigenTensorTestSingle(const char * MyTypeName, typename EigenIO::Traits::scalar_type Flag, + unsigned short Precision, std::string &filename, const char * pszExtension, unsigned int &TestNum, + IndexTypes... otherDims) +{ + using Traits = EigenIO::Traits; + using scalar_type = typename Traits::scalar_type; + std::unique_ptr pTensor{new T(otherDims...)}; + SequentialInit( * pTensor, Flag, Precision ); + filename = "iotest_" + std::to_string(++TestNum) + "_" + MyTypeName + pszExtension; + ioTest(filename, * pTensor, MyTypeName, MyTypeName); +} template -void EigenHdf5IOTest(const char * pszExtension, unsigned short Precision = 0) +void EigenTensorTest(const char * pszExtension, unsigned short Precision = 0) { unsigned int TestNum = 0; std::string filename; { int Flag = 7; - unsigned short Precision = 0; using TensorSingle = Eigen::TensorFixedSize>; - TensorWriteRead( TensorSingle ) + EigenTensorTestSingle(TEST_PARAMS( TensorSingle )); } TestScalar Flag{1,-3.1415927}; using TensorSimple = Eigen::Tensor, 6>; - TensorWriteReadV( TensorSimple, 1, 1, 1, 1, 1, 1 ) - TensorWriteReadV( TensorRank3, 6, 3, 2 ) - TensorWriteRead ( Tensor_9_4_2 ) + using I = typename TensorSimple::Index; + EigenTensorTestSingle( TEST_PARAMS( TensorSimple ), 1, 1, 1, 1, 1, 1 ); + EigenTensorTestSingle( TEST_PARAMS( TensorRank3 ), 6, 3, 2 ); + EigenTensorTestSingle(TEST_PARAMS( Tensor_9_4_2 )); { unsigned short Flag = 1; - unsigned short Precision = 0; TensorRank5UShort t; - TensorWriteReadInner ( TensorRank5UShort ); + EigenTensorTestSingle(TEST_PARAMS( TensorRank5UShort )); std::cout << " Testing alternate memory order read ... "; TensorRank5UShortAlt t2; RDR_ reader(filename); @@ -201,23 +204,20 @@ void EigenHdf5IOTest(const char * pszExtension, unsigned short Precision = 0) } std::cout << " done." << std::endl; } - TensorWriteRead ( LSCTensor ) - TensorWriteReadLarge( PerambIOTestClass ) -#ifndef NO_STRESS_TESTS - std::cout << "sizeof( LCMTensor ) = " << sizeof( LCMTensor ) / 1024 / 1024 << " MB" << std::endl; - TensorWriteReadLarge ( LCMTensor ) - // Also write > 4GB of complex numbers (I suspect this will fail inside Hdf5) + EigenTensorTestSingle(TEST_PARAMS( LSCTensor )); { - static constexpr size_t Num = 0x11000000; - std::cout << "Stress test: " << Num * sizeof( Grid_complex ) / 1024 / 1024 - << " MB array of complex" << std::endl; - using Stress = std::vector>; - Stress t (Num); - TensorWriteReadInnerNoInit( Stress ); + static const char MyTypeName[] = "PerambIOTestClass"; + std::unique_ptr pObj{new PerambIOTestClass()}; + filename = "iotest_" + std::to_string(++TestNum) + "_" + MyTypeName + pszExtension; + ioTest(filename, * pObj, MyTypeName, MyTypeName); } +#ifdef STRESS_TESTS + using LCMTensor = Eigen::TensorFixedSize,2>,7>,3>, + Eigen::Sizes<2,4,11,10,9>, Eigen::StorageOptions::RowMajor>; + std::cout << "sizeof( LCMTensor ) = " << sizeof( LCMTensor ) / 1024 / 1024 << " MB" << std::endl; + EigenTensorTestSingle(TEST_PARAMS( LCMTensor )); #endif } -#endif template void tensorConvTestFn(GridSerialRNG &rng, const std::string label) @@ -295,24 +295,22 @@ int main(int argc,char **argv) ioTest("iotest.dat", obj, "text (object) "); ioTest("iotest.dat", vec, "text (vector of objects)"); //// text - //ioTest("iotest.json", obj, "JSON (object) "); - //ioTest("iotest.json", vec, "JSON (vector of objects)"); + ioTest("iotest.json", obj, "JSON (object) "); + ioTest("iotest.json", vec, "JSON (vector of objects)"); //// HDF5 #ifdef HAVE_HDF5 ioTest("iotest.h5", obj, "HDF5 (object) "); ioTest("iotest.h5", vec, "HDF5 (vector of objects)"); -#endif - std::cout << "\n==== detailed text tensor tests (Grid::EigenIO)" << std::endl; - EigenHdf5IOTest(".dat", 6); - std::cout << "\n==== detailed xml tensor tests (Grid::EigenIO)" << std::endl; - EigenHdf5IOTest(".xml", 4); - std::cout << "\n==== detailed binary tensor tests (Grid::EigenIO)" << std::endl; - EigenHdf5IOTest(".bin"); -#ifdef HAVE_HDF5 std::cout << "\n==== detailed Hdf5 tensor tests (Grid::EigenIO)" << std::endl; - EigenHdf5IOTest(".h5"); + EigenTensorTest(".h5"); #endif + std::cout << "\n==== detailed binary tensor tests (Grid::EigenIO)" << std::endl; + EigenTensorTest(".bin"); + std::cout << "\n==== detailed xml tensor tests (Grid::EigenIO)" << std::endl; + EigenTensorTest(".xml", 6); + std::cout << "\n==== detailed text tensor tests (Grid::EigenIO)" << std::endl; + EigenTensorTest(".dat", 5); std::cout << "\n==== vector flattening/reconstruction" << std::endl; typedef std::vector>> vec3d;