From 35818fdf6c5eac5e42a81c02c2820497dd7e56bd Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Thu, 20 Aug 2015 23:04:38 +0100 Subject: [PATCH] Text and Binary readers --- lib/Grid.h | 2 +- lib/Make.inc | 2 +- lib/Serialisation.h | 6 - lib/parallelIO/NerscIO.h | 2 +- lib/serialisation/BinaryIO.h | 124 +++++++++++++++ .../{Reader.h => Serialisation.h} | 25 ++- lib/serialisation/TextIO.h | 147 ++++++++++++++++++ lib/serialisation/{XMLReader.h => XmlIO.h} | 24 +-- tests/Test_serialisation.cc | 43 +++-- 9 files changed, 332 insertions(+), 43 deletions(-) delete mode 100644 lib/Serialisation.h create mode 100644 lib/serialisation/BinaryIO.h rename lib/serialisation/{Reader.h => Serialisation.h} (89%) create mode 100644 lib/serialisation/TextIO.h rename lib/serialisation/{XMLReader.h => XmlIO.h} (95%) diff --git a/lib/Grid.h b/lib/Grid.h index 619f6208..052916f2 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -30,7 +30,7 @@ /////////////////// // Grid headers /////////////////// -#include +#include #include #include #include diff --git a/lib/Make.inc b/lib/Make.inc index 6fb4a651..eba274ca 100644 --- a/lib/Make.inc +++ b/lib/Make.inc @@ -1,4 +1,4 @@ -HFILES=./algorithms/approx/bigfloat.h ./algorithms/approx/bigfloat_double.h ./algorithms/approx/Chebyshev.h ./algorithms/approx/MultiShiftFunction.h ./algorithms/approx/Remez.h ./algorithms/approx/Zolotarev.h ./algorithms/CoarsenedMatrix.h ./algorithms/iterative/AdefGeneric.h ./algorithms/iterative/ConjugateGradient.h ./algorithms/iterative/ConjugateGradientMultiShift.h ./algorithms/iterative/ConjugateResidual.h ./algorithms/iterative/NormalEquations.h ./algorithms/iterative/PrecConjugateResidual.h ./algorithms/iterative/PrecGeneralisedConjugateResidual.h ./algorithms/iterative/SchurRedBlack.h ./algorithms/LinearOperator.h ./algorithms/Preconditioner.h ./algorithms/SparseMatrix.h ./Algorithms.h ./AlignedAllocator.h ./cartesian/Cartesian_base.h ./cartesian/Cartesian_full.h ./cartesian/Cartesian_red_black.h ./Cartesian.h ./communicator/Communicator_base.h ./Communicator.h ./Config.h ./cshift/Cshift_common.h ./cshift/Cshift_mpi.h ./cshift/Cshift_none.h ./Cshift.h ./Grid.h ./Init.h ./lattice/Lattice_arith.h ./lattice/Lattice_base.h ./lattice/Lattice_comparison.h ./lattice/Lattice_comparison_utils.h ./lattice/Lattice_conformable.h ./lattice/Lattice_coordinate.h ./lattice/Lattice_ET.h ./lattice/Lattice_local.h ./lattice/Lattice_overload.h ./lattice/Lattice_peekpoke.h ./lattice/Lattice_reality.h ./lattice/Lattice_reduction.h ./lattice/Lattice_rng.h ./lattice/Lattice_trace.h ./lattice/Lattice_transfer.h ./lattice/Lattice_transpose.h ./lattice/Lattice_unary.h ./lattice/Lattice_where.h ./Lattice.h ./Log.h ./Old/Tensor_peek.h ./Old/Tensor_poke.h ./parallelIO/NerscIO.h ./pugixml/pugixml.h ./qcd/action/ActionBase.h ./qcd/action/ActionParams.h ./qcd/action/Actions.h ./qcd/action/fermion/CayleyFermion5D.h ./qcd/action/fermion/ContinuedFractionFermion5D.h ./qcd/action/fermion/DomainWallFermion.h ./qcd/action/fermion/FermionOperator.h ./qcd/action/fermion/FermionOperatorImpl.h ./qcd/action/fermion/g5HermitianLinop.h ./qcd/action/fermion/MobiusFermion.h ./qcd/action/fermion/MobiusZolotarevFermion.h ./qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h ./qcd/action/fermion/OverlapWilsonCayleyZolotarevFermion.h ./qcd/action/fermion/OverlapWilsonContfracTanhFermion.h ./qcd/action/fermion/OverlapWilsonContfracZolotarevFermion.h ./qcd/action/fermion/OverlapWilsonPartialFractionTanhFermion.h ./qcd/action/fermion/OverlapWilsonPartialFractionZolotarevFermion.h ./qcd/action/fermion/PartialFractionFermion5D.h ./qcd/action/fermion/ScaledShamirFermion.h ./qcd/action/fermion/ShamirZolotarevFermion.h ./qcd/action/fermion/WilsonCompressor.h ./qcd/action/fermion/WilsonFermion.h ./qcd/action/fermion/WilsonFermion5D.h ./qcd/action/fermion/WilsonKernels.h ./qcd/action/gauge/WilsonGaugeAction.h ./qcd/action/pseudofermion/EvenOddSchurDifferentiable.h ./qcd/action/pseudofermion/OneFlavourEvenOddRational.h ./qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h ./qcd/action/pseudofermion/OneFlavourRational.h ./qcd/action/pseudofermion/OneFlavourRationalRatio.h ./qcd/action/pseudofermion/TwoFlavour.h ./qcd/action/pseudofermion/TwoFlavourEvenOdd.h ./qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h ./qcd/action/pseudofermion/TwoFlavourRatio.h ./qcd/hmc/HMC.h ./qcd/hmc/integrators/Integrator.h ./qcd/hmc/integrators/Integrator_algorithm.h ./qcd/QCD.h ./qcd/spin/Dirac.h ./qcd/spin/TwoSpinor.h ./qcd/utils/CovariantCshift.h ./qcd/utils/LinalgUtils.h ./qcd/utils/SpaceTimeGrid.h ./qcd/utils/SUn.h ./qcd/utils/WilsonLoops.h ./serialisation/MacroMagic.h ./serialisation/Reader.h ./serialisation/XMLReader.h ./simd/Grid_avx.h ./simd/Grid_avx512.h ./simd/Grid_empty.h ./simd/Grid_neon.h ./simd/Grid_qpx.h ./simd/Grid_sse4.h ./simd/Grid_vector_types.h ./simd/Grid_vector_unops.h ./Simd.h ./stencil/Lebesgue.h ./Stencil.h ./tensors/Tensor_arith.h ./tensors/Tensor_arith_add.h ./tensors/Tensor_arith_mac.h ./tensors/Tensor_arith_mul.h ./tensors/Tensor_arith_scalar.h ./tensors/Tensor_arith_sub.h ./tensors/Tensor_class.h ./tensors/Tensor_determinant.h ./tensors/Tensor_exp.h ./tensors/Tensor_extract_merge.h ./tensors/Tensor_index.h ./tensors/Tensor_inner.h ./tensors/Tensor_logical.h ./tensors/Tensor_outer.h ./tensors/Tensor_reality.h ./tensors/Tensor_Ta.h ./tensors/Tensor_trace.h ./tensors/Tensor_traits.h ./tensors/Tensor_transpose.h ./tensors/Tensor_unary.h ./Tensors.h ./Threads.h ./Timer.h +HFILES=./algorithms/approx/bigfloat.h ./algorithms/approx/bigfloat_double.h ./algorithms/approx/Chebyshev.h ./algorithms/approx/MultiShiftFunction.h ./algorithms/approx/Remez.h ./algorithms/approx/Zolotarev.h ./algorithms/CoarsenedMatrix.h ./algorithms/iterative/AdefGeneric.h ./algorithms/iterative/ConjugateGradient.h ./algorithms/iterative/ConjugateGradientMultiShift.h ./algorithms/iterative/ConjugateResidual.h ./algorithms/iterative/NormalEquations.h ./algorithms/iterative/PrecConjugateResidual.h ./algorithms/iterative/PrecGeneralisedConjugateResidual.h ./algorithms/iterative/SchurRedBlack.h ./algorithms/LinearOperator.h ./algorithms/Preconditioner.h ./algorithms/SparseMatrix.h ./Algorithms.h ./AlignedAllocator.h ./cartesian/Cartesian_base.h ./cartesian/Cartesian_full.h ./cartesian/Cartesian_red_black.h ./Cartesian.h ./communicator/Communicator_base.h ./Communicator.h ./Config.h ./cshift/Cshift_common.h ./cshift/Cshift_mpi.h ./cshift/Cshift_none.h ./Cshift.h ./Grid.h ./Init.h ./lattice/Lattice_arith.h ./lattice/Lattice_base.h ./lattice/Lattice_comparison.h ./lattice/Lattice_comparison_utils.h ./lattice/Lattice_conformable.h ./lattice/Lattice_coordinate.h ./lattice/Lattice_ET.h ./lattice/Lattice_local.h ./lattice/Lattice_overload.h ./lattice/Lattice_peekpoke.h ./lattice/Lattice_reality.h ./lattice/Lattice_reduction.h ./lattice/Lattice_rng.h ./lattice/Lattice_trace.h ./lattice/Lattice_transfer.h ./lattice/Lattice_transpose.h ./lattice/Lattice_unary.h ./lattice/Lattice_where.h ./Lattice.h ./Log.h ./Old/Tensor_peek.h ./Old/Tensor_poke.h ./parallelIO/NerscIO.h ./pugixml/pugixml.h ./qcd/action/ActionBase.h ./qcd/action/ActionParams.h ./qcd/action/Actions.h ./qcd/action/fermion/CayleyFermion5D.h ./qcd/action/fermion/ContinuedFractionFermion5D.h ./qcd/action/fermion/DomainWallFermion.h ./qcd/action/fermion/FermionOperator.h ./qcd/action/fermion/FermionOperatorImpl.h ./qcd/action/fermion/g5HermitianLinop.h ./qcd/action/fermion/MobiusFermion.h ./qcd/action/fermion/MobiusZolotarevFermion.h ./qcd/action/fermion/OverlapWilsonCayleyTanhFermion.h ./qcd/action/fermion/OverlapWilsonCayleyZolotarevFermion.h ./qcd/action/fermion/OverlapWilsonContfracTanhFermion.h ./qcd/action/fermion/OverlapWilsonContfracZolotarevFermion.h ./qcd/action/fermion/OverlapWilsonPartialFractionTanhFermion.h ./qcd/action/fermion/OverlapWilsonPartialFractionZolotarevFermion.h ./qcd/action/fermion/PartialFractionFermion5D.h ./qcd/action/fermion/ScaledShamirFermion.h ./qcd/action/fermion/ShamirZolotarevFermion.h ./qcd/action/fermion/WilsonCompressor.h ./qcd/action/fermion/WilsonFermion.h ./qcd/action/fermion/WilsonFermion5D.h ./qcd/action/fermion/WilsonKernels.h ./qcd/action/gauge/WilsonGaugeAction.h ./qcd/action/pseudofermion/EvenOddSchurDifferentiable.h ./qcd/action/pseudofermion/OneFlavourEvenOddRational.h ./qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h ./qcd/action/pseudofermion/OneFlavourRational.h ./qcd/action/pseudofermion/OneFlavourRationalRatio.h ./qcd/action/pseudofermion/TwoFlavour.h ./qcd/action/pseudofermion/TwoFlavourEvenOdd.h ./qcd/action/pseudofermion/TwoFlavourEvenOddRatio.h ./qcd/action/pseudofermion/TwoFlavourRatio.h ./qcd/hmc/HMC.h ./qcd/hmc/integrators/Integrator.h ./qcd/hmc/integrators/Integrator_algorithm.h ./qcd/QCD.h ./qcd/spin/Dirac.h ./qcd/spin/TwoSpinor.h ./qcd/utils/CovariantCshift.h ./qcd/utils/LinalgUtils.h ./qcd/utils/SpaceTimeGrid.h ./qcd/utils/SUn.h ./qcd/utils/WilsonLoops.h ./serialisation/BinaryIO.h ./serialisation/MacroMagic.h ./serialisation/Serialisation.h ./serialisation/XmlIO.h ./simd/Grid_avx.h ./simd/Grid_avx512.h ./simd/Grid_empty.h ./simd/Grid_neon.h ./simd/Grid_qpx.h ./simd/Grid_sse4.h ./simd/Grid_vector_types.h ./simd/Grid_vector_unops.h ./Simd.h ./stencil/Lebesgue.h ./Stencil.h ./tensors/Tensor_arith.h ./tensors/Tensor_arith_add.h ./tensors/Tensor_arith_mac.h ./tensors/Tensor_arith_mul.h ./tensors/Tensor_arith_scalar.h ./tensors/Tensor_arith_sub.h ./tensors/Tensor_class.h ./tensors/Tensor_determinant.h ./tensors/Tensor_exp.h ./tensors/Tensor_extract_merge.h ./tensors/Tensor_index.h ./tensors/Tensor_inner.h ./tensors/Tensor_logical.h ./tensors/Tensor_outer.h ./tensors/Tensor_reality.h ./tensors/Tensor_Ta.h ./tensors/Tensor_trace.h ./tensors/Tensor_traits.h ./tensors/Tensor_transpose.h ./tensors/Tensor_unary.h ./Tensors.h ./Threads.h ./Timer.h CCFILES=./algorithms/approx/MultiShiftFunction.cc ./algorithms/approx/Remez.cc ./algorithms/approx/Zolotarev.cc ./Init.cc ./Log.cc ./pugixml/pugixml.cc ./qcd/action/fermion/CayleyFermion5D.cc ./qcd/action/fermion/ContinuedFractionFermion5D.cc ./qcd/action/fermion/PartialFractionFermion5D.cc ./qcd/action/fermion/WilsonFermion.cc ./qcd/action/fermion/WilsonFermion5D.cc ./qcd/action/fermion/WilsonKernels.cc ./qcd/action/fermion/WilsonKernelsHand.cc ./qcd/hmc/HMC.cc ./qcd/hmc/integrators/Integrator.cc ./qcd/spin/Dirac.cc ./qcd/utils/SpaceTimeGrid.cc ./stencil/Lebesgue.cc ./stencil/Stencil_common.cc diff --git a/lib/Serialisation.h b/lib/Serialisation.h deleted file mode 100644 index da90eae1..00000000 --- a/lib/Serialisation.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef GRID_SERIALISATION_H -#define GRID_SERIALISATION_H - -#include // subdir aggregate - -#endif diff --git a/lib/parallelIO/NerscIO.h b/lib/parallelIO/NerscIO.h index 4db182e0..5339e5c8 100644 --- a/lib/parallelIO/NerscIO.h +++ b/lib/parallelIO/NerscIO.h @@ -441,7 +441,7 @@ inline void readNerscObject(Lattice &Umu,std::string file,munger munge,int template inline void writeNerscObject(Lattice &Umu,std::string file,munger munge,int offset, - int sequence,double lt,double pl) + int sequence,double lt,double pl) { GridBase *grid = Umu._grid; NerscField header; diff --git a/lib/serialisation/BinaryIO.h b/lib/serialisation/BinaryIO.h new file mode 100644 index 00000000..62633d29 --- /dev/null +++ b/lib/serialisation/BinaryIO.h @@ -0,0 +1,124 @@ +#ifndef GRID_SERIALISATION_BINARY_READER_H +#define GRID_SERIALISATION_BINARY_READER_H + +#include +#include +#include +#include +#include +#include +#include + + +namespace Grid { + +class BinaryWriter { +private: + + std::ofstream file; + +public: + + BinaryWriter(const std::string &_file) : file(_file,std::ios::binary|std::ios::out) {} + + ~BinaryWriter() {} + + // Binary is scopeless + void push(const std::string &s) {} + void pop(void) {} + + void iwrite(const std::string& s,const std::string &output) { + uint32_t sz = output.size(); + iwrite(s,sz); + const char * cstr = output.c_str(); + for(int c=0;c void writeInternal( const std::string& s, T output ){ + // FIXME --- htons, htonl, htno64 etc.. + file.write((char *)&output,sizeof(T)); + } + +}; + + +class BinaryReader { +private: + + std::ifstream file; + +public: + + + BinaryReader(const std::string &_file) : file(_file,std::ios::binary|std::ios::in) {} + + ~BinaryReader() {} + + // Binary is scopeless + void push(const std::string &s) { } + void pop(void) { } + + void iread( const std::string& s,std::string &output ) { + + output.clear(); + + uint32_t sz; + file.read((char *)&sz,sizeof(sz)); + + for(int c=0;c void iread( const std::string& s, std::vector &output ) { + + T tmp; + uint64_t n; + + iread("N",n); + output.resize(0); + for(int i=0;i void readInternal( const std::string& path, T &output ){ + file.read((char *)&output,sizeof(output)); // byte order?? + } + +}; + +} +#endif diff --git a/lib/serialisation/Reader.h b/lib/serialisation/Serialisation.h similarity index 89% rename from lib/serialisation/Reader.h rename to lib/serialisation/Serialisation.h index 9ba3192a..d530c9cf 100644 --- a/lib/serialisation/Reader.h +++ b/lib/serialisation/Serialisation.h @@ -35,9 +35,10 @@ namespace Grid { template< class Reader> void read(Reader& rd, const std::string& s, bool &output ) { rd.iread(s,output); }; - template - void write(Writer& wr, const std::string& s,const std::vector output ) { + template void write(Writer& wr, const std::string& s,const std::vector output ) { push(wr,s); + uint64_t sz =output.size(); + write(wr,"N",sz); for(int i=0;i -//#include -//#include -//#include - -#include +#include +#include +//#include +//#include +#include ////////////////////////////////////////// -// Select the default serialiser +// Select the default serialiser use ifdef's ////////////////////////////////////////// namespace Grid { - - using XMLPolicy::Reader; - using XMLPolicy::Writer; - + typedef XMLReader Reader; + typedef XMLWriter Writer; } - #endif diff --git a/lib/serialisation/TextIO.h b/lib/serialisation/TextIO.h new file mode 100644 index 00000000..a653fc16 --- /dev/null +++ b/lib/serialisation/TextIO.h @@ -0,0 +1,147 @@ +#ifndef GRID_SERIALISATION_TEXT_READER_H +#define GRID_SERIALISATION_TEXT_READER_H + +#include +#include +#include +#include +#include +#include +#include + +namespace Grid { + +class TextWriter { +private: + + std::ofstream file; + int level; + void indent(void) { + for(int i=0;i void writeInternal( const std::string& s, T output ){ + indent(); + file << std::boolalpha << output< + void iread( const std::string& s, std::vector &output ) { + + push(s); + + uint64_t n; iread("N",n); + + // skip the vector length + T tmp; + output.resize(0); + for(int i=0;i void readInternal( const std::string& path, T &output ){ + std::string asString; + iread(path,asString); + convert(asString,output); + } + + template void convert(const std::string &asString,T &output) + { + std::istringstream is(asString); is.exceptions(std::ios::failbit); + try { + is >> std::boolalpha >> output; + } catch(std::istringstream::failure e) { + std::cerr << "XML read failure on "<<" "< void writeInternal( const std::string& s, T output ){ std::ostringstream os; os << std::boolalpha << output; @@ -95,9 +96,7 @@ public: node= doc.child("document"); } - ~XMLReader() - { - } + ~XMLReader() { } void iread( const std::string& s,std::string &output ) { output=node.child(s.c_str()).first_child().value(); @@ -112,17 +111,25 @@ public: template void iread( const std::string& s, std::vector &output ) { - output.resize(0); - T tmp; push(s); + + uint64_t n; + + pugi::xml_node it=node.first_child(); + + // skip the vector length + T tmp; int i=0; - for(pugi::xml_node it=node.first_child(); it; it = it.next_sibling() ){ + output.resize(0); + for(it = it.next_sibling(); it; it = it.next_sibling() ){ std::ostringstream oss; oss << "elem" << i; read(*this,oss.str(),tmp); output.push_back(tmp); i++; } + + assert(i == n ); pop(); }; @@ -159,10 +166,5 @@ private: }; -namespace XMLPolicy -{ - typedef XMLReader Reader; - typedef XMLWriter Writer; -}; } #endif diff --git a/tests/Test_serialisation.cc b/tests/Test_serialisation.cc index 47328d45..91e37902 100644 --- a/tests/Test_serialisation.cc +++ b/tests/Test_serialisation.cc @@ -14,10 +14,11 @@ public: std::vector >, twodimarray, ); - myclass() : array(4,5.0), twodimarray(3,std::vector(2,2.0)) { - x=1; - y=2; - b=false; + myclass(){} + myclass(int i) : array(4,5.1), twodimarray(3,std::vector(2,1.23456)) { + x=i; + y=2*i; + b=true; name="bother said pooh"; } @@ -51,15 +52,39 @@ int main(int argc,char **argv) write(WR,"b",b); pop(WR); - myclass obj; + myclass obj(1234); // non-trivial constructor write(WR,"obj",obj); }; - Reader RD("bother2.xml"); + Reader RD("bother.xml"); - myclass copy; + myclass copy1; + myclass copy2; + myclass copy3; - read(RD,"obj",copy); - std::cout << "Loaded " << copy<