From 3bfd1f13e67735d2273127eefabafb779c00996d Mon Sep 17 00:00:00 2001 From: paboyle Date: Sun, 11 Jun 2017 23:14:10 +0100 Subject: [PATCH] I/O improvements --- benchmarks/Benchmark_memory_bandwidth.cc | 4 +- benchmarks/Benchmark_su3.cc | 4 +- configure.ac | 2 +- extras/Hadrons/Modules/MGauge/Load.cc | 4 +- lib/Grid.h | 1 + lib/GridStd.h | 1 + lib/cartesian/Cartesian_base.h | 9 +- lib/cartesian/Cartesian_full.h | 8 +- lib/cartesian/Cartesian_red_black.h | 4 + lib/communicator/Communicator_base.h | 2 + lib/communicator/Communicator_mpi.cc | 8 + lib/communicator/Communicator_mpi3.cc | 8 + lib/communicator/Communicator_none.cc | 2 + lib/parallelIO/BinaryIO.h | 249 ++++++--- lib/parallelIO/IldgIO.h | 472 ++++++++++++------ lib/parallelIO/IldgIOtypes.h | 110 ++-- lib/parallelIO/NerscIO.h | 301 +++-------- .../hmc/checkpointers/BinaryCheckpointer.h | 38 +- lib/qcd/hmc/checkpointers/ILDGCheckpointer.h | 32 +- lib/qcd/hmc/checkpointers/NerscCheckpointer.h | 2 +- lib/qcd/utils/Utils.h | 3 - lib/serialisation/XmlIO.cc | 58 ++- lib/serialisation/XmlIO.h | 11 +- tests/IO/Test_nersc_io.cc | 4 +- tests/IO/Test_nersc_read.cc | 2 +- tests/IO/Test_serialisation.cc | 19 +- 26 files changed, 779 insertions(+), 579 deletions(-) diff --git a/benchmarks/Benchmark_memory_bandwidth.cc b/benchmarks/Benchmark_memory_bandwidth.cc index d57c4df5..1aa088f8 100644 --- a/benchmarks/Benchmark_memory_bandwidth.cc +++ b/benchmarks/Benchmark_memory_bandwidth.cc @@ -55,8 +55,8 @@ int main (int argc, char ** argv) std::cout< latt_size ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]}); diff --git a/benchmarks/Benchmark_su3.cc b/benchmarks/Benchmark_su3.cc index 1321715a..3d7f9bc9 100644 --- a/benchmarks/Benchmark_su3.cc +++ b/benchmarks/Benchmark_su3.cc @@ -35,9 +35,9 @@ using namespace Grid::QCD; int main (int argc, char ** argv) { Grid_init(&argc,&argv); -#define LMAX (32) +#define LMAX (64) - int Nloop=200; + int Nloop=20; std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); diff --git a/configure.ac b/configure.ac index 62b7545b..2fc9dfec 100644 --- a/configure.ac +++ b/configure.ac @@ -27,7 +27,7 @@ AX_GXX_VERSION AC_DEFINE_UNQUOTED([GXX_VERSION],["$GXX_VERSION"], [version of g++ that will compile the code]) -CXXFLAGS="-O3 $CXXFLAGS" +CXXFLAGS="-g $CXXFLAGS" ############### Checks for typedefs, structures, and compiler characteristics diff --git a/extras/Hadrons/Modules/MGauge/Load.cc b/extras/Hadrons/Modules/MGauge/Load.cc index e5ee8abb..062e7e98 100644 --- a/extras/Hadrons/Modules/MGauge/Load.cc +++ b/extras/Hadrons/Modules/MGauge/Load.cc @@ -65,7 +65,7 @@ void TLoad::setup(void) // execution /////////////////////////////////////////////////////////////////// void TLoad::execute(void) { - NerscField header; + FieldMetaData header; std::string fileName = par().file + "." + std::to_string(env().getTrajectory()); @@ -74,5 +74,5 @@ void TLoad::execute(void) LatticeGaugeField &U = *env().createLattice(getName()); NerscIO::readConfiguration(U, header, fileName); LOG(Message) << "NERSC header:" << std::endl; - dump_nersc_header(header, LOG(Message)); + dump_meta_data(header, LOG(Message)); } diff --git a/lib/Grid.h b/lib/Grid.h index 543b0330..ce16894f 100644 --- a/lib/Grid.h +++ b/lib/Grid.h @@ -42,6 +42,7 @@ Author: paboyle #include #include #include +#include #include #endif diff --git a/lib/GridStd.h b/lib/GridStd.h index fb5e5b21..959ba9ac 100644 --- a/lib/GridStd.h +++ b/lib/GridStd.h @@ -18,6 +18,7 @@ #include #include #include +#include /////////////////// // Grid config diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h index b31b3b5f..0db6ce0d 100644 --- a/lib/cartesian/Cartesian_base.h +++ b/lib/cartesian/Cartesian_base.h @@ -50,7 +50,6 @@ public: GridBase(const std::vector & processor_grid) : CartesianCommunicator(processor_grid) {}; - // Physics Grid information. std::vector _simd_layout;// Which dimensions get relayed out over simd lanes. std::vector _fdimensions;// (full) Global dimensions of array prior to cb removal @@ -63,13 +62,12 @@ public: int _isites; int _fsites; // _isites*_osites = product(dimensions). int _gsites; - std::vector _slice_block; // subslice information + std::vector _slice_block;// subslice information std::vector _slice_stride; std::vector _slice_nblock; - // Might need these at some point - // std::vector _lstart; // local start of array in gcoors. _processor_coor[d]*_ldimensions[d] - // std::vector _lend; // local end of array in gcoors _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1 + std::vector _lstart; // local start of array in gcoors _processor_coor[d]*_ldimensions[d] + std::vector _lend ; // local end of array in gcoors _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1 public: @@ -176,6 +174,7 @@ public: inline int gSites(void) const { return _isites*_osites*_Nprocessors; }; inline int Nd (void) const { return _ndimension;}; + inline const std::vector LocalStarts(void) { return _lstart; }; inline const std::vector &FullDimensions(void) { return _fdimensions;}; inline const std::vector &GlobalDimensions(void) { return _gdimensions;}; inline const std::vector &LocalDimensions(void) { return _ldimensions;}; diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h index 7e29d311..b0e47fa4 100644 --- a/lib/cartesian/Cartesian_full.h +++ b/lib/cartesian/Cartesian_full.h @@ -76,6 +76,8 @@ public: _ldimensions.resize(_ndimension); _rdimensions.resize(_ndimension); _simd_layout.resize(_ndimension); + _lstart.resize(_ndimension); + _lend.resize(_ndimension); _ostride.resize(_ndimension); _istride.resize(_ndimension); @@ -94,8 +96,10 @@ public: // Use a reduced simd grid _ldimensions[d]= _gdimensions[d]/_processors[d]; //local dimensions _rdimensions[d]= _ldimensions[d]/_simd_layout[d]; //overdecomposition - _osites *= _rdimensions[d]; - _isites *= _simd_layout[d]; + _lstart[d] = _processor_coor[d]*_ldimensions[d]; + _lend[d] = _processor_coor[d]*_ldimensions[d]+_ldimensions[d]-1; + _osites *= _rdimensions[d]; + _isites *= _simd_layout[d]; // Addressing support if ( d==0 ) { diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h index 2f132c19..3037de00 100644 --- a/lib/cartesian/Cartesian_red_black.h +++ b/lib/cartesian/Cartesian_red_black.h @@ -151,6 +151,8 @@ public: _ldimensions.resize(_ndimension); _rdimensions.resize(_ndimension); _simd_layout.resize(_ndimension); + _lstart.resize(_ndimension); + _lend.resize(_ndimension); _ostride.resize(_ndimension); _istride.resize(_ndimension); @@ -169,6 +171,8 @@ public: _gdimensions[d] = _gdimensions[d]/2; // Remove a checkerboard } _ldimensions[d] = _gdimensions[d]/_processors[d]; + _lstart[d] = _processor_coor[d]*_ldimensions[d]; + _lend[d] = _processor_coor[d]*_ldimensions[d]+_ldimensions[d]-1; // Use a reduced simd grid _simd_layout[d] = simd_layout[d]; diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h index 23d4f647..12a8429f 100644 --- a/lib/communicator/Communicator_base.h +++ b/lib/communicator/Communicator_base.h @@ -177,6 +177,8 @@ class CartesianCommunicator { void GlobalSumVector(ComplexF *c,int N); void GlobalSum(ComplexD &c); void GlobalSumVector(ComplexD *c,int N); + void GlobalXOR(uint32_t &); + void GlobalXOR(uint64_t &); template void GlobalSum(obj &o){ typedef typename obj::scalar_type scalar_type; diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index 470a06c7..bd2a62fb 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -83,6 +83,14 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); assert(ierr==0); } +void CartesianCommunicator::GlobalXOR(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalXOR(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator); + assert(ierr==0); +} void CartesianCommunicator::GlobalSum(float &f){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc index 54a0f9b5..632eb991 100644 --- a/lib/communicator/Communicator_mpi3.cc +++ b/lib/communicator/Communicator_mpi3.cc @@ -510,6 +510,14 @@ void CartesianCommunicator::GlobalSum(uint64_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator); assert(ierr==0); } +void CartesianCommunicator::GlobalXOR(uint32_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator); + assert(ierr==0); +} +void CartesianCommunicator::GlobalXOR(uint64_t &u){ + int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator); + assert(ierr==0); +} void CartesianCommunicator::GlobalSum(float &f){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc index ace2868b..5319ab93 100644 --- a/lib/communicator/Communicator_none.cc +++ b/lib/communicator/Communicator_none.cc @@ -59,6 +59,8 @@ void CartesianCommunicator::GlobalSum(double &){} void CartesianCommunicator::GlobalSum(uint32_t &){} void CartesianCommunicator::GlobalSum(uint64_t &){} void CartesianCommunicator::GlobalSumVector(double *,int N){} +void CartesianCommunicator::GlobalXOR(uint32_t &){} +void CartesianCommunicator::GlobalXOR(uint64_t &){} void CartesianCommunicator::SendRecvPacket(void *xmit, void *recv, diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h index 8b8d4165..bc3da38b 100644 --- a/lib/parallelIO/BinaryIO.h +++ b/lib/parallelIO/BinaryIO.h @@ -125,57 +125,94 @@ class BinaryIO { ///////////////////////////////////////////////////////////////////////////// // more byte manipulation helpers ///////////////////////////////////////////////////////////////////////////// - static inline void Uint32Checksum(uint32_t *buf,uint64_t buf_size_bytes,uint32_t &csum) + + template static inline void Uint32Checksum(Lattice &lat, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) + { + typedef typename vobj::scalar_object sobj; + + GridBase *grid = lat._grid; + int lsites = grid->lSites(); + + std::vector scalardata(lsites); + unvectorizeToLexOrdArray(scalardata,lat); + + Uint32Checksum(grid,scalardata,nersc_csum,scidac_csuma,scidac_csumb); + } + + template + static inline void Uint32Checksum(GridBase *grid, + std::vector &fbuf, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) + { + const uint64_t size32 = sizeof(fobj)/sizeof(uint32_t); + + + int nd = grid->_ndimension; + + uint64_t lsites =grid->lSites(); + std::vector local_vol =grid->LocalDimensions(); + std::vector local_start =grid->LocalStarts(); + std::vector global_vol =grid->FullDimensions(); + #pragma omp parallel { - uint32_t csum_thr=0; - uint64_t count = buf_size_bytes/sizeof(uint32_t); + std::vector coor(nd); + uint32_t nersc_csum_thr=0; + uint32_t scidac_csuma_thr=0; + uint32_t scidac_csumb_thr=0; + uint32_t site_crc=0; + uint32_t zcrc = crc32(0L, Z_NULL, 0); + #pragma omp for - for(uint64_t i=0;i>(32-gsite29); + scidac_csumb_thr ^= site_crc<>(32-gsite31); } + #pragma omp critical - csum = csum + csum_thr; + { + nersc_csum += nersc_csum_thr; + scidac_csuma^= scidac_csuma_thr; + scidac_csumb^= scidac_csumb_thr; + } } } + // Network is big endian - static inline void htobe32_v(void *file_object,uint64_t bytes,uint32_t &csum){ - Uint32Checksum((uint32_t *)file_object,bytes,csum); - htobe32_v(file_object,bytes); - } - static inline void htobe64_v(void *file_object,uint64_t bytes,uint32_t &csum){ - Uint32Checksum((uint32_t *)file_object,bytes,csum); - htobe64_v(file_object,bytes); - } - static inline void htole32_v(void *file_object,uint64_t bytes,uint32_t &csum){ - Uint32Checksum((uint32_t *)file_object,bytes,csum); - htole32_v(file_object,bytes); - } - static inline void htole64_v(void *file_object,uint64_t bytes,uint32_t &csum){ - Uint32Checksum((uint32_t *)file_object,bytes,csum); - htole64_v(file_object,bytes); - } - static inline void be32toh_v(void *file_object,uint64_t bytes,uint32_t &csum){ - be32toh_v(file_object,bytes); - Uint32Checksum((uint32_t *)file_object,bytes,csum); - } - static inline void be64toh_v(void *file_object,uint64_t bytes,uint32_t &csum){ - be64toh_v(file_object,bytes); - Uint32Checksum((uint32_t *)file_object,bytes,csum); - } - static inline void le32toh_v(void *file_object,uint64_t bytes,uint32_t &csum){ - le32toh_v(file_object,bytes); - Uint32Checksum((uint32_t *)file_object,bytes,csum); - } - static inline void le64toh_v(void *file_object,uint64_t bytes,uint32_t &csum){ - le64toh_v(file_object,bytes); - Uint32Checksum((uint32_t *)file_object,bytes,csum); - } - static inline void htobe32_v(void *file_object,uint64_t bytes){ be32toh_v(file_object,bytes);} - static inline void htobe64_v(void *file_object,uint64_t bytes){ be64toh_v(file_object,bytes);} - static inline void htole32_v(void *file_object,uint64_t bytes){ le32toh_v(file_object,bytes);} - static inline void htole64_v(void *file_object,uint64_t bytes){ le64toh_v(file_object,bytes);} + static inline void htobe32_v(void *file_object,uint32_t bytes){ be32toh_v(file_object,bytes);} + static inline void htobe64_v(void *file_object,uint32_t bytes){ be64toh_v(file_object,bytes);} + static inline void htole32_v(void *file_object,uint32_t bytes){ le32toh_v(file_object,bytes);} + static inline void htole64_v(void *file_object,uint32_t bytes){ le64toh_v(file_object,bytes);} static inline void be32toh_v(void *file_object,uint64_t bytes) { @@ -199,6 +236,7 @@ class BinaryIO { fp[i] = ntohl(f); } } + // BE is same as network static inline void be64toh_v(void *file_object,uint64_t bytes) { @@ -238,18 +276,23 @@ class BinaryIO { static const int BINARYIO_WRITE = 0x01; template - static inline uint32_t IOobject(word w, - GridBase *grid, - std::vector &iodata, - std::string file, - int offset, - const std::string &format, int control) + static inline void IOobject(word w, + GridBase *grid, + std::vector &iodata, + std::string file, + int offset, + const std::string &format, int control, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) { grid->Barrier(); GridStopWatch timer; GridStopWatch bstimer; - uint32_t csum=0; + nersc_csum=0; + scidac_csuma=0; + scidac_csumb=0; int ndim = grid->Dimensions(); int nrank = grid->ProcessorCount(); @@ -359,20 +402,22 @@ class BinaryIO { grid->Barrier(); bstimer.Start(); - if (ieee32big) be32toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); - if (ieee32) le32toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); - if (ieee64big) be64toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); - if (ieee64) le64toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); + if (ieee32big) be32toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee32) le32toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64big) be64toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64) le64toh_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + Uint32Checksum(grid,iodata,nersc_csum,scidac_csuma,scidac_csumb); bstimer.Stop(); } if ( control & BINARYIO_WRITE ) { bstimer.Start(); - if (ieee32big) htobe32_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); - if (ieee32) htole32_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); - if (ieee64big) htobe64_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); - if (ieee64) htole64_v((void *)&iodata[0], sizeof(fobj)*iodata.size(),csum); + Uint32Checksum(grid,iodata,nersc_csum,scidac_csuma,scidac_csumb); + if (ieee32big) htobe32_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee32) htole32_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64big) htobe64_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); + if (ieee64) htole64_v((void *)&iodata[0], sizeof(fobj)*iodata.size()); bstimer.Stop(); grid->Barrier(); @@ -418,17 +463,27 @@ class BinaryIO { // Safety check ////////////////////////////////////////////////////////////////////////////// grid->Barrier(); - grid->GlobalSum(csum); + grid->GlobalSum(nersc_csum); + grid->GlobalXOR(scidac_csuma); + grid->GlobalXOR(scidac_csumb); grid->Barrier(); - - return csum; + // std::cout << "Binary IO NERSC checksum 0x"< - static inline uint32_t readLatticeObject(Lattice &Umu,std::string file,munger munge,int offset,const std::string &format) + static inline void readLatticeObject(Lattice &Umu, + std::string file, + munger munge, + int offset, + const std::string &format, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) { typedef typename vobj::scalar_object sobj; typedef typename vobj::Realified::scalar_type word; word w=0; @@ -439,7 +494,8 @@ class BinaryIO { std::vector scalardata(lsites); std::vector iodata(lsites); // Munge, checksum, byte order in here - uint32_t csum= IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); GridStopWatch timer; timer.Start(); @@ -451,15 +507,20 @@ class BinaryIO { timer.Stop(); std::cout< - static inline uint32_t writeLatticeObject(Lattice &Umu,std::string file,munger munge,int offset,const std::string &format) + static inline void writeLatticeObject(Lattice &Umu, + std::string file, + munger munge, + int offset, + const std::string &format, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) { typedef typename vobj::scalar_object sobj; typedef typename vobj::Realified::scalar_type word; word w=0; @@ -480,36 +541,45 @@ class BinaryIO { grid->Barrier(); timer.Stop(); - uint32_t csum= IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); std::cout< RNGstate; typedef RngStateType word; word w=0; - uint32_t csum = 0; std::string format = "IEEE32BIG"; GridBase *grid = parallel._grid; int gsites = grid->gSites(); int lsites = grid->lSites(); + uint32_t nersc_csum_tmp; + uint32_t scidac_csuma_tmp; + uint32_t scidac_csumb_tmp; + GridStopWatch timer; std::cout << GridLogMessage << "RNG read I/O on file " << file << std::endl; std::vector iodata(lsites); - csum= IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_READ|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); timer.Start(); parallel_for(int lidx=0;lidx tmp(RngStateCount); std::copy(iodata[0].begin(),iodata[0].end(),tmp.begin()); serial.SetState(tmp,0); } - std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl; + nersc_csum = nersc_csum + nersc_csum_tmp; + scidac_csuma = scidac_csuma ^ scidac_csuma_tmp; + scidac_csumb = scidac_csumb ^ scidac_csumb_tmp; + + // std::cout << GridLogMessage << "RNG file nersc_checksum " << std::hex << nersc_csum << std::dec << std::endl; + // std::cout << GridLogMessage << "RNG file scidac_checksuma " << std::hex << scidac_csuma << std::dec << std::endl; + // std::cout << GridLogMessage << "RNG file scidac_checksumb " << std::hex << scidac_csumb << std::dec << std::endl; + std::cout << GridLogMessage << "RNG state overhead " << timer.Elapsed() << std::endl; - return csum; } ///////////////////////////////////////////////////////////////////////////// // Write a RNG; lexico map to an array of state and use IOobject ////////////////////////////////////////////////////////////////////////////////////// - static inline uint32_t writeRNG(GridSerialRNG &serial,GridParallelRNG ¶llel,std::string file,int offset) + static inline void writeRNG(GridSerialRNG &serial, + GridParallelRNG ¶llel, + std::string file, + int offset, + uint32_t &nersc_csum, + uint32_t &scidac_csuma, + uint32_t &scidac_csumb) { typedef typename GridSerialRNG::RngStateType RngStateType; typedef RngStateType word; word w=0; const int RngStateCount = GridSerialRNG::RngStateCount; typedef std::array RNGstate; - uint32_t csum = 0; - GridBase *grid = parallel._grid; int gsites = grid->gSites(); int lsites = grid->lSites(); + uint32_t nersc_csum_tmp; + uint32_t scidac_csuma_tmp; + uint32_t scidac_csumb_tmp; + GridStopWatch timer; std::string format = "IEEE32BIG"; @@ -561,7 +647,8 @@ class BinaryIO { } timer.Stop(); - csum= IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_LEXICOGRAPHIC, + nersc_csum,scidac_csuma,scidac_csumb); iodata.resize(1); { @@ -569,11 +656,11 @@ class BinaryIO { serial.GetState(tmp,0); std::copy(tmp.begin(),tmp.end(),iodata[0].begin()); } - csum+= IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_MASTER_APPEND); + IOobject(w,grid,iodata,file,offset,format,BINARYIO_WRITE|BINARYIO_MASTER_APPEND, + nersc_csum_tmp,scidac_csuma_tmp,scidac_csumb_tmp); - std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl; + // std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl; std::cout << GridLogMessage << "RNG state overhead " << timer.Elapsed() << std::endl; - return csum; } }; } diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h index 0912e2f6..237edf43 100644 --- a/lib/parallelIO/IldgIO.h +++ b/lib/parallelIO/IldgIO.h @@ -43,201 +43,351 @@ extern "C" { // for linkage #include "lime.h" } + +// Unused SCIDAC records names +// SCIDAC_PRIVATE_FILE_XML "scidac-private-file-xml" +// SCIDAC_SITELIST "scidac-sitelist" +// SCIDAC_FILE_XML "scidac-file-xml" +// SCIDAC_RIVATE_RECORD_XML "scidac-private-record-xml" +// SCIDAC_RECORD_XML "scidac-record-xml" +// SCIDAC_BINARY_DATA "scidac-binary-data" +// +// Scidac checksum: CRC32 every site, xor reduce some hash of this. +// https://github.com/usqcd-software/qio/blob/master/lib/dml/DML_utils.c + namespace Grid { namespace QCD { -inline void ILDGGrid(GridBase *grid, ILDGField &header) { - assert(grid->_ndimension == 4); // emit error if not - header.dimension.resize(4); - header.boundary.resize(4); - for (int d = 0; d < 4; d++) { - header.dimension[d] = grid->_fdimensions[d]; - // Read boundary conditions from ... ? - header.boundary[d] = std::string("periodic"); - } -} - -inline void ILDGChecksum(uint32_t *buf, uint32_t buf_size_bytes, - uint32_t &csum) { - BinaryIO::Uint32Checksum(buf, buf_size_bytes, csum); -} - -////////////////////////////////////////////////////////////////////// -// Utilities ; these are QCD aware -////////////////////////////////////////////////////////////////////// -template -inline void ILDGStatistics(GaugeField &data, ILDGField &header) { - // How to convert data precision etc... - header.link_trace = Grid::QCD::WilsonLoops::linkTrace(data); - header.plaquette = Grid::QCD::WilsonLoops::avgPlaquette(data); - // header.polyakov = -} - -// Forcing QCD here -template -struct ILDGMunger { - void operator()(fobj &in, sobj &out, uint32_t &csum) { - for (int mu = 0; mu < 4; mu++) { - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - } - } - } - ILDGChecksum((uint32_t *)&in, sizeof(in), csum); - }; -}; - -template -struct ILDGUnmunger { - void operator()(sobj &in, fobj &out, uint32_t &csum) { - for (int mu = 0; mu < 4; mu++) { - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - } - } - } - ILDGChecksum((uint32_t *)&out, sizeof(out), csum); - }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// Write and read from fstream; compute header offset for payload -//////////////////////////////////////////////////////////////////////////////// -enum ILDGstate {ILDGread, ILDGwrite}; - -class ILDGIO : public BinaryIO { - FILE *File; - LimeWriter *LimeW; - LimeRecordHeader *LimeHeader; - LimeReader *LimeR; - std::string filename; - - +class IldgIO : public BinaryIO { public: - ILDGIO(std::string file, ILDGstate RW) { - filename = file; - if (RW == ILDGwrite){ - File = fopen(file.c_str(), "w"); - // check if opened correctly - LimeW = limeCreateWriter(File); - } else { - File = fopen(file.c_str(), "r"); - // check if opened correctly - - LimeR = limeCreateReader(File); - } - } - - ~ILDGIO() { fclose(File); } - - int createHeader(std::string message, int MB, int ME, size_t PayloadSize, LimeWriter* L){ + static int createHeader(std::string message, int MB, int ME, size_t PayloadSize, LimeWriter* L) + { LimeRecordHeader *h; h = limeCreateHeader(MB, ME, const_cast(message.c_str()), PayloadSize); - int status = limeWriteRecordHeader(h, L); - if (status < 0) { - std::cerr << "ILDG Header error\n"; - return status; - } + assert(limeWriteRecordHeader(h, L) >= 0); limeDestroyHeader(h); return LIME_SUCCESS; } - unsigned int writeHeader(ILDGField &header) { - // write header in LIME - n_uint64_t nbytes; - int MB_flag = 1, ME_flag = 0; - - char message[] = "ildg-format"; - nbytes = strlen(message); - LimeHeader = limeCreateHeader(MB_flag, ME_flag, message, nbytes); - limeWriteRecordHeader(LimeHeader, LimeW); - limeDestroyHeader(LimeHeader); - // save the xml header here - // use the xml_writer to c++ streams in pugixml - // and convert to char message - limeWriteRecordData(message, &nbytes, LimeW); + template + static void writeLimeObject(int MB,int ME,serialisable_object &object,std::string object_name,std::string record_name, LimeWriter *LimeW) + { + std::string xmlstring; + { + XmlWriter WR("",""); + write(WR,object_name,object); + xmlstring = WR.XmlString(); + } + uint64_t nbytes = xmlstring.size(); + LimeRecordHeader *h = limeCreateHeader(MB, ME,(char *)record_name.c_str(), nbytes); + assert(limeWriteRecordHeader(h, LimeW)>=0); + assert(limeWriteRecordData(&xmlstring[0], &nbytes, LimeW)>=0); limeWriterCloseRecord(LimeW); - - return 0; + limeDestroyHeader(h); } - unsigned int readHeader(ILDGField &header) { + static unsigned int writeHeader(FieldMetaData &header, LimeWriter *LimeW) { + + uint64_t nbytes; + + ildgFormat ildgfmt ; + usqcdInfo info; + + ////////////////////////////////////////////////////// + // Fill ILDG header data struct + ////////////////////////////////////////////////////// + ildgfmt.field = std::string("su3gauge"); + ildgfmt.precision = 64; + ildgfmt.version = 1.0; + ildgfmt.lx = header.dimension[0]; + ildgfmt.ly = header.dimension[1]; + ildgfmt.lz = header.dimension[2]; + ildgfmt.lt = header.dimension[3]; + assert(header.nd==4); + assert(header.nd==header.dimension.size()); + + info.version=1.0; + info.plaq = header.plaquette; + info.linktr = header.link_trace; + + // Following scidac file downloaded from NERSC under MILC + // Begin message, keep open on successive records + //Message 1 + // Type: scidac-private-file-xml 1.1416 16 16 48 0 + // Type: scidac-file-xml MILC ILDG archival gauge configuration + //Message 2 + // Type: scidac-private-record-xml 1.0Thu May 11 00:11:33 2006 UTC0 + // QDP_F3_ColorMatrixF3724 + // Type: scidac-record-xml + // Type: ildg-format + // Type: ildg-data-lfn + // Type: ildg-binary-data + // Type: scidac-checksum + + writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT),LimeW); + writeLimeObject(0,0,info ,std::string("usqcdInfo" ),std::string(USQCD_INFO ),LimeW); + writeLimeObject(0,0,ildgfmt,std::string("ildgFormat") ,std::string(ILDG_FORMAT),LimeW); + // LFN is not a serializable object + { + std::string LFN = header.ildg_lfn; + uint64_t PayloadSize = LFN.size(); + createHeader(ILDG_DATA_LFN, 0 , 0, PayloadSize, LimeW); + limeWriteRecordData(const_cast(LFN.c_str()), &PayloadSize, LimeW); + limeWriterCloseRecord(LimeW); + } return 0; } template - uint32_t readConfiguration(Lattice > &Umu) { - typedef Lattice > GaugeField; - typedef LorentzColourMatrixD sobjd; - typedef LorentzColourMatrixF sobjf; - typedef iLorentzColourMatrix itype; - typedef LorentzColourMatrix sobj; - GridBase *grid = Umu._grid; + static void writeConfiguration(std::string filename,Lattice > &Umu, std::string format) { - ILDGField header; - readHeader(header); + FILE *File = fopen(filename.c_str(), "w"); + LimeWriter *LimeW = limeCreateWriter(File); - // now just the conf, ignore the header - std::string format = std::string("IEEE64BIG"); - do {limeReaderNextRecord(LimeR);} - while (strncmp(limeReaderType(LimeR), "ildg-binary-data",16)); - - n_uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration) - - - ILDGtype ILDGt(true, LimeR); - // this is special for double prec data, just for the moment - uint32_t csum = BinaryIO::readObjectParallel< itype, sobjd >( - Umu, filename, ILDGMunger(), 0, format, ILDGt); - - // Check configuration - // todo - - return csum; - } - - template - uint32_t writeConfiguration(Lattice > &Umu, std::string format) { typedef Lattice > GaugeField; typedef iLorentzColourMatrix vobj; typedef typename vobj::scalar_object sobj; typedef LorentzColourMatrixD fobj; - ILDGField header; - // fill the header + GridBase * grid = Umu._grid; + + //////////////////////////////////////// + // fill the headers + //////////////////////////////////////// + FieldMetaData header; + + GridMetaData(grid,header); + GaugeStatistics(Umu,header); + MachineCharacteristics(header); + + assert( (format=="IEEE64BIG") || (format=="IEEE32BIG")); header.floating_point = format; + header.checksum = 0x0; // unused in ILDG + writeHeader(header,LimeW); - ILDGUnmunger munge; - unsigned int offset = writeHeader(header); - - BinaryIO::Uint32Checksum(Umu, munge, header.checksum); - + //////////////////////////////////////// // Write data record header - n_uint64_t PayloadSize = sizeof(fobj) * Umu._grid->_gsites; - createHeader("ildg-binary-data", 0, 1, PayloadSize, LimeW); - - ILDGtype ILDGt(true, LimeW); - uint32_t csum = BinaryIO::writeObjectParallel( - Umu, filename, munge, 0, header.floating_point, ILDGt); - + //////////////////////////////////////// + uint64_t PayloadSize = sizeof(fobj) * Umu._grid->_gsites; + createHeader(ILDG_BINARY_DATA, 0, 0, PayloadSize, LimeW); + + off_t offset = ftell(File); + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + GaugeSimpleMunger munge; + BinaryIO::writeLatticeObject(Umu, filename, munge, offset, header.floating_point, + nersc_csum,scidac_csuma,scidac_csumb); limeWriterCloseRecord(LimeW); - // Last record - // the logical file name LNF - // look into documentation on how to generate this string - std::string LNF = "empty"; + //////////////////////////////////////// + // Write checksum element, propagaing forward from the BinaryIO + //////////////////////////////////////// + scidacChecksum checksum; + checksum.suma= scidac_csuma; + checksum.sumb= scidac_csumb; + // std::cout << " writing scidac checksums "< + static void readConfiguration(std::string filename,Lattice > &Umu, FieldMetaData &FieldMetaData_) { + + typedef Lattice > GaugeField; + typedef LorentzColourMatrixD sobjd; + typedef LorentzColourMatrixF sobjf; + typedef iLorentzColourMatrix itype; + typedef LorentzColourMatrix sobj; + + GridBase *grid = Umu._grid; + + std::vector dims = Umu._grid->FullDimensions(); + assert(dims.size()==4); + + FILE *File = fopen(filename.c_str(), "r"); + LimeReader *LimeR = limeCreateReader(File); - PayloadSize = sizeof(LNF); - createHeader("ildg-binary-lfn", 1 , 1, PayloadSize, LimeW); - limeWriteRecordData(const_cast(LNF.c_str()), &PayloadSize, LimeW); + // Metadata holders + ildgFormat ildgFormat_ ; + std::string ildgLFN_ ; + scidacChecksum scidacChecksum_; + usqcdInfo usqcdInfo_ ; - limeWriterCloseRecord(LimeW); + // track what we read from file + int found_ildgFormat =0; + int found_ildgLFN =0; + int found_scidacChecksum=0; + int found_usqcdInfo =0; + int found_ildgBinary =0; + int found_FieldMetaData =0; - return csum; + uint32_t nersc_csum; + uint32_t scidac_csuma; + uint32_t scidac_csumb; + + // Binary format + std::string format; + + ////////////////////////////////////////////////////////////////////////// + // Loop over all records + // -- Order is poorly guaranteed except ILDG header preceeds binary section. + // -- Run like an event loop. + // -- Impose trust hierarchy. Grid takes precedence & look for ILDG, and failing + // that Scidac. + // -- Insist on Scidac checksum record. + ////////////////////////////////////////////////////////////////////////// + + while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { + + uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration) + + ////////////////////////////////////////////////////////////////// + // If not BINARY_DATA read a string and parse + ////////////////////////////////////////////////////////////////// + if ( strncmp(limeReaderType(LimeR), ILDG_BINARY_DATA,strlen(ILDG_BINARY_DATA) ) ) { + + // Copy out the string + std::vector xmlc(nbytes+1,'\0'); + limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); + std::cout << GridLogMessage<< "Non binary record :" < munge; + BinaryIO::readLatticeObject< itype, sobjd >(Umu, filename, munge, offset, format, + nersc_csum,scidac_csuma,scidac_csumb); + found_ildgBinary = 1; + } + + } + + ////////////////////////////////////////////////////// + // Minimally must find binary segment and checksum + ////////////////////////////////////////////////////// + assert(found_ildgBinary); + assert(found_scidacChecksum); + + // Must find something with the lattice dimensions + assert(found_FieldMetaData||found_ildgFormat); + + if ( found_FieldMetaData ) { + + std::cout << GridLogMessage<<"a Grid MetaData was record found: configuration was probably written by Grid ! Yay ! "<(Umu,checker); + assert(fabs(checker.plaquette - FieldMetaData_.plaquette )<1.0e-5); + assert(fabs(checker.link_trace - FieldMetaData_.link_trace)<1.0e-5); + std::cout << GridLogMessage<<"Plaquette and link trace match " << std::endl; + } } // format for RNG? Now just binary out diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h index 4c7a1edd..8e1316eb 100644 --- a/lib/parallelIO/IldgIOtypes.h +++ b/lib/parallelIO/IldgIOtypes.h @@ -34,47 +34,83 @@ extern "C" { // for linkage namespace Grid { -struct ILDGtype { - bool is_ILDG; - LimeWriter* LW; - LimeReader* LR; +#define GRID_FORMAT "grid-format" +#define ILDG_FORMAT "ildg-format" +#define ILDG_BINARY_DATA "ildg-binary-data" +#define ILDG_DATA_LFN "ildg-data-lfn" +#define USQCD_INFO "usqcdInfo" +#define SCIDAC_CHECKSUM "scidac-checksum" - ILDGtype(bool is, LimeWriter* L) : is_ILDG(is), LW(L), LR(NULL) {} - ILDGtype(bool is, LimeReader* L) : is_ILDG(is), LW(NULL), LR(L) {} - ILDGtype() : is_ILDG(false), LW(NULL), LR(NULL) {} +///////////////////////////////////////////////////////////////////////////////// +// Data representation of records that enter ILDG and SciDac formats +///////////////////////////////////////////////////////////////////////////////// +struct ildgFormat : Serializable { +public: + GRID_SERIALIZABLE_CLASS_MEMBERS(ildgFormat, + double, version, + std::string, field, + int, precision, + int, lx, + int, ly, + int, lz, + int, lt); + ildgFormat() { + version=1.0; + }; }; - -class ILDGField { +struct usqcdInfo : Serializable { public: - // header strings (not in order) - std::vector dimension; - std::vector boundary; - int data_start; - std::string hdr_version; - std::string storage_format; - // Checks on data - double link_trace; - double plaquette; - uint32_t checksum; - unsigned int sequence_number; - std::string data_type; - std::string ensemble_id; - std::string ensemble_label; - std::string creator; - std::string creator_hardware; - std::string creation_date; - std::string archive_date; - std::string floating_point; + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdInfo, + double, version, + double, plaq, + double, linktr, + std::string, info); + usqcdInfo() { + version=1.0; + }; +}; + +struct usqcdPropFile : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdPropFile, + double, version, + std::string, type, + std::string, info); + usqcdPropFile() { + version=1.0; + }; +}; +struct usqcdSourceInfo : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdSourceInfo, + double, version, + std::string, info); + usqcdSourceInfo() { + version=1.0; + }; +}; +struct usqcdPropInfo : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(usqcdPropInfo, + double, version, + int, spin, + int, color, + std::string, info); + usqcdPropInfo() { + version=1.0; + }; +}; +struct scidacChecksum : Serializable { + public: + GRID_SERIALIZABLE_CLASS_MEMBERS(scidacChecksum, + double, version, + uint32_t, suma, + uint32_t, sumb); + scidacChecksum() { + version=1.0; + suma=sumb=0; + }; }; } -#else -namespace Grid { - -struct ILDGtype { - bool is_ILDG; - ILDGtype() : is_ILDG(false) {} -}; -} - #endif #endif diff --git a/lib/parallelIO/NerscIO.h b/lib/parallelIO/NerscIO.h index ba9d23de..cc37b537 100644 --- a/lib/parallelIO/NerscIO.h +++ b/lib/parallelIO/NerscIO.h @@ -30,168 +30,11 @@ #ifndef GRID_NERSC_IO_H #define GRID_NERSC_IO_H -#include -#include -#include -#include -#include - -#include -#include -#include - namespace Grid { namespace QCD { using namespace Grid; - //////////////////////////////////////////////////////////////////////////////// - // Some data types for intermediate storage - //////////////////////////////////////////////////////////////////////////////// - template using iLorentzColour2x3 = iVector, 2>, 4 >; - - typedef iLorentzColour2x3 LorentzColour2x3; - typedef iLorentzColour2x3 LorentzColour2x3F; - typedef iLorentzColour2x3 LorentzColour2x3D; - - //////////////////////////////////////////////////////////////////////////////// - // header specification/interpretation - //////////////////////////////////////////////////////////////////////////////// - class NerscField { - public: - // header strings (not in order) - int dimension[4]; - std::string boundary[4]; - int data_start; - std::string hdr_version; - std::string storage_format; - // Checks on data - double link_trace; - double plaquette; - uint32_t checksum; - unsigned int sequence_number; - std::string data_type; - std::string ensemble_id ; - std::string ensemble_label ; - std::string creator ; - std::string creator_hardware ; - std::string creation_date ; - std::string archive_date ; - std::string floating_point; - }; - - ////////////////////////////////////////////////////////////////////// - // Bit and Physical Checksumming and QA of data - ////////////////////////////////////////////////////////////////////// - - inline void NerscGrid(GridBase *grid,NerscField &header) - { - assert(grid->_ndimension==4); - for(int d=0;d<4;d++) { - header.dimension[d] = grid->_fdimensions[d]; - } - for(int d=0;d<4;d++) { - header.boundary[d] = std::string("PERIODIC"); - } - } - template - inline void NerscStatistics(GaugeField & data,NerscField &header) - { - // How to convert data precision etc... - header.link_trace=Grid::QCD::WilsonLoops::linkTrace(data); - header.plaquette =Grid::QCD::WilsonLoops::avgPlaquette(data); - } - - inline void NerscMachineCharacteristics(NerscField &header) - { - // Who - struct passwd *pw = getpwuid (getuid()); - if (pw) header.creator = std::string(pw->pw_name); - - // When - std::time_t t = std::time(nullptr); - std::tm tm = *std::localtime(&t); - std::ostringstream oss; - // oss << std::put_time(&tm, "%c %Z"); - header.creation_date = oss.str(); - header.archive_date = header.creation_date; - - // What - struct utsname name; uname(&name); - header.creator_hardware = std::string(name.nodename)+"-"; - header.creator_hardware+= std::string(name.machine)+"-"; - header.creator_hardware+= std::string(name.sysname)+"-"; - header.creator_hardware+= std::string(name.release); - - } - ////////////////////////////////////////////////////////////////////// - // Utilities ; these are QCD aware - ////////////////////////////////////////////////////////////////////// - inline void reconstruct3(LorentzColourMatrix & cm) - { - const int x=0; - const int y=1; - const int z=2; - for(int mu=0;mu<4;mu++){ - cm(mu)()(2,x) = adj(cm(mu)()(0,y)*cm(mu)()(1,z)-cm(mu)()(0,z)*cm(mu)()(1,y)); //x= yz-zy - cm(mu)()(2,y) = adj(cm(mu)()(0,z)*cm(mu)()(1,x)-cm(mu)()(0,x)*cm(mu)()(1,z)); //y= zx-xz - cm(mu)()(2,z) = adj(cm(mu)()(0,x)*cm(mu)()(1,y)-cm(mu)()(0,y)*cm(mu)()(1,x)); //z= xy-yx - } - } - - template - struct NerscSimpleMunger{ - void operator()(fobj &in, sobj &out) { - for (int mu = 0; mu < Nd; mu++) { - for (int i = 0; i < Nc; i++) { - for (int j = 0; j < Nc; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - }} - } - }; - }; - - template - struct NerscSimpleUnmunger { - - void operator()(sobj &in, fobj &out) { - for (int mu = 0; mu < Nd; mu++) { - for (int i = 0; i < Nc; i++) { - for (int j = 0; j < Nc; j++) { - out(mu)()(i, j) = in(mu)()(i, j); - }} - } - }; - }; - - template - struct Nersc3x2munger{ - - void operator() (fobj &in,sobj &out){ - for(int mu=0;mu<4;mu++){ - for(int i=0;i<2;i++){ - for(int j=0;j<3;j++){ - out(mu)()(i,j) = in(mu)(i)(j); - }} - } - reconstruct3(out); - } - }; - - template - struct Nersc3x2unmunger{ - - void operator() (sobj &in,fobj &out){ - for(int mu=0;mu<4;mu++){ - for(int i=0;i<2;i++){ - for(int j=0;j<3;j++){ - out(mu)(i)(j) = in(mu)()(i,j); - }} - } - } - }; - - //////////////////////////////////////////////////////////////////////////////// // Write and read from fstream; comput header offset for payload //////////////////////////////////////////////////////////////////////////////// @@ -202,42 +45,17 @@ namespace Grid { std::ofstream fout(file,std::ios::out); } -#define dump_nersc_header(field, s) \ - s << "BEGIN_HEADER" << std::endl; \ - s << "HDR_VERSION = " << field.hdr_version << std::endl; \ - s << "DATATYPE = " << field.data_type << std::endl; \ - s << "STORAGE_FORMAT = " << field.storage_format << std::endl; \ - for(int i=0;i<4;i++){ \ - s << "DIMENSION_" << i+1 << " = " << field.dimension[i] << std::endl ; \ - } \ - s << "LINK_TRACE = " << std::setprecision(10) << field.link_trace << std::endl; \ - s << "PLAQUETTE = " << std::setprecision(10) << field.plaquette << std::endl; \ - for(int i=0;i<4;i++){ \ - s << "BOUNDARY_"< header; @@ -309,19 +127,21 @@ namespace Grid { return field.data_start; } - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Now the meat: the object readers - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Now the meat: the object readers + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template - static inline void readConfiguration(Lattice > &Umu,NerscField& header,std::string file) + static inline void readConfiguration(Lattice > &Umu, + FieldMetaData& header, + std::string file) { typedef Lattice > GaugeField; GridBase *grid = Umu._grid; int offset = readHeader(file,Umu._grid,header); - NerscField clone(header); + FieldMetaData clone(header); std::string format(header.floating_point); @@ -330,34 +150,38 @@ namespace Grid { int ieee64big = (format == std::string("IEEE64BIG")); int ieee64 = (format == std::string("IEEE64")); - uint32_t csum; + uint32_t nersc_csum,scidac_csuma,scidac_csumb; // depending on datatype, set up munger; // munger is a function of if ( header.data_type == std::string("4D_SU3_GAUGE") ) { if ( ieee32 || ieee32big ) { - csum=BinaryIO::readLatticeObject, LorentzColour2x3F> - (Umu,file,Nersc3x2munger(), offset,format); + BinaryIO::readLatticeObject, LorentzColour2x3F> + (Umu,file,Gauge3x2munger(), offset,format, + nersc_csum,scidac_csuma,scidac_csumb); } if ( ieee64 || ieee64big ) { - csum=BinaryIO::readLatticeObject, LorentzColour2x3D> - (Umu,file,Nersc3x2munger(),offset,format); + BinaryIO::readLatticeObject, LorentzColour2x3D> + (Umu,file,Gauge3x2munger(),offset,format, + nersc_csum,scidac_csuma,scidac_csumb); } } else if ( header.data_type == std::string("4D_SU3_GAUGE_3x3") ) { if ( ieee32 || ieee32big ) { - csum=BinaryIO::readLatticeObject,LorentzColourMatrixF> - (Umu,file,NerscSimpleMunger(),offset,format); + BinaryIO::readLatticeObject,LorentzColourMatrixF> + (Umu,file,GaugeSimpleMunger(),offset,format, + nersc_csum,scidac_csuma,scidac_csumb); } if ( ieee64 || ieee64big ) { - csum=BinaryIO::readLatticeObject,LorentzColourMatrixD> - (Umu,file,NerscSimpleMunger(),offset,format); + BinaryIO::readLatticeObject,LorentzColourMatrixD> + (Umu,file,GaugeSimpleMunger(),offset,format, + nersc_csum,scidac_csuma,scidac_csumb); } } else { assert(0); } - NerscStatistics(Umu,clone); + GaugeStatistics(Umu,clone); - std::cout< - static inline void writeConfiguration(Lattice > &Umu,std::string file, int two_row,int bits32) + static inline void writeConfiguration(Lattice > &Umu, + std::string file, + int two_row, + int bits32) { typedef Lattice > GaugeField; typedef iLorentzColourMatrix vobj; typedef typename vobj::scalar_object sobj; + FieldMetaData header; + /////////////////////////////////////////// // Following should become arguments - NerscField header; + /////////////////////////////////////////// header.sequence_number = 1; header.ensemble_id = "UKQCD"; header.ensemble_label = "DWF"; @@ -402,32 +231,31 @@ namespace Grid { GridBase *grid = Umu._grid; - NerscGrid(grid,header); - NerscStatistics(Umu,header); - NerscMachineCharacteristics(header); + GridMetaData(grid,header); + assert(header.nd==4); + GaugeStatistics(Umu,header); + MachineCharacteristics(header); int offset; truncate(file); - if ( two_row ) { - header.floating_point = std::string("IEEE64BIG"); - header.data_type = std::string("4D_SU3_GAUGE"); - Nersc3x2unmunger munge; - offset = writeHeader(header,file); - header.checksum=BinaryIO::writeLatticeObject(Umu,file,munge,offset,header.floating_point); - writeHeader(header,file); - } else { - header.floating_point = std::string("IEEE64BIG"); - header.data_type = std::string("4D_SU3_GAUGE_3x3"); - NerscSimpleUnmunger munge; - offset = writeHeader(header,file); - header.checksum=BinaryIO::writeLatticeObject(Umu,file,munge,offset,header.floating_point); - writeHeader(header,file); - } + // Sod it -- always write 3x3 double + header.floating_point = std::string("IEEE64BIG"); + header.data_type = std::string("4D_SU3_GAUGE_3x3"); + GaugeSimpleUnmunger munge; + offset = writeHeader(header,file); + + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::writeLatticeObject(Umu,file,munge,offset,header.floating_point, + nersc_csum,scidac_csuma,scidac_csumb); + header.checksum = nersc_csum; + writeHeader(header,file); + std::cout< - uint32_t csum=BinaryIO::readRNG(serial,parallel,file,offset); + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::readRNG(serial,parallel,file,offset,nersc_csum,scidac_csuma,scidac_csumb); - if ( csum != header.checksum ) { - std::cerr << "checksum mismatch "< { fout.close(); } - void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, - GridParallelRNG &pRNG) { + void TrajectoryComplete(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { + if ((traj % Params.saveInterval) == 0) { std::string config, rng; this->build_filenames(traj, Params, config, rng); + uint32_t nersc_csum; + uint32_t scidac_csuma; + uint32_t scidac_csumb; + BinarySimpleUnmunger munge; truncate(rng); - BinaryIO::writeRNG(sRNG, pRNG, rng, 0); + BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); truncate(config); - uint32_t csum = BinaryIO::writeLatticeObject( - U, config, munge, 0, Params.format); + + BinaryIO::writeLatticeObject(U, config, munge, 0, Params.format, + nersc_csum,scidac_csuma,scidac_csumb); std::cout << GridLogMessage << "Written Binary Configuration " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksum " << std::hex + << nersc_csum <<"/" + << scidac_csuma <<"/" + << scidac_csumb + << std::dec << std::endl; } + }; - void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG, - GridParallelRNG &pRNG) { + void CheckpointRestore(int traj, Field &U, GridSerialRNG &sRNG, GridParallelRNG &pRNG) { std::string config, rng; this->build_filenames(traj, Params, config, rng); BinarySimpleMunger munge; - BinaryIO::readRNG(sRNG, pRNG, rng, 0); - uint32_t csum = BinaryIO::readLatticeObject( - U, config, munge, 0, Params.format); + uint32_t nersc_csum; + uint32_t scidac_csuma; + uint32_t scidac_csumb; + BinaryIO::readRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + BinaryIO::readLatticeObject(U, config, munge, 0, Params.format, + nersc_csum,scidac_csuma,scidac_csumb); + std::cout << GridLogMessage << "Read Binary Configuration " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksums " << std::hex << nersc_csum<<"/"< { // check here that the format is valid int ieee32big = (Params.format == std::string("IEEE32BIG")); - int ieee32 = (Params.format == std::string("IEEE32")); + int ieee32 = (Params.format == std::string("IEEE32")); int ieee64big = (Params.format == std::string("IEEE64BIG")); - int ieee64 = (Params.format == std::string("IEEE64")); + int ieee64 = (Params.format == std::string("IEEE64")); if (!(ieee64big || ieee32 || ieee32big || ieee64)) { std::cout << GridLogError << "Unrecognized file format " << Params.format @@ -74,13 +74,17 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer { if ((traj % Params.saveInterval) == 0) { std::string config, rng; this->build_filenames(traj, Params, config, rng); - - ILDGIO IO(config, ILDGwrite); - BinaryIO::writeRNGSerial(sRNG, pRNG, rng, 0); - uint32_t csum = IO.writeConfiguration(U, Params.format); + + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + IldgIO::writeConfiguration(config,U, Params.format); std::cout << GridLogMessage << "Written ILDG Configuration on " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksum " << std::hex + << nersc_csum<<"/" + << scidac_csuma<<"/" + << scidac_csumb + << std::dec << std::endl; } }; @@ -89,12 +93,18 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer { std::string config, rng; this->build_filenames(traj, Params, config, rng); - ILDGIO IO(config, ILDGread); - BinaryIO::readRNGSerial(sRNG, pRNG, rng, 0); - uint32_t csum = IO.readConfiguration(U); // format from the header + uint32_t nersc_csum,scidac_csuma,scidac_csumb; + BinaryIO::readRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb); + + FieldMetaData header; + IldgIO::readConfiguration(config,U,header); // format from the header std::cout << GridLogMessage << "Read ILDG Configuration from " << config - << " checksum " << std::hex << csum << std::dec << std::endl; + << " checksum " << std::hex + << nersc_csum<<"/" + << scidac_csuma<<"/" + << scidac_csumb + << std::dec << std::endl; }; }; } diff --git a/lib/qcd/hmc/checkpointers/NerscCheckpointer.h b/lib/qcd/hmc/checkpointers/NerscCheckpointer.h index 395369a0..a4b1b480 100644 --- a/lib/qcd/hmc/checkpointers/NerscCheckpointer.h +++ b/lib/qcd/hmc/checkpointers/NerscCheckpointer.h @@ -70,7 +70,7 @@ class NerscHmcCheckpointer : public BaseHmcCheckpointer { std::string config, rng; this->build_filenames(traj, Params, config, rng); - NerscField header; + FieldMetaData header; NerscIO::readRNGState(sRNG, pRNG, header, rng); NerscIO::readConfiguration(U, header, config); }; diff --git a/lib/qcd/utils/Utils.h b/lib/qcd/utils/Utils.h index 61c81cb5..1786db54 100644 --- a/lib/qcd/utils/Utils.h +++ b/lib/qcd/utils/Utils.h @@ -12,7 +12,4 @@ #include #include - - - #endif diff --git a/lib/serialisation/XmlIO.cc b/lib/serialisation/XmlIO.cc index b04263c9..a132a2f0 100644 --- a/lib/serialisation/XmlIO.cc +++ b/lib/serialisation/XmlIO.cc @@ -32,16 +32,21 @@ using namespace Grid; using namespace std; // Writer implementation /////////////////////////////////////////////////////// -XmlWriter::XmlWriter(const string &fileName) -: fileName_(fileName) +XmlWriter::XmlWriter(const string &fileName, string toplev) : fileName_(fileName) { - node_ = doc_.append_child(); - node_.set_name("grid"); + if ( toplev == std::string("") ) { + node_=doc_; + } else { + node_=doc_.append_child(); + node_.set_name(toplev.c_str()); + } } XmlWriter::~XmlWriter(void) { - doc_.save_file(fileName_.c_str(), " "); + if ( fileName_ != std::string("") ) { + doc_.save_file(fileName_.c_str(), " "); + } } void XmlWriter::push(const string &s) @@ -53,21 +58,44 @@ void XmlWriter::pop(void) { node_ = node_.parent(); } - -// Reader implementation /////////////////////////////////////////////////////// -XmlReader::XmlReader(const string &fileName) -: fileName_(fileName) +std::string XmlWriter::XmlString(void) { - pugi::xml_parse_result result = doc_.load_file(fileName_.c_str()); - - if ( !result ) - { + std::ostringstream oss; + doc_.save(oss); + return oss.str(); +} + +XmlReader::XmlReader(const char *xmlstring,string toplev) : fileName_("") +{ + pugi::xml_parse_result result; + result = doc_.load_string(xmlstring); + if ( !result ) { cerr << "XML error description: " << result.description() << "\n"; cerr << "XML error offset : " << result.offset << "\n"; abort(); } - - node_ = doc_.child("grid"); + if ( toplev == std::string("") ) { + node_ = doc_; + } else { + node_ = doc_.child(toplev.c_str()); + } +} + +// Reader implementation /////////////////////////////////////////////////////// +XmlReader::XmlReader(const string &fileName,string toplev) : fileName_(fileName) +{ + pugi::xml_parse_result result; + result = doc_.load_file(fileName_.c_str()); + if ( !result ) { + cerr << "XML error description: " << result.description() << "\n"; + cerr << "XML error offset : " << result.offset << "\n"; + abort(); + } + if ( toplev == std::string("") ) { + node_ = doc_; + } else { + node_ = doc_.child(toplev.c_str()); + } } bool XmlReader::push(const string &s) diff --git a/lib/serialisation/XmlIO.h b/lib/serialisation/XmlIO.h index f333b9aa..fcdbf1e4 100644 --- a/lib/serialisation/XmlIO.h +++ b/lib/serialisation/XmlIO.h @@ -44,10 +44,9 @@ namespace Grid { class XmlWriter: public Writer - { - + { public: - XmlWriter(const std::string &fileName); + XmlWriter(const std::string &fileName,std::string toplev = std::string("grid") ); virtual ~XmlWriter(void); void push(const std::string &s); void pop(void); @@ -55,6 +54,7 @@ namespace Grid void writeDefault(const std::string &s, const U &x); template void writeDefault(const std::string &s, const std::vector &x); + std::string XmlString(void); private: pugi::xml_document doc_; pugi::xml_node node_; @@ -64,7 +64,8 @@ namespace Grid class XmlReader: public Reader { public: - XmlReader(const std::string &fileName); + XmlReader(const char *xmlstring,std::string toplev = std::string("grid") ); + XmlReader(const std::string &fileName,std::string toplev = std::string("grid") ); virtual ~XmlReader(void) = default; bool push(const std::string &s); void pop(void); @@ -118,7 +119,7 @@ namespace Grid std::string buf; readDefault(s, buf); - std::cout << s << " " << buf << std::endl; + // std::cout << s << " " << buf << std::endl; fromString(output, buf); } diff --git a/tests/IO/Test_nersc_io.cc b/tests/IO/Test_nersc_io.cc index 14c6080d..ca04e623 100644 --- a/tests/IO/Test_nersc_io.cc +++ b/tests/IO/Test_nersc_io.cc @@ -64,8 +64,8 @@ int main (int argc, char ** argv) std::cout < U(4,&Fine); - NerscField header; + FieldMetaData header; std::string file("./ckpoint_lat"); NerscIO::readConfiguration(Umu,header,file); diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc index 7d911dfd..ceddee77 100644 --- a/tests/IO/Test_serialisation.cc +++ b/tests/IO/Test_serialisation.cc @@ -31,6 +31,7 @@ Author: Peter Boyle using namespace Grid; +using namespace Grid::QCD; GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3); @@ -62,6 +63,7 @@ public: } }; + int16_t i16 = 1; uint16_t u16 = 2; int32_t i32 = 3; @@ -237,7 +239,22 @@ int main(int argc,char **argv) std::cout << "Loaded (JSON) -----------------" << std::endl; std::cout << jcopy1 << std::endl << jveccopy1 << std::endl; } - + + { + ildgFormat format; + format.version =1.0; + format.field =std::string("su3gauge"); + format.precision =32; + format.lx =24; + format.ly =24; + format.lz =24; + format.lt =48; + XmlWriter WR("ildg-format.xml",""); + XmlWriter WRs("",""); + write(WR,"ildgFormat",format); + write(WRs,"ildgFormat",format); + std::cout << " XmlString: " <