From c48ae4f3adf50701380af544d62367d7ec77377d Mon Sep 17 00:00:00 2001 From: Michael Marshall <43034299+mmphys@users.noreply.github.com> Date: Sun, 28 Apr 2019 23:24:57 +0100 Subject: [PATCH] 1) Only the boss should write the perambulator - possibly was a source of intermittent corruption? 2) Implemented and test a perambulator conversion utility in Test_distil (commented out near the start of main) --- Hadrons/Distil.hpp | 55 +++++++++++++++------ Hadrons/Modules/MDistil/PerambFromSolve.hpp | 37 +++++++------- Hadrons/Modules/MDistil/Perambulator.hpp | 17 ++++--- tests/hadrons/Test_distil.cc | 11 ++++- 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/Hadrons/Distil.hpp b/Hadrons/Distil.hpp index 9e8b873d..3daead34 100644 --- a/Hadrons/Distil.hpp +++ b/Hadrons/Distil.hpp @@ -440,11 +440,11 @@ void NamedTensor::ReadBinary(const std assert((sizeof(Scalar_) % Endian_Scalar_Size) == 0 && "NamedTensor error: Scalar_ is not composed of Endian_Scalar_Size" ); // Size of the data in bytes const uint32_t Scalar_Size{sizeof(Scalar_)}; - const auto NumElements{tensor.size()}; - const std::streamsize TotalDataSize{static_cast(NumElements * Scalar_Size)}; + Index NumElements{tensor.size()}; + std::streamsize TotalDataSize{static_cast(NumElements * Scalar_Size)}; uint64_t u64; r.read(reinterpret_cast(&u64), sizeof(u64)); - assert( TotalDataSize == be64toh( u64 ) && "NamedTensor error: Size of the data in bytes" ); + assert( TotalDataSize == 0 || TotalDataSize == be64toh( u64 ) && "NamedTensor error: Size of the data in bytes" ); // Size of a Scalar_ uint32_t u32; r.read(reinterpret_cast(&u32), sizeof(u32)); @@ -454,19 +454,47 @@ void NamedTensor::ReadBinary(const std r.read(reinterpret_cast(&u16), sizeof(u16)); assert( Endian_Scalar_Size == be16toh( u16 ) && "NamedTensor error: Scalar_Unit_size"); // number of dimensions which aren't 1 - r.read(reinterpret_cast(&u16), sizeof(u16)); - u16 = be16toh( u16 ); - for( auto dim : tensor.dimensions() ) + uint16_t NumFileDimensions; + r.read(reinterpret_cast(&NumFileDimensions), sizeof(NumFileDimensions)); + NumFileDimensions = be16toh( NumFileDimensions ); + /*for( auto dim : tensor.dimensions() ) if( dim == 1 ) - u16++; - assert( this->NumIndices == u16 && "NamedTensor error: number of dimensions which aren't 1" ); - // dimensions together with names - int d = 0; - for( auto dim : tensor.dimensions() ) { - if( dim != 1 ) { + u16++;*/ + assert( ( TotalDataSize == 0 && this->NumIndices >= NumFileDimensions || this->NumIndices == NumFileDimensions ) + && "NamedTensor error: number of dimensions which aren't 1" ); + if( TotalDataSize == 0 ) { + // Read each dimension, using names to skip past dimensions == 1 + std::array NewDimensions; + for( Index &i : NewDimensions ) i = 1; + int d = 0; + for( int FileDimension = 0; FileDimension < NumFileDimensions; FileDimension++ ) { + // read dimension + uint16_t thisDim; + r.read(reinterpret_cast(&thisDim), sizeof(thisDim)); + // read dimension name + r.read(reinterpret_cast(&u16), sizeof(u16)); + size_t l = be16toh( u16 ); + std::string s( l, '?' ); + r.read(&s[0], l); + // skip forward to matching name + while( IndexNames[d].size() > 0 && s != IndexNames[d] ) + assert(++d < NumIndices && "NamedTensor error: dimension name" ); + if( IndexNames[d].size() == 0 ) + IndexNames[d] = s; + NewDimensions[d++] = be16toh( thisDim ); + } + tensor.resize(NewDimensions); + NumElements = 1; + for( Index i : NewDimensions ) NumElements *= i; + TotalDataSize = NumElements * Scalar_Size; + } else { + // dimensions together with names + const auto & TensorDims{tensor.dimensions()}; + for( int d = 0; d < NumIndices_; d++ ) { // size of dimension r.read(reinterpret_cast(&u16), sizeof(u16)); - assert( dim == be16toh( u16 ) && "size of dimension" ); + u16 = be16toh( u16 ); + assert( TensorDims[d] == u16 && "size of dimension" ); // length of dimension name r.read(reinterpret_cast(&u16), sizeof(u16)); size_t l = be16toh( u16 ); @@ -476,7 +504,6 @@ void NamedTensor::ReadBinary(const std r.read(&s[0], l); assert( s == IndexNames[d] && "NamedTensor error: dimension name" ); } - d++; } // Actual data char * const pStart{reinterpret_cast(tensor.data())}; diff --git a/Hadrons/Modules/MDistil/PerambFromSolve.hpp b/Hadrons/Modules/MDistil/PerambFromSolve.hpp index 2f9b4ebc..14774a2d 100644 --- a/Hadrons/Modules/MDistil/PerambFromSolve.hpp +++ b/Hadrons/Modules/MDistil/PerambFromSolve.hpp @@ -215,31 +215,32 @@ void TPerambFromSolve::execute(void) int Ntlocal = grid4d->LocalDimensions()[3]; int Ntfirst = grid4d->LocalStarts()[3]; - const std::string &PerambFileName{par().PerambFileName}; - - - for (int inoise = 0; inoise < nnoise; inoise++) { - for (int dk = 0; dk < LI_reduced; dk++) { - for (int dt = 0; dt < Nt_inv; dt++) { - for (int ds = 0; ds < SI; ds++) { - for (int is = 0; is < Ns; is++) { - result_nospin = peekSpin(solve[inoise+nnoise*(dk+LI*(dt+Nt_inv*ds))],is); - for (int t = Ntfirst; t < Ntfirst + Ntlocal; t++) { - ExtractSliceLocal(result_3d,result_nospin,0,t-Ntfirst,Grid::QCD::Tdir); - for (int ivec = 0; ivec < nvec_reduced; ivec++) { - ExtractSliceLocal(evec3d,epack.evec[ivec],0,t,3); - pokeSpin(perambulator(t, ivec, dk, inoise,dt,ds),static_cast(innerProduct(evec3d, result_3d)),is); - std::cout << "perambulator(t, ivec, dk, inoise,dt,ds)(is) = (" << t << "," << ivec << "," << dk << "," << inoise << "," << dt << "," << ds << ")(" << is << ") = " << perambulator(t, ivec, dk, inoise,dt,ds)()(is)() << std::endl; - } + for (int inoise = 0; inoise < nnoise; inoise++) { + for (int dk = 0; dk < LI_reduced; dk++) { + for (int dt = 0; dt < Nt_inv; dt++) { + for (int ds = 0; ds < SI; ds++) { + for (int is = 0; is < Ns; is++) { + result_nospin = peekSpin(solve[inoise+nnoise*(dk+LI*(dt+Nt_inv*ds))],is); + for (int t = Ntfirst; t < Ntfirst + Ntlocal; t++) { + ExtractSliceLocal(result_3d,result_nospin,0,t-Ntfirst,Grid::QCD::Tdir); + for (int ivec = 0; ivec < nvec_reduced; ivec++) { + ExtractSliceLocal(evec3d,epack.evec[ivec],0,t,3); + pokeSpin(perambulator(t, ivec, dk, inoise,dt,ds),static_cast(innerProduct(evec3d, result_3d)),is); + std::cout << "perambulator(t, ivec, dk, inoise,dt,ds)(is) = (" << t << "," << ivec << "," << dk << "," << inoise << "," << dt << "," << ds << ")(" << is << ") = " << perambulator(t, ivec, dk, inoise,dt,ds)()(is)() << std::endl; } } } } } } + } - if(PerambFileName.length()) { - std::string sPerambName{PerambFileName + "." + std::to_string(vm().getTrajectory())}; + if(grid4d->IsBoss()) { + std::string sPerambName{par().PerambFileName}; + if( sPerambName.length() == 0 ) + sPerambName = getName(); + sPerambName.append( "." ); + sPerambName.append( std::to_string(vm().getTrajectory())); //perambulator.WriteBinary(sPerambName); perambulator.write(sPerambName.c_str()); } diff --git a/Hadrons/Modules/MDistil/Perambulator.hpp b/Hadrons/Modules/MDistil/Perambulator.hpp index 3e16c805..06a5f36d 100644 --- a/Hadrons/Modules/MDistil/Perambulator.hpp +++ b/Hadrons/Modules/MDistil/Perambulator.hpp @@ -215,7 +215,6 @@ void TPerambulator::execute(void) // Load perambulator if it exists on disk instead of creating it // Not sure this is how we want it - rather specify an input flag 'read' // and assert that the file is there. - const std::string &PerambFileName{par().PerambFileName}; envGetTmp(LatticeSpinColourVector, dist_source); envGetTmp(LatticeSpinColourVector, tmp2); @@ -276,18 +275,22 @@ void TPerambulator::execute(void) ExtractSliceLocal(evec3d,epack.evec[ivec],0,t,3); pokeSpin(perambulator(t, ivec, dk, inoise,dt,ds),static_cast(innerProduct(evec3d, result_3d)),is); } + } + } } } } } - } -} } - std::cout << "perambulator done" << std::endl; - perambulator.SliceShare( grid3d, grid4d ); + std::cout << "perambulator done" << std::endl; + perambulator.SliceShare( grid3d, grid4d ); - if(PerambFileName.length()) { - std::string sPerambName{PerambFileName + "." + std::to_string(vm().getTrajectory())}; + if(grid4d->IsBoss()) { + std::string sPerambName{par().PerambFileName}; + if( sPerambName.length() == 0 ) + sPerambName = getName(); + sPerambName.append( "." ); + sPerambName.append( std::to_string(vm().getTrajectory())); //perambulator.WriteBinary(sPerambName); perambulator.write(sPerambName.c_str()); } diff --git a/tests/hadrons/Test_distil.cc b/tests/hadrons/Test_distil.cc index aada8d4f..d5ac269c 100644 --- a/tests/hadrons/Test_distil.cc +++ b/tests/hadrons/Test_distil.cc @@ -1016,6 +1016,14 @@ bool DebugGridTensorTest( void ) return true; } + +bool ConvertPeramb(const char * pszSource, const char * pszDest) { + std::array sIndexNames{"Nt", "nvec", "LI", "nnoise", "Nt_inv", "SI"}; + Grid::Hadrons::MDistil::Perambulator p(sIndexNames); + p.ReadBinary( pszSource ); + p.write(pszDest); + return true; +} #endif int main(int argc, char *argv[]) @@ -1031,8 +1039,9 @@ int main(int argc, char *argv[]) << ", sizeof(hsize_t) = " << sizeof(hsize_t) << ", sizeof(unsigned long long) = " << sizeof(unsigned long long) << std::endl; - if( DebugEigenTest() ) return 0; + //if( DebugEigenTest() ) return 0; //if(DebugGridTensorTest()) return 0; + //if(ConvertPeramb("PerambL_100_tsrc0.3000","PerambL_100_tsrc0.3000")) return 0; #endif // Decode command-line parameters. 1st one is which test to run