From c519aab19dcfb559ab21585d9f9221b1fc193a60 Mon Sep 17 00:00:00 2001 From: Guido Cossu Date: Tue, 7 Nov 2017 13:55:37 +0000 Subject: [PATCH 1/5] Fixing the MPI memory leak in the communicators --- lib/communicator/Communicator_mpi.cc | 2 +- lib/communicator/Communicator_mpi3.cc | 8 ++++++++ lib/communicator/Communicator_mpit.cc | 5 +++-- lib/communicator/Communicator_shmem.cc | 2 ++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index 5593aa8b..1c1ae6c5 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -57,7 +57,7 @@ CartesianCommunicator::~CartesianCommunicator() { int MPI_is_finalised; MPI_Finalized(&MPI_is_finalised); - if (communicator && MPI_is_finalised) + if (communicator && !MPI_is_finalised) MPI_Comm_free(&communicator); } diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc index 3cac726c..52f65c34 100644 --- a/lib/communicator/Communicator_mpi3.cc +++ b/lib/communicator/Communicator_mpi3.cc @@ -596,6 +596,14 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors) } } }; +CartesianCommunicator::~CartesianCommunicator() +{ + int MPI_is_finalised; + MPI_Finalized(&MPI_is_finalised); + if (communicator && !MPI_is_finalised) + MPI_Comm_free(&communicator); +} + void CartesianCommunicator::GlobalSum(uint32_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc index 56f96c20..1c24433b 100644 --- a/lib/communicator/Communicator_mpit.cc +++ b/lib/communicator/Communicator_mpit.cc @@ -55,11 +55,12 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { CartesianCommunicator::~CartesianCommunicator() { - if (communicator && !MPI::Is_finalized()) + int MPI_is_finalised; + MPI_Finalized(&MPI_is_finalised); + if (communicator && !MPI_is_finalised) MPI_Comm_free(&communicator); } - void CartesianCommunicator::GlobalSum(uint32_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc index ed49285d..03e3173e 100644 --- a/lib/communicator/Communicator_shmem.cc +++ b/lib/communicator/Communicator_shmem.cc @@ -75,6 +75,8 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { ShmInitGeneric(); } +CartesianCommunicator::~CartesianCommunicator(){} + CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent) : CartesianCommunicator(processors) { From 9b8d1cc3da4769f250665cc8e05fb305c794bc5d Mon Sep 17 00:00:00 2001 From: Azusa Yamaguchi Date: Tue, 7 Nov 2017 14:48:45 +0000 Subject: [PATCH 2/5] Staggered Schur decomposed matrix norm changed to not be the Schur anymore :( Carleton wanted this for multimass / multishift --- lib/algorithms/LinearOperator.h | 12 ++++++++++++ lib/algorithms/iterative/SchurRedBlack.h | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/algorithms/LinearOperator.h b/lib/algorithms/LinearOperator.h index 9b0e4942..0fa039c8 100644 --- a/lib/algorithms/LinearOperator.h +++ b/lib/algorithms/LinearOperator.h @@ -317,11 +317,23 @@ namespace Grid { } virtual RealD Mpc (const Field &in, Field &out) { Field tmp(in._grid); + Field tmp2(in._grid); + + _Mat.Mooee(in,out); + _Mat.Mooee(out,tmp); + + _Mat.Meooe(in,out); + _Mat.Meooe(out,tmp2); + + return axpy_norm(out,-1.0,tmp2,tmp); +#if 0 + //... much prefer conventional Schur norm _Mat.Meooe(in,tmp); _Mat.MooeeInv(tmp,out); _Mat.Meooe(out,tmp); _Mat.Mooee(in,out); return axpy_norm(out,-1.0,tmp,out); +#endif } virtual RealD MpcDag (const Field &in, Field &out){ return Mpc(in,out); diff --git a/lib/algorithms/iterative/SchurRedBlack.h b/lib/algorithms/iterative/SchurRedBlack.h index a0fd86a6..b9767aa8 100644 --- a/lib/algorithms/iterative/SchurRedBlack.h +++ b/lib/algorithms/iterative/SchurRedBlack.h @@ -90,7 +90,7 @@ namespace Grid { // Take a matrix and form a Red Black solver calling a Herm solver // Use of RB info prevents making SchurRedBlackSolve conform to standard interface /////////////////////////////////////////////////////////////////////////////////////////////////////// - + // Now make the norm reflect extra factor of Mee template class SchurRedBlackStaggeredSolve { private: OperatorFunction & _HermitianRBSolver; @@ -136,8 +136,8 @@ namespace Grid { _Matrix.Meooe (tmp,Mtmp); assert( Mtmp.checkerboard ==Odd); tmp=src_o-Mtmp; assert( tmp.checkerboard ==Odd); - src_o = tmp; assert(src_o.checkerboard ==Odd); - // _Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source + //src_o = tmp; assert(src_o.checkerboard ==Odd); + _Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source from dumb choice of matrix norm. ////////////////////////////////////////////////////////////// // Call the red-black solver From 1860b1698c4b204a332e2fad4dea86c97ffb0abc Mon Sep 17 00:00:00 2001 From: Azusa Yamaguchi Date: Wed, 8 Nov 2017 09:03:01 +0000 Subject: [PATCH 3/5] Fixed the bag on MPI_T at Cam --- lib/communicator/Communicator_mpit.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc index b2fb71a6..bceea0d8 100644 --- a/lib/communicator/Communicator_mpit.cc +++ b/lib/communicator/Communicator_mpit.cc @@ -57,7 +57,7 @@ CartesianCommunicator::~CartesianCommunicator() { int MPI_is_finalised; MPI_Finalized(&MPI_is_finalised); - if (communicator && !MPI_is_finalised) + if (communicator && !MPI_is_finalised){ MPI_Comm_free(&communicator); for(int i=0;i< communicator_halo.size();i++){ MPI_Comm_free(&communicator_halo[i]); @@ -246,7 +246,7 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector Date: Thu, 9 Nov 2017 19:46:57 +0000 Subject: [PATCH 4/5] Declaring virtual functions as pure virtual functions. --- lib/algorithms/iterative/ImplicitlyRestartedLanczos.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h index 7a0760c9..7b85c095 100644 --- a/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h +++ b/lib/algorithms/iterative/ImplicitlyRestartedLanczos.h @@ -168,8 +168,8 @@ void basisDeflate(const std::vector &_v,const std::vector& eval,co template class ImplicitlyRestartedLanczosTester { public: - virtual int TestConvergence(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox); - virtual int ReconstructEval(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox); + virtual int TestConvergence(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox)=0; + virtual int ReconstructEval(int j,RealD resid,Field &evec, RealD &eval,RealD evalMaxApprox)=0; }; enum IRLdiagonalisation { From e55397bc134ead26dbac8e2ef244406a8b9d6a3b Mon Sep 17 00:00:00 2001 From: azusayamaguchi Date: Fri, 24 Nov 2017 14:18:30 +0000 Subject: [PATCH 5/5] Staggerd cg --- lib/algorithms/LinearOperator.h | 20 +++++++++++--------- lib/algorithms/iterative/SchurRedBlack.h | 9 +++++++-- lib/lattice/Lattice_transfer.h | 16 ++++++---------- tests/solver/Test_staggered_cg_schur.cc | 14 ++++++++++++++ 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/lib/algorithms/LinearOperator.h b/lib/algorithms/LinearOperator.h index 0fa039c8..26746e6e 100644 --- a/lib/algorithms/LinearOperator.h +++ b/lib/algorithms/LinearOperator.h @@ -308,32 +308,34 @@ namespace Grid { public: SchurStaggeredOperator (Matrix &Mat): _Mat(Mat){}; virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ + GridLogIterative.TimingMode(1); + std::cout << GridLogIterative << " HermOpAndNorm "< inline void pickCheckerboard(int cb,Lattice &half,const Lattice &full){ half.checkerboard = cb; - int ssh=0; - //parallel_for - for(int ss=0;ssoSites();ss++){ - std::vector coor; + + parallel_for(int ss=0;ssoSites();ss++){ int cbos; - + std::vector coor; full._grid->oCoorFromOindex(coor,ss); cbos=half._grid->CheckerBoard(coor); if (cbos==cb) { + int ssh=half._grid->oIndex(coor); half._odata[ssh] = full._odata[ss]; - ssh++; } } } template inline void setCheckerboard(Lattice &full,const Lattice &half){ int cb = half.checkerboard; - int ssh=0; - //parallel_for - for(int ss=0;ssoSites();ss++){ + parallel_for(int ss=0;ssoSites();ss++){ std::vector coor; int cbos; @@ -77,8 +73,8 @@ inline void subdivides(GridBase *coarse,GridBase *fine) cbos=half._grid->CheckerBoard(coor); if (cbos==cb) { + int ssh=half._grid->oIndex(coor); full._odata[ss]=half._odata[ssh]; - ssh++; } } } diff --git a/tests/solver/Test_staggered_cg_schur.cc b/tests/solver/Test_staggered_cg_schur.cc index 09044995..a5c25b85 100644 --- a/tests/solver/Test_staggered_cg_schur.cc +++ b/tests/solver/Test_staggered_cg_schur.cc @@ -70,7 +70,21 @@ int main (int argc, char ** argv) ConjugateGradient CG(1.0e-8,10000); SchurRedBlackStaggeredSolve SchurSolver(CG); + double volume=1.0; + for(int mu=0;mu volume * 1146 + double ncall=CG.IterationsToComplete; + double flops=(16*(3*(6+8+8)) + 15*3*2)*volume*ncall; // == 66*16 + == 1146 + + std::cout<