From b9104f307241558b2d8ec47ee9814a162da5bda8 Mon Sep 17 00:00:00 2001 From: paboyle Date: Wed, 21 Jun 2017 21:08:03 +0100 Subject: [PATCH 01/18] Block CG --- lib/algorithms/Algorithms.h | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/lib/algorithms/Algorithms.h b/lib/algorithms/Algorithms.h index 5123c7a1..3b62c753 100644 --- a/lib/algorithms/Algorithms.h +++ b/lib/algorithms/Algorithms.h @@ -44,30 +44,15 @@ Author: Peter Boyle #include #include #include - -// Lanczos support -//#include +#include #include #include #include -// Eigen/lanczos // EigCg -// MCR // Pcg -// Multishift CG // Hdcg // GCR // etc.. -// integrator/Leapfrog -// integrator/Omelyan -// integrator/ForceGradient - -// montecarlo/hmc -// montecarlo/rhmc -// montecarlo/metropolis -// etc... - - #endif From 1d7aa673a4cebfde47afb7da8ec9b10a3f681b28 Mon Sep 17 00:00:00 2001 From: paboyle Date: Wed, 21 Jun 2017 21:08:53 +0100 Subject: [PATCH 02/18] Include BlockCG by default --- tests/solver/Test_staggered_block_cg_unprec.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/solver/Test_staggered_block_cg_unprec.cc b/tests/solver/Test_staggered_block_cg_unprec.cc index 8db41e98..5d449f73 100644 --- a/tests/solver/Test_staggered_block_cg_unprec.cc +++ b/tests/solver/Test_staggered_block_cg_unprec.cc @@ -27,7 +27,6 @@ Author: Peter Boyle *************************************************************************************/ /* END LEGAL */ #include -#include using namespace std; using namespace Grid; From 6ebf9f15b7de6b074feb74af0c5037da0fd0730d Mon Sep 17 00:00:00 2001 From: paboyle Date: Thu, 22 Jun 2017 08:14:34 +0100 Subject: [PATCH 03/18] Splitting communicators first cut --- benchmarks/Benchmark_staggered.cc | 2 +- benchmarks/Benchmark_wilson.cc | 2 +- benchmarks/Benchmark_wilson_sweep.cc | 2 +- lib/cartesian/Cartesian_base.h | 2 + lib/cartesian/Cartesian_full.h | 23 +++++- lib/cartesian/Cartesian_red_black.h | 45 +++++++++-- lib/communicator/Communicator_base.cc | 2 +- lib/communicator/Communicator_base.h | 18 ++++- lib/communicator/Communicator_mpi.cc | 76 ++++++++++++++++--- lib/communicator/Communicator_mpi3.cc | 9 +++ lib/communicator/Communicator_none.cc | 3 + lib/communicator/Communicator_shmem.cc | 5 ++ lib/qcd/utils/SpaceTimeGrid.cc | 45 ++++++----- lib/util/Lexicographic.h | 4 +- tests/Test_stencil.cc | 2 +- tests/core/Test_cshift_red_black.cc | 2 +- tests/core/Test_cshift_red_black_rotate.cc | 2 +- tests/core/Test_fft.cc | 2 +- tests/core/Test_gpwilson_even_odd.cc | 2 +- tests/core/Test_main.cc | 2 +- tests/core/Test_staggered.cc | 2 +- tests/core/Test_wilson_even_odd.cc | 2 +- .../core/Test_wilson_twisted_mass_even_odd.cc | 2 +- tests/forces/Test_gp_rect_force.cc | 2 +- tests/forces/Test_laplacian_force.cc | 2 +- tests/forces/Test_rect_force.cc | 2 +- tests/forces/Test_wilson_force.cc | 2 +- tests/smearing/Test_WilsonFlow.cc | 4 +- tests/solver/Test_laplacian.cc | 2 +- tests/solver/Test_staggered_cg_unprec.cc | 2 +- tests/solver/Test_wilson_cg_prec.cc | 2 +- tests/solver/Test_wilson_cg_schur.cc | 2 +- tests/solver/Test_wilson_cg_unprec.cc | 2 +- tests/solver/Test_wilson_cr_unprec.cc | 2 +- 34 files changed, 215 insertions(+), 65 deletions(-) diff --git a/benchmarks/Benchmark_staggered.cc b/benchmarks/Benchmark_staggered.cc index dc2dcf91..f5325b28 100644 --- a/benchmarks/Benchmark_staggered.cc +++ b/benchmarks/Benchmark_staggered.cc @@ -40,7 +40,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< friend class Lattice; GridBase(const std::vector & processor_grid) : CartesianCommunicator(processor_grid) {}; + GridBase(const std::vector & processor_grid, + const CartesianCommunicator &parent) : CartesianCommunicator(processor_grid,parent) {}; // Physics Grid information. std::vector _simd_layout;// Which dimensions get relayed out over simd lanes. diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h index b0e47fa4..bced0791 100644 --- a/lib/cartesian/Cartesian_full.h +++ b/lib/cartesian/Cartesian_full.h @@ -61,10 +61,29 @@ public: virtual int CheckerBoardShift(int source_cb,int dim,int shift, int osite){ return shift; } + ///////////////////////////////////////////////////////////////////////// + // Constructor takes a parent grid and possibly subdivides communicator. + ///////////////////////////////////////////////////////////////////////// GridCartesian(const std::vector &dimensions, const std::vector &simd_layout, - const std::vector &processor_grid - ) : GridBase(processor_grid) + const std::vector &processor_grid, + GridCartesian &parent) : GridBase(processor_grid,parent) + { + Init(dimensions,simd_layout,processor_grid); + } + ///////////////////////////////////////////////////////////////////////// + // Construct from comm world + ///////////////////////////////////////////////////////////////////////// + GridCartesian(const std::vector &dimensions, + const std::vector &simd_layout, + const std::vector &processor_grid) : GridBase(processor_grid) + { + Init(dimensions,simd_layout,processor_grid); + } + + void Init(const std::vector &dimensions, + const std::vector &simd_layout, + const std::vector &processor_grid) { /////////////////////// // Grid information diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h index 3037de00..1f831bc4 100644 --- a/lib/cartesian/Cartesian_red_black.h +++ b/lib/cartesian/Cartesian_red_black.h @@ -112,24 +112,57 @@ public: } }; - GridRedBlackCartesian(const GridBase *base) : GridRedBlackCartesian(base->_fdimensions,base->_simd_layout,base->_processors) {}; + //////////////////////////////////////////////////////////// + // Create Redblack from original grid; require full grid pointer ? + //////////////////////////////////////////////////////////// + GridRedBlackCartesian(const GridBase *base) : GridBase(base->_processors,*base) + { + int dims = base->_ndimension; + std::vector checker_dim_mask(dims,1); + int checker_dim = 0; + Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim); + }; - GridRedBlackCartesian(const std::vector &dimensions, + //////////////////////////////////////////////////////////// + // Create redblack from original grid, with non-trivial checker dim mask + //////////////////////////////////////////////////////////// + GridRedBlackCartesian(const GridBase *base, + const std::vector &checker_dim_mask, + int checker_dim + ) : GridBase(base->_processors,*base) + { + Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim) ; + } +#if 0 + //////////////////////////////////////////////////////////// + // Create redblack grid ;; deprecate these. Should not + // need direct creation of redblack without a full grid to base on + //////////////////////////////////////////////////////////// + GridRedBlackCartesian(const GridBase *base, + const std::vector &dimensions, const std::vector &simd_layout, const std::vector &processor_grid, const std::vector &checker_dim_mask, int checker_dim - ) : GridBase(processor_grid) + ) : GridBase(processor_grid,*base) { Init(dimensions,simd_layout,processor_grid,checker_dim_mask,checker_dim); } - GridRedBlackCartesian(const std::vector &dimensions, + + //////////////////////////////////////////////////////////// + // Create redblack grid + //////////////////////////////////////////////////////////// + GridRedBlackCartesian(const GridBase *base, + const std::vector &dimensions, const std::vector &simd_layout, - const std::vector &processor_grid) : GridBase(processor_grid) + const std::vector &processor_grid) : GridBase(processor_grid,*base) { std::vector checker_dim_mask(dimensions.size(),1); - Init(dimensions,simd_layout,processor_grid,checker_dim_mask,0); + int checker_dim = 0; + Init(dimensions,simd_layout,processor_grid,checker_dim_mask,checker_dim); } +#endif + void Init(const std::vector &dimensions, const std::vector &simd_layout, const std::vector &processor_grid, diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc index 557fef48..b042646f 100644 --- a/lib/communicator/Communicator_base.cc +++ b/lib/communicator/Communicator_base.cc @@ -60,7 +60,7 @@ void CartesianCommunicator::ShmBufferFreeAll(void) { ///////////////////////////////// // Grid information queries ///////////////////////////////// -int CartesianCommunicator::Dimensions(void) { return _ndimension; }; +int CartesianCommunicator::Dimensions(void) { return _ndimension; }; int CartesianCommunicator::IsBoss(void) { return _processor==0; }; int CartesianCommunicator::BossRank(void) { return 0; }; int CartesianCommunicator::ThisRank(void) { return _processor; }; diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h index 12a8429f..ecbb2061 100644 --- a/lib/communicator/Communicator_base.h +++ b/lib/communicator/Communicator_base.h @@ -68,6 +68,7 @@ class CartesianCommunicator { static MPI_Comm communicator_world; MPI_Comm communicator; typedef MPI_Request CommsRequest_t; + #else typedef int CommsRequest_t; #endif @@ -135,11 +136,24 @@ class CartesianCommunicator { // Must call in Grid startup //////////////////////////////////////////////// static void Init(int *argc, char ***argv); - + //////////////////////////////////////////////// - // Constructor of any given grid + // Constructors to sub-divide a parent communicator + // and default to comm world //////////////////////////////////////////////// + CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent); CartesianCommunicator(const std::vector &pdimensions_in); + + private: +#if defined (GRID_COMMS_MPI) + //|| defined (GRID_COMMS_MPI3) + //////////////////////////////////////////////// + // Private initialise from an MPI communicator + // Can use after an MPI_Comm_split, but hidden from user so private + //////////////////////////////////////////////// + void InitFromMPICommunicator(const std::vector &processors, MPI_Comm communicator_base); +#endif + public: //////////////////////////////////////////////////////////////////////////////////////// // Wraps MPI_Cart routines, or implements equivalent on other impls diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index bd2a62fb..b24b60bd 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -53,24 +53,80 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { ShmInitGeneric(); } -CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +{ + InitFromMPICommunicator(processors,communicator_world); + std::cout << "Passed communicator world to a new communicator" < &processors,const CartesianCommunicator &parent) { _ndimension = processors.size(); - std::vector periodic(_ndimension,1); + assert(_ndimension = parent._ndimension); + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // split the communicator + ////////////////////////////////////////////////////////////////////////////////////////////////////// + std::vector ratio(_ndimension); + std::vector rcoor(_ndimension); + std::vector scoor(_ndimension); + + int Nsubcomm=1; + int Nsubrank=1; + for(int d=0;d<_ndimension;d++) { + ratio[d] = parent._processors[d] / processors[d]; + rcoor[d] = parent._processor_coor[d] / processors[d]; + scoor[d] = parent._processor_coor[d] % processors[d]; + assert(ratio[d] * processors[d] == parent._processors[d]); // must exactly subdivide + Nsubcomm *= ratio[d]; + Nsubrank *= processors[d]; + } + + int rlex, slex; + Lexicographic::IndexFromCoor(rcoor,rlex,ratio); + Lexicographic::IndexFromCoor(scoor,slex,processors); + + MPI_Comm comm_split; + int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split); + assert(ierr==0); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // Set up from the new split communicator + ////////////////////////////////////////////////////////////////////////////////////////////////////// + InitFromMPICommunicator(processors,comm_split); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // Declare victory + ////////////////////////////////////////////////////////////////////////////////////////////////////// + std::cout << "Divided communicator "<< parent._Nprocessors<<" into " + < &processors, MPI_Comm communicator_base) +{ + if ( communicator_base != communicator_world ) { + std::cout << "Cartesian communicator created with a non-world communicator"< periodic(_ndimension,1); + MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator); + MPI_Comm_rank(communicator,&_processor); + MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); + + int Size; MPI_Comm_size(communicator,&Size); assert(Size==_Nprocessors); diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc index 632eb991..6d7f301c 100644 --- a/lib/communicator/Communicator_mpi3.cc +++ b/lib/communicator/Communicator_mpi3.cc @@ -371,6 +371,15 @@ void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &c assert(lr!=-1); Lexicographic::CoorFromIndex(coor,lr,_processors); } + +////////////////////////////////// +// Try to subdivide communicator +////////////////////////////////// +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,CartesianCommunicator &parent) + : CartesianCommunicator(processors) +{ + std::cout << "Attempts to split MPI3 communicators will fail until implemented" < &processors) { int ierr; diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc index 5319ab93..a7484782 100644 --- a/lib/communicator/Communicator_none.cc +++ b/lib/communicator/Communicator_none.cc @@ -38,6 +38,9 @@ void CartesianCommunicator::Init(int *argc, char *** arv) ShmInitGeneric(); } +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,CartesianCommunicator &parent) + : CartesianCommunicator(processors) {} + CartesianCommunicator::CartesianCommunicator(const std::vector &processors) { _processors = processors; diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc index 3c76c808..49d32697 100644 --- a/lib/communicator/Communicator_shmem.cc +++ b/lib/communicator/Communicator_shmem.cc @@ -75,6 +75,11 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { ShmInitGeneric(); } +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,CartesianCommunicator &parent) + : CartesianCommunicator(processors) +{ + std::cout << "Attempts to split SHMEM communicators will fail " < &processors) { _ndimension = processors.size(); diff --git a/lib/qcd/utils/SpaceTimeGrid.cc b/lib/qcd/utils/SpaceTimeGrid.cc index 3ada4a3b..cbbe0aee 100644 --- a/lib/qcd/utils/SpaceTimeGrid.cc +++ b/lib/qcd/utils/SpaceTimeGrid.cc @@ -68,18 +68,21 @@ GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimRedBlackGrid(int Ls,const GridC { int N4=FourDimGrid->_ndimension; int cbd=1; - std::vector latt5(1,Ls); - std::vector simd5(1,1); - std::vector mpi5(1,1); + // std::vector latt5(1,Ls); + // std::vector simd5(1,1); + // std::vector mpi5(1,1); std::vector cb5(1,0); for(int d=0;d_fdimensions[d]); - simd5.push_back(FourDimGrid->_simd_layout[d]); - mpi5.push_back(FourDimGrid->_processors[d]); + // latt5.push_back(FourDimGrid->_fdimensions[d]); + // simd5.push_back(FourDimGrid->_simd_layout[d]); + // mpi5.push_back(FourDimGrid->_processors[d]); cb5.push_back( 1); - } - return new GridRedBlackCartesian(latt5,simd5,mpi5,cb5,cbd); + } + GridCartesian *tmp = makeFiveDimGrid(Ls,FourDimGrid); + GridRedBlackCartesian *ret = new GridRedBlackCartesian(tmp,cb5,cbd); + delete tmp; + return ret; } @@ -99,24 +102,30 @@ GridCartesian *SpaceTimeGrid::makeFiveDimDWFGrid(int Ls,const GridCartes } return new GridCartesian(latt5,simd5,mpi5); } - +/////////////////////////////////////////////////// +// Interface is inefficient and forces the deletion +// Pass in the non-redblack grid +/////////////////////////////////////////////////// GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(int Ls,const GridCartesian *FourDimGrid) { int N4=FourDimGrid->_ndimension; - int nsimd = FourDimGrid->Nsimd(); int cbd=1; - std::vector latt5(1,Ls); - std::vector simd5(1,nsimd); - std::vector mpi5(1,1); std::vector cb5(1,0); + // int nsimd = FourDimGrid->Nsimd(); + // std::vector latt5(1,Ls); + // std::vector simd5(1,nsimd); + // std::vector mpi5(1,1); for(int d=0;d_fdimensions[d]); - simd5.push_back(1); - mpi5.push_back(FourDimGrid->_processors[d]); + // latt5.push_back(FourDimGrid->_fdimensions[d]); + // simd5.push_back(1); + // mpi5.push_back(FourDimGrid->_processors[d]); cb5.push_back(1); - } - return new GridRedBlackCartesian(latt5,simd5,mpi5,cb5,cbd); + } + GridCartesian *tmp = makeFiveDimDWFGrid(Ls,FourDimGrid); + GridRedBlackCartesian *ret = new GridRedBlackCartesian(tmp,cb5,cbd); + delete tmp; + return ret; } diff --git a/lib/util/Lexicographic.h b/lib/util/Lexicographic.h index 2d4e5df5..b922dba5 100644 --- a/lib/util/Lexicographic.h +++ b/lib/util/Lexicographic.h @@ -7,7 +7,7 @@ namespace Grid{ class Lexicographic { public: - static inline void CoorFromIndex (std::vector& coor,int index,std::vector &dims){ + static inline void CoorFromIndex (std::vector& coor,int index,const std::vector &dims){ int nd= dims.size(); coor.resize(nd); for(int d=0;d& coor,int &index,std::vector &dims){ + static inline void IndexFromCoor (const std::vector& coor,int &index,const std::vector &dims){ int nd=dims.size(); int stride=1; index=0; diff --git a/tests/Test_stencil.cc b/tests/Test_stencil.cc index d7bc5a6c..be8ddfd8 100644 --- a/tests/Test_stencil.cc +++ b/tests/Test_stencil.cc @@ -49,7 +49,7 @@ int main (int argc, char ** argv) double volume = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3]; GridCartesian Fine(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian rbFine(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian rbFine(&Fine); GridParallelRNG fRNG(&Fine); // fRNG.SeedFixedIntegers(std::vector({45,12,81,9}); diff --git a/tests/core/Test_cshift_red_black.cc b/tests/core/Test_cshift_red_black.cc index f9269709..c7b0c2f1 100644 --- a/tests/core/Test_cshift_red_black.cc +++ b/tests/core/Test_cshift_red_black.cc @@ -47,7 +47,7 @@ int main (int argc, char ** argv) mask[0]=0; GridCartesian Fine (latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBFine(latt_size,simd_layout,mpi_layout,mask,1); + GridRedBlackCartesian RBFine(&Fine,mask,1); GridParallelRNG FineRNG(&Fine); FineRNG.SeedFixedIntegers(std::vector({45,12,81,9})); diff --git a/tests/core/Test_cshift_red_black_rotate.cc b/tests/core/Test_cshift_red_black_rotate.cc index 3ef1cd21..aa9e6104 100644 --- a/tests/core/Test_cshift_red_black_rotate.cc +++ b/tests/core/Test_cshift_red_black_rotate.cc @@ -47,7 +47,7 @@ int main (int argc, char ** argv) mask[0]=0; GridCartesian Fine (latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBFine(latt_size,simd_layout,mpi_layout,mask,1); + GridRedBlackCartesian RBFine(&Fine,mask,1); GridParallelRNG FineRNG(&Fine); FineRNG.SeedFixedIntegers(std::vector({45,12,81,9})); diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc index 877683f0..b2336cfa 100644 --- a/tests/core/Test_fft.cc +++ b/tests/core/Test_fft.cc @@ -47,7 +47,7 @@ int main (int argc, char ** argv) vol = vol * latt_size[d]; } GridCartesian GRID(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGRID(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGRID(&GRID); LatticeComplexD one(&GRID); LatticeComplexD zz(&GRID); diff --git a/tests/core/Test_gpwilson_even_odd.cc b/tests/core/Test_gpwilson_even_odd.cc index fc12fe75..2069eb40 100644 --- a/tests/core/Test_gpwilson_even_odd.cc +++ b/tests/core/Test_gpwilson_even_odd.cc @@ -40,7 +40,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); int threads = GridThread::GetThreads(); std::cout< simd_layout = GridDefaultSimd(Nd, vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size, simd_layout, mpi_layout); - GridRedBlackCartesian RBGrid(latt_size, simd_layout, mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1, 2, 3, 4, 5}); GridSerialRNG sRNG; @@ -149,4 +149,4 @@ JSON } -*/ \ No newline at end of file +*/ diff --git a/tests/solver/Test_laplacian.cc b/tests/solver/Test_laplacian.cc index ce4307ab..f163fe54 100644 --- a/tests/solver/Test_laplacian.cc +++ b/tests/solver/Test_laplacian.cc @@ -40,7 +40,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4,5}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); diff --git a/tests/solver/Test_staggered_cg_unprec.cc b/tests/solver/Test_staggered_cg_unprec.cc index 5e0358d7..eb33c004 100644 --- a/tests/solver/Test_staggered_cg_unprec.cc +++ b/tests/solver/Test_staggered_cg_unprec.cc @@ -57,7 +57,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); diff --git a/tests/solver/Test_wilson_cg_prec.cc b/tests/solver/Test_wilson_cg_prec.cc index 011bc70b..99ddfceb 100644 --- a/tests/solver/Test_wilson_cg_prec.cc +++ b/tests/solver/Test_wilson_cg_prec.cc @@ -52,7 +52,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); diff --git a/tests/solver/Test_wilson_cg_schur.cc b/tests/solver/Test_wilson_cg_schur.cc index 7bbf74d3..13ac0090 100644 --- a/tests/solver/Test_wilson_cg_schur.cc +++ b/tests/solver/Test_wilson_cg_schur.cc @@ -52,7 +52,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); diff --git a/tests/solver/Test_wilson_cg_unprec.cc b/tests/solver/Test_wilson_cg_unprec.cc index 19c5f854..db227ec8 100644 --- a/tests/solver/Test_wilson_cg_unprec.cc +++ b/tests/solver/Test_wilson_cg_unprec.cc @@ -52,7 +52,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); diff --git a/tests/solver/Test_wilson_cr_unprec.cc b/tests/solver/Test_wilson_cr_unprec.cc index 4182c04e..eccd7e74 100644 --- a/tests/solver/Test_wilson_cr_unprec.cc +++ b/tests/solver/Test_wilson_cr_unprec.cc @@ -52,7 +52,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); From 5e4bea8f2022fcb720bd887a754ee8943b3f38cd Mon Sep 17 00:00:00 2001 From: paboyle Date: Thu, 22 Jun 2017 08:38:54 +0100 Subject: [PATCH 04/18] Benchmark DWF works --- lib/communicator/Communicator_mpi.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index b24b60bd..28a270a0 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -86,19 +86,23 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors, Lexicographic::IndexFromCoor(scoor,slex,processors); MPI_Comm comm_split; - int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split); - assert(ierr==0); + if ( Nsubcomm > 1 ) { + int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split); + assert(ierr==0); + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // Declare victory + ////////////////////////////////////////////////////////////////////////////////////////////////////// + std::cout << "Divided communicator "<< parent._Nprocessors<<" into " + < Date: Thu, 22 Jun 2017 18:53:11 +0100 Subject: [PATCH 05/18] Able to run a test job splitting into multiple MPI subdomains. --- lib/algorithms/iterative/ConjugateGradient.h | 4 +- lib/cartesian/Cartesian_base.h | 3 - lib/cartesian/Cartesian_full.h | 2 +- lib/communicator/Communicator_mpi.cc | 54 +++++---- lib/parallelIO/IldgIO.h | 115 ++++++++++++++++--- lib/parallelIO/IldgIOtypes.h | 4 + lib/parallelIO/MetaData.h | 1 + lib/qcd/utils/SpaceTimeGrid.cc | 19 +-- 8 files changed, 138 insertions(+), 64 deletions(-) diff --git a/lib/algorithms/iterative/ConjugateGradient.h b/lib/algorithms/iterative/ConjugateGradient.h index ed453161..5c968e04 100644 --- a/lib/algorithms/iterative/ConjugateGradient.h +++ b/lib/algorithms/iterative/ConjugateGradient.h @@ -52,8 +52,8 @@ class ConjugateGradient : public OperatorFunction { MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv){}; - void operator()(LinearOperatorBase &Linop, const Field &src, - Field &psi) { + void operator()(LinearOperatorBase &Linop, const Field &src, Field &psi) { + psi.checkerboard = src.checkerboard; conformable(psi, src); diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h index 25041d17..a7719ec4 100644 --- a/lib/cartesian/Cartesian_base.h +++ b/lib/cartesian/Cartesian_base.h @@ -211,9 +211,6 @@ public: assert(lidx & gcoor,int & gidx){ gidx=0; int mult=1; diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h index bced0791..eb388f84 100644 --- a/lib/cartesian/Cartesian_full.h +++ b/lib/cartesian/Cartesian_full.h @@ -67,7 +67,7 @@ public: GridCartesian(const std::vector &dimensions, const std::vector &simd_layout, const std::vector &processor_grid, - GridCartesian &parent) : GridBase(processor_grid,parent) + const GridCartesian &parent) : GridBase(processor_grid,parent) { Init(dimensions,simd_layout,processor_grid); } diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index 28a270a0..7879f518 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -56,46 +56,52 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { CartesianCommunicator::CartesianCommunicator(const std::vector &processors) { InitFromMPICommunicator(processors,communicator_world); - std::cout << "Passed communicator world to a new communicator" < &processors,const CartesianCommunicator &parent) { _ndimension = processors.size(); assert(_ndimension = parent._ndimension); - + ////////////////////////////////////////////////////////////////////////////////////////////////////// // split the communicator ////////////////////////////////////////////////////////////////////////////////////////////////////// - std::vector ratio(_ndimension); - std::vector rcoor(_ndimension); - std::vector scoor(_ndimension); + int Nparent; + MPI_Comm_size(parent.communicator,&Nparent); - int Nsubcomm=1; - int Nsubrank=1; + int childsize=1; for(int d=0;d<_ndimension;d++) { - ratio[d] = parent._processors[d] / processors[d]; - rcoor[d] = parent._processor_coor[d] / processors[d]; - scoor[d] = parent._processor_coor[d] % processors[d]; - assert(ratio[d] * processors[d] == parent._processors[d]); // must exactly subdivide - Nsubcomm *= ratio[d]; - Nsubrank *= processors[d]; + childsize *= processors[d]; } + int Nchild = Nparent/childsize; + assert (childsize * Nchild == Nparent); - int rlex, slex; - Lexicographic::IndexFromCoor(rcoor,rlex,ratio); - Lexicographic::IndexFromCoor(scoor,slex,processors); + int prank; MPI_Comm_rank(parent.communicator,&prank); + int crank = prank % childsize; + int ccomm = prank / childsize; MPI_Comm comm_split; - if ( Nsubcomm > 1 ) { - int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split); + if ( Nchild > 1 ) { + + std::cout << GridLogMessage<<"Child communicator of "<< parent.communicator< &processors, ////////////////////////////////////////////////////////////////////////////////////////////////////// void CartesianCommunicator::InitFromMPICommunicator(const std::vector &processors, MPI_Comm communicator_base) { - if ( communicator_base != communicator_world ) { - std::cout << "Cartesian communicator created with a non-world communicator"<(); - // std::cout << " Lorentz N/S/V/M : " << _LorentzN<<" "<<_LorentzScalar<<"/"<<_LorentzVector<<"/"<<_LorentzMatrix<_gsites; + + // std::cout << "R sizeof(sobj)= " <_gsites< munge; - BinaryIO::readLatticeObject< sobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); + BinaryIO::readLatticeObject< vobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); ///////////////////////////////////////////// // Insist checksum is next record ///////////////////////////////////////////// - readLimeObject(scidacChecksum_,std::string("scidacChecksum"),record_name); + readLimeObject(scidacChecksum_,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM)); ///////////////////////////////////////////// // Verify checksums @@ -242,9 +252,14 @@ class GridLimeReader : public BinaryIO { // should this be a do while; can we miss a first record?? while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { + // std::cout << GridLogMessage<< " readLimeObject seeking "<< record_name <<" found record :" < xmlc(nbytes+1,'\0'); limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); XmlReader RD(&xmlc[0],""); @@ -302,14 +317,18 @@ class GridLimeWriter : public BinaryIO { write(WR,object_name,object); xmlstring = WR.XmlString(); } + // std::cout << "WriteLimeObject" << record_name <(record_name.c_str()), nbytes); + assert(h!= NULL); err=limeWriteRecordHeader(h, LimeW); assert(err>=0); err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0); err=limeWriterCloseRecord(LimeW); assert(err>=0); limeDestroyHeader(h); + // std::cout << " File offset is now"<_gsites; createLimeRecordHeader(record_name, 0, 0, PayloadSize); + + // std::cout << "W sizeof(sobj)" <_gsites<(); BinarySimpleMunger munge; BinaryIO::writeLatticeObject(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb); @@ -354,7 +379,7 @@ class GridLimeWriter : public BinaryIO { checksum.suma= streama.str(); checksum.sumb= streamb.str(); std::cout << GridLogMessage<<" writing scidac checksums "< + template void writeScidacFieldRecord(Lattice &field,userRecord _userRecord) { - typedef typename vobj::scalar_object sobj; - uint64_t nbytes; GridBase * grid = field._grid; //////////////////////////////////////// @@ -397,6 +420,66 @@ class ScidacWriter : public GridLimeWriter { } }; + +class ScidacReader : public GridLimeReader { + public: + + template + void readScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile) + { + scidacFile _scidacFile(grid); + readLimeObject(_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML)); + readLimeObject(_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML)); + } + //////////////////////////////////////////////// + // Write generic lattice field in scidac format + //////////////////////////////////////////////// + template + void readScidacFieldRecord(Lattice &field,userRecord &_userRecord) + { + typedef typename vobj::scalar_object sobj; + GridBase * grid = field._grid; + + //////////////////////////////////////// + // fill the Grid header + //////////////////////////////////////// + FieldMetaData header; + scidacRecord _scidacRecord; + scidacFile _scidacFile; + + ////////////////////////////////////////////// + // Fill the Lime file record by record + ////////////////////////////////////////////// + readLimeObject(header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message + readLimeObject(_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML)); + readLimeObject(_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML)); + readLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA)); + } + void skipPastBinaryRecord(void) { + std::string rec_name(ILDG_BINARY_DATA); + while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { + if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) ) ) { + skipPastObjectRecord(std::string(SCIDAC_CHECKSUM)); + return; + } + } + } + void skipPastObjectRecord(std::string rec_name) { + while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { + if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) ) ) { + return; + } + } + } + void skipScidacFieldRecord() { + skipPastObjectRecord(std::string(GRID_FORMAT)); + skipPastObjectRecord(std::string(SCIDAC_RECORD_XML)); + skipPastObjectRecord(std::string(SCIDAC_PRIVATE_RECORD_XML)); + skipPastBinaryRecord(); + } +}; + + class IldgWriter : public ScidacWriter { public: @@ -425,8 +508,6 @@ class IldgWriter : public ScidacWriter { typedef iLorentzColourMatrix vobj; typedef typename vobj::scalar_object sobj; - uint64_t nbytes; - //////////////////////////////////////// // fill the Grid header //////////////////////////////////////// diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h index c3a5321c..53664b49 100644 --- a/lib/parallelIO/IldgIOtypes.h +++ b/lib/parallelIO/IldgIOtypes.h @@ -64,6 +64,10 @@ namespace Grid { // file compatability, so should be correct to assume the undocumented but defacto file structure. ///////////////////////////////////////////////////////////////////////////////// +struct emptyUserRecord : Serializable { + GRID_SERIALIZABLE_CLASS_MEMBERS(emptyUserRecord,int,dummy); +}; + //////////////////////// // Scidac private file xml // 1.1416 16 16 32 0 diff --git a/lib/parallelIO/MetaData.h b/lib/parallelIO/MetaData.h index 6d45d0a5..8fc0d777 100644 --- a/lib/parallelIO/MetaData.h +++ b/lib/parallelIO/MetaData.h @@ -104,6 +104,7 @@ namespace Grid { header.nd = nd; header.dimension.resize(nd); header.boundary.resize(nd); + header.data_start = 0; for(int d=0;d_fdimensions[d]; } diff --git a/lib/qcd/utils/SpaceTimeGrid.cc b/lib/qcd/utils/SpaceTimeGrid.cc index cbbe0aee..b2b5d9c8 100644 --- a/lib/qcd/utils/SpaceTimeGrid.cc +++ b/lib/qcd/utils/SpaceTimeGrid.cc @@ -60,7 +60,7 @@ GridCartesian *SpaceTimeGrid::makeFiveDimGrid(int Ls,const GridCartesian simd5.push_back(FourDimGrid->_simd_layout[d]); mpi5.push_back(FourDimGrid->_processors[d]); } - return new GridCartesian(latt5,simd5,mpi5); + return new GridCartesian(latt5,simd5,mpi5,*FourDimGrid); } @@ -68,15 +68,8 @@ GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimRedBlackGrid(int Ls,const GridC { int N4=FourDimGrid->_ndimension; int cbd=1; - // std::vector latt5(1,Ls); - // std::vector simd5(1,1); - // std::vector mpi5(1,1); std::vector cb5(1,0); - for(int d=0;d_fdimensions[d]); - // simd5.push_back(FourDimGrid->_simd_layout[d]); - // mpi5.push_back(FourDimGrid->_processors[d]); cb5.push_back( 1); } GridCartesian *tmp = makeFiveDimGrid(Ls,FourDimGrid); @@ -100,7 +93,7 @@ GridCartesian *SpaceTimeGrid::makeFiveDimDWFGrid(int Ls,const GridCartes simd5.push_back(1); mpi5.push_back(FourDimGrid->_processors[d]); } - return new GridCartesian(latt5,simd5,mpi5); + return new GridCartesian(latt5,simd5,mpi5,*FourDimGrid); } /////////////////////////////////////////////////// // Interface is inefficient and forces the deletion @@ -111,15 +104,7 @@ GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(int Ls,const Gr int N4=FourDimGrid->_ndimension; int cbd=1; std::vector cb5(1,0); - // int nsimd = FourDimGrid->Nsimd(); - // std::vector latt5(1,Ls); - // std::vector simd5(1,nsimd); - // std::vector mpi5(1,1); - for(int d=0;d_fdimensions[d]); - // simd5.push_back(1); - // mpi5.push_back(FourDimGrid->_processors[d]); cb5.push_back(1); } GridCartesian *tmp = makeFiveDimDWFGrid(Ls,FourDimGrid); From 600d7ddc2ea54c956a63d9cf0de448909130f51e Mon Sep 17 00:00:00 2001 From: paboyle Date: Thu, 22 Jun 2017 18:54:34 +0100 Subject: [PATCH 06/18] Proof of concept : Multi RHS solver, running independent solves on different ranks --- tests/solver/Test_dwf_mrhs_cg.cc | 175 +++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 tests/solver/Test_dwf_mrhs_cg.cc diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc new file mode 100644 index 00000000..fa940390 --- /dev/null +++ b/tests/solver/Test_dwf_mrhs_cg.cc @@ -0,0 +1,175 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/Test_dwf_mrhs_cg.cc + + Copyright (C) 2015 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + +int main (int argc, char ** argv) +{ + typedef typename DomainWallFermionR::FermionField FermionField; + typedef typename DomainWallFermionR::ComplexField ComplexField; + typename DomainWallFermionR::ImplParams params; + + const int Ls=8; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + std::vector mpi_split (mpi_layout.size(),1); + + std::cout << "UGrid (world root)"<RankCount() ; + + ///////////////////////////////////////////// + // Split into 1^4 mpi communicators + ///////////////////////////////////////////// + std::cout << "SGrid (world root)"< seeds({1,2,3,4}); + + GridParallelRNG pRNG(UGrid ); pRNG.SeedFixedIntegers(seeds); + GridParallelRNG pRNG5(FGrid); pRNG5.SeedFixedIntegers(seeds); + std::vector src(nrhs,FGrid); + std::vector result(nrhs,FGrid); + + for(int s=0;sThisRank(); + LatticeGaugeField s_Umu(SGrid); + FermionField s_src(SFGrid); + FermionField s_res(SFGrid); + { + ScidacReader _ScidacReader; + _ScidacReader.open(file); + std::cout << GridLogMessage << " Opened file "< HermOp(Ddwf); + ConjugateGradient CG((1.0e-8/(me+1)),10000); + s_res = zero; + CG(HermOp,s_src,s_res); + + /////////////////////////////////////// + // Share the information + /////////////////////////////////////// + std::vector iterations(nrhs,0); + iterations[me] = CG.IterationsToComplete; + + for(int n=0;nGlobalSum(iterations[n]); + } + + ///////////////////////////////////////////////////////////// + // Report how long they all took + ///////////////////////////////////////////////////////////// + for(int r=0;r Date: Thu, 22 Jun 2017 19:32:41 +0100 Subject: [PATCH 07/18] const fixes --- lib/communicator/Communicator_mpi3.cc | 2 +- lib/communicator/Communicator_none.cc | 2 +- lib/communicator/Communicator_shmem.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc index 6d7f301c..22445f60 100644 --- a/lib/communicator/Communicator_mpi3.cc +++ b/lib/communicator/Communicator_mpi3.cc @@ -375,7 +375,7 @@ void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &c ////////////////////////////////// // Try to subdivide communicator ////////////////////////////////// -CartesianCommunicator::CartesianCommunicator(const std::vector &processors,CartesianCommunicator &parent) +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent) : CartesianCommunicator(processors) { std::cout << "Attempts to split MPI3 communicators will fail until implemented" < &processors,CartesianCommunicator &parent) +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent) : CartesianCommunicator(processors) {} CartesianCommunicator::CartesianCommunicator(const std::vector &processors) diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc index 49d32697..ed49285d 100644 --- a/lib/communicator/Communicator_shmem.cc +++ b/lib/communicator/Communicator_shmem.cc @@ -75,7 +75,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { ShmInitGeneric(); } -CartesianCommunicator::CartesianCommunicator(const std::vector &processors,CartesianCommunicator &parent) +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent) : CartesianCommunicator(processors) { std::cout << "Attempts to split SHMEM communicators will fail " < Date: Fri, 23 Jun 2017 09:42:21 +0100 Subject: [PATCH 08/18] Ticking off lots on the TODO list --- TODO | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/TODO b/TODO index 001c6c0c..f0c022ca 100644 --- a/TODO +++ b/TODO @@ -2,18 +2,24 @@ TODO: --------------- Large item work list: -1)- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O +1)- BG/Q port and check 2)- Christoph's local basis expansion Lanczos -3)- BG/Q port and check -4)- Precision conversion and sort out localConvert <-- partial - - Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet -5)- Physical propagator interface -6)- Conserved currents -7)- Multigrid Wilson and DWF, compare to other Multigrid implementations -8)- HDCR resume +-- +3a)- RNG I/O in ILDG/SciDAC (minor) +3b)- Precision conversion and sort out localConvert <-- partial/easy +3c)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet + +4)- Physical propagator interface + +5)- Conserved currents + +6)- Multigrid Wilson and DWF, compare to other Multigrid implementations + +7)- HDCR resume Recent DONE +-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O ; <-- DONE ; bmark cori -- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE -- GaugeFix into central location <-- DONE -- Scidac and Ildg metadata handling <-- DONE From 349d75e48379c353c886966cebcb0eeb944bb624 Mon Sep 17 00:00:00 2001 From: paboyle Date: Fri, 23 Jun 2017 02:57:59 -0700 Subject: [PATCH 09/18] Precision fix --- lib/lattice/Lattice_reduction.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lattice/Lattice_reduction.h b/lib/lattice/Lattice_reduction.h index c5b20f3c..fc1ccadb 100644 --- a/lib/lattice/Lattice_reduction.h +++ b/lib/lattice/Lattice_reduction.h @@ -540,7 +540,7 @@ static void sliceInnerProductMatrix( Eigen::MatrixXcd &mat, const Lattice for(int i=0;i Date: Fri, 23 Jun 2017 23:10:24 +0100 Subject: [PATCH 10/18] Added an update to TODO list --- TODO | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/TODO b/TODO index f0c022ca..a778f476 100644 --- a/TODO +++ b/TODO @@ -3,19 +3,17 @@ TODO: Large item work list: -1)- BG/Q port and check +1)- BG/Q port and check ; Andrew says ok. 2)- Christoph's local basis expansion Lanczos -- 3a)- RNG I/O in ILDG/SciDAC (minor) 3b)- Precision conversion and sort out localConvert <-- partial/easy 3c)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet - +-- 4)- Physical propagator interface - 5)- Conserved currents - +-- 6)- Multigrid Wilson and DWF, compare to other Multigrid implementations - 7)- HDCR resume Recent DONE From 57002924bc3232308cd1add6610b632a3b055fe9 Mon Sep 17 00:00:00 2001 From: paboyle Date: Sun, 2 Jul 2017 14:58:30 -0700 Subject: [PATCH 11/18] NERSC shakeout of this --- lib/communicator/Communicator_mpi.cc | 8 ++-- tests/solver/Test_dwf_mrhs_cg.cc | 65 ++++++++++++++-------------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index 7879f518..1a7a0c05 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -70,7 +70,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors, MPI_Comm_size(parent.communicator,&Nparent); int childsize=1; - for(int d=0;d<_ndimension;d++) { + for(int d=0;d &processors, MPI_Comm comm_split; if ( Nchild > 1 ) { - std::cout << GridLogMessage<<"Child communicator of "<< parent.communicator<ThisRank(); LatticeGaugeField s_Umu(SGrid); FermionField s_src(SFGrid); FermionField s_res(SFGrid); + { + ScidacWriter _ScidacWriter; + _ScidacWriter.open(file); + std::cout << GridLogMessage << " Writing out gauge field "<Barrier(); + } + } + + /////////////////////////////////////////////////////////////// // Set up N-solvers as trivially parallel /////////////////////////////////////////////////////////////// From dee68fc7287a2af2d2acba8f72f96760722d275a Mon Sep 17 00:00:00 2001 From: paboyle Date: Sun, 2 Jul 2017 23:33:48 +0100 Subject: [PATCH 12/18] IO working multiple nodes again. Strategy of all nodes writing metadata is unsafe. Only one rank should do this. must identify this rank. Means pass communicator to the Objects. --- lib/parallelIO/IldgIO.h | 8 ++++++-- lib/parallelIO/IldgIOtypes.h | 1 + lib/parallelIO/MetaData.h | 3 +++ tests/solver/Test_dwf_mrhs_cg.cc | 21 ++++++++++++++++++--- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h index 542f80ef..7031f110 100644 --- a/lib/parallelIO/IldgIO.h +++ b/lib/parallelIO/IldgIO.h @@ -258,10 +258,13 @@ class GridLimeReader : public BinaryIO { if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) ) ) { - // std::cout << GridLogMessage<< " readLimeObject matches ! " < xmlc(nbytes+1,'\0'); limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); + + std::cout << GridLogMessage<< " readLimeObject matches XML " << &xmlc[0] <ThisRank(); LatticeGaugeField s_Umu(SGrid); FermionField s_src(SFGrid); FermionField s_res(SFGrid); { + FGrid->Barrier(); ScidacWriter _ScidacWriter; _ScidacWriter.open(file); + std::cout << GridLogMessage << "****************************************************************** "<Barrier(); + std::cout << GridLogMessage << "****************************************************************** "<Barrier(); + std::cout << GridLogMessage << "****************************************************************** "<Barrier(); + std::cout << GridLogMessage << "****************************************************************** "<Barrier(); + std::cout << GridLogMessage << "****************************************************************** "< Date: Sun, 2 Jul 2017 16:47:42 -0700 Subject: [PATCH 13/18] Working on Cori --- lib/parallelIO/IldgIO.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h index 7031f110..ba71153d 100644 --- a/lib/parallelIO/IldgIO.h +++ b/lib/parallelIO/IldgIO.h @@ -178,7 +178,7 @@ class GridLimeReader : public BinaryIO { ///////////////////////////////////////////// // Open the file ///////////////////////////////////////////// - void open(std::string &_filename) + void open(const std::string &_filename) { filename= _filename; File = fopen(filename.c_str(), "r"); @@ -258,12 +258,12 @@ class GridLimeReader : public BinaryIO { if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) ) ) { - std::cout << GridLogMessage<< " readLimeObject matches ! " << record_name < xmlc(nbytes+1,'\0'); limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR); - std::cout << GridLogMessage<< " readLimeObject matches XML " << &xmlc[0] < Date: Sun, 2 Jul 2017 16:47:58 -0700 Subject: [PATCH 14/18] Works on Cori --- tests/solver/Test_dwf_mrhs_cg.cc | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc index 72d8d8f4..b4880148 100644 --- a/tests/solver/Test_dwf_mrhs_cg.cc +++ b/tests/solver/Test_dwf_mrhs_cg.cc @@ -128,29 +128,36 @@ int main (int argc, char ** argv) { for(int n=0;nBarrier(); + std::stringstream filefn; + filefn << filef << "."<< n; + std::cout << GridLogMessage << "****************************************************************** "<Barrier(); + FGrid->Barrier(); - std::cout << GridLogMessage << "****************************************************************** "<Barrier(); } + FGrid->Barrier(); } @@ -185,7 +192,7 @@ int main (int argc, char ** argv) // Report how long they all took ///////////////////////////////////////////////////////////// for(int r=0;r Date: Mon, 2 Oct 2017 07:13:40 +0100 Subject: [PATCH 15/18] Solving an annoying compilation error in json --- lib/json/json.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json/json.hpp b/lib/json/json.hpp index e7c42920..9d589120 100644 --- a/lib/json/json.hpp +++ b/lib/json/json.hpp @@ -63,7 +63,7 @@ SOFTWARE. #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" #endif #elif defined(__GNUC__) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40805 #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" #endif #endif From 073525c5b3a32e6ab4567246893c08d8da609ac9 Mon Sep 17 00:00:00 2001 From: paboyle Date: Mon, 2 Oct 2017 03:38:21 -0700 Subject: [PATCH 16/18] Small patch from cori --- tests/solver/Test_dwf_mrhs_cg.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc index b4880148..2d2cfcb1 100644 --- a/tests/solver/Test_dwf_mrhs_cg.cc +++ b/tests/solver/Test_dwf_mrhs_cg.cc @@ -128,12 +128,11 @@ int main (int argc, char ** argv) { for(int n=0;n Date: Mon, 2 Oct 2017 12:25:52 +0100 Subject: [PATCH 17/18] RB constructor change --- tests/solver/Test_staggered_cg_prec.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/solver/Test_staggered_cg_prec.cc b/tests/solver/Test_staggered_cg_prec.cc index 66f11d3d..0e359c2d 100644 --- a/tests/solver/Test_staggered_cg_prec.cc +++ b/tests/solver/Test_staggered_cg_prec.cc @@ -57,7 +57,7 @@ int main (int argc, char ** argv) std::vector simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd()); std::vector mpi_layout = GridDefaultMpi(); GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout); + GridRedBlackCartesian RBGrid(&Grid); std::vector seeds({1,2,3,4}); GridParallelRNG pRNG(&Grid); pRNG.SeedFixedIntegers(seeds); From 1edcf902b73b66240b293f6fae94b991117da1b7 Mon Sep 17 00:00:00 2001 From: paboyle Date: Mon, 2 Oct 2017 12:41:02 +0100 Subject: [PATCH 18/18] Macos ANON --- lib/communicator/Communicator_base.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc index c60f6c6d..836bc01e 100644 --- a/lib/communicator/Communicator_base.cc +++ b/lib/communicator/Communicator_base.cc @@ -147,8 +147,13 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) { } void CartesianCommunicator::ShmInitGeneric(void){ #if 1 - - int mmap_flag = MAP_SHARED | MAP_ANONYMOUS; + int mmap_flag =0; +#ifdef MAP_ANONYMOUS + mmap_flag = mmap_flag| MAP_SHARED | MAP_ANONYMOUS; +#endif +#ifdef MAP_ANON + mmap_flag = mmap_flag| MAP_SHARED | MAP_ANON; +#endif #ifdef MAP_HUGETLB if ( Hugepages ) mmap_flag |= MAP_HUGETLB; #endif