diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc index 836bc01e..bcf429ab 100644 --- a/lib/communicator/Communicator_base.cc +++ b/lib/communicator/Communicator_base.cc @@ -96,6 +96,105 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N) GlobalSumVector((double *)c,2*N); } + +#if defined( GRID_COMMS_MPI) || defined (GRID_COMMS_MPIT) + +CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent) +{ + _ndimension = processors.size(); + assert(_ndimension = parent._ndimension); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // split the communicator + ////////////////////////////////////////////////////////////////////////////////////////////////////// + int Nparent; + MPI_Comm_size(parent.communicator,&Nparent); + + int childsize=1; + for(int d=0;d 1 ) { + + std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec< &processors, MPI_Comm communicator_base) +{ + // if ( communicator_base != communicator_world ) { + // std::cout << "Cartesian communicator created with a non-world communicator"< periodic(_ndimension,1); + MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator); + MPI_Comm_rank(communicator,&_processor); + MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); + + int Size; + MPI_Comm_size(communicator,&Size); + +#ifdef GRID_COMMS_MPIT + communicator_halo.resize (2*_ndimension); + for(int i=0;i<_ndimension*2;i++){ + MPI_Comm_dup(communicator,&communicator_halo[i]); + } +#endif + + assert(Size==_Nprocessors); +} + +CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +{ + InitFromMPICommunicator(processors,communicator_world); +} + +#endif + #if !defined( GRID_COMMS_MPI3) int CartesianCommunicator::NodeCount(void) { return ProcessorCount();}; diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h index e8791cd8..bfdb0da1 100644 --- a/lib/communicator/Communicator_base.h +++ b/lib/communicator/Communicator_base.h @@ -157,8 +157,7 @@ class CartesianCommunicator { CartesianCommunicator(const std::vector &pdimensions_in); private: -#if defined (GRID_COMMS_MPI) - //|| defined (GRID_COMMS_MPI3) +#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPIT) //////////////////////////////////////////////// // Private initialise from an MPI communicator // Can use after an MPI_Comm_split, but hidden from user so private diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index 1a7a0c05..a55c0164 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -52,95 +52,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world); ShmInitGeneric(); } - -CartesianCommunicator::CartesianCommunicator(const std::vector &processors) -{ - InitFromMPICommunicator(processors,communicator_world); - // std::cout << "Passed communicator world to a new communicator" < &processors,const CartesianCommunicator &parent) -{ - _ndimension = processors.size(); - assert(_ndimension = parent._ndimension); - - ////////////////////////////////////////////////////////////////////////////////////////////////////// - // split the communicator - ////////////////////////////////////////////////////////////////////////////////////////////////////// - int Nparent; - MPI_Comm_size(parent.communicator,&Nparent); - - int childsize=1; - for(int d=0;d 1 ) { - - std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec< &processors, MPI_Comm communicator_base) -{ - // if ( communicator_base != communicator_world ) { - // std::cout << "Cartesian communicator created with a non-world communicator"< periodic(_ndimension,1); - MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator); - MPI_Comm_rank(communicator,&_processor); - MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); - - int Size; - MPI_Comm_size(communicator,&Size); - - assert(Size==_Nprocessors); -} void CartesianCommunicator::GlobalSum(uint32_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc index eb6ef87d..5137c27b 100644 --- a/lib/communicator/Communicator_mpit.cc +++ b/lib/communicator/Communicator_mpit.cc @@ -53,33 +53,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) { ShmInitGeneric(); } -CartesianCommunicator::CartesianCommunicator(const std::vector &processors) -{ - _ndimension = processors.size(); - std::vector periodic(_ndimension,1); - - _Nprocessors=1; - _processors = processors; - _processor_coor.resize(_ndimension); - - MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator); - MPI_Comm_rank(communicator,&_processor); - MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); - - for(int i=0;i<_ndimension;i++){ - _Nprocessors*=_processors[i]; - } - - communicator_halo.resize (2*_ndimension); - for(int i=0;i<_ndimension*2;i++){ - MPI_Comm_dup(communicator,&communicator_halo[i]); - } - - int Size; - MPI_Comm_size(communicator,&Size); - - assert(Size==_Nprocessors); -} void CartesianCommunicator::GlobalSum(uint32_t &u){ int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator); assert(ierr==0); diff --git a/lib/qcd/action/fermion/CayleyFermion5D.cc b/lib/qcd/action/fermion/CayleyFermion5D.cc index 838b1c3d..eace6484 100644 --- a/lib/qcd/action/fermion/CayleyFermion5D.cc +++ b/lib/qcd/action/fermion/CayleyFermion5D.cc @@ -77,7 +77,6 @@ void CayleyFermion5D::DminusDag(const FermionField &psi, FermionField &chi } } - template void CayleyFermion5D::CayleyReport(void) { this->Report(); @@ -119,7 +118,6 @@ template void CayleyFermion5D::CayleyZeroCounters(void) MooeeInvTime=0; } - template void CayleyFermion5D::M5D (const FermionField &psi, FermionField &chi) {