diff --git a/lib/communicator/Communicator.h b/lib/communicator/Communicator.h index c332c04a..76d6f37a 100644 --- a/lib/communicator/Communicator.h +++ b/lib/communicator/Communicator.h @@ -28,6 +28,7 @@ Author: Peter Boyle #ifndef GRID_COMMUNICATOR_H #define GRID_COMMUNICATOR_H +#include #include #include diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc index b10c1a04..dfa4846b 100644 --- a/lib/communicator/Communicator_base.cc +++ b/lib/communicator/Communicator_base.cc @@ -47,8 +47,8 @@ int CartesianCommunicator::Dimensions(void) { return int CartesianCommunicator::IsBoss(void) { return _processor==0; }; int CartesianCommunicator::BossRank(void) { return 0; }; int CartesianCommunicator::ThisRank(void) { return _processor; }; -const std::vector & CartesianCommunicator::ThisProcessorCoor(void) { return _processor_coor; }; -const std::vector & CartesianCommunicator::ProcessorGrid(void) { return _processors; }; +const Coordinate & CartesianCommunicator::ThisProcessorCoor(void) { return _processor_coor; }; +const Coordinate & CartesianCommunicator::ProcessorGrid(void) { return _processors; }; int CartesianCommunicator::ProcessorCount(void) { return _Nprocessors; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h index b6e50c24..0eebd305 100644 --- a/lib/communicator/Communicator_base.h +++ b/lib/communicator/Communicator_base.h @@ -52,9 +52,9 @@ public: // Communicator should know nothing of the physics grid, only processor grid. //////////////////////////////////////////// int _Nprocessors; // How many in all - std::vector _processors; // Which dimensions get relayed out over processors lanes. + Coordinate _processors; // Which dimensions get relayed out over processors lanes. int _processor; // linear processor rank - std::vector _processor_coor; // linear processor coordinate + Coordinate _processor_coor; // linear processor coordinate unsigned long _ndimension; static Grid_MPI_Comm communicator_world; Grid_MPI_Comm communicator; @@ -69,8 +69,8 @@ public: // Constructors to sub-divide a parent communicator // and default to comm world //////////////////////////////////////////////// - CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent,int &srank); - CartesianCommunicator(const std::vector &pdimensions_in); + CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank); + CartesianCommunicator(const Coordinate &pdimensions_in); virtual ~CartesianCommunicator(); private: @@ -79,7 +79,7 @@ private: // Private initialise from an MPI communicator // Can use after an MPI_Comm_split, but hidden from user so private //////////////////////////////////////////////// - void InitFromMPICommunicator(const std::vector &processors, Grid_MPI_Comm communicator_base); + void InitFromMPICommunicator(const Coordinate &processors, Grid_MPI_Comm communicator_base); public: @@ -87,15 +87,15 @@ public: // Wraps MPI_Cart routines, or implements equivalent on other impls //////////////////////////////////////////////////////////////////////////////////////// void ShiftedRanks(int dim,int shift,int & source, int & dest); - int RankFromProcessorCoor(std::vector &coor); - void ProcessorCoorFromRank(int rank,std::vector &coor); + int RankFromProcessorCoor(Coordinate &coor); + void ProcessorCoorFromRank(int rank,Coordinate &coor); int Dimensions(void) ; int IsBoss(void) ; int BossRank(void) ; int ThisRank(void) ; - const std::vector & ThisProcessorCoor(void) ; - const std::vector & ProcessorGrid(void) ; + const Coordinate & ThisProcessorCoor(void) ; + const Coordinate & ProcessorGrid(void) ; int ProcessorCount(void) ; //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc index dbe570f8..ad268dbb 100644 --- a/lib/communicator/Communicator_mpi3.cc +++ b/lib/communicator/Communicator_mpi3.cc @@ -65,14 +65,14 @@ void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest); assert(ierr==0); } -int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) +int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor) { int rank; int ierr=MPI_Cart_rank (communicator, &coor[0], &rank); assert(ierr==0); return rank; } -void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor) +void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor) { coor.resize(_ndimension); int ierr=MPI_Cart_coords (communicator, rank, _ndimension,&coor[0]); @@ -82,7 +82,7 @@ void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &c //////////////////////////////////////////////////////////////////////////////////////////////////////// // Initialises from communicator_world //////////////////////////////////////////////////////////////////////////////////////////////////////// -CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +CartesianCommunicator::CartesianCommunicator(const Coordinate &processors) { MPI_Comm optimal_comm; GlobalSharedMemory::OptimalCommunicator (processors,optimal_comm); // Remap using the shared memory optimising routine @@ -93,13 +93,13 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors) ////////////////////////////////// // Try to subdivide communicator ////////////////////////////////// -CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent,int &srank) +CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank) { _ndimension = processors.size(); int parent_ndimension = parent._ndimension; assert(_ndimension >= parent._ndimension); - std::vector parent_processor_coor(_ndimension,0); - std::vector parent_processors (_ndimension,1); + Coordinate parent_processor_coor(_ndimension,0); + Coordinate parent_processors (_ndimension,1); // Can make 5d grid from 4d etc... int pad = _ndimension-parent_ndimension; @@ -126,9 +126,9 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors, // std::cout << " child size "< ccoor(_ndimension); // coor within subcommunicator - std::vector scoor(_ndimension); // coor of split within parent - std::vector ssize(_ndimension); // coor of split within parent + Coordinate ccoor(_ndimension); // coor within subcommunicator + Coordinate scoor(_ndimension); // coor of split within parent + Coordinate ssize(_ndimension); // coor of split within parent for(int d=0;d<_ndimension;d++){ ccoor[d] = parent_processor_coor[d] % processors[d]; @@ -208,7 +208,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors, } } -void CartesianCommunicator::InitFromMPICommunicator(const std::vector &processors, MPI_Comm communicator_base) +void CartesianCommunicator::InitFromMPICommunicator(const Coordinate &processors, MPI_Comm communicator_base) { _ndimension = processors.size(); _processor_coor.resize(_ndimension); @@ -222,7 +222,7 @@ void CartesianCommunicator::InitFromMPICommunicator(const std::vector &proc _Nprocessors*=_processors[i]; } - std::vector periodic(_ndimension,1); + Coordinate periodic(_ndimension,1); MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],0,&communicator); MPI_Comm_rank(communicator,&_processor); MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]); @@ -459,7 +459,7 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,uint64_t bytes) { - std::vector row(_ndimension,1); + Coordinate row(_ndimension,1); assert(dim>=0 && dim<_ndimension); // Split the communicator diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc index 12a84995..d9c4e558 100644 --- a/lib/communicator/Communicator_none.cc +++ b/lib/communicator/Communicator_none.cc @@ -42,14 +42,14 @@ void CartesianCommunicator::Init(int *argc, char *** arv) GlobalSharedMemory::Hugepages); } -CartesianCommunicator::CartesianCommunicator(const std::vector &processors,const CartesianCommunicator &parent,int &srank) +CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank) : CartesianCommunicator(processors) { srank=0; SetCommunicator(communicator_world); } -CartesianCommunicator::CartesianCommunicator(const std::vector &processors) +CartesianCommunicator::CartesianCommunicator(const Coordinate &processors) { _processors = processors; _ndimension = processors.size(); @@ -122,8 +122,8 @@ int CartesianCommunicator::RankWorld(void){return 0;} void CartesianCommunicator::Barrier(void){} void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {} void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { } -int CartesianCommunicator::RankFromProcessorCoor(std::vector &coor) { return 0;} -void CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector &coor){ coor = _processor_coor; } +int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor) { return 0;} +void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor){ coor = _processor_coor; } void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest) { source =0; diff --git a/lib/communicator/SharedMemory.h b/lib/communicator/SharedMemory.h index 1e213b86..7ce14d9b 100644 --- a/lib/communicator/SharedMemory.h +++ b/lib/communicator/SharedMemory.h @@ -90,7 +90,7 @@ public: // Create an optimal reordered communicator that makes MPI_Cart_create get it right ////////////////////////////////////////////////////////////////////////////////////// static void Init(Grid_MPI_Comm comm); // Typically MPI_COMM_WORLD - static void OptimalCommunicator(const std::vector &processors,Grid_MPI_Comm & optimal_comm); // Turns MPI_COMM_WORLD into right layout for Cartesian + static void OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm); // Turns MPI_COMM_WORLD into right layout for Cartesian /////////////////////////////////////////////////// // Provide shared memory facilities off comm world /////////////////////////////////////////////////// diff --git a/lib/communicator/SharedMemoryMPI.cc b/lib/communicator/SharedMemoryMPI.cc index 3184d071..4b74f3e9 100644 --- a/lib/communicator/SharedMemoryMPI.cc +++ b/lib/communicator/SharedMemoryMPI.cc @@ -114,7 +114,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm) _ShmSetup=1; } -void GlobalSharedMemory::OptimalCommunicator(const std::vector &processors,Grid_MPI_Comm & optimal_comm) +void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm) { //////////////////////////////////////////////////////////////// // Assert power of two shm_size. @@ -133,9 +133,9 @@ void GlobalSharedMemory::OptimalCommunicator(const std::vector &processors, // in a maximally symmetrical way //////////////////////////////////////////////////////////////// int ndimension = processors.size(); - std::vector processor_coor(ndimension); - std::vector WorldDims = processors; std::vector ShmDims (ndimension,1); std::vector NodeDims (ndimension); - std::vector ShmCoor (ndimension); std::vector NodeCoor (ndimension); std::vector WorldCoor(ndimension); + Coordinate processor_coor(ndimension); + Coordinate WorldDims = processors; Coordinate ShmDims(ndimension,1); Coordinate NodeDims (ndimension); + Coordinate ShmCoor(ndimension); Coordinate NodeCoor(ndimension); Coordinate WorldCoor(ndimension); int dim = 0; for(int l2=0;l2 &processors,Grid_MPI_Comm & optimal_comm) +void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm) { optimal_comm = WorldComm; }