diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc index bcf429ab..ce9a3cf0 100644 --- a/lib/communicator/Communicator_base.cc +++ b/lib/communicator/Communicator_base.cc @@ -117,32 +117,40 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors, int Nchild = Nparent/childsize; assert (childsize * Nchild == Nparent); - int prank; MPI_Comm_rank(parent.communicator,&prank); - int crank = prank % childsize; - int ccomm = prank / childsize; + std::vector ccoor(_ndimension); // coor within subcommunicator + std::vector scoor(_ndimension); // coor of split within parent + std::vector ssize(_ndimension); // coor of split within parent + + for(int d=0;d<_ndimension;d++){ + ccoor[d] = parent._processor_coor[d] % processors[d]; + scoor[d] = parent._processor_coor[d] / processors[d]; + ssize[d] = parent._processors[d]/ processors[d]; + } + int crank,srank; // rank within subcomm ; rank of subcomm within blocks of subcomms + Lexicographic::IndexFromCoor(ccoor,crank,processors); + Lexicographic::IndexFromCoor(scoor,srank,ssize); MPI_Comm comm_split; if ( Nchild > 1 ) { - std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec< void AllToAll(int dim,std::vector &in, std::vector &out){ + assert(dim>=0); + assert(dim<_ndimension); + int numnode = _processors[dim]; + // std::cerr << " AllToAll in.size() "< void Broadcast(int root,obj &data) { diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc index a55c0164..678e4517 100644 --- a/lib/communicator/Communicator_mpi.cc +++ b/lib/communicator/Communicator_mpi.cc @@ -187,6 +187,21 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes) root, communicator); assert(ierr==0); +} +void CartesianCommunicator::AllToAll(int dim,void *in,void *out,int bytes) +{ + std::vector row(_ndimension,1); + assert(dim>=0 && dim<_ndimension); + + // Split the communicator + row[dim] = _processors[dim]; + + CartesianCommunicator Comm(row,*this); + Comm.AllToAll(in,out,bytes); +} +void CartesianCommunicator::AllToAll(void *in,void *out,int bytes) +{ + MPI_Alltoall(in ,bytes,MPI_BYTE,out,bytes,MPI_BYTE,communicator); } /////////////////////////////////////////////////////// // Should only be used prior to Grid Init finished. @@ -207,5 +222,7 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) assert(ierr==0); } + + } diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc index 40feefec..e9d71a15 100644 --- a/lib/communicator/Communicator_none.cc +++ b/lib/communicator/Communicator_none.cc @@ -98,6 +98,10 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector & { assert(0); } +void CartesianCommunicator::AllToAll(int dim,void *in,void *out,int bytes) +{ + bcopy(in,out,bytes); +} int CartesianCommunicator::RankWorld(void){return 0;} void CartesianCommunicator::Barrier(void){}