portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-04-25 05:05:56 +01:00
Code Releases Activity

Moving multicommunicator into mpi3 also for threading

Browse Source
This commit is contained in:
paboyle 2017-08-20 02:39:10 +01:00
parent 11062fb686
commit 1cdf999668
2 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/communicator/Communicator_base.h
View File

@ -78,15 +78,15 @@ class CartesianCommunicator {
#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT) #if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT)
static MPI_Comm communicator_world; static MPI_Comm communicator_world;
MPI_Comm communicator;
MPI_Comm communicator;
std::vector<MPI_Comm> communicator_halo;
typedef MPI_Request CommsRequest_t; typedef MPI_Request CommsRequest_t;
#else #else
typedef int CommsRequest_t; typedef int CommsRequest_t;
#endif #endif
#if defined (GRID_COMMS_MPIT)
std::vector<MPI_Comm> communicator_halo;
#endif
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
// Helper functionality for SHM Windows common to all other impls // Helper functionality for SHM Windows common to all other impls

12
lib/communicator/Communicator_mpi3.cc
View File

@ -405,8 +405,14 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
{ {
int ierr; int ierr;
communicator=communicator_world; communicator=communicator_world;
_ndimension = processors.size(); _ndimension = processors.size();
communicator_halo.resize (2*_ndimension);
for(int i=0;i<_ndimension*2;i++){
MPI_Comm_dup(communicator,&communicator_halo[i]);
}
//////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////
// Assert power of two shm_size. // Assert power of two shm_size.
//////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////
@ -648,6 +654,8 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
int from, int from,
int bytes,int dir) int bytes,int dir)
{ {
assert(dir < communicator_halo.size());
MPI_Request xrq; MPI_Request xrq;
MPI_Request rrq; MPI_Request rrq;
@ -666,14 +674,14 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
gfrom = MPI_UNDEFINED; gfrom = MPI_UNDEFINED;
#endif #endif
if ( gfrom ==MPI_UNDEFINED) { if ( gfrom ==MPI_UNDEFINED) {
ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq); ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator_halo[dir],&rrq);
assert(ierr==0); assert(ierr==0);
list.push_back(rrq); list.push_back(rrq);
off_node_bytes+=bytes; off_node_bytes+=bytes;
} }
if ( gdest == MPI_UNDEFINED ) { if ( gdest == MPI_UNDEFINED ) {
ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq); ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator_halo[dir],&xrq);
assert(ierr==0); assert(ierr==0);
list.push_back(xrq); list.push_back(xrq);
off_node_bytes+=bytes; off_node_bytes+=bytes;