portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2024-11-10 07:55:35 +00:00
Code Releases Activity

Better mpi request completion

Browse Source
This commit is contained in:
Peter Boyle 2022-07-28 12:18:35 -04:00
parent 9c21add0c6
commit a93d5459d4
1 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Grid/stencil/Stencil.h
View File

@ -290,6 +290,8 @@ public:
std::vector<Decompress> DecompressionsSHM; std::vector<Decompress> DecompressionsSHM;
std::vector<CopyReceiveBuffer> CopyReceiveBuffers ; std::vector<CopyReceiveBuffer> CopyReceiveBuffers ;
std::vector<CachedTransfer> CachedTransfers; std::vector<CachedTransfer> CachedTransfers;
std::vector<CommsRequest_t> MpiReqs;
/////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////
// Unified Comms buffers for all directions // Unified Comms buffers for all directions
/////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////
@ -357,9 +359,8 @@ public:
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs) void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
{ {
reqs.resize(Packets.size());
for(int i=0;i<Packets.size();i++){ for(int i=0;i<Packets.size();i++){
_grid->StencilSendToRecvFromBegin(reqs[i], _grid->StencilSendToRecvFromBegin(MpiReqs,
Packets[i].send_buf, Packets[i].send_buf,
Packets[i].to_rank,Packets[i].do_send, Packets[i].to_rank,Packets[i].do_send,
Packets[i].recv_buf, Packets[i].recv_buf,
@ -370,9 +371,7 @@ public:
void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs) void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
{ {
for(int i=0;i<Packets.size();i++){ _grid->StencilSendToRecvFromComplete(MpiReqs,i);
_grid->StencilSendToRecvFromComplete(reqs[i],i);
}
} }
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
// Blocking send and receive. Either sequential or parallel. // Blocking send and receive. Either sequential or parallel.
@ -499,6 +498,7 @@ public:
Packets.resize(0); Packets.resize(0);
CopyReceiveBuffers.resize(0); CopyReceiveBuffers.resize(0);
CachedTransfers.resize(0); CachedTransfers.resize(0);
MpiReqs.resize(0);
} }
void AddCopy(void *from,void * to, Integer bytes) void AddCopy(void *from,void * to, Integer bytes)
{ {
@ -795,7 +795,6 @@ public:
u_simd_recv_buf[l] = (cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj)); u_simd_recv_buf[l] = (cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj));
u_simd_send_buf[l] = (cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj)); u_simd_send_buf[l] = (cobj *)_grid->ShmBufferMalloc(_unified_buffer_size*sizeof(cobj));
} }
PrecomputeByteOffsets(); PrecomputeByteOffsets();
} }
@ -1107,7 +1106,6 @@ public:
// Gather locally // Gather locally
//////////////////////////////////////////////////////// ////////////////////////////////////////////////////////
assert(send_buf!=NULL); assert(send_buf!=NULL);
Gather_plane_simple_table(face_table[face_idx],rhs,send_buf,compress,comm_off,so); Gather_plane_simple_table(face_table[face_idx],rhs,send_buf,compress,comm_off,so);
} }
@ -1214,8 +1212,9 @@ public:
face_table[face_idx].size()*sizeof(face_table_host[0])); face_table[face_idx].size()*sizeof(face_table_host[0]));
} }
if ( comms_send || comms_recv ) if ( comms_send || comms_recv ) {
Gather_plane_exchange_table(face_table[face_idx],rhs,spointers,dimension,sx,cbmask,compress,permute_type); Gather_plane_exchange_table(face_table[face_idx],rhs,spointers,dimension,sx,cbmask,compress,permute_type);
}
face_idx++; face_idx++;
//spointers[0] -- low //spointers[0] -- low