diff --git a/Grid/communicator/Communicator_base.cc b/Grid/communicator/Communicator_base.cc index 90bdf4fe4..da74b09f2 100644 --- a/Grid/communicator/Communicator_base.cc +++ b/Grid/communicator/Communicator_base.cc @@ -34,6 +34,9 @@ Author: Peter Boyle NAMESPACE_BEGIN(Grid); bool Stencil_force_mpi = true; +#ifdef GRID_FJMPI +bool Stencil_persistent = false; +#endif bool Stencil_force_barrier = false; /////////////////////////////////////////////////////////////// diff --git a/Grid/communicator/Communicator_base.h b/Grid/communicator/Communicator_base.h index fdf35f2e6..200601905 100644 --- a/Grid/communicator/Communicator_base.h +++ b/Grid/communicator/Communicator_base.h @@ -38,6 +38,10 @@ Author: Peter Boyle NAMESPACE_BEGIN(Grid); extern bool Stencil_force_mpi ; +#ifdef GRID_FJMPI +// Fujitsu MPI persistent requests for the halo exchange with other nodes (set by --comms-overlap) +extern bool Stencil_persistent ; +#endif extern bool Stencil_force_barrier ; class CartesianCommunicator : public SharedMemory { @@ -201,6 +205,11 @@ public: // Could do a PollHtoD and have a CommsMerge dependence void StencilSendToRecvFromPollDtoH (std::vector &list); +#ifdef A64FX + // Make progress on outstanding requests without blocking (MPI only progresses + // inside MPI calls on Fugaku); safe to call repeatedly before Complete. + void StencilSendToRecvFromProgress(std::vector &list); +#endif void StencilSendToRecvFromPollIRecv(std::vector &list); double StencilSendToRecvFromBegin(std::vector &list, diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc index e352db21d..5461902ae 100644 --- a/Grid/communicator/Communicator_mpi3.cc +++ b/Grid/communicator/Communicator_mpi3.cc @@ -28,6 +28,90 @@ Author: Peter Boyle #include #include +// Fujitsu MPI extended persistent requests (FJMPI_Prequest_*, Fujitsu MPI User's Guide J2UL-2565 5.3). +// Fujitsu MPI moves the body of a large non-blocking message only inside MPI calls (6.2), so the halo +// exchange does not overlap the interior kernel. With these requests the partners exchange buffer +// information once, at initialisation, and FJMPI_Prequest_start starts the transfer of the message body. +// GRID_FJMPI is set by configure (--enable-fjmpi) when mpi-ext.h declares the Fujitsu extensions. +#ifdef GRID_FJMPI +#include + +NAMESPACE_BEGIN(Grid); +namespace { +// Persistent halo requests are used with --comms-overlap (Stencil_persistent); they bring nothing +// without overlap. Only for peers on other nodes: persistent transfers to peers on the same node delay +// the transfers to other nodes. +bool PersistentHaloOn(void) { return Stencil_persistent; } +bool PersistentHaloPeer(int grank) { return grank==MPI_UNDEFINED; } +struct PersistentHaloReq { + void *buf; int count; MPI_Request req; +}; +// Requests are keyed by (send, communicator, peer, tag): Fujitsu MPI allows one request per +// (peer, tag, communicator). Several stencils can use the same communicator (e.g. even and odd +// checkerboards), so each distinct pair of halo buffers of a direction gets a slot, in order of +// first use, and the slot enters the tag. All ranks run the same sequence of stencils, so partners +// agree on the slot. +struct PersistentHaloCache { + std::map, std::vector > > slots; + std::map, PersistentHaloReq> reqs; + int tag_ub = -1; +}; +PersistentHaloCache &PersistentHalo(void) { static PersistentHaloCache c; return c; } + +int PersistentHaloSlot(MPI_Comm comm,int dir,void *xmit,void *recv) +{ + auto &v = PersistentHalo().slots[std::make_pair(comm,dir)]; + for(int k=0;ksecond.buf==buf && it->second.count==count ) return it->second.req; + MPI_Request_free(&it->second.req); + reqs.erase(it); + } + PersistentHaloReq r = {buf,count,MPI_REQUEST_NULL}; + int ierr = send ? FJMPI_Prequest_send_init(buf,count,MPI_INT32_T,peer,tag,comm,&r.req) + : FJMPI_Prequest_recv_init(buf,count,MPI_INT32_T,peer,tag,comm,&r.req); + GRID_ASSERT(ierr==0); + reqs[key] = r; + return r.req; +} +void PersistentHaloFree(MPI_Comm comm) +{ + auto &c = PersistentHalo(); + for(auto it=c.reqs.begin();it!=c.reqs.end();) { + if ( std::get<1>(it->first)==comm ) { MPI_Request_free(&it->second.req); it=c.reqs.erase(it); } + else ++it; + } + for(auto it=c.slots.begin();it!=c.slots.end();) { + if ( it->first.first==comm ) it=c.slots.erase(it); else ++it; + } +} +} +NAMESPACE_END(Grid); +#endif + void GridAbort(void) { MPI_Abort(MPI_COMM_WORLD,SIGABRT); } extern void * Grid_backtrace_buffer[_NBACKTRACE]; @@ -262,6 +346,9 @@ CartesianCommunicator::~CartesianCommunicator() if (communicator && !MPI_is_finalised) { MPI_Comm_free(&communicator); for(int i=0;i &list) {}; void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector &list) {}; +#ifdef A64FX +void CartesianCommunicator::StencilSendToRecvFromProgress(std::vector &list) +{ + int nreq=list.size(); + if (nreq==0) return; + int flag; + // completed requests become MPI_REQUEST_NULL (persistent ones become inactive), which the final + // Waitall handles + int ierr = MPI_Testall(nreq,&list[0],&flag,MPI_STATUSES_IGNORE); + GRID_ASSERT(ierr==0); +} +#endif double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector &list, void *xmit, int dest,int dox, @@ -428,6 +527,34 @@ double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector= 0 && PersistentHaloPeer(gfrom); + bool mpi_send = dox && ( (gdest ==MPI_UNDEFINED) || Stencil_force_mpi ) && xtag >= 0 && PersistentHaloPeer(gdest); + MPI_Request rrq; + if ( mpi_recv ) rrq = PersistentHaloGet(0,recv,(int)(rbytes/sizeof(int32_t)),from,rtag,comm); + if ( mpi_send ) PersistentHaloGet(1,xmit,(int)(xbytes/sizeof(int32_t)),dest,xtag,comm); + if ( mpi_recv ) { + int ierr = FJMPI_Prequest_start(&rrq); + GRID_ASSERT(ierr==0); + list.push_back(rrq); + return rbytes; + } + } +#endif return 0.0; // Do nothing -- no preparation required } double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector &list, @@ -454,7 +581,19 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector= 0 ) { + xrq = PersistentHaloGet(1,xmit_comp,(int)(xbytes/sizeof(int32_t)),dest,ptag_send,communicator_halo[commdir]); + ierr = FJMPI_Prequest_start(&xrq); + } else { + tag= dir+_processor*32; + ierr =MPI_Isend(xmit_comp,(int)(xbytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator_halo[commdir],&xrq); + } +#else tag= dir+_processor*32; ierr =MPI_Isend(xmit_comp,(int)(xbytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator_halo[commdir],&xrq); +#endif GRID_ASSERT(ierr==0); list.push_back(xrq); off_node_bytes+=xbytes; @@ -506,6 +655,9 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector &list) {}; +#endif /////////////////////////////////////////// // Pipeline mode through host memory /////////////////////////////////////////// diff --git a/Grid/communicator/Communicator_none.cc b/Grid/communicator/Communicator_none.cc index e1891f8f6..740257e90 100644 --- a/Grid/communicator/Communicator_none.cc +++ b/Grid/communicator/Communicator_none.cc @@ -167,6 +167,9 @@ double CartesianCommunicator::StencilSendToRecvFrom( void *xmit, } void CartesianCommunicator::StencilSendToRecvFromPollIRecv(std::vector &list) {}; void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector &list) {}; +#ifdef A64FX +void CartesianCommunicator::StencilSendToRecvFromProgress(std::vector &list) {}; +#endif double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector &list, void *xmit, int xmit_to_rank,int dox, diff --git a/Grid/qcd/action/fermion/WilsonKernels.h b/Grid/qcd/action/fermion/WilsonKernels.h index ad077dd36..1573f1266 100644 --- a/Grid/qcd/action/fermion/WilsonKernels.h +++ b/Grid/qcd/action/fermion/WilsonKernels.h @@ -42,6 +42,12 @@ public: enum { CommsAndCompute, CommsThenCompute }; static int Opt; static int Comms; +#ifdef A64FX + // If set, called by thread 0 every few sites of the interior kernel so that + // communications started before it progress during the compute. + static void (*CommsProgress)(void *); + static void *CommsProgressArg; +#endif }; template class WilsonKernels : public FermionOperator , public WilsonKernelsStatic { diff --git a/Grid/qcd/action/fermion/implementation/WilsonFermion5DImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonFermion5DImplementation.h index 48ec71bf1..5f68c0743 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonFermion5DImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonFermion5DImplementation.h @@ -346,6 +346,11 @@ void WilsonFermion5D::DhopInternalOverlappedComms(StencilImpl & st, // do the compute interior ///////////////////////////// int Opt = WilsonKernelsStatic::Opt; // Why pass this. Kernels should know +#ifdef A64FX + // MPI does not progress outside MPI calls: let the interior kernel poll + WilsonKernelsStatic::CommsProgress = [](void *s) { ((StencilImpl *)s)->CommunicateProgress(); }; + WilsonKernelsStatic::CommsProgressArg = (void *)&st; +#endif if (dag == DaggerYes) { GRID_TRACE("DhopDagInterior"); Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),LLs,U.oSites(),in,out,1,0); @@ -353,6 +358,10 @@ void WilsonFermion5D::DhopInternalOverlappedComms(StencilImpl & st, GRID_TRACE("DhopInterior"); Kernels::DhopKernel (Opt,st,U,st.CommBuf(),LLs,U.oSites(),in,out,1,0); } +#ifdef A64FX + WilsonKernelsStatic::CommsProgress = nullptr; + WilsonKernelsStatic::CommsProgressArg = nullptr; +#endif //ifdef GRID_ACCELERATED #if 0 diff --git a/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h index cba9b48c7..0fc17ed3e 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonFermionImplementation.h @@ -436,6 +436,11 @@ void WilsonFermion::DhopInternalOverlappedComms(StencilImpl &st, // do the compute interior ///////////////////////////// int Opt = WilsonKernelsStatic::Opt; +#ifdef A64FX + // MPI does not progress outside MPI calls: let the interior kernel poll + WilsonKernelsStatic::CommsProgress = [](void *s) { ((StencilImpl *)s)->CommunicateProgress(); }; + WilsonKernelsStatic::CommsProgressArg = (void *)&st; +#endif if (dag == DaggerYes) { GRID_TRACE("DhopDagInterior"); Kernels::DhopDagKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0); @@ -443,6 +448,10 @@ void WilsonFermion::DhopInternalOverlappedComms(StencilImpl &st, GRID_TRACE("DhopInterior"); Kernels::DhopKernel(Opt,st,U,st.CommBuf(),1,U.oSites(),in,out,1,0); } +#ifdef A64FX + WilsonKernelsStatic::CommsProgress = nullptr; + WilsonKernelsStatic::CommsProgressArg = nullptr; +#endif ///////////////////////////// // Complete comms diff --git a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h index 961dad1d2..da64752de 100644 --- a/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h +++ b/Grid/qcd/action/fermion/implementation/WilsonKernelsImplementation.h @@ -480,6 +480,33 @@ extern "C" { WilsonKernels::A(st_v,U_v,buf,sF,sU,Ls,1,in_v,out_v); \ }); #ifdef A64FX +// Interior kernel with communication progress: thread 0 calls the progress hook +// between chunks of sites; dynamic scheduling lets the other threads pick up the +// sites thread 0 has no time for while it drives the communications. Chunks of +// 8 4d sites for 5d operators, 32 sites for 4d operators (measured on Fugaku). +#define ASM_CALL_PROGRESS(A) \ + if ( WilsonKernelsStatic::CommsProgress ) { \ + auto progress = WilsonKernelsStatic::CommsProgress; \ + auto progress_arg = WilsonKernelsStatic::CommsProgressArg; \ + const uint64_t chunk = std::max(8,32/Ls); \ + const uint64_t nchunk = (Nsite + chunk - 1)/chunk; \ + thread_region { \ + const bool master = (thread_num() == 0); \ + DO_PRAGMA(omp for schedule(dynamic,1) nowait) \ + for(uint64_t c=0;c::A(st_v,U_v,buf,sF,sU,Ls,1,in_v,out_v); \ + } \ + if ( master ) progress(progress_arg); \ + } \ + } \ + } else { \ + ASM_CALL(A); \ + } + // Exterior kernel on the surface 4d sites only (the surface list holds Ls // consecutive 5d entries per surface 4d site) #define ASM_CALL_EXT(A) \ @@ -491,6 +518,7 @@ extern "C" { WilsonKernels::A(st_v,U_v,buf,sF,sU,Ls,1,in_v,out_v); \ }); #else +#define ASM_CALL_PROGRESS(A) ASM_CALL(A) #define ASM_CALL_EXT(A) ASM_CALL(A) #endif @@ -529,7 +557,7 @@ void WilsonKernels::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALLNB(GenericDhopSiteInt); return;} if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteInt); return;} #ifndef GRID_CUDA - if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteInt); return;} + if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL_PROGRESS(AsmDhopSiteInt); return;} #endif } else if( exterior ) { // // dependent on result of merge @@ -575,7 +603,7 @@ void WilsonKernels::DhopKernel(int Opt,StencilImpl &st, DoubledGaugeField if (Opt == WilsonKernelsStatic::OptGeneric ) { KERNEL_CALLNB(GenericDhopSiteDagInt); return;} if (Opt == WilsonKernelsStatic::OptHandUnroll ) { KERNEL_CALLNB(HandDhopSiteDagInt); return;} #ifndef GRID_CUDA - if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL(AsmDhopSiteDagInt); return;} + if (Opt == WilsonKernelsStatic::OptInlineAsm ) { ASM_CALL_PROGRESS(AsmDhopSiteDagInt); return;} #endif } else if( exterior ) { // Dependent on result of merge diff --git a/Grid/qcd/action/fermion/instantiation/WilsonKernelsInstantiation.cc b/Grid/qcd/action/fermion/instantiation/WilsonKernelsInstantiation.cc index c02da6c88..008aba69b 100644 --- a/Grid/qcd/action/fermion/instantiation/WilsonKernelsInstantiation.cc +++ b/Grid/qcd/action/fermion/instantiation/WilsonKernelsInstantiation.cc @@ -34,6 +34,10 @@ NAMESPACE_BEGIN(Grid); // Move these int WilsonKernelsStatic::Opt = WilsonKernelsStatic::OptGeneric; +#ifdef A64FX +void (*WilsonKernelsStatic::CommsProgress)(void *) = nullptr; +void *WilsonKernelsStatic::CommsProgressArg = nullptr; +#endif int WilsonKernelsStatic::Comms = WilsonKernelsStatic::CommsAndCompute; NAMESPACE_END(Grid); diff --git a/Grid/stencil/Stencil.h b/Grid/stencil/Stencil.h index 918888fd7..02f9d7355 100644 --- a/Grid/stencil/Stencil.h +++ b/Grid/stencil/Stencil.h @@ -630,6 +630,13 @@ public: } } +#ifdef A64FX + // Progress outstanding communications while computing (see CommunicateBegin) + void CommunicateProgress(void) + { + _grid->StencilSendToRecvFromProgress(MpiReqs); + } +#endif void CommunicateComplete(std::vector > &reqs) { // std::cout << "Communicate Complete "< +#include ]], + [[MPI_Request r; int c[3]; + FJMPI_Prequest_start(&r); + FJMPI_Topology_get_coords(MPI_COMM_WORLD,0,FJMPI_LOGICAL,3,c);]])], + [fjmpi=yes],[fjmpi=no]) + AC_MSG_RESULT([${fjmpi}]) + CXXFLAGS=$ac_save_CXXFLAGS +fi +if test "${ac_FJMPI}X" == "yesX" && test "${fjmpi}X" != "yesX"; then + AC_MSG_ERROR([--enable-fjmpi=yes but the Fujitsu MPI extensions were not found (needs --enable-comms=mpi and Fujitsu MPI)]) +fi +if test "${fjmpi}X" == "yesX"; then + AC_DEFINE([GRID_FJMPI],[1],[Fujitsu MPI extensions]) +fi + ############### RNG selection AC_ARG_ENABLE([rng],[AS_HELP_STRING([--enable-rng=ranlux48|mt19937|sitmo],[\ Select Random Number Generator to be used])],\ @@ -955,6 +982,7 @@ Threading : ${ac_openmp} Acceleration : ${ac_ACCELERATOR} Unified virtual memory : ${ac_UNIFIED} Communications type : ${comms_type} +Fujitsu MPI extensions : ${fjmpi} Shared memory allocator : ${ac_SHM} Shared memory mmap path : ${ac_SHMPATH} Default precision : ${ac_PRECISION}