diff --git a/Grid/communicator/SharedMemory.cc b/Grid/communicator/SharedMemory.cc index 662f319f1..87247efd5 100644 --- a/Grid/communicator/SharedMemory.cc +++ b/Grid/communicator/SharedMemory.cc @@ -33,6 +33,7 @@ NAMESPACE_BEGIN(Grid); // static data int GlobalSharedMemory::HPEhypercube = 1; +int GlobalSharedMemory::TofuPlacement = 0; uint64_t GlobalSharedMemory::MAX_MPI_SHM_BYTES = 1024LL*1024LL*1024LL; int GlobalSharedMemory::Hugepages = 0; int GlobalSharedMemory::_ShmSetup; diff --git a/Grid/communicator/SharedMemory.h b/Grid/communicator/SharedMemory.h index cdaae51c2..1628e6b08 100644 --- a/Grid/communicator/SharedMemory.h +++ b/Grid/communicator/SharedMemory.h @@ -99,6 +99,10 @@ public: // HPE 8600 hypercube optimisation /////////////////////////////////////// static int HPEhypercube; + /////////////////////////////////////// + // Tofu (Fujitsu MPI) node placement, --tofu + /////////////////////////////////////// + static int TofuPlacement; static int ShmSetup(void) { return _ShmSetup; } static int ShmAlloc(void) { return _ShmAlloc; } @@ -131,6 +135,7 @@ public: static void OptimalCommunicator (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims); static void OptimalCommunicatorHypercube (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims); static void OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims); + static void OptimalCommunicatorTofu (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims); static void GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims); /////////////////////////////////////////////////// // Provide shared memory facilities off comm world diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index 213543ccd..346e2d5d1 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -32,6 +32,10 @@ Author: Christoph Lehner #include #include +#ifdef GRID_FJMPI +#include +#endif + #ifdef GRID_CUDA #include #endif @@ -278,6 +282,18 @@ void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_M gethostname(name,namelen); int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ; + if ( TofuPlacement ) { +#ifdef GRID_FJMPI + OptimalCommunicatorTofu(processors,optimal_comm,SHM); +#else + if ( WorldRank==0 ) { + std::cerr << "Grid --tofu: not built with Fujitsu MPI" << std::endl; + } + MPI_Barrier(WorldComm); + MPI_Abort(WorldComm,EXIT_FAILURE); +#endif + return; + } if(nscan==3 && HPEhypercube ) OptimalCommunicatorHypercube(processors,optimal_comm,SHM); else OptimalCommunicatorSharedMemory(processors,optimal_comm,SHM); } @@ -450,6 +466,202 @@ void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &proce int ierr= MPI_Comm_split(WorldComm,0,rank,&optimal_comm); GRID_ASSERT(ierr==0); } +#ifdef GRID_FJMPI +//////////////////////////////////////////////////////////////////////////////////////////// +// Tofu placement (--tofu): lay the node grid onto the job's logical torus (pjsub -L node=AxBxC:torus), +// each direction along a job axis of the same extent. Errors are detected on all ranks alike. +//////////////////////////////////////////////////////////////////////////////////////////// +static void TofuPlacementError(const std::string &msg) +{ + if ( GlobalSharedMemory::WorldRank==0 ) std::cerr << "Grid --tofu: " << msg << std::endl; + MPI_Barrier(GlobalSharedMemory::WorldComm); + MPI_Abort(GlobalSharedMemory::WorldComm,EXIT_FAILURE); +} +static std::string TofuJoin(const std::vector &v,const char *sep) +{ + std::ostringstream os; + for(int i=0;i pvec(processors.begin(),processors.end()); + std::vector nvec(NodeDims.begin(),NodeDims.end()); + std::vector svec(ShmDims.begin(),ShmDims.end()); + std::string layout = "--mpi " + TofuJoin(pvec,".") + " gives a node grid " + TofuJoin(nvec,".") + + " (" + std::to_string(nodes) + " nodes, ranks per node " + TofuJoin(svec,".") + ")"; + + //////////////////////////////////////////////////////////////// + // Shape of the job's logical node torus and this node's coordinate in it + //////////////////////////////////////////////////////////////// + int jdim=0; + if ( FJMPI_Topology_get_dimension(&jdim)!=FJMPI_SUCCESS || jdim<1 || jdim>3 ) { + TofuPlacementError("cannot read the job's node shape (FJMPI_Topology_get_dimension); " + "is this a Fujitsu MPI job on whole nodes?"); + } + std::vector jshape(3,1), jcoor(3,0); + if ( FJMPI_Topology_get_shape(&jshape[0],&jshape[1],&jshape[2])!=FJMPI_SUCCESS ) { + TofuPlacementError("cannot read the job's node shape (FJMPI_Topology_get_shape)"); + } + for(int a=jdim;a<3;a++) jshape[a]=1; + if ( FJMPI_Topology_get_coords(WorldComm,WorldRank,FJMPI_LOGICAL,jdim,&jcoor[0])!=FJMPI_SUCCESS ) { + TofuPlacementError("cannot read the node coordinates (FJMPI_Topology_get_coords)"); + } + std::vector jvec(jshape.begin(),jshape.begin()+jdim); + int jnodes = jshape[0]*jshape[1]*jshape[2]; + + //////////////////////////////////////////////////////////////// + // Lay each node grid direction of extent > 1 along an unused job axis of the same extent + //////////////////////////////////////////////////////////////// + std::vector axis(ndimension,-1); + std::vector used(3,0); + bool solvable = (nodes==WorldNodes) && (jnodes==WorldNodes); + for(int d=0;d1 && !used[a] ) solvable=false; + if ( !solvable ) { + std::vector want; + for(int d=0;d1 ) want.push_back(NodeDims[d]); + std::ostringstream os; + os << layout << ", which cannot be laid onto the job's node shape " << TofuJoin(jvec,"x") + << " (" << jnodes << " nodes, " << WorldNodes << " in use): every node grid extent larger than 1" + << " must equal a different extent of the job shape. "; + if ( want.size()>=1 && want.size()<=3 ) { + while ( want.size()<3 ) want.push_back(1); + os << "Request a job shape with the extents " << TofuJoin(want,", ") << " in any order" + << " (pjsub -L \"node=" << TofuJoin(want,"x") << ":torus\"), or change --mpi."; + } else { + os << "At most 3 node grid directions may have more than one node; change --mpi."; + } + TofuPlacementError(os.str()); + } + + //////////////////////////////////////////////////////////////// + // Processor coordinate: node block from the torus, on-node block as in the default layout + //////////////////////////////////////////////////////////////// + for(int d=0;d=0) ? jcoor[axis[d]] : 0; + Lexicographic::CoorFromIndexReversed(ShmCoor ,WorldShmRank,ShmDims); + for(int d=0;d ranks(WorldSize); + MPI_Allgather(&rank,1,MPI_INT,&ranks[0],1,MPI_INT,WorldComm); + std::vector owner(WorldSize,-1); + for(int r=0;r logged; + if ( std::find(logged.begin(),logged.end(),processors)==logged.end() ) { + logged.push_back(processors); + struct TofuRankInfo { + int tofu[6], logical[3]; + char host[32]; + } me; + for(int i=0;i<6;i++) me.tofu[i]=-1; + FJMPI_Topology_get_coords(WorldComm,WorldRank,FJMPI_TOFU_SYS,6,me.tofu); + for(int i=0;i<3;i++) me.logical[i]=jcoor[i]; + memset(me.host,0,sizeof(me.host)); + gethostname(me.host,sizeof(me.host)-1); + std::vector all(WorldSize); + MPI_Allgather(&me,sizeof(me),MPI_BYTE,&all[0],sizeof(me),MPI_BYTE,WorldComm); + + std::vector along; + for(int d=0;d=0 ) along.push_back("dimension " + std::to_string(d) + " along job axis " + std::to_string(axis[d])); + } + std::ostringstream os; + for(int i=0;i cv(c.begin(),c.end()); + char line[256]; + snprintf(line,sizeof(line),"rank %5d coor %-14s world %5d host %-16s tofu %2d %2d %2d %d %d %d logical %2d %2d %2d", + k,TofuJoin(cv,".").c_str(),owner[k],r.host,r.tofu[0],r.tofu[1],r.tofu[2],r.tofu[3],r.tofu[4],r.tofu[5], + r.logical[0],r.logical[1],r.logical[2]); + std::cout << GridLogDebug << "Tofu placement: " << line << std::endl; + } + for(int d=0;d hist; + int onnode=0; + for(int k=0;k