diff --git a/Grid/communicator/SharedMemory.h b/Grid/communicator/SharedMemory.h index 439d0ae2..2f89f571 100644 --- a/Grid/communicator/SharedMemory.h +++ b/Grid/communicator/SharedMemory.h @@ -59,12 +59,18 @@ class GlobalSharedMemory { private: static const int MAXLOG2RANKSPERNODE = 16; + // Init once lock on the buffer allocation static int _ShmSetup; static int _ShmAlloc; static uint64_t _ShmAllocBytes; public: + /////////////////////////////////////// + // HPE 8600 hypercube optimisation + /////////////////////////////////////// + static int HPEhypercube; + static int ShmSetup(void) { return _ShmSetup; } static int ShmAlloc(void) { return _ShmAlloc; } static uint64_t ShmAllocBytes(void) { return _ShmAllocBytes; } diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index 926c7c94..512822ab 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -36,6 +36,7 @@ Author: Peter Boyle NAMESPACE_BEGIN(Grid); #define header "SharedMemoryMpi: " /*Construct from an MPI communicator*/ +int GlobalSharedMemory::HPEhypercube = 1; void GlobalSharedMemory::Init(Grid_MPI_Comm comm) { assert(_ShmSetup==0); @@ -152,8 +153,8 @@ void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_M gethostname(name,namelen); int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ; - if(nscan==3) OptimalCommunicatorHypercube(processors,optimal_comm); - else OptimalCommunicatorSharedMemory(processors,optimal_comm); + if(nscan==3 && HPEhypercube ) OptimalCommunicatorHypercube(processors,optimal_comm); + else OptimalCommunicatorSharedMemory(processors,optimal_comm); } void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processors,Grid_MPI_Comm & optimal_comm) { diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc index 48250e6a..a8410d0f 100644 --- a/Grid/util/Init.cc +++ b/Grid/util/Init.cc @@ -366,6 +366,13 @@ void Grid_init(int *argc,char ***argv) GlobalSharedMemory::MAX_MPI_SHM_BYTES = MB64*1024LL*1024LL; } + if( GridCmdOptionExists(*argv,*argv+*argc,"--hypercube") ){ + int enable; + arg= GridCmdOptionPayload(*argv,*argv+*argc,"--hypercube"); + GridCmdOptionInt(arg,enable); + GlobalSharedMemory::HPEhypercube = enable; + } + if( GridCmdOptionExists(*argv,*argv+*argc,"--shm-hugepages") ){ GlobalSharedMemory::Hugepages = 1; } diff --git a/HMC/Mobius2p1fEOFA_F1.cc b/HMC/Mobius2p1fEOFA_F1.cc index 672c343a..3d51b16c 100644 --- a/HMC/Mobius2p1fEOFA_F1.cc +++ b/HMC/Mobius2p1fEOFA_F1.cc @@ -252,7 +252,6 @@ int main(int argc, char **argv) { double ActionStoppingCondition = 1e-10; double DerivativeStoppingCondition = 1e-7; - double DerivativeStoppingConditionLoose = 3e-7; double MaxCGIterations = 30000; //////////////////////////////////// @@ -289,8 +288,9 @@ int main(int argc, char **argv) { ConjugateGradient ActionCG(ActionStoppingCondition,MaxCGIterations); ConjugateGradient DerivativeCG(DerivativeStoppingCondition,MaxCGIterations); - const int MX_inner = 5000; #ifdef MIXED_PRECISION + const int MX_inner = 5000; + // Mixed precision EOFA LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L); LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R); @@ -386,6 +386,7 @@ int main(int argc, char **argv) { //////////////////////////////////////////////////////////////////////////// // Mixed precision CG for 2f force //////////////////////////////////////////////////////////////////////////// + double DerivativeStoppingConditionLoose = 3e-7; DenominatorsF.push_back(new FermionActionF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[h],M5,b,c, ParamsF)); LinOpD.push_back(new LinearOperatorD(*Denominators[h]));