From 1c096626cbeab7af27e293193a8b4ae07a55390f Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 20 Jul 2019 16:06:16 +0100
Subject: [PATCH] Hypercube defaults to on if HPE detected, but override to off
 possible

---
 Grid/communicator/SharedMemory.h     | 6 ++++++
 Grid/communicator/SharedMemoryMPI.cc | 5 +++--
 Grid/util/Init.cc                    | 7 +++++++
 HMC/Mobius2p1fEOFA_F1.cc             | 5 +++--
 4 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/Grid/communicator/SharedMemory.h b/Grid/communicator/SharedMemory.h
index 439d0ae2..2f89f571 100644
--- a/Grid/communicator/SharedMemory.h
+++ b/Grid/communicator/SharedMemory.h
@@ -59,12 +59,18 @@ class GlobalSharedMemory {
 private:
   static const int     MAXLOG2RANKSPERNODE = 16;            
 
+
   // Init once lock on the buffer allocation
   static int      _ShmSetup;
   static int      _ShmAlloc;
   static uint64_t _ShmAllocBytes;
 
 public:
+  ///////////////////////////////////////
+  // HPE 8600 hypercube optimisation
+  ///////////////////////////////////////
+  static int HPEhypercube;
+
   static int      ShmSetup(void)      { return _ShmSetup; }
   static int      ShmAlloc(void)      { return _ShmAlloc; }
   static uint64_t ShmAllocBytes(void) { return _ShmAllocBytes; }
diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc
index 926c7c94..512822ab 100644
--- a/Grid/communicator/SharedMemoryMPI.cc
+++ b/Grid/communicator/SharedMemoryMPI.cc
@@ -36,6 +36,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 NAMESPACE_BEGIN(Grid); 
 #define header "SharedMemoryMpi: "
 /*Construct from an MPI communicator*/
+int GlobalSharedMemory::HPEhypercube = 1;
 void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
 {
   assert(_ShmSetup==0);
@@ -152,8 +153,8 @@ void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_M
   gethostname(name,namelen);
   int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ;
 
-  if(nscan==3) OptimalCommunicatorHypercube(processors,optimal_comm);
-  else         OptimalCommunicatorSharedMemory(processors,optimal_comm);
+  if(nscan==3 && HPEhypercube ) OptimalCommunicatorHypercube(processors,optimal_comm);
+  else                          OptimalCommunicatorSharedMemory(processors,optimal_comm);
 }
 void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processors,Grid_MPI_Comm & optimal_comm)
 {
diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc
index 48250e6a..a8410d0f 100644
--- a/Grid/util/Init.cc
+++ b/Grid/util/Init.cc
@@ -366,6 +366,13 @@ void Grid_init(int *argc,char ***argv)
     GlobalSharedMemory::MAX_MPI_SHM_BYTES = MB64*1024LL*1024LL;
   }
 
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--hypercube") ){
+    int enable;
+    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--hypercube");
+    GridCmdOptionInt(arg,enable);
+    GlobalSharedMemory::HPEhypercube = enable;
+  }
+
   if( GridCmdOptionExists(*argv,*argv+*argc,"--shm-hugepages") ){
     GlobalSharedMemory::Hugepages = 1;
   }
diff --git a/HMC/Mobius2p1fEOFA_F1.cc b/HMC/Mobius2p1fEOFA_F1.cc
index 672c343a..3d51b16c 100644
--- a/HMC/Mobius2p1fEOFA_F1.cc
+++ b/HMC/Mobius2p1fEOFA_F1.cc
@@ -252,7 +252,6 @@ int main(int argc, char **argv) {
   
   double ActionStoppingCondition     = 1e-10;
   double DerivativeStoppingCondition = 1e-7;
-  double DerivativeStoppingConditionLoose = 3e-7;
   double MaxCGIterations = 30000;
 
   ////////////////////////////////////
@@ -289,8 +288,9 @@ int main(int argc, char **argv) {
 
   ConjugateGradient<FermionField>      ActionCG(ActionStoppingCondition,MaxCGIterations);
   ConjugateGradient<FermionField>  DerivativeCG(DerivativeStoppingCondition,MaxCGIterations);
-  const int MX_inner = 5000;
 #ifdef MIXED_PRECISION
+  const int MX_inner = 5000;
+
   // Mixed precision EOFA
   LinearOperatorEOFAD Strange_LinOp_L (Strange_Op_L);
   LinearOperatorEOFAD Strange_LinOp_R (Strange_Op_R);
@@ -386,6 +386,7 @@ int main(int argc, char **argv) {
     ////////////////////////////////////////////////////////////////////////////
     // Mixed precision CG for 2f force
     ////////////////////////////////////////////////////////////////////////////
+    double DerivativeStoppingConditionLoose = 3e-7;
 
     DenominatorsF.push_back(new FermionActionF(UF,*FGridF,*FrbGridF,*GridPtrF,*GridRBPtrF,light_den[h],M5,b,c, ParamsF));
     LinOpD.push_back(new LinearOperatorD(*Denominators[h]));