From 32bc7a6ab893e79ba033fb29ca4618e59278b2c5 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 5 Aug 2016 10:36:00 +0100
Subject: [PATCH] MPI back out of change that hangs AVX2 for clang, gcc needs
 the -mfma flag.

---
 benchmarks/Benchmark_comms.cc          |  4 +--
 configure.ac                           |  2 +-
 lib/communicator/Communicator_base.h   |  9 ------
 lib/communicator/Communicator_mpi.cc   | 39 ++++++++------------------
 lib/communicator/Communicator_none.cc  | 13 ---------
 lib/communicator/Communicator_shmem.cc | 13 ---------
 6 files changed, 14 insertions(+), 66 deletions(-)

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index 7162a0f9..d0cd96c7 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -194,7 +194,7 @@ int main (int argc, char ** argv)
     }
   }  
 
-
+#if 0
 
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= Benchmarking sequential persistent halo exchange in "<<nmu<<" dimensions"<<std::endl;
@@ -315,7 +315,7 @@ int main (int argc, char ** argv)
     }
   }
 
-
+#endif
 
   Grid_finalize();
 }
diff --git a/configure.ac b/configure.ac
index 905a665b..89bb3568 100644
--- a/configure.ac
+++ b/configure.ac
@@ -108,7 +108,7 @@ case ${ax_cv_cxx_compiler_vendor} in
         SIMD_FLAGS='-mavx -mfma4';;
       AVX2)
         AC_DEFINE([AVX2],[1],[AVX2 intrinsics])
-        SIMD_FLAGS='-mavx2';;
+        SIMD_FLAGS='-mavx2 -mfma';;
       AVX512|AVX512MIC|KNL)
         AC_DEFINE([AVX512],[1],[AVX512 intrinsics])
         SIMD_FLAGS='-mavx512f -mavx512pf -mavx512er -mavx512cd';;
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index dd19c4f4..94d277e9 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -127,21 +127,12 @@ class CartesianCommunicator {
 			int recv_from_rank,
 			int bytes);
 
-    void SendToRecvFromInit(std::vector<CommsRequest_t> &list,
-			    void *xmit,
-			    int xmit_to_rank,
-			    void *recv,
-			    int recv_from_rank,
-			    int bytes);
-
     void SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 			 void *xmit,
 			 int xmit_to_rank,
 			 void *recv,
 			 int recv_from_rank,
 			 int bytes);
-
-    void SendToRecvFromBegin(std::vector<CommsRequest_t> &list);
     void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
 
     ////////////////////////////////////////////////////////////
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 6830c14a..dff9811a 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -144,28 +144,6 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
 }
 
 // Basic Halo comms primitive
-// Basic Halo comms primitive
-void CartesianCommunicator::SendToRecvFromInit(std::vector<CommsRequest_t> &list,
-					       void *xmit,
-					       int dest,
-					       void *recv,
-					       int from,
-					       int bytes)
-{
-  MPI_Request xrq;
-  MPI_Request rrq;
-  int rank = _processor;
-  int ierr;
-  ierr =MPI_Send_init(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
-  ierr|=MPI_Recv_init(recv, bytes, MPI_CHAR,dest,_processor,communicator,&rrq);
-  assert(ierr==0);
-  list.push_back(xrq);
-  list.push_back(rrq);
-}
-void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list)
-{
-  MPI_Startall(list.size(),&list[0]);
-}
 void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 						void *xmit,
 						int dest,
@@ -173,12 +151,17 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 						int from,
 						int bytes)
 {
-  std::vector<CommsRequest_t> reqs(0);
-  SendToRecvFromInit(reqs,xmit,dest,recv,from,bytes);
-  SendToRecvFromBegin(reqs);
-  for(int i=0;i<reqs.size();i++){
-    list.push_back(reqs[i]);
-  }
+  MPI_Request xrq;
+  MPI_Request rrq;
+  int rank = _processor;
+  int ierr;
+  ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+  ierr|=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+  
+  assert(ierr==0);
+
+  list.push_back(xrq);
+  list.push_back(rrq);
 }
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 2873462a..8601255a 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -84,19 +84,6 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 {
   assert(0);
 }
-void CartesianCommunicator::SendToRecvFromInit(std::vector<CommsRequest_t> &list,
-						void *xmit,
-						int dest,
-						void *recv,
-						int from,
-						int bytes)
-{
-  assert(0);
-}
-void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list)
-{
-  assert(0);
-}
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
   assert(0);
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index c415b13d..091e266e 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -268,10 +268,6 @@ void CartesianCommunicator::SendRecvPacket(void *xmit,
 }
 
 // Basic Halo comms primitive
-void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list)
-{
-  assert(0); //unimplemented
-}
 void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 						void *xmit,
 						int dest,
@@ -284,15 +280,6 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   //  shmem_putmem_nb(recv,xmit,bytes,dest,NULL);
   shmem_putmem(recv,xmit,bytes,dest);
 }
-void CartesianCommunicator::SendToRecvFromInit(std::vector<CommsRequest_t> &list,
-						void *xmit,
-						int dest,
-						void *recv,
-						int from,
-						int bytes)
-{
-  assert(0); // Unimplemented
-}
 void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
 {
   //  shmem_quiet();      // I'm done