From b4967f0231050fcb2dd68ac69c1db075fd4baa0e Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sun, 9 Sep 2018 14:28:02 +0100
Subject: [PATCH] Verbose and error trapping cleaner

---
 lib/communicator/Communicator_mpi3.cc | 2 +-
 lib/communicator/SharedMemoryMPI.cc   | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 2b8753a7..c13bc2e4 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -44,7 +44,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv)
   MPI_Initialized(&flag); // needed to coexist with other libs apparently
   if ( !flag ) {
     MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided);
-    assert (provided == MPI_THREAD_MULTIPLE);
+    //assert (provided == MPI_THREAD_MULTIPLE);
   }
 
   Grid_quiesce_nodes();
diff --git a/lib/communicator/SharedMemoryMPI.cc b/lib/communicator/SharedMemoryMPI.cc
index 4a288979..dea913bc 100644
--- a/lib/communicator/SharedMemoryMPI.cc
+++ b/lib/communicator/SharedMemoryMPI.cc
@@ -49,6 +49,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
 #endif
   MPI_Comm_rank(WorldShmComm     ,&WorldShmRank);
   MPI_Comm_size(WorldShmComm     ,&WorldShmSize);
+  std::cout << " World  communicator of size " <<WorldSize << std::endl;
   std::cout << " Shared communicator of size " <<WorldShmSize << std::endl;
   // WorldShmComm, WorldShmSize, WorldShmRank
 
@@ -193,13 +194,14 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
 
 
   //std::cerr << " allocating "<<bytes <<"  bytes "<< std::endl;
-  if ( cudaMallocManaged(&ShmCommBuf, bytes) !=  cudaSuccess) {
-    perror("cudaMallocManaged failed ");
+  auto err =  cudaMallocManaged(&ShmCommBuf, bytes);
+  if ( err !=  cudaSuccess) {
+    std::cerr << " SharedMemoryMPI.cc cudaMallocManaged failed for " << bytes<<" bytes " <<cudaGetErrorString(err)<< std::endl;
     exit(EXIT_FAILURE);  
   }
 
   if (ShmCommBuf == (void *)NULL ) {
-    perror("cudaMallocManaged failed ");
+    std::cerr << " SharedMemoryMPI.cc cudaMallocManaged failed NULL pointer for " << bytes<<" bytes " << std::endl;
     exit(EXIT_FAILURE);  
   }
   std::cerr << " Cuda allocated managed memory at "<<std::hex<<ShmCommBuf <<" - " << ((uint64_t)ShmCommBuf + bytes)<<std::dec<< std::endl;