From 6365a89ba3f1ba37f121a2c6ff1c844d8d6e9704 Mon Sep 17 00:00:00 2001
From: Christoph Lehner <christoph@lhnr.de>
Date: Thu, 30 Jul 2020 07:25:05 -0400
Subject: [PATCH] create separate InitMessage for MemoryManager that can be
 called after communicator setup

---
 Grid/allocator/MemoryManager.cc | 36 ++++++++++++++++++---------------
 Grid/allocator/MemoryManager.h  |  1 +
 Grid/util/Init.cc               |  4 +---
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/Grid/allocator/MemoryManager.cc b/Grid/allocator/MemoryManager.cc
index a2866507..b79c7df3 100644
--- a/Grid/allocator/MemoryManager.cc
+++ b/Grid/allocator/MemoryManager.cc
@@ -137,38 +137,42 @@ void MemoryManager::Init(void)
     }
   }
 
-  // only root node delivers messages, this is called before communicator is initialized,
-  // so need a manual restriction
-  if ( CartesianCommunicator::RankWorld() == 0 ) {
-    std::cout << GridLogMessage<< "MemoryManager::Init() setting up"<<std::endl;
-#ifdef ALLOCATION_CACHE
-    std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent allocations: SMALL "<<Ncache[CpuSmall]<<" LARGE "<<Ncache[Cpu]<<std::endl;
+}
+
+void MemoryManager::InitMessage(void) {
+#ifndef GRID_UVM
+  std::cout << GridLogMessage << "MemoryManager Cache "<< MemoryManager::DeviceMaxBytes <<" bytes "<<std::endl;
 #endif
-    
+  
+  std::cout << GridLogMessage<< "MemoryManager::Init() setting up"<<std::endl;
+#ifdef ALLOCATION_CACHE
+  std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent allocations: SMALL "<<Ncache[CpuSmall]<<" LARGE "<<Ncache[Cpu]<<std::endl;
+#endif
+  
 #ifdef GRID_UVM
-    std::cout << GridLogMessage<< "MemoryManager::Init() Unified memory space"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Unified memory space"<<std::endl;
 #ifdef GRID_CUDA
-    std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMallocManaged"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMallocManaged"<<std::endl;
 #endif
 #ifdef GRID_HIP
-    std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMallocManaged"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMallocManaged"<<std::endl;
 #endif
 #ifdef GRID_SYCL
-    std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_shared"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_shared"<<std::endl;
 #endif
 #else
-    std::cout << GridLogMessage<< "MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory"<<std::endl;
 #ifdef GRID_CUDA
-    std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMalloc"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMalloc"<<std::endl;
 #endif
 #ifdef GRID_HIP
-    std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMalloc"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMalloc"<<std::endl;
 #endif
 #ifdef GRID_SYCL
-    std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
+  std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
 #endif
 #endif
-  }
+
 }
 
 void *MemoryManager::Insert(void *ptr,size_t bytes,int type) 
diff --git a/Grid/allocator/MemoryManager.h b/Grid/allocator/MemoryManager.h
index f158bade..23065c58 100644
--- a/Grid/allocator/MemoryManager.h
+++ b/Grid/allocator/MemoryManager.h
@@ -98,6 +98,7 @@ private:
   static void PrintBytes(void);
  public:
   static void Init(void);
+  static void InitMessage(void);
   static void *SharedAllocate(size_t bytes);
   static void  SharedFree    (void *ptr,size_t bytes);
   static void *CpuAllocate(size_t bytes);
diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc
index 6a84a2f2..8e870525 100644
--- a/Grid/util/Init.cc
+++ b/Grid/util/Init.cc
@@ -377,9 +377,7 @@ void Grid_init(int *argc,char ***argv)
     std::cout << GridLogMessage << "Mapped stencil comms buffers as MAP_HUGETLB "<<std::endl;
   }
 
-#ifndef GRID_UVM
-  std::cout << GridLogMessage << "MemoryManager Cache "<< MemoryManager::DeviceMaxBytes <<" bytes "<<std::endl;
-#endif
+  MemoryManager::InitMessage();
 
   if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-mem") ){
     MemoryProfiler::debug = true;