From 639dc1ab21552237ec3cc6b6538ca14f0f9c84df Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Wed, 31 Jul 2019 01:23:23 +0100
Subject: [PATCH] GPU reduction fix and also exit backtrace option

---
 Grid/util/Init.cc | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/Grid/util/Init.cc b/Grid/util/Init.cc
index 4fa3e963..f9da7532 100644
--- a/Grid/util/Init.cc
+++ b/Grid/util/Init.cc
@@ -320,15 +320,15 @@ void GridGpuInit(void)
     printf("GpuInit: Cuda reports %d GPUs on MPI rank 0\n",nDevices);
   }
 
-  if ( world_rank == 0) {
-    for (int i = 0; i < nDevices; i++) {
+  for (int i = 0; i < nDevices; i++) {
 
 #define GPU_PROP_FMT(canMapHostMemory,FMT)     printf("GpuInit:   " #canMapHostMemory ": " FMT" \n",prop.canMapHostMemory);
 #define GPU_PROP(canMapHostMemory)             GPU_PROP_FMT(canMapHostMemory,"%d");
-
+    
+    //      cudaGetDeviceProperties(&prop, i);
+    cudaGetDeviceProperties(&gpu_props[i], i);
+    if ( world_rank == 0) {
       cudaDeviceProp prop; 
-      //      cudaGetDeviceProperties(&prop, i);
-      cudaGetDeviceProperties(&gpu_props[i], i);
       prop = gpu_props[i];
       printf("GpuInit: ========================\n");
       printf("GpuInit: Device Number    : %d\n", i);
@@ -344,8 +344,8 @@ void GridGpuInit(void)
       GPU_PROP(singleToDoublePrecisionPerfRatio);
 #endif
     }
-    printf("GpuInit: ================================================\n");
   }
+  printf("GpuInit: ================================================\n");
 #endif
 }
 
@@ -622,6 +622,11 @@ void Grid_sa_signal_handler(int sig,siginfo_t *si,void * ptr)
   return;
 };
 
+void Grid_exit_handler(void)
+{
+  BACKTRACEFP(stdout);
+  fflush(stdout);
+}
 void Grid_debug_handler_init(void)
 {
   struct sigaction sa;
@@ -637,6 +642,8 @@ void Grid_debug_handler_init(void)
   sigaction(SIGFPE,&sa,NULL);
   sigaction(SIGKILL,&sa,NULL);
   sigaction(SIGILL,&sa,NULL);
+
+  atexit(Grid_exit_handler);
 }
 
 NAMESPACE_END(Grid);