From e8057d6b4ae2e4eb7c2be4d931f65d532be61892 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ed.ac.uk>
Date: Tue, 2 Dec 2025 23:15:32 +0000
Subject: [PATCH] Updated for verbose on host vs. device side csum

---
 Grid/communicator/Communicator_mpi3.cc  | 3 +++
 systems/Aurora/tests/reproBigJob.pbs    | 2 +-
 systems/Aurora/tests/reproBigJob256.pbs | 2 +-
 tests/Test_dwf_mixedcg_prec.cc          | 2 ++
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc
index f72b5773..c7212df9 100644
--- a/Grid/communicator/Communicator_mpi3.cc
+++ b/Grid/communicator/Communicator_mpi3.cc
@@ -828,6 +828,7 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsReque
 #define AUDIT_FLIGHT_RECORDER_ERRORS
 #ifdef  AUDIT_FLIGHT_RECORDER_ERRORS
   uint64_t EC = FlightRecorder::CommsErrorCount();
+  if (EC) std::cerr << " global sum error count "<<EC<<std::endl;
   this->GlobalSum(EC);
   if (EC) {
     for(int r=0;r<list.size();r++){
@@ -837,10 +838,12 @@ void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsReque
 	uint64_t rbytes_data = list[r].bytes;
 #endif
       if (list[r].PacketType == InterNodeReceiveHtoD) {
+	std::cerr << " calling xor reduce "<<std::endl;
 	uint64_t csg = gpu_xor((uint64_t*)list[r].device_buf,rbytes_data/8);
 	uint64_t csh = cpu_xor((uint64_t*)list[r].host_buf,rbytes_data/8);
 	std::cerr << " Packet "<<r<<" Receive from " <<list[r].dest<<" host csum "<<csh<<" gpu csum "<<csg<<std::endl;
       } if (list[r].PacketType == InterNodeXmitISend ) {
+	std::cerr << " calling xor reduce "<<std::endl;
 	uint64_t csg = gpu_xor((uint64_t*)list[r].device_buf,rbytes_data/8);
 	uint64_t csh = cpu_xor((uint64_t*)list[r].host_buf,rbytes_data/8);
 	std::cerr << " Packet "<<r<<" Send to " <<list[r].dest<<" host csum "<<csh<<" gpu csum "<<csg<<std::endl;
diff --git a/systems/Aurora/tests/reproBigJob.pbs b/systems/Aurora/tests/reproBigJob.pbs
index d455c590..afbee41c 100644
--- a/systems/Aurora/tests/reproBigJob.pbs
+++ b/systems/Aurora/tests/reproBigJob.pbs
@@ -51,7 +51,7 @@ EOF
 CMD="mpiexec -np 384 -ppn 12  -envall --hostfile nodefile \
 	     ../gpu_tile.sh \
 	     $BINARY --mpi 4.4.4.6 --grid 64.64.64.96  \
-		--shm-mpi 0 --comms-overlap --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 6000 --log Message "
+		--shm-mpi 0 --comms-overlap --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 6000 --log Message --debug-stdout "
 
 echo $CMD > command-line
 env > environment
diff --git a/systems/Aurora/tests/reproBigJob256.pbs b/systems/Aurora/tests/reproBigJob256.pbs
index 8cf75d75..0a2de8d4 100644
--- a/systems/Aurora/tests/reproBigJob256.pbs
+++ b/systems/Aurora/tests/reproBigJob256.pbs
@@ -53,7 +53,7 @@ EOF
 CMD="mpiexec -np 3072 -ppn 12  -envall --hostfile nodefile \
 	     ../gpu_tile.sh \
 	     $BINARY --mpi 8.8.8.6 --grid 128.128.128.288  \
-		--shm-mpi 0 --comms-overlap --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 18000 --log Message --debug-stdout --heartbeat"
+		--shm-mpi 0 --comms-overlap --shm 4096 --device-mem 32000 --accelerator-threads 32 --seconds 18000 --log Message --debug-stdout "
 
 echo $CMD > command-line
 env > environment
diff --git a/tests/Test_dwf_mixedcg_prec.cc b/tests/Test_dwf_mixedcg_prec.cc
index c87908a3..99a5f4be 100644
--- a/tests/Test_dwf_mixedcg_prec.cc
+++ b/tests/Test_dwf_mixedcg_prec.cc
@@ -82,6 +82,7 @@ int main (int argc, char ** argv)
   
   Grid_init(&argc,&argv);
 
+  std::cout << GridLogMessage<<" in main(): Grid is initialised"<<std::endl;
   const int Ls=12;
 
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()),GridDefaultMpi());
@@ -94,6 +95,7 @@ int main (int argc, char ** argv)
   GridCartesian         * FGrid_f   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_f);
   GridRedBlackCartesian * FrbGrid_f = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_f);
   
+  std::cout << GridLogMessage<<" in main(): making RNGs"<<std::endl;
   std::vector<int> seeds4({1,2,3,4});
   std::vector<int> seeds5({5,6,7,8});
   GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);