From 57002924bc3232308cd1add6610b632a3b055fe9 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 2 Jul 2017 14:58:30 -0700
Subject: [PATCH] NERSC shakeout of this

---
 lib/communicator/Communicator_mpi.cc |  8 ++--
 tests/solver/Test_dwf_mrhs_cg.cc     | 65 ++++++++++++++--------------
 2 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 7879f518..1a7a0c05 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -70,7 +70,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
   MPI_Comm_size(parent.communicator,&Nparent);
 
   int childsize=1;
-  for(int d=0;d<_ndimension;d++) {
+  for(int d=0;d<processors.size();d++) {
     childsize *= processors[d];
   }
   int Nchild = Nparent/childsize;
@@ -83,12 +83,12 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
   MPI_Comm comm_split;
   if ( Nchild > 1 ) { 
 
-    std::cout << GridLogMessage<<"Child communicator of "<< parent.communicator<<std::endl;
-    std::cout << GridLogMessage<<" parent grid    ";
+    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
+    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
     for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
     std::cout<<std::endl;
 
-    std::cout << GridLogMessage<<" child grid    ";
+    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
     for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
     std::cout<<std::endl;
 
diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index fa940390..e44c15d0 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -95,49 +95,50 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   emptyUserRecord record;
   std::string file("./scratch.scidac");
-  {
-    ScidacWriter _ScidacWriter;
-    _ScidacWriter.open(file);
-    _ScidacWriter.writeScidacFieldRecord(Umu,record);
-    for(int n=0;n<nrhs;n++){
-      _ScidacWriter.writeScidacFieldRecord(src[n],record);
-      
-    }
-    _ScidacWriter.close();
-  }
-
-
-
-  //////////////////////////////////////////
-  // Read back into single rank fields
-  //////////////////////////////////////////
-
-  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
-  std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
-  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
-
   int me = UGrid->ThisRank();
   LatticeGaugeField s_Umu(SGrid);
   FermionField s_src(SFGrid);
   FermionField s_res(SFGrid);
+
   {
+    ScidacWriter _ScidacWriter;
+    _ScidacWriter.open(file);
+    std::cout << GridLogMessage << " Writing out gauge field "<<std::endl;
+    _ScidacWriter.writeScidacFieldRecord(Umu,record);
+    _ScidacWriter.close();
+
+    std::cout << GridLogMessage << " Reading in gauge field "<<std::endl;
     ScidacReader  _ScidacReader;
     _ScidacReader.open(file);
-    std::cout << GridLogMessage << " Opened file "<<std::endl;
     _ScidacReader.readScidacFieldRecord(s_Umu,record);
-    std::cout << GridLogMessage << " Read gauge field "<<std::endl;
-    for(int n=0;n<nrhs;n++){
-      if ( n==me ) { 
-	std::cout << GridLogMessage << " Read record  "<<n<<std::endl;
-	_ScidacReader.readScidacFieldRecord(s_src,record);
-      } else { 
-	std::cout << GridLogMessage << " Skip record  "<<n<<std::endl;
-	_ScidacReader.skipScidacFieldRecord();
-      }      
-    }
     _ScidacReader.close();
   }
 
+
+  {
+    for(int n=0;n<nrhs;n++){
+
+      std::cout << GridLogMessage << " Writing out record "<<n<<std::endl;
+      ScidacWriter _ScidacWriter;
+      _ScidacWriter.open(file);
+      _ScidacWriter.writeScidacFieldRecord(src[n],record);
+      _ScidacWriter.close();
+      
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+      std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+      
+      if ( n==me ) { 
+	ScidacReader  _ScidacReader;
+	_ScidacReader.open(file);
+	_ScidacReader.readScidacFieldRecord(s_src,record);
+	_ScidacReader.close();
+      }
+      FGrid->Barrier();
+    }
+  }
+
+
   ///////////////////////////////////////////////////////////////
   // Set up N-solvers as trivially parallel
   ///////////////////////////////////////////////////////////////