From dee68fc7287a2af2d2acba8f72f96760722d275a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 2 Jul 2017 23:33:48 +0100
Subject: [PATCH] IO working multiple nodes again. Strategy of all nodes
 writing metadata is unsafe. Only one rank should do this. must identify this
 rank. Means pass communicator to the Objects.

---
 lib/parallelIO/IldgIO.h          |  8 ++++++--
 lib/parallelIO/IldgIOtypes.h     |  1 +
 lib/parallelIO/MetaData.h        |  3 +++
 tests/solver/Test_dwf_mrhs_cg.cc | 21 ++++++++++++++++++---
 4 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h
index 542f80ef..7031f110 100644
--- a/lib/parallelIO/IldgIO.h
+++ b/lib/parallelIO/IldgIO.h
@@ -258,10 +258,13 @@ class GridLimeReader : public BinaryIO {
 
       if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
 
-	//	std::cout << GridLogMessage<< " readLimeObject matches ! " <<std::endl;
+	std::cout << GridLogMessage<< " readLimeObject matches ! " << record_name <<std::endl;
 
 	std::vector<char> xmlc(nbytes+1,'\0');
 	limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR);    
+
+	std::cout << GridLogMessage<< " readLimeObject matches XML " << &xmlc[0] <<std::endl;
+
 	XmlReader RD(&xmlc[0],"");
 	read(RD,object_name,object);
 	return;
@@ -276,8 +279,9 @@ class GridLimeWriter : public BinaryIO {
  public:
    ///////////////////////////////////////////////////
    // FIXME: format for RNG? Now just binary out instead
+   // FIXME: collective calls or not ?
+   //      : must know if I am the I/O boss
    ///////////////////////////////////////////////////
-
    FILE       *File;
    LimeWriter *LimeW;
    std::string filename;
diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h
index 53664b49..5b397e14 100644
--- a/lib/parallelIO/IldgIOtypes.h
+++ b/lib/parallelIO/IldgIOtypes.h
@@ -66,6 +66,7 @@ namespace Grid {
 
 struct emptyUserRecord : Serializable { 
   GRID_SERIALIZABLE_CLASS_MEMBERS(emptyUserRecord,int,dummy);
+  emptyUserRecord() { dummy=0; };
 };
 
 ////////////////////////
diff --git a/lib/parallelIO/MetaData.h b/lib/parallelIO/MetaData.h
index 8fc0d777..ccc8b18f 100644
--- a/lib/parallelIO/MetaData.h
+++ b/lib/parallelIO/MetaData.h
@@ -85,6 +85,9 @@ namespace Grid {
 	nd=4;
 	dimension.resize(4);
 	boundary.resize(4);
+	scidac_checksuma=0;
+	scidac_checksumb=0;
+	checksum=0;
       }
     };
 
diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index e44c15d0..72d8d8f4 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -95,42 +95,57 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   emptyUserRecord record;
   std::string file("./scratch.scidac");
+  std::string filef("./scratch.scidac.ferm");
   int me = UGrid->ThisRank();
   LatticeGaugeField s_Umu(SGrid);
   FermionField s_src(SFGrid);
   FermionField s_res(SFGrid);
 
   {
+    FGrid->Barrier();
     ScidacWriter _ScidacWriter;
     _ScidacWriter.open(file);
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     std::cout << GridLogMessage << " Writing out gauge field "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     _ScidacWriter.writeScidacFieldRecord(Umu,record);
     _ScidacWriter.close();
-
+    FGrid->Barrier();
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     std::cout << GridLogMessage << " Reading in gauge field "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     ScidacReader  _ScidacReader;
     _ScidacReader.open(file);
     _ScidacReader.readScidacFieldRecord(s_Umu,record);
     _ScidacReader.close();
+    FGrid->Barrier();
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+    std::cout << GridLogMessage << " Read in gauge field "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   }
 
 
   {
     for(int n=0;n<nrhs;n++){
 
+      FGrid->Barrier();
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       std::cout << GridLogMessage << " Writing out record "<<n<<std::endl;
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       ScidacWriter _ScidacWriter;
-      _ScidacWriter.open(file);
+      _ScidacWriter.open(filef);
       _ScidacWriter.writeScidacFieldRecord(src[n],record);
       _ScidacWriter.close();
       
+      FGrid->Barrier();
+
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       
       if ( n==me ) { 
 	ScidacReader  _ScidacReader;
-	_ScidacReader.open(file);
+	_ScidacReader.open(filef);
 	_ScidacReader.readScidacFieldRecord(s_src,record);
 	_ScidacReader.close();
       }