Merge branch 'develop' into feature/hadrons

benchmarks/Benchmark_IO.cc

@@ -1,5 +1,5 @@
-#ifdef HAVE_LIME
 #include <Grid/Grid.h>
+#ifdef HAVE_LIME
 
 using namespace std;
 using namespace Grid;
@@ -23,7 +23,7 @@ string filestem(const int l)
 void limeWrite(const string filestem, LatticeFermion &vec)
 {
   emptyUserRecord record;
-  ScidacWriter    binWriter;
+  ScidacWriter    binWriter(vec._grid->IsBoss());
 
   binWriter.open(filestem + ".bin");
   binWriter.writeScidacFieldRecord(vec, record);
@@ -50,7 +50,7 @@ void writeBenchmark(const int l, const WriterFn &write)
   GridParallelRNG           rng(g);
   LatticeFermion            vec(g);
   emptyUserRecord           record;
-  ScidacWriter              binWriter;
+  ScidacWriter              binWriter(g->IsBoss());
 
   cout << "-- Local volume " << l << "^4" << endl;
   random(rng, vec);
@@ -101,7 +101,6 @@ int main (int argc, char ** argv)
   return EXIT_SUCCESS;
 }
 #else
-#include <Grid/Grid.h>
 int main (int argc, char ** argv)
 {
   return EXIT_SUCCESS;

lib/parallelIO/IldgIO.h

@@ -272,8 +272,10 @@ class GridLimeReader : public BinaryIO {
   }
 };
 
-class GridLimeWriter : public BinaryIO {
+class GridLimeWriter : public BinaryIO 
+{
  public:
+
    ///////////////////////////////////////////////////
    // FIXME: format for RNG? Now just binary out instead
    // FIXME: collective calls or not ?
@@ -282,17 +284,24 @@ class GridLimeWriter : public BinaryIO {
    FILE       *File;
    LimeWriter *LimeW;
    std::string filename;
-
+   bool        boss_node;
+   GridLimeWriter( bool isboss = true) {
+     boss_node = isboss;
+   }
    void open(const std::string &_filename) { 
      filename= _filename;
-     File = fopen(filename.c_str(), "w");
+     if ( boss_node ) {
-     LimeW = limeCreateWriter(File); assert(LimeW != NULL );
+       File = fopen(filename.c_str(), "w");
+       LimeW = limeCreateWriter(File); assert(LimeW != NULL );
+     }
    }
    /////////////////////////////////////////////
    // Close the file
    /////////////////////////////////////////////
    void close(void) {
-     fclose(File);
+     if ( boss_node ) {
+       fclose(File);
+     }
      //  limeDestroyWriter(LimeW);
    }
   ///////////////////////////////////////////////////////
@@ -300,10 +309,12 @@ class GridLimeWriter : public BinaryIO {
   ///////////////////////////////////////////////////////
   int createLimeRecordHeader(std::string message, int MB, int ME, size_t PayloadSize)
   {
-    LimeRecordHeader *h;
+    if ( boss_node ) {
-    h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
+      LimeRecordHeader *h;
-    assert(limeWriteRecordHeader(h, LimeW) >= 0);
+      h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
-    limeDestroyHeader(h);
+      assert(limeWriteRecordHeader(h, LimeW) >= 0);
+      limeDestroyHeader(h);
+    }
     return LIME_SUCCESS;
   }
   ////////////////////////////////////////////
@@ -312,28 +323,31 @@ class GridLimeWriter : public BinaryIO {
   template<class serialisable_object>
   void writeLimeObject(int MB,int ME,serialisable_object &object,std::string object_name,std::string record_name)
   {
-    std::string xmlstring;
+    if ( boss_node ) {
-    {
+      std::string xmlstring;
-      XmlWriter WR("","");
+      {
-      write(WR,object_name,object);
+	XmlWriter WR("","");
-      xmlstring = WR.XmlString();
+	write(WR,object_name,object);
+	xmlstring = WR.XmlString();
+      }
+      //    std::cout << "WriteLimeObject" << record_name <<std::endl;
+      uint64_t nbytes = xmlstring.size();
+      //    std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
+      int err;
+      LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes); 
+      assert(h!= NULL);
+      
+      err=limeWriteRecordHeader(h, LimeW);                    assert(err>=0);
+      err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
+      err=limeWriterCloseRecord(LimeW);                       assert(err>=0);
+      limeDestroyHeader(h);
     }
-    //    std::cout << "WriteLimeObject" << record_name <<std::endl;
-    uint64_t nbytes = xmlstring.size();
-    //    std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
-    int err;
-    LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes); 
-    assert(h!= NULL);
-
-    err=limeWriteRecordHeader(h, LimeW);                    assert(err>=0);
-    err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
-    err=limeWriterCloseRecord(LimeW);                       assert(err>=0);
-    limeDestroyHeader(h);
-    //    std::cout << " File offset is now"<<ftello(File) << std::endl;
   }
-  ////////////////////////////////////////////
+  ////////////////////////////////////////////////////
   // Write a generic lattice field and csum
-  ////////////////////////////////////////////
+  // This routine is Collectively called by all nodes
+  // in communicator used by the field._grid
+  ////////////////////////////////////////////////////
   template<class vobj>
   void writeLimeLatticeBinaryObject(Lattice<vobj> &field,std::string record_name)
   {
@@ -352,6 +366,8 @@ class GridLimeWriter : public BinaryIO {
     ////////////////////////////////////////////////////////////////////
     
     GridBase *grid = field._grid;
+    assert(boss_node == field._grid->IsBoss() );
+
     ////////////////////////////////////////////
     // Create record header
     ////////////////////////////////////////////
@@ -359,8 +375,10 @@ class GridLimeWriter : public BinaryIO {
     int err;
     uint32_t nersc_csum,scidac_csuma,scidac_csumb;
     uint64_t PayloadSize = sizeof(sobj) * grid->_gsites;
-    createLimeRecordHeader(record_name, 0, 0, PayloadSize);
+    if ( boss_node ) {
-    fflush(File);
+      createLimeRecordHeader(record_name, 0, 0, PayloadSize);
+      fflush(File);
+    }
     
     //    std::cout << "W sizeof(sobj)"      <<sizeof(sobj)<<std::endl;
     //    std::cout << "W Gsites "           <<field._grid->_gsites<<std::endl;
@@ -369,17 +387,15 @@ class GridLimeWriter : public BinaryIO {
     ////////////////////////////////////////////////
     // Check all nodes agree on file position
     ////////////////////////////////////////////////
-    uint64_t offset1 = ftello(File);    
+    uint64_t offset1;
-
+    if ( boss_node ) {
-    uint64_t compare = offset1;
+      offset1 = ftello(File);    
-    grid->Broadcast(0,(void *)&compare,sizeof(compare));
+    }
-
+    grid->Broadcast(0,(void *)&offset1,sizeof(offset1));
-    assert(compare == offset1 ); 
 
     ///////////////////////////////////////////
-    // Write by other means into the binary record
+    // The above is collective. Write by other means into the binary record
     ///////////////////////////////////////////
-
     std::string format = getFormatString<vobj>();
     BinarySimpleMunger<sobj,sobj> munge;
     BinaryIO::writeLatticeObject<vobj,sobj>(field, filename, munge, offset1, format,nersc_csum,scidac_csuma,scidac_csumb);
@@ -387,21 +403,19 @@ class GridLimeWriter : public BinaryIO {
     ///////////////////////////////////////////
     // Wind forward and close the record
     ///////////////////////////////////////////
-    fseek(File,0,SEEK_END);             
+    if ( boss_node ) {
-    uint64_t offset2 = ftello(File);     //    std::cout << " now at offset "<<offset2 << std::endl;
+      fseek(File,0,SEEK_END);             
-
+      uint64_t offset2 = ftello(File);     //    std::cout << " now at offset "<<offset2 << std::endl;
-    /////////////////////////////////////////////////////////////
+      assert( (offset2-offset1) == PayloadSize);
-    // Must synchronise the nodes so no race between nodes
+    }
-    /////////////////////////////////////////////////////////////
-    grid->Barrier();
 
     /////////////////////////////////////////////////////////////
     // Check MPI-2 I/O did what we expect to file
     /////////////////////////////////////////////////////////////
-    assert( (offset2-offset1) == PayloadSize);
-
-    err=limeWriterCloseRecord(LimeW);  assert(err>=0);
 
+    if ( boss_node ) { 
+      err=limeWriterCloseRecord(LimeW);  assert(err>=0);
+    }
     ////////////////////////////////////////
     // Write checksum element, propagaing forward from the BinaryIO
     // Always pair a checksum with a binary object, and close message
@@ -411,21 +425,26 @@ class GridLimeWriter : public BinaryIO {
     std::stringstream streamb; streamb << std::hex << scidac_csumb;
     checksum.suma= streama.str();
     checksum.sumb= streamb.str();
-    //    std::cout << GridLogMessage<<" writing scidac checksums "<<std::hex<<scidac_csuma<<"/"<<scidac_csumb<<std::dec<<std::endl;
+    if ( boss_node ) { 
-    writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
+      writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
+    }
   }
 };
 
 class ScidacWriter : public GridLimeWriter {
  public:
 
-   template<class SerialisableUserFile>
+  ScidacWriter(bool isboss =true ) : GridLimeWriter(isboss)  { };
-   void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
+
-   {
+  template<class SerialisableUserFile>
-     scidacFile    _scidacFile(grid);
+  void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
-     writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+  {
-     writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+    scidacFile    _scidacFile(grid);
-   }
+    if ( this->boss_node ) {
+      writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+      writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+    }
+  }
   ////////////////////////////////////////////////
   // Write generic lattice field in scidac format
   ////////////////////////////////////////////////
@@ -446,9 +465,12 @@ class ScidacWriter : public GridLimeWriter {
     //////////////////////////////////////////////
     // Fill the Lime file record by record
     //////////////////////////////////////////////
-    writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
+    if ( this->boss_node ) {
-    writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+      writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
-    writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+      writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+      writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+    }
+    // Collective call
     writeLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA));      // Closes message with checksum
   }
 };
@@ -515,6 +537,8 @@ class ScidacReader : public GridLimeReader {
 
 class IldgWriter : public ScidacWriter {
  public:
+  
+  IldgWriter(bool isboss) : ScidacWriter(isboss) {};
 
   ///////////////////////////////////
   // A little helper

lib/qcd/hmc/checkpointers/ILDGCheckpointer.h

@@ -74,10 +74,10 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer<Implementation> {
     if ((traj % Params.saveInterval) == 0) {
       std::string config, rng;
       this->build_filenames(traj, Params, config, rng);
-      
+      GridBase *grid = U._grid;
       uint32_t nersc_csum,scidac_csuma,scidac_csumb;
       BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb);
-      IldgWriter _IldgWriter;
+      IldgWriter _IldgWriter(grid->IsBoss());
       _IldgWriter.open(config);
       _IldgWriter.writeConfiguration(U, traj, config, config);
       _IldgWriter.close();

tests/IO/Test_ildg_io.cc

@@ -79,7 +79,7 @@ int main (int argc, char ** argv)
   std::cout <<GridLogMessage<<"** Writing out  ILDG conf    *********"<<std::endl;
   std::cout <<GridLogMessage<<"**************************************"<<std::endl;
   std::string file("./ckpoint_ildg.4000");
-  IldgWriter _IldgWriter;
+  IldgWriter _IldgWriter(Fine.IsBoss());
   _IldgWriter.open(file);
   _IldgWriter.writeConfiguration(Umu,4000,std::string("dummy_ildg_LFN"),std::string("dummy_config"));
   _IldgWriter.close();

tests/lanczos/Test_dwf_compressed_lanczos_reorg.cc

@@ -58,7 +58,7 @@ public:
   {
     assert(this->subspace.size()==nbasis);
     emptyUserRecord record;
-    Grid::QCD::ScidacWriter WR;
+    Grid::QCD::ScidacWriter WR(this->_FineGrid->IsBoss());
     WR.open(evecs_file);
     for(int k=0;k<nbasis;k++) {
       WR.writeScidacFieldRecord(this->subspace[k],record);
@@ -96,7 +96,7 @@ public:
   {
     int n = this->evec_coarse.size();
     emptyUserRecord record;
-    Grid::QCD::ScidacWriter WR;
+    Grid::QCD::ScidacWriter WR(this->_CoarseGrid->IsBoss());
     WR.open(evecs_file);
     for(int k=0;k<n;k++) {
       WR.writeScidacFieldRecord(this->evec_coarse[k],record);

tests/solver/Test_dwf_mrhs_cg.cc

@@ -114,7 +114,7 @@ int main (int argc, char ** argv)
 
   {
     FGrid->Barrier();
-    ScidacWriter _ScidacWriter;
+    ScidacWriter _ScidacWriter(FGrid->IsBoss());
     _ScidacWriter.open(file);
     std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     std::cout << GridLogMessage << " Writing out gauge field "<<std::endl;
@@ -144,7 +144,7 @@ int main (int argc, char ** argv)
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
 
       std::stringstream filefn;      filefn << filef << "."<< n;
-      ScidacWriter _ScidacWriter;
+      ScidacWriter _ScidacWriter(FGrid->IsBoss());
       _ScidacWriter.open(filefn.str());
       _ScidacWriter.writeScidacFieldRecord(src[n],record);
       _ScidacWriter.close();