Merge remote-tracking branch 'upstream/develop' into feature/wilsonmg

benchmarks/Benchmark_IO.cc

@@ -0,0 +1,108 @@
+#include <Grid/Grid.h>
+#ifdef HAVE_LIME
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+#define MSG cout << GridLogMessage
+#define SEP \
+"============================================================================="
+#ifndef BENCH_IO_LMAX
+#define BENCH_IO_LMAX 40
+#endif
+
+typedef function<void(const string, LatticeFermion &)> WriterFn;
+typedef function<void(LatticeFermion &, const string)> ReaderFn;
+
+string filestem(const int l)
+{
+  return "iobench_l" + to_string(l);
+}
+
+void limeWrite(const string filestem, LatticeFermion &vec)
+{
+  emptyUserRecord record;
+  ScidacWriter    binWriter(vec._grid->IsBoss());
+
+  binWriter.open(filestem + ".bin");
+  binWriter.writeScidacFieldRecord(vec, record);
+  binWriter.close();
+}
+
+void limeRead(LatticeFermion &vec, const string filestem)
+{
+  emptyUserRecord record;
+  ScidacReader    binReader;
+
+  binReader.open(filestem + ".bin");
+  binReader.readScidacFieldRecord(vec, record);
+  binReader.close();
+}
+
+void writeBenchmark(const int l, const WriterFn &write)
+{
+  auto                      mpi  = GridDefaultMpi();
+  auto                      simd = GridDefaultSimd(Nd, vComplex::Nsimd());
+  vector<int>               latt = {l*mpi[0], l*mpi[1], l*mpi[2], l*mpi[3]};
+  unique_ptr<GridCartesian> gPt(SpaceTimeGrid::makeFourDimGrid(latt, simd, mpi));
+  GridCartesian             *g = gPt.get();
+  GridParallelRNG           rng(g);
+  LatticeFermion            vec(g);
+  emptyUserRecord           record;
+  ScidacWriter              binWriter(g->IsBoss());
+
+  cout << "-- Local volume " << l << "^4" << endl;
+  random(rng, vec);
+  write(filestem(l), vec);
+}
+
+void readBenchmark(const int l, const ReaderFn &read)
+{
+  auto                      mpi  = GridDefaultMpi();
+  auto                      simd = GridDefaultSimd(Nd, vComplex::Nsimd());
+  vector<int>               latt = {l*mpi[0], l*mpi[1], l*mpi[2], l*mpi[3]};
+  unique_ptr<GridCartesian> gPt(SpaceTimeGrid::makeFourDimGrid(latt, simd, mpi));
+  GridCartesian             *g = gPt.get();
+  LatticeFermion            vec(g);
+  emptyUserRecord           record;
+  ScidacReader              binReader;
+
+  cout << "-- Local volume " << l << "^4" << endl;
+  read(vec, filestem(l));
+}
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  auto simd = GridDefaultSimd(Nd,vComplex::Nsimd());
+  auto mpi  = GridDefaultMpi();
+
+  int64_t threads = GridThread::GetThreads();
+  MSG << "Grid is setup to use " << threads << " threads" << endl;
+  MSG << SEP << endl;
+  MSG << "Benchmark Lime write" << endl;
+  MSG << SEP << endl;
+  for (int l = 4; l <= BENCH_IO_LMAX; l += 2)
+  {
+    writeBenchmark(l, limeWrite);
+  }
+
+  MSG << "Benchmark Lime read" << endl;
+  MSG << SEP << endl;
+  for (int l = 4; l <= BENCH_IO_LMAX; l += 2)
+  {
+    readBenchmark(l, limeRead);
+  }
+
+  Grid_finalize();
+
+  return EXIT_SUCCESS;
+}
+#else
+int main (int argc, char ** argv)
+{
+  return EXIT_SUCCESS;
+}
+#endif

lib/parallelIO/BinaryIO.h

@@ -372,7 +372,7 @@ class BinaryIO {
 	std::cout << GridLogMessage <<"IOobject: C++ read I/O " << file << " : "
                   << iodata.size() * sizeof(fobj) << " bytes" << std::endl;
         std::ifstream fin;
-        fin.open(file, std::ios::binary | std::ios::in);
+	fin.open(file, std::ios::binary | std::ios::in);
         if (control & BINARYIO_MASTER_APPEND)
         {
           fin.seekg(-sizeof(fobj), fin.end);
@@ -453,11 +453,15 @@ class BinaryIO {
 	std::ofstream fout; 
 	fout.exceptions ( std::fstream::failbit | std::fstream::badbit );
 	try {
-	  fout.open(file,std::ios::binary|std::ios::out|std::ios::in);
+	  if (offset) { // Must already exist and contain data
+	    fout.open(file,std::ios::binary|std::ios::out|std::ios::in);
+	  } else {     // Allow create
+	    fout.open(file,std::ios::binary|std::ios::out);
+	  }
 	} catch (const std::fstream::failure& exc) {
 	  std::cout << GridLogError << "Error in opening the file " << file << " for output" <<std::endl;
 	  std::cout << GridLogError << "Exception description: " << exc.what() << std::endl;
-	  std::cout << GridLogError << "Probable cause: wrong path, inaccessible location "<< std::endl;
+	  //	  std::cout << GridLogError << "Probable cause: wrong path, inaccessible location "<< std::endl;
 #ifdef USE_MPI_IO
 	  MPI_Abort(MPI_COMM_WORLD,1);
 #else

lib/parallelIO/IldgIO.h

@@ -272,8 +272,10 @@ class GridLimeReader : public BinaryIO {
   }
 };
 
-class GridLimeWriter : public BinaryIO {
+class GridLimeWriter : public BinaryIO 
+{
  public:
+
    ///////////////////////////////////////////////////
    // FIXME: format for RNG? Now just binary out instead
    // FIXME: collective calls or not ?
@@ -282,17 +284,24 @@ class GridLimeWriter : public BinaryIO {
    FILE       *File;
    LimeWriter *LimeW;
    std::string filename;
-
+   bool        boss_node;
+   GridLimeWriter( bool isboss = true) {
+     boss_node = isboss;
+   }
    void open(const std::string &_filename) { 
      filename= _filename;
-     File = fopen(filename.c_str(), "w");
+     if ( boss_node ) {
-     LimeW = limeCreateWriter(File); assert(LimeW != NULL );
+       File = fopen(filename.c_str(), "w");
+       LimeW = limeCreateWriter(File); assert(LimeW != NULL );
+     }
    }
    /////////////////////////////////////////////
    // Close the file
    /////////////////////////////////////////////
    void close(void) {
-     fclose(File);
+     if ( boss_node ) {
+       fclose(File);
+     }
      //  limeDestroyWriter(LimeW);
    }
   ///////////////////////////////////////////////////////
@@ -300,10 +309,12 @@ class GridLimeWriter : public BinaryIO {
   ///////////////////////////////////////////////////////
   int createLimeRecordHeader(std::string message, int MB, int ME, size_t PayloadSize)
   {
-    LimeRecordHeader *h;
+    if ( boss_node ) {
-    h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
+      LimeRecordHeader *h;
-    assert(limeWriteRecordHeader(h, LimeW) >= 0);
+      h = limeCreateHeader(MB, ME, const_cast<char *>(message.c_str()), PayloadSize);
-    limeDestroyHeader(h);
+      assert(limeWriteRecordHeader(h, LimeW) >= 0);
+      limeDestroyHeader(h);
+    }
     return LIME_SUCCESS;
   }
   ////////////////////////////////////////////
@@ -312,28 +323,31 @@ class GridLimeWriter : public BinaryIO {
   template<class serialisable_object>
   void writeLimeObject(int MB,int ME,serialisable_object &object,std::string object_name,std::string record_name)
   {
-    std::string xmlstring;
+    if ( boss_node ) {
-    {
+      std::string xmlstring;
-      XmlWriter WR("","");
+      {
-      write(WR,object_name,object);
+	XmlWriter WR("","");
-      xmlstring = WR.XmlString();
+	write(WR,object_name,object);
+	xmlstring = WR.XmlString();
+      }
+      //    std::cout << "WriteLimeObject" << record_name <<std::endl;
+      uint64_t nbytes = xmlstring.size();
+      //    std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
+      int err;
+      LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes); 
+      assert(h!= NULL);
+      
+      err=limeWriteRecordHeader(h, LimeW);                    assert(err>=0);
+      err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
+      err=limeWriterCloseRecord(LimeW);                       assert(err>=0);
+      limeDestroyHeader(h);
     }
-    //    std::cout << "WriteLimeObject" << record_name <<std::endl;
-    uint64_t nbytes = xmlstring.size();
-    //    std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
-    int err;
-    LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes); 
-    assert(h!= NULL);
-
-    err=limeWriteRecordHeader(h, LimeW);                    assert(err>=0);
-    err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
-    err=limeWriterCloseRecord(LimeW);                       assert(err>=0);
-    limeDestroyHeader(h);
-    //    std::cout << " File offset is now"<<ftello(File) << std::endl;
   }
-  ////////////////////////////////////////////
+  ////////////////////////////////////////////////////
   // Write a generic lattice field and csum
-  ////////////////////////////////////////////
+  // This routine is Collectively called by all nodes
+  // in communicator used by the field._grid
+  ////////////////////////////////////////////////////
   template<class vobj>
   void writeLimeLatticeBinaryObject(Lattice<vobj> &field,std::string record_name)
   {
@@ -350,6 +364,9 @@ class GridLimeWriter : public BinaryIO {
     // iv) fseek on FILE * to end of this disjoint section.
     //  v) Continue writing scidac record.
     ////////////////////////////////////////////////////////////////////
+    
+    GridBase *grid = field._grid;
+    assert(boss_node == field._grid->IsBoss() );
 
     ////////////////////////////////////////////
     // Create record header
@@ -357,19 +374,28 @@ class GridLimeWriter : public BinaryIO {
     typedef typename vobj::scalar_object sobj;
     int err;
     uint32_t nersc_csum,scidac_csuma,scidac_csumb;
-    uint64_t PayloadSize = sizeof(sobj) * field._grid->_gsites;
+    uint64_t PayloadSize = sizeof(sobj) * grid->_gsites;
-    createLimeRecordHeader(record_name, 0, 0, PayloadSize);
+    if ( boss_node ) {
-
+      createLimeRecordHeader(record_name, 0, 0, PayloadSize);
+      fflush(File);
+    }
+    
     //    std::cout << "W sizeof(sobj)"      <<sizeof(sobj)<<std::endl;
     //    std::cout << "W Gsites "           <<field._grid->_gsites<<std::endl;
     //    std::cout << "W Payload expected " <<PayloadSize<<std::endl;
 
-    fflush(File);
+    ////////////////////////////////////////////////
+    // Check all nodes agree on file position
+    ////////////////////////////////////////////////
+    uint64_t offset1;
+    if ( boss_node ) {
+      offset1 = ftello(File);    
+    }
+    grid->Broadcast(0,(void *)&offset1,sizeof(offset1));
 
     ///////////////////////////////////////////
-    // Write by other means into the binary record
+    // The above is collective. Write by other means into the binary record
     ///////////////////////////////////////////
-    uint64_t offset1 = ftello(File);    //    std::cout << " Writing to offset "<<offset1 << std::endl;
     std::string format = getFormatString<vobj>();
     BinarySimpleMunger<sobj,sobj> munge;
     BinaryIO::writeLatticeObject<vobj,sobj>(field, filename, munge, offset1, format,nersc_csum,scidac_csuma,scidac_csumb);
@@ -377,13 +403,19 @@ class GridLimeWriter : public BinaryIO {
     ///////////////////////////////////////////
     // Wind forward and close the record
     ///////////////////////////////////////////
-    fseek(File,0,SEEK_END);             
+    if ( boss_node ) {
-    uint64_t offset2 = ftello(File);     //    std::cout << " now at offset "<<offset2 << std::endl;
+      fseek(File,0,SEEK_END);             
+      uint64_t offset2 = ftello(File);     //    std::cout << " now at offset "<<offset2 << std::endl;
+      assert( (offset2-offset1) == PayloadSize);
+    }
 
-    assert((offset2-offset1) == PayloadSize);
+    /////////////////////////////////////////////////////////////
-
+    // Check MPI-2 I/O did what we expect to file
-    err=limeWriterCloseRecord(LimeW);  assert(err>=0);
+    /////////////////////////////////////////////////////////////
 
+    if ( boss_node ) { 
+      err=limeWriterCloseRecord(LimeW);  assert(err>=0);
+    }
     ////////////////////////////////////////
     // Write checksum element, propagaing forward from the BinaryIO
     // Always pair a checksum with a binary object, and close message
@@ -393,21 +425,26 @@ class GridLimeWriter : public BinaryIO {
     std::stringstream streamb; streamb << std::hex << scidac_csumb;
     checksum.suma= streama.str();
     checksum.sumb= streamb.str();
-    //    std::cout << GridLogMessage<<" writing scidac checksums "<<std::hex<<scidac_csuma<<"/"<<scidac_csumb<<std::dec<<std::endl;
+    if ( boss_node ) { 
-    writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
+      writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
+    }
   }
 };
 
 class ScidacWriter : public GridLimeWriter {
  public:
 
-   template<class SerialisableUserFile>
+  ScidacWriter(bool isboss =true ) : GridLimeWriter(isboss)  { };
-   void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
+
-   {
+  template<class SerialisableUserFile>
-     scidacFile    _scidacFile(grid);
+  void writeScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
-     writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+  {
-     writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+    scidacFile    _scidacFile(grid);
-   }
+    if ( this->boss_node ) {
+      writeLimeObject(1,0,_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+      writeLimeObject(0,1,_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+    }
+  }
   ////////////////////////////////////////////////
   // Write generic lattice field in scidac format
   ////////////////////////////////////////////////
@@ -428,9 +465,12 @@ class ScidacWriter : public GridLimeWriter {
     //////////////////////////////////////////////
     // Fill the Lime file record by record
     //////////////////////////////////////////////
-    writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
+    if ( this->boss_node ) {
-    writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+      writeLimeObject(1,0,header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
-    writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+      writeLimeObject(0,0,_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+      writeLimeObject(0,0,_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+    }
+    // Collective call
     writeLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA));      // Closes message with checksum
   }
 };
@@ -497,6 +537,8 @@ class ScidacReader : public GridLimeReader {
 
 class IldgWriter : public ScidacWriter {
  public:
+  
+  IldgWriter(bool isboss) : ScidacWriter(isboss) {};
 
   ///////////////////////////////////
   // A little helper

lib/parallelIO/IldgIOtypes.h

@@ -136,8 +136,9 @@ struct scidacRecord : Serializable {
 				  int, typesize,
 				  int, datacount);
 
-  scidacRecord() { version =1.0; }
+  scidacRecord()
-
+  : version(1.0), recordtype(0), colors(0), spins(0), typesize(0), datacount(0)
+  {}
 };
 
 ////////////////////////

lib/parallelIO/MetaData.h

@@ -81,18 +81,16 @@ namespace Grid {
 				      std::string, creation_date,
 				      std::string, archive_date,
 				      std::string, floating_point);
-      FieldMetaData(void) { 
+      // WARNING: non-initialised values might lead to twisted parallel IO
-	nd=4;
+      // issues, std::string are fine because they initliase to size 0
-	dimension.resize(4);
+      // as per C++ standard.
-	boundary.resize(4);
+      FieldMetaData(void) 
-	scidac_checksuma=0;
+      : nd(4), dimension(4,0), boundary(4, ""), data_start(0),
-	scidac_checksumb=0;
+      link_trace(0.), plaquette(0.), checksum(0),
-	checksum=0;
+      scidac_checksuma(0), scidac_checksumb(0), sequence_number(0)
-      }
+      {}
     };
 
-
-
   namespace QCD {
 
     using namespace Grid;

lib/qcd/hmc/checkpointers/ILDGCheckpointer.h

@@ -74,10 +74,10 @@ class ILDGHmcCheckpointer : public BaseHmcCheckpointer<Implementation> {
     if ((traj % Params.saveInterval) == 0) {
       std::string config, rng;
       this->build_filenames(traj, Params, config, rng);
-      
+      GridBase *grid = U._grid;
       uint32_t nersc_csum,scidac_csuma,scidac_csumb;
       BinaryIO::writeRNG(sRNG, pRNG, rng, 0,nersc_csum,scidac_csuma,scidac_csumb);
-      IldgWriter _IldgWriter;
+      IldgWriter _IldgWriter(grid->IsBoss());
       _IldgWriter.open(config);
       _IldgWriter.writeConfiguration(U, traj, config, config);
       _IldgWriter.close();

tests/IO/Test_ildg_io.cc

@@ -79,7 +79,7 @@ int main (int argc, char ** argv)
   std::cout <<GridLogMessage<<"** Writing out  ILDG conf    *********"<<std::endl;
   std::cout <<GridLogMessage<<"**************************************"<<std::endl;
   std::string file("./ckpoint_ildg.4000");
-  IldgWriter _IldgWriter;
+  IldgWriter _IldgWriter(Fine.IsBoss());
   _IldgWriter.open(file);
   _IldgWriter.writeConfiguration(Umu,4000,std::string("dummy_ildg_LFN"),std::string("dummy_config"));
   _IldgWriter.close();

tests/lanczos/Test_dwf_compressed_lanczos_reorg.cc

@@ -58,7 +58,7 @@ public:
   {
     assert(this->subspace.size()==nbasis);
     emptyUserRecord record;
-    Grid::QCD::ScidacWriter WR;
+    Grid::QCD::ScidacWriter WR(this->_FineGrid->IsBoss());
     WR.open(evecs_file);
     for(int k=0;k<nbasis;k++) {
       WR.writeScidacFieldRecord(this->subspace[k],record);
@@ -96,7 +96,7 @@ public:
   {
     int n = this->evec_coarse.size();
     emptyUserRecord record;
-    Grid::QCD::ScidacWriter WR;
+    Grid::QCD::ScidacWriter WR(this->_CoarseGrid->IsBoss());
     WR.open(evecs_file);
     for(int k=0;k<n;k++) {
       WR.writeScidacFieldRecord(this->evec_coarse[k],record);

tests/solver/Test_dwf_mrhs_cg.cc

@@ -114,7 +114,7 @@ int main (int argc, char ** argv)
 
   {
     FGrid->Barrier();
-    ScidacWriter _ScidacWriter;
+    ScidacWriter _ScidacWriter(FGrid->IsBoss());
     _ScidacWriter.open(file);
     std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     std::cout << GridLogMessage << " Writing out gauge field "<<std::endl;
@@ -144,7 +144,7 @@ int main (int argc, char ** argv)
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
 
       std::stringstream filefn;      filefn << filef << "."<< n;
-      ScidacWriter _ScidacWriter;
+      ScidacWriter _ScidacWriter(FGrid->IsBoss());
       _ScidacWriter.open(filefn.str());
       _ScidacWriter.writeScidacFieldRecord(src[n],record);
       _ScidacWriter.close();