Adding a barrier in the RNG save

configure.ac

@@ -359,4 +359,5 @@ Summary of configuration for $PACKAGE v$VERSION
 - LIBS:
 `echo ${LIBS} | tr ' ' '\n' | sed 's/^-/    -/g'`
 -------------------------------------------------------
-"
+" > grid.configure.summary
+cat grid.configure.summary

lib/lattice/Lattice_rng.h

@@ -291,24 +291,24 @@ namespace Grid {
 
       for(int gidx=0;gidx<gsites;gidx++){
 
-  int rank,o_idx,i_idx;
+        int rank,o_idx,i_idx;
-  _grid->GlobalIndexToGlobalCoor(gidx,gcoor);
+      	_grid->GlobalIndexToGlobalCoor(gidx,gcoor);
-  _grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor);
+      	_grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor);
 
-  int l_idx=generator_idx(o_idx,i_idx);
+      	int l_idx=generator_idx(o_idx,i_idx);
-  
+      	
-  std::vector<int> site_seeds(4);
+      	std::vector<int> site_seeds(4);
-  for(int i=0;i<4;i++){
+      	for(int i=0;i<4;i++){
-    site_seeds[i]= ui(pseeder);
+          site_seeds[i]= ui(pseeder);
-  }
+      	}
 
-  _grid->Broadcast(0,(void *)&site_seeds[0],sizeof(int)*site_seeds.size());
+      	_grid->Broadcast(0,(void *)&site_seeds[0],sizeof(int)*site_seeds.size());
 
-  if( rank == _grid->ThisRank() ){
+      	if( rank == _grid->ThisRank() ){
-    fixedSeed ssrc(site_seeds);
+          fixedSeed ssrc(site_seeds);
-    typename source::result_type sinit = ssrc();
+          typename source::result_type sinit = ssrc();
-    _generators[l_idx] = RngEngine(sinit);
+          _generators[l_idx] = RngEngine(sinit);
-  }
+      	}
       }
       _seeded=1;
     }    

lib/parallelIO/BinaryIO.h

@@ -258,7 +258,7 @@ class BinaryIO {
   template<class vobj,class fobj,class munger> 
   static inline uint32_t writeObjectSerial(Lattice<vobj> &Umu,std::string file,munger munge,int offset,const std::string & format)
   {
-  	typedef typename vobj::scalar_object sobj;
+        typedef typename vobj::scalar_object sobj;
 
     GridBase *grid = Umu._grid;
 
@@ -296,14 +296,14 @@ class BinaryIO {
       
       if ( grid->IsBoss() ) {
 
-      	if(ieee32big) htobe32_v((void *)&file_object,sizeof(file_object));
+        if(ieee32big) htobe32_v((void *)&file_object,sizeof(file_object));
-      	if(ieee32)    htole32_v((void *)&file_object,sizeof(file_object));
+        if(ieee32)    htole32_v((void *)&file_object,sizeof(file_object));
-      	if(ieee64big) htobe64_v((void *)&file_object,sizeof(file_object));
+        if(ieee64big) htobe64_v((void *)&file_object,sizeof(file_object));
-      	if(ieee64)    htole64_v((void *)&file_object,sizeof(file_object));
+        if(ieee64)    htole64_v((void *)&file_object,sizeof(file_object));
 
-  			// NB could gather an xstrip as an optimisation.
+                        // NB could gather an xstrip as an optimisation.
-      	fout.write((char *)&file_object,sizeof(file_object));
+        fout.write((char *)&file_object,sizeof(file_object));
-      	bytes+=sizeof(file_object);
+        bytes+=sizeof(file_object);
       }
     }}}}
     timer.Stop();
@@ -326,7 +326,7 @@ class BinaryIO {
     //////////////////////////////////////////////////
     std::ofstream fout;
     if (grid->IsBoss()) {
-      fout.open(file, std::ios::binary | std::ios::out | std::ios::in);
+      fout.open(file, std::ios::binary | std::ios::out);
       if (!fout.is_open()) {
         std::cout << GridLogMessage << "writeRNGSerial: Error opening file " << file << std::endl;
         exit(0);// write better error handling
@@ -348,20 +348,23 @@ class BinaryIO {
       grid->GlobalIndexToGlobalCoor(gidx, gcoor);
       grid->GlobalCoorToRankIndex(rank, o_idx, i_idx, gcoor);
       int l_idx = parallel.generator_idx(o_idx, i_idx);
-      std::cout << GridLogDebug << "l_idx "<< l_idx << " o_idx " << o_idx << " i_idx " << i_idx << std::endl;
+      std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx
-
+                << " i_idx " << i_idx << " rank " << rank << std::endl;
       if (rank == grid->ThisRank()) {
         parallel.GetState(saved, l_idx);
       }
       grid->Broadcast(rank, (void *)&saved[0], bytes);
 
+      grid->Barrier();  // necessary?
       if (grid->IsBoss()) {
         std::cout << "Saved: " << saved << std::endl;
         Uint32Checksum((uint32_t *)&saved[0], bytes, csum);
         fout.write((char *)&saved[0], bytes);
       }
+      grid->Barrier();  // this can be necessary
     }
 
+  
     if (grid->IsBoss()) {
       serial.GetState(saved, 0);
       Uint32Checksum((uint32_t *)&saved[0], bytes, csum);
@@ -376,8 +379,7 @@ class BinaryIO {
     timer.Stop();
 
     std::cout << GridLogMessage   << "RNG file checksum " << std::hex << csum << std::dec << std::endl;
-    std::cout << GridLogMessage << "RNG state saved in " << timer.Elapsed() << " "
+    std::cout << GridLogMessage << "RNG state saved in " << timer.Elapsed() << std::endl;
-              << timer.useconds() <<" us"  <<std::endl;
     return csum;
   }
 
@@ -420,17 +422,19 @@ class BinaryIO {
       grid->GlobalIndexToGlobalCoor(gidx,gcoor);
       grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor);
       int l_idx=parallel.generator_idx(o_idx,i_idx);
-      std::cout << GridLogDebug << "l_idx "<< l_idx << " o_idx " << o_idx << " i_idx " << i_idx << std::endl;
+      std::cout << GridLogDebug << "l_idx " << l_idx << " o_idx " << o_idx
+                << " i_idx " << i_idx << " rank " << rank << std::endl;
 
       if ( grid->IsBoss() ) {
         fin.read((char *)&saved[0],bytes);
+        std::cout << "Saved: " << saved << std::endl;
         Uint32Checksum((uint32_t *)&saved[0],bytes,csum);
       }
+      
       grid->Broadcast(0,(void *)&saved[0],bytes);
 
       if( rank == grid->ThisRank() ){
         parallel.SetState(saved,l_idx);
-        std::cout << "Saved: " << saved << std::endl;
       }
     }
 

lib/qcd/hmc/GenericHMCrunner.h

@@ -139,10 +139,12 @@ namespace QCD {
       GridParallelRNG pRNG(UGrid);
       Field           U(UGrid);
 
+
       typedef MinimumNorm2<Implementation,
                            SmearingPolicy,
                            RepresentationsPolicy>
           IntegratorType; // change here to change the algorithm
+          
       IntegratorType MDynamics(UGrid, MDparameters, TheAction, Smearing);
 
       HMCparameters HMCpar;

lib/serialisation/TextIO.cc

@@ -6,8 +6,9 @@
 
     Copyright (C) 2015
 
-Author: Antonin Portelli <antonin.portelli@me.com>
+    Author: Antonin Portelli <antonin.portelli@me.com>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu<guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -50,14 +51,19 @@ void TextWriter::indent(void)
 {
   for (int i = 0; i < level_; ++i)
   {
-    file_ << '\t';
+    file_ << '\t';//is this portable?
   }
 };
 
 // Reader implementation ///////////////////////////////////////////////////////
-TextReader::TextReader(const string &fileName)
+TextReader::TextReader(const string &fileName) 
-: file_(fileName, ios::in)
+{
-{}
+    file_.open(fileName, ios::in);
+    if (!file_.is_open()) {
+        std::cout << GridLogMessage << "TextReader: Error opening file " << fileName << std::endl;
+        exit(0);// write better error handling
+    } 
+}
 
 void TextReader::push(const string &s)
 {
@@ -78,7 +84,7 @@ void TextReader::checkIndent(void)
     file_.get(c);
     if (c != '\t')
     {
-      cerr << "mismatch on tab " << c << " level " << level_;
+      cerr << "TextReader: mismatch on tab " << c << " level " << level_;
       cerr << " i "<< i << endl;
       abort();
     }

tests/hmc/Test_hmc_EODWFRatio_Binary.cc

@@ -37,28 +37,28 @@ using namespace Grid::QCD;
 namespace Grid {
 namespace QCD {
 
-class HMCRunnerParameters : Serializable {
+  class HMCRunnerParameters : Serializable {
- public:
+  public:
-  GRID_SERIALIZABLE_CLASS_MEMBERS(HMCRunnerParameters,
+    GRID_SERIALIZABLE_CLASS_MEMBERS(HMCRunnerParameters,
-                                  double, beta,
+      double, beta,
-                                  double, mass,
+      double, mass,
-                                  int, MaxCGIterations,
+      int, MaxCGIterations,
-                                  double, StoppingCondition,
+      double, StoppingCondition,
-                                  bool, smearedAction,
+      bool, smearedAction,
-                                  int, SaveInterval,
+      int, SaveInterval,
-                                  std::string, format,
+      std::string, format,
-                                  std::string, conf_prefix,
+      std::string, conf_prefix,
-                                  std::string, rng_prefix,
+      std::string, rng_prefix,
-                                  double, rho,
+      double, rho,
-                                  int, SmearingLevels,
+      int, SmearingLevels,
-                                  );
+      );
 
-  HMCRunnerParameters() {}
+    HMCRunnerParameters() {}
-};
+  };
 
 // Derive from the BinaryHmcRunner (templated for gauge fields)
 class HmcRunner : public BinaryHmcRunner {
- public:
+public:
   void BuildTheAction(int argc, char **argv)
 
   {
@@ -70,7 +70,7 @@ class HmcRunner : public BinaryHmcRunner {
 
     UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
     UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
-  
+
     FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
     FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
@@ -87,12 +87,12 @@ class HmcRunner : public BinaryHmcRunner {
     RealD scale = 2.0;
     FermionAction DenOp(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,scale);
     FermionAction NumOp(U,*FGrid,*FrbGrid,*UGrid,*UrbGrid,pv,M5,scale);
-  
+
     double StoppingCondition = 1.0e-8;
     double MaxCGIterations = 10000;
     ConjugateGradient<FermionField>  CG(StoppingCondition,MaxCGIterations);
     TwoFlavourEvenOddRatioPseudoFermionAction<ImplPolicy> Nf2(NumOp, DenOp,CG,CG);
-  
+
     // Set smearing (true/false), default: false
     Nf2.is_smeared = true;
 
@@ -115,8 +115,7 @@ class HmcRunner : public BinaryHmcRunner {
     std::string format = std::string("IEEE64BIG");
     std::string conf_prefix = std::string("DWF_ckpoint_lat");
     std::string rng_prefix = std::string("DWF_ckpoint_rng");
-    BinaryHmcCheckpointer<BinaryHmcRunner::ImplPolicy> Checkpoint(
+    BinaryHmcCheckpointer<BinaryHmcRunner::ImplPolicy> Checkpoint(conf_prefix, rng_prefix, SaveInterval, format);
-        conf_prefix, rng_prefix, SaveInterval, format);
     // Can implement also a specific function in the hmcrunner
     // AddCheckpoint (...) that takes the same parameters + a string/tag
     // defining the type of the checkpointer
@@ -124,8 +123,7 @@ class HmcRunner : public BinaryHmcRunner {
     // Then force all checkpoint to have few common functions
     // return an object that is then passed to the Run function
 
-    PlaquetteLogger<BinaryHmcRunner::ImplPolicy> PlaqLog(
+    PlaquetteLogger<BinaryHmcRunner::ImplPolicy> PlaqLog(std::string("Plaquette"));
-        std::string("Plaquette"));
     ObservablesList.push_back(&PlaqLog);
     ObservablesList.push_back(&Checkpoint);
 
@@ -139,7 +137,7 @@ class HmcRunner : public BinaryHmcRunner {
 
     Run(argc, argv, Checkpoint, SmearingPolicy); 
     //Run(argc, argv, Checkpoint);  // no smearing
-  };
+};
 };
 }
 }
@@ -153,6 +151,7 @@ int main(int argc, char **argv) {
 
   HmcRunner TheHMC;
 
+
   // Seeds for the random number generators
   std::vector<int> SerSeed({1, 2, 3, 4, 5});
   std::vector<int> ParSeed({6, 7, 8, 9, 10});

tests/hmc/Test_hmc_WilsonGauge_Binary.cc

@@ -37,20 +37,22 @@ using namespace Grid::QCD;
 namespace Grid {
 namespace QCD {
 
+//Change here the type of reader
+typedef Grid::TextReader InputFileReader; 
+
+
 class HMCRunnerParameters : Serializable {
  public:
   GRID_SERIALIZABLE_CLASS_MEMBERS(HMCRunnerParameters,
                                   double, beta,
-                                  double, mass,
+                                  int, MDsteps,
-                                  int, MaxCGIterations,
+                                  double, TrajectorLength,
-                                  double, StoppingCondition,
-                                  bool, smearedAction,
                                   int, SaveInterval,
                                   std::string, format,
                                   std::string, conf_prefix,
                                   std::string, rng_prefix,
-                                  double, rho,
+                                  std::string, serial_seeds,
-                                  int, SmearingLevels,
+                                  std::string, parallel_seeds,
                                   );
 
   HMCRunnerParameters() {}
@@ -59,7 +61,8 @@ class HMCRunnerParameters : Serializable {
 // Derive from the BinaryHmcRunner (templated for gauge fields)
 class HmcRunner : public BinaryHmcRunner {
  public:
-  void BuildTheAction(int argc, char **argv)
+    HMCRunnerParameters HMCPar;
+    void BuildTheAction(int argc, char **argv)
 
   {
     typedef WilsonImplR ImplPolicy;
@@ -70,7 +73,7 @@ class HmcRunner : public BinaryHmcRunner {
     
 
     // Gauge action
-    WilsonGaugeActionR Waction(5.6);
+    WilsonGaugeActionR Waction(HMCPar.beta);
 
     // Collect actions
     ActionLevel<Field> Level1(1);
@@ -78,12 +81,8 @@ class HmcRunner : public BinaryHmcRunner {
     TheAction.push_back(Level1);
 
     // Add observables
-    int SaveInterval = 1;
-    std::string format = std::string("IEEE64BIG");
-    std::string conf_prefix = std::string("ckpoint_lat");
-    std::string rng_prefix = std::string("ckpoint_rng");
     BinaryHmcCheckpointer<BinaryHmcRunner::ImplPolicy> Checkpoint(
-        conf_prefix, rng_prefix, SaveInterval, format);
+        HMCPar.conf_prefix, HMCPar.rng_prefix, HMCPar.SaveInterval, HMCPar.format);
     // Can implement also a specific function in the hmcrunner
     // AddCheckpoint (...) that takes the same parameters + a string/tag
     // defining the type of the checkpointer
@@ -110,13 +109,18 @@ int main(int argc, char **argv) {
             << " threads" << std::endl;
 
   HmcRunner TheHMC;
+  InputFileReader Reader("input.wilson_gauge.params");
+  read(Reader, "HMC", TheHMC.HMCPar);
+
+  std::cout << GridLogMessage << TheHMC.HMCPar << std::endl;
 
   // Seeds for the random number generators
-  std::vector<int> SerSeed({1, 2, 3, 4, 5});
+  std::vector<int> SerSeed = strToVec<int>(TheHMC.HMCPar.serial_seeds);
-  std::vector<int> ParSeed({6, 7, 8, 9, 10});
+  std::vector<int> ParSeed = strToVec<int>(TheHMC.HMCPar.parallel_seeds);
+
   TheHMC.RNGSeeds(SerSeed, ParSeed);
 
-  TheHMC.MDparameters.set(20, 1.0);// MDsteps, traj length
+  TheHMC.MDparameters.set(TheHMC.HMCPar.MDsteps, TheHMC.HMCPar.TrajectorLength);
 
   TheHMC.BuildTheAction(argc, argv);