Debugging the RNG state save

lib/parallelIO/BinaryIO.h

@@ -7,7 +7,7 @@
     Copyright (C) 2015
 
     Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: paboyle <paboyle@ph.ed.ac.uk>
+    Author: Guido Cossu<guido.cossu@ed.ac.uk>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -313,15 +313,14 @@ class BinaryIO {
     return csum;
   }
 
-  static inline uint32_t writeRNGSerial(GridSerialRNG &serial,
+  static inline uint32_t writeRNGSerial(GridSerialRNG &serial, GridParallelRNG &parallel, std::string file, int offset) {
-                                        GridParallelRNG &parallel,
-                                        std::string file, int offset) {
     typedef typename GridSerialRNG::RngStateType RngStateType;
     const int RngStateCount = GridSerialRNG::RngStateCount;
 
     GridBase *grid = parallel._grid;
     int gsites = grid->_gsites;
 
+    GridStopWatch timer; timer.Start();
     //////////////////////////////////////////////////
     // Serialise through node zero
     //////////////////////////////////////////////////
@@ -329,19 +328,21 @@ class BinaryIO {
     if (grid->IsBoss()) {
       fout.open(file, std::ios::binary | std::ios::out | std::ios::in);
       if (!fout.is_open()) {
-        std::cout << GridLogMessage << "writeRNGSerial: Error opening file "
+        std::cout << GridLogMessage << "writeRNGSerial: Error opening file " << file << std::endl;
-                  << file << std::endl;
+        exit(0);// write better error handling
-        exit(0);
       }
       fout.seekp(offset);
     }
 
-    std::cout << GridLogMessage << "Serial RNG write I/O " << file << std::endl;
+    std::cout << GridLogMessage << "Serial RNG write I/O on file " << file << std::endl;
     uint32_t csum = 0;
     std::vector<RngStateType> saved(RngStateCount);
     int bytes = sizeof(RngStateType) * saved.size();
+    std::cout << GridLogDebug << "RngStateCount: " << RngStateCount << std::endl;
+    std::cout << GridLogDebug << "Type has " << bytes << " bytes" << std::endl;
     std::vector<int> gcoor;
 
+    std::cout << GridLogDebug << "gsites: " << gsites << " loop" << std::endl;
     for (int gidx = 0; gidx < gsites; gidx++) {
       int rank, o_idx, i_idx;
       grid->GlobalIndexToGlobalCoor(gidx, gcoor);
@@ -349,12 +350,10 @@ class BinaryIO {
       int l_idx = parallel.generator_idx(o_idx, i_idx);
 
       if (rank == grid->ThisRank()) {
-        //  std::cout << "rank" << rank<<" Getting state for index
-        //  "<<l_idx<<std::endl;
         parallel.GetState(saved, l_idx);
-      }
 
         grid->Broadcast(rank, (void *)&saved[0], bytes);
+      }
 
       if (grid->IsBoss()) {
         Uint32Checksum((uint32_t *)&saved[0], bytes, csum);
@@ -367,15 +366,21 @@ class BinaryIO {
       Uint32Checksum((uint32_t *)&saved[0], bytes, csum);
       fout.write((char *)&saved[0], bytes);
     }
+
     grid->Broadcast(0, (void *)&csum, sizeof(csum));
 
     if (grid->IsBoss()) 
       fout.close();
 
-    std::cout << GridLogMessage << "RNG file checksum " << std::hex << csum << std::dec << std::endl;
+    timer.Stop();
 
+    std::cout << GridLogMessage   << "RNG file checksum " << std::hex << csum << std::dec << std::endl;
+    std::cout << GridLogMessage << "RNG state saved in " << timer.Elapsed() << " "
+              << timer.useconds() <<" us"  <<std::endl;
     return csum;
   }
+
+
   static inline uint32_t readRNGSerial(GridSerialRNG &serial,GridParallelRNG &parallel,std::string file,int offset)
   {
     typedef typename GridSerialRNG::RngStateType RngStateType;
@@ -387,16 +392,27 @@ class BinaryIO {
     //////////////////////////////////////////////////
     // Serialise through node zero
     //////////////////////////////////////////////////
-    std::cout<< GridLogMessage<< "Serial RNG read I/O "<< file<<std::endl;
+    std::cout<< GridLogMessage<< "Serial RNG read I/O of file "<<file<<std::endl;
 
-    std::ifstream fin(file,std::ios::binary|std::ios::in);
+    std::ifstream fin;
+    if (grid->IsBoss()) {
+      fin.open(file, std::ios::binary | std::ios::in);
+      if (!fin.is_open()) {
+        std::cout << GridLogMessage << "readRNGSerial: Error opening file " << file << std::endl;
+        exit(0);// write better error handling
+      }
       fin.seekg(offset);
+    }
+
     
     uint32_t csum=0;
     std::vector<RngStateType> saved(RngStateCount);
     int bytes = sizeof(RngStateType)*saved.size();
+    std::cout << GridLogDebug << "RngStateCount: " << RngStateCount << std::endl;
+    std::cout << GridLogDebug << "Type has " << bytes << " bytes" << std::endl;
     std::vector<int> gcoor;
 
+    std::cout << GridLogDebug << "gsites: " << gsites << " loop" << std::endl;
     for(int gidx=0;gidx<gsites;gidx++){
 
       int rank,o_idx,i_idx;
@@ -407,14 +423,14 @@ class BinaryIO {
       if ( grid->IsBoss() ) {
         fin.read((char *)&saved[0],bytes);
         Uint32Checksum((uint32_t *)&saved[0],bytes,csum);
-      }
+
 
         grid->Broadcast(0,(void *)&saved[0],bytes);
+      }
 
       if( rank == grid->ThisRank() ){
         parallel.SetState(saved,l_idx);
       }
-
     }
 
     if ( grid->IsBoss() ) {
@@ -523,11 +539,14 @@ class BinaryIO {
 
       Lexicographic::CoorFromIndex(tsite,tlex,range);
 
-      for(int d=0;d<nd;d++){
+      for(int d=0; d<nd; d++)
+      {
+
         lsite[d] = tsite[d]%grid->_ldimensions[d];  // local site
         gsite[d] = tsite[d]+start[d];               // global site
       }
 
+
       /////////////////////////
       // Get the rank of owner of data
       /////////////////////////

lib/qcd/QCD.h

@@ -46,7 +46,7 @@ namespace QCD {
 
     static const int Nc=3;
     static const int Ns=4;
-    static const int Nd=5;
+    static const int Nd=4;
     static const int Nhs=2; // half spinor
     static const int Nds=8; // double stored gauge field
     static const int Ngp=2; // gparity index range