From 571096632421651b623a9516902d5dce9eeeb4e4 Mon Sep 17 00:00:00 2001 From: paboyle Date: Sat, 19 Dec 2015 18:32:25 +0000 Subject: [PATCH] Options to use mersenne twister OR ranlux48 via --enable-rng flag at configure time. Can save and restore RNG state via new (serial) I/O routines in a NERSC header style file. Store a Parallel (one per site) and a single serial RNG file. --- configure | 40 +++++++++++++++ configure.ac | 28 ++++++++++ lib/Config.h.in | 6 +++ lib/lattice/Lattice_rng.h | 40 ++++++++++++--- lib/parallelIO/BinaryIO.h | 105 ++++++++++++++++++++++++++++++++++++++ lib/parallelIO/NerscIO.h | 90 +++++++++++++++++++++++++++++--- tests/Test_main.cc | 20 +++++--- tests/Test_nersc_io.cc | 24 +++++++++ 8 files changed, 331 insertions(+), 22 deletions(-) diff --git a/configure b/configure index 2f14088e..cf70e492 100755 --- a/configure +++ b/configure @@ -749,6 +749,7 @@ enable_openmp enable_simd enable_precision enable_comms +enable_rng enable_chroma ' ac_precious_vars='build_alias @@ -1395,6 +1396,8 @@ Optional Features: --enable-precision=single|double Select default word size of Real --enable-comms=none|mpi Select communications + --enable-rng=ranlux48|mt19937 + Select Random Number Generator to be used --enable-chroma Expect chroma compiled under c++11 Some influential environment variables: @@ -6387,6 +6390,10 @@ done #Please install or provide the correct path to your installation #Info at: http://www.mpfr.org/)]) +# +# SIMD instructions selection +# + # Check whether --enable-simd was given. if test "${enable_simd+set}" = set; then : enableval=$enable_simd; \ @@ -6519,6 +6526,10 @@ $as_echo "#define GRID_DEFAULT_PRECISION_DOUBLE 1" >>confdefs.h ;; esac +# +# Comms selection +# + # Check whether --enable-comms was given. if test "${enable_comms+set}" = set; then : enableval=$enable_comms; ac_COMMS=${enable_comms} @@ -6562,6 +6573,35 @@ else fi +# +# RNG selection +# +# Check whether --enable-rng was given. +if test "${enable_rng+set}" = set; then : + enableval=$enable_rng; \ + ac_RNG=${enable_rng} +else + ac_RNG=ranlux48 +fi + +case ${ac_RNG} in + ranlux48) + +$as_echo "#define RNG_RANLUX 1" >>confdefs.h + + ;; + mt19937) + +$as_echo "#define RNG_MT19937 1" >>confdefs.h + + ;; + *) + as_fn_error $? "${ac_RNG} unsupported --enable-rng option" "$LINENO" 5; + ;; +esac +# +# Chroma regression tests +# # Check whether --enable-chroma was given. if test "${enable_chroma+set}" = set; then : enableval=$enable_chroma; ac_CHROMA=yes diff --git a/configure.ac b/configure.ac index 094a70bd..6631a12a 100644 --- a/configure.ac +++ b/configure.ac @@ -66,6 +66,10 @@ AC_CHECK_FUNCS([gettimeofday]) #Please install or provide the correct path to your installation #Info at: http://www.mpfr.org/)]) +# +# SIMD instructions selection +# + AC_ARG_ENABLE([simd],[AC_HELP_STRING([--enable-simd=SSE4|AVX|AVXFMA4|AVX2|AVX512|IMCI],\ [Select instructions to be SSE4.0, AVX 1.0, AVX 2.0+FMA, AVX 512, IMCI])],\ [ac_SIMD=${enable_simd}],[ac_SIMD=AVX2]) @@ -160,6 +164,10 @@ case ${ac_PRECISION} in ;; esac +# +# Comms selection +# + AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi],[Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none]) case ${ac_COMMS} in @@ -179,6 +187,26 @@ esac AM_CONDITIONAL(BUILD_COMMS_MPI,[ test "X${ac_COMMS}X" == "XmpiX" ]) AM_CONDITIONAL(BUILD_COMMS_NONE,[ test "X${ac_COMMS}X" == "XnoneX" ]) +# +# RNG selection +# +AC_ARG_ENABLE([rng],[AC_HELP_STRING([--enable-rng=ranlux48|mt19937],\ + [Select Random Number Generator to be used])],\ + [ac_RNG=${enable_rng}],[ac_RNG=ranlux48]) +case ${ac_RNG} in + ranlux48) + AC_DEFINE([RNG_RANLUX],[1],[RNG_RANLUX] ) + ;; + mt19937) + AC_DEFINE([RNG_MT19937],[1],[RNG_MT19937] ) + ;; + *) + AC_MSG_ERROR([${ac_RNG} unsupported --enable-rng option]); + ;; +esac +# +# Chroma regression tests +# AC_ARG_ENABLE([chroma],[AC_HELP_STRING([--enable-chroma],[Expect chroma compiled under c++11 ])],ac_CHROMA=yes,ac_CHROMA=no) case ${ac_CHROMA} in diff --git a/lib/Config.h.in b/lib/Config.h.in index 4704273d..de782626 100644 --- a/lib/Config.h.in +++ b/lib/Config.h.in @@ -143,6 +143,12 @@ /* Define to the version of this package. */ #undef PACKAGE_VERSION +/* RNG_MT19937 */ +#undef RNG_MT19937 + +/* RNG_RANLUX */ +#undef RNG_RANLUX + /* SSE4 Intrinsics */ #undef SSE4 diff --git a/lib/lattice/Lattice_rng.h b/lib/lattice/Lattice_rng.h index 895d49ad..b35bad30 100644 --- a/lib/lattice/Lattice_rng.h +++ b/lib/lattice/Lattice_rng.h @@ -81,11 +81,39 @@ namespace Grid { int _seeded; // One generator per site. // Uniform and Gaussian distributions from these generators. - std::vector _generators; +#ifdef RNG_RANLUX + typedef uint64_t RngStateType; + typedef std::ranlux48 RngEngine; + static const int RngStateCount = 15; +#else + typedef std::mt19937 RngEngine; + typedef uint32_t RngStateType; + static const int RngStateCount = std::mt19937::state_size; +#endif + std::vector _generators; std::uniform_real_distribution _uniform; std::normal_distribution _gaussian; - + void GetState(std::vector & saved,int gen) { + saved.resize(RngStateCount); + std::stringstream ss; + // std::cout << _generators[gen]<>saved[i]; + } + } + void SetState(std::vector & saved,int gen){ + assert(saved.size()==RngStateCount); + std::stringstream ss; + for(int i=0;i>_generators[gen]; + // std::cout << _generators[gen]< _generators; // Uniform and Gaussian distributions from these generators. std::uniform_real_distribution _uniform; @@ -220,7 +246,7 @@ namespace Grid { int gsites = _grid->_gsites; typename source::result_type init = src(); - std::ranlux48 pseeder(init); + RngEngine pseeder(init); std::uniform_int_distribution ui; for(int gidx=0;gidxThisRank() ){ fixedSeed ssrc(site_seeds); typename source::result_type sinit = ssrc(); - _generators[l_idx] = std::ranlux48(sinit); + _generators[l_idx] = RngEngine(sinit); } } _seeded=1; diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h index 853544e1..d67588e9 100644 --- a/lib/parallelIO/BinaryIO.h +++ b/lib/parallelIO/BinaryIO.h @@ -232,6 +232,110 @@ class BinaryIO { return csum; } + static inline uint32_t writeRNGSerial(GridSerialRNG &serial,GridParallelRNG ¶llel,std::string file,int offset) + { + typedef typename GridSerialRNG::RngStateType RngStateType; + const int RngStateCount = GridSerialRNG::RngStateCount; + + GridBase *grid = parallel._grid; + int gsites = grid->_gsites; + + ////////////////////////////////////////////////// + // Serialise through node zero + ////////////////////////////////////////////////// + std::cout<< GridLogMessage<< "Serial RNG write I/O "<< file<IsBoss() ) { + fout.open(file,std::ios::binary|std::ios::out|std::ios::in); + fout.seekp(offset); + } + + uint32_t csum=0; + std::vector saved(RngStateCount); + int bytes = sizeof(RngStateType)*saved.size(); + std::vector gcoor; + + for(int gidx=0;gidxGlobalIndexToGlobalCoor(gidx,gcoor); + grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor); + int l_idx=parallel.generator_idx(o_idx,i_idx); + + if( rank == grid->ThisRank() ){ + // std::cout << "rank" << rank<<" Getting state for index "<Broadcast(rank,(void *)&saved[0],bytes); + + if ( grid->IsBoss() ) { + Uint32Checksum((uint32_t *)&saved[0],bytes,csum); + fout.write((char *)&saved[0],bytes); + } + + } + + if ( grid->IsBoss() ) { + serial.GetState(saved,0); + Uint32Checksum((uint32_t *)&saved[0],bytes,csum); + fout.write((char *)&saved[0],bytes); + } + + return csum; + } + static inline uint32_t readRNGSerial(GridSerialRNG &serial,GridParallelRNG ¶llel,std::string file,int offset) + { + typedef typename GridSerialRNG::RngStateType RngStateType; + const int RngStateCount = GridSerialRNG::RngStateCount; + + GridBase *grid = parallel._grid; + int gsites = grid->_gsites; + + ////////////////////////////////////////////////// + // Serialise through node zero + ////////////////////////////////////////////////// + std::cout<< GridLogMessage<< "Serial RNG read I/O "<< file< saved(RngStateCount); + int bytes = sizeof(RngStateType)*saved.size(); + std::vector gcoor; + + for(int gidx=0;gidxGlobalIndexToGlobalCoor(gidx,gcoor); + grid->GlobalCoorToRankIndex(rank,o_idx,i_idx,gcoor); + int l_idx=parallel.generator_idx(o_idx,i_idx); + + if ( grid->IsBoss() ) { + fin.read((char *)&saved[0],bytes); + Uint32Checksum((uint32_t *)&saved[0],bytes,csum); + } + + grid->Broadcast(0,(void *)&saved[0],bytes); + + if( rank == grid->ThisRank() ){ + parallel.SetState(saved,l_idx); + } + + } + + if ( grid->IsBoss() ) { + fin.read((char *)&saved[0],bytes); + serial.SetState(saved,0); + Uint32Checksum((uint32_t *)&saved[0],bytes,csum); + } + + return csum; + } + + template static inline uint32_t readObjectParallel(Lattice &Umu,std::string file,munger munge,int offset,const std::string &format) { @@ -284,6 +388,7 @@ class BinaryIO { { uint32_t tmp = IOnode; grid->GlobalSum(tmp); + std::cout<< std::dec ; std::cout<< GridLogMessage<< "Parallel read I/O to "<< file << " with " <_ndimension;d++){ std::cout<< range[d]; diff --git a/lib/parallelIO/NerscIO.h b/lib/parallelIO/NerscIO.h index a442f20b..4af7cad7 100644 --- a/lib/parallelIO/NerscIO.h +++ b/lib/parallelIO/NerscIO.h @@ -181,9 +181,12 @@ inline void NerscMachineCharacteristics(NerscField &header) class NerscIO : public BinaryIO { public: + static inline void truncate(std::string file){ + std::ofstream fout(file,std::ios::out); + } static inline unsigned int writeHeader(NerscField &field,std::string file) { - std::ofstream fout(file,std::ios::out); + std::ofstream fout(file,std::ios::out|std::ios::in); fout.seekp(0,std::ios::beg); fout << "BEGIN_HEADER" << std::endl; @@ -201,8 +204,7 @@ class NerscIO : public BinaryIO { fout << "BOUNDARY_"< > &Umu std::cout << GridLogMessage << " TESTING PARALLEL WRITE offsets " << offset1 << " "<< offset << std::endl; - std::cout << GridLogMessage < > &Umu std::cout< + uint32_t csum=BinaryIO::readRNGSerial(serial,parallel,file,offset); + + assert(csum == header.checksum ); + + std::cout< saved; + SerialRNG.GetState(saved,0); + SerialRNG1.SetState(saved,0); + + RealD dd1,dd2; + + std::cout << "Testing RNG state save restore"<