diff --git a/lib/Grid_communicator.h b/lib/Grid_communicator.h index f5baa413..84d080ae 100644 --- a/lib/Grid_communicator.h +++ b/lib/Grid_communicator.h @@ -99,6 +99,8 @@ class CartesianCommunicator { Broadcast(root,(void *)&data,sizeof(data)); }; + static void BroadcastWorld(int root,void* data, int bytes); + }; } diff --git a/lib/communicator/Grid_communicator_fake.cc b/lib/communicator/Grid_communicator_fake.cc index 5f4a4024..c5b77620 100644 --- a/lib/communicator/Grid_communicator_fake.cc +++ b/lib/communicator/Grid_communicator_fake.cc @@ -37,6 +37,9 @@ void CartesianCommunicator::Barrier(void) void CartesianCommunicator::Broadcast(int root,void* data, int bytes) { } +void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) +{ +} void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest) diff --git a/lib/communicator/Grid_communicator_mpi.cc b/lib/communicator/Grid_communicator_mpi.cc index bd53bd9c..6a3f69f2 100644 --- a/lib/communicator/Grid_communicator_mpi.cc +++ b/lib/communicator/Grid_communicator_mpi.cc @@ -92,5 +92,14 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes) communicator); } +void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) +{ + MPI_Bcast(data, + bytes, + MPI_BYTE, + root, + MPI_COMM_WORLD); +} + } diff --git a/lib/lattice/Grid_lattice_rng.h b/lib/lattice/Grid_lattice_rng.h index 24627c92..85d8c471 100644 --- a/lib/lattice/Grid_lattice_rng.h +++ b/lib/lattice/Grid_lattice_rng.h @@ -26,7 +26,70 @@ namespace Grid { } }; - class GridRNG { + class GridRNGbase { + + public: + + GridRNGbase() : _uniform{0,1}, _gaussian(0.0,1.0) {}; + + int _seeded; + // One generator per site. + // Uniform and Gaussian distributions from these generators. + std::vector _generators; + std::uniform_real_distribution _uniform; + std::normal_distribution _gaussian; + + + }; + + class GridSerialRNG : public GridRNGbase { + public: + + // FIXME ... do we require lockstep draws of randoms + // from all nodes keeping seeds consistent. + // place a barrier/broadcast in the fill routine + template void Seed(source &src) + { + typename source::result_type init = src(); + CartesianCommunicator::BroadcastWorld(0,(void *)&init,sizeof(init)); + _generators[0] = std::ranlux48(init); + _seeded=1; + } + + GridSerialRNG() : GridRNGbase() { + _generators.resize(1); + _seeded=0; + } + + + template inline void fill(sobj &l,distribution &dist){ + + typedef typename sobj::scalar_type scalar_type; + + int words = sizeof(sobj)/sizeof(scalar_type); + + scalar_type *buf = (scalar_type *) & l; + + for(int idx=0;idx &seeds){ + fixedSeed src(seeds); + Seed(src); + } + + }; + + class GridParallelRNG : public GridRNGbase { public: // One generator per site. std::vector _generators; @@ -42,25 +105,13 @@ namespace Grid { return is*_grid->oSites()+os; } - GridRNG(GridBase *grid) : _uniform{0,1}, _gaussian(0.0,1.0) { + GridParallelRNG(GridBase *grid) : GridRNGbase() { _grid=grid; _vol =_grid->iSites()*_grid->oSites(); _generators.resize(_vol); - // SeedFixedIntegers(seeds); - // worst case we seed properly but non-deterministically - SeedRandomDevice(); + _seeded=0; } - // FIXME: drive seeding from node zero and transmit to all - // to get unique randoms on each node - void SeedRandomDevice(void){ - std::random_device rd; - Seed(rd); - } - void SeedFixedIntegers(std::vector &seeds){ - fixedSeed src(seeds); - Seed(src); - } // This loop could be made faster to avoid the Ahmdahl by // i) seed generators on each timeslice, for x=y=z=0; @@ -86,6 +137,7 @@ namespace Grid { _generators[l_idx] = std::ranlux48(init); } } + _seeded=1; } //FIXME implement generic IO and create state save/restore @@ -122,15 +174,34 @@ namespace Grid { merge(l._odata[ss],pointers); } }; + + void SeedRandomDevice(void){ + std::random_device rd; + Seed(rd); + } + void SeedFixedIntegers(std::vector &seeds){ + fixedSeed src(seeds); + Seed(src); + } + }; - // FIXME Implement a consistent seed management strategy - template inline void random(GridRNG &rng,Lattice &l){ + + template inline void random(GridParallelRNG &rng,Lattice &l){ rng.fill(l,rng._uniform); } - template inline void gaussian(GridRNG &rng,Lattice &l){ + template inline void gaussian(GridParallelRNG &rng,Lattice &l){ rng.fill(l,rng._gaussian); } + + + template inline void random(GridSerialRNG &rng,sobj &l){ + rng.fill(l,rng._uniform); + } + template inline void gaussian(GridSerialRNG &rng,sobj &l){ + rng.fill(l,rng._gaussian); + } + } #endif diff --git a/lib/math/Grid_math_inner.h b/lib/math/Grid_math_inner.h index 88387db7..465501e1 100644 --- a/lib/math/Grid_math_inner.h +++ b/lib/math/Grid_math_inner.h @@ -6,6 +6,13 @@ namespace Grid { // innerProduct Vector x Vector -> Scalar // innerProduct Matrix x Matrix -> Scalar /////////////////////////////////////////////////////////////////////////////////////// + template inline RealD norm2l(sobj &arg){ + typedef typename sobj::scalar_type scalar; + decltype(innerProduct(arg,arg)) nrm; + nrm = innerProduct(arg,arg); + return real(nrm); + } + template inline auto innerProduct (const iVector& lhs,const iVector& rhs) -> iScalar { diff --git a/lib/qcd/Grid_qcd_dirac.h b/lib/qcd/Grid_qcd_dirac.h index 945178e9..03dc9cc5 100644 --- a/lib/qcd/Grid_qcd_dirac.h +++ b/lib/qcd/Grid_qcd_dirac.h @@ -76,15 +76,15 @@ namespace QCD { ret(i,0) = timesMinusI(rhs(i,3)); ret(i,1) = timesMinusI(rhs(i,2)); ret(i,2) = timesI(rhs(i,1)); - ret(i,3) = timesI(rhs(i,1)); + ret(i,3) = timesI(rhs(i,0)); } }; template inline void multGammaX(iMatrix &ret, const iMatrix &rhs){ for(int i=0;i inline void multMinusGammaX(iMatrix &ret, const iMatrix &rhs){ diff --git a/tests/Grid_cshift.cc b/tests/Grid_cshift.cc index 3fe2b9cc..995756d6 100644 --- a/tests/Grid_cshift.cc +++ b/tests/Grid_cshift.cc @@ -14,7 +14,8 @@ int main (int argc, char ** argv) std::vector latt_size ({8,8,8,16}); GridCartesian Fine(latt_size,simd_layout,mpi_layout); - GridRNG FineRNG(&Fine); + GridParallelRNG FineRNG(&Fine); + FineRNG.SeedRandomDevice(); LatticeComplex U(&Fine); LatticeComplex ShiftU(&Fine); diff --git a/tests/Grid_gamma.cc b/tests/Grid_gamma.cc index d3a33039..758f9faa 100644 --- a/tests/Grid_gamma.cc +++ b/tests/Grid_gamma.cc @@ -16,9 +16,15 @@ int main (int argc, char ** argv) GridCartesian Grid(latt_size,simd_layout,mpi_layout); - GridRNG RNG(&Grid); + GridParallelRNG pRNG(&Grid); + pRNG.SeedRandomDevice(); + + GridSerialRNG sRNG; + sRNG.SeedRandomDevice(); SpinMatrix ident=zero; + SpinMatrix rnd ; random(sRNG,rnd); + SpinMatrix ll=zero; SpinMatrix rr=zero; SpinMatrix result; @@ -94,6 +100,29 @@ int main (int argc, char ** argv) } - + std::cout << "Testing Gamma^2 - 1 = 0"< U(4,&Fine); diff --git a/tests/Grid_stencil.cc b/tests/Grid_stencil.cc index 917cb558..f35581f6 100644 --- a/tests/Grid_stencil.cc +++ b/tests/Grid_stencil.cc @@ -48,7 +48,8 @@ int main (int argc, char ** argv) GridCartesian Fine(latt_size,simd_layout,mpi_layout); GridRedBlackCartesian rbFine(latt_size,simd_layout,mpi_layout); - GridRNG fRNG(&Fine); + GridParallelRNG fRNG(&Fine); + fRNG.SeedRandomDevice(); LatticeColourMatrix Foo(&Fine); LatticeColourMatrix Bar(&Fine);