From 839605c45c4da742f074f807d481d3d05761a27c Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 7 Nov 2018 23:38:46 +0000 Subject: [PATCH 1/4] Verbose reduce --- Grid/communicator/Communicator_mpi3.cc | 6 ------ Grid/communicator/SharedMemoryMPI.cc | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Grid/communicator/Communicator_mpi3.cc b/Grid/communicator/Communicator_mpi3.cc index 424b7973..e6800afe 100644 --- a/Grid/communicator/Communicator_mpi3.cc +++ b/Grid/communicator/Communicator_mpi3.cc @@ -50,8 +50,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) assert(0); } - Grid_quiesce_nodes(); - // Never clean up as done once. MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world); @@ -124,10 +122,8 @@ CartesianCommunicator::CartesianCommunicator(const std::vector &processors, // split the communicator ////////////////////////////////////////////////////////////////////////////////////////////////////// // int Nparent = parent._processors ; - // std::cout << " splitting from communicator "< &processors, int Nchild = Nparent/childsize; assert (childsize * Nchild == Nparent); - // std::cout << " child size "< ccoor(_ndimension); // coor within subcommunicator std::vector scoor(_ndimension); // coor of split within parent std::vector ssize(_ndimension); // coor of split within parent diff --git a/Grid/communicator/SharedMemoryMPI.cc b/Grid/communicator/SharedMemoryMPI.cc index b425d97b..d3b094ad 100644 --- a/Grid/communicator/SharedMemoryMPI.cc +++ b/Grid/communicator/SharedMemoryMPI.cc @@ -413,7 +413,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags) assert(((uint64_t)ptr&0x3F)==0); close(fd); WorldShmCommBufs[r] =ptr; - std::cout << "Set WorldShmCommBufs["< Date: Wed, 7 Nov 2018 23:39:18 +0000 Subject: [PATCH 2/4] Allow shrinking machine in orthog direction for extract slice local --- Grid/lattice/Lattice_transfer.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Grid/lattice/Lattice_transfer.h b/Grid/lattice/Lattice_transfer.h index f988f310..4eca50d6 100644 --- a/Grid/lattice/Lattice_transfer.h +++ b/Grid/lattice/Lattice_transfer.h @@ -464,8 +464,10 @@ void InsertSliceLocal(const Lattice &lowDim, Lattice & higherDim,int assert(orthog>=0); for(int d=0;d_processors[d] == hg->_processors[d]); - assert(lg->_ldimensions[d] == hg->_ldimensions[d]); + if ( d!=orthog ) { + assert(lg->_processors[d] == hg->_processors[d]); + assert(lg->_ldimensions[d] == hg->_ldimensions[d]); + } } // the above should guarantee that the operations are local @@ -499,8 +501,10 @@ void ExtractSliceLocal(Lattice &lowDim, Lattice & higherDim,int slic assert(orthog>=0); for(int d=0;d_processors[d] == hg->_processors[d]); - assert(lg->_ldimensions[d] == hg->_ldimensions[d]); + if ( d!=orthog ) { + assert(lg->_processors[d] == hg->_processors[d]); + assert(lg->_ldimensions[d] == hg->_ldimensions[d]); + } } // the above should guarantee that the operations are local From 68c13045d6ce9982f9bc760c085e29f2c3eb8f2e Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 7 Nov 2018 23:40:15 +0000 Subject: [PATCH 3/4] Added a test for Felix and Michael to look at --- tests/debug/Test_split_laplacian.cc | 104 ++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/debug/Test_split_laplacian.cc diff --git a/tests/debug/Test_split_laplacian.cc b/tests/debug/Test_split_laplacian.cc new file mode 100644 index 00000000..174dc1b1 --- /dev/null +++ b/tests/debug/Test_split_laplacian.cc @@ -0,0 +1,104 @@ + /************************************************************************************* + + Grid physics library, www.github.com/paboyle/Grid + + Source file: ./tests/Test_dwf_mrhs_cg.cc + + Copyright (C) 2015 + +Author: Peter Boyle + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + See the full license in the file "LICENSE" in the top level distribution directory + *************************************************************************************/ + /* END LEGAL */ +#include + +using namespace std; +using namespace Grid; +using namespace Grid::QCD; + + +int main (int argc, char ** argv) +{ + typedef LatticeComplex ComplexField; + + Grid_init(&argc,&argv); + + std::vector latt_size = GridDefaultLatt(); + int nd = latt_size.size(); + int ndm1 = nd-1; + + std::vector simd_layout = GridDefaultSimd(nd,vComplex::Nsimd()); + std::vector mpi_layout = GridDefaultMpi(); + std::vector mpi_split (mpi_layout.size(),1); + + std::cout << " Full " << GridCmdVectorIntToString(latt_size) << " subgrid" < latt_m = latt_size; latt_m[nd-1] = 1; + std::vector mpi_m = mpi_layout; mpi_m [nd-1] = 1; + std::vector simd_m = GridDefaultSimd(ndm1,vComplex::Nsimd()); simd_m.push_back(1); + + + std::cout << " Requesting " << GridCmdVectorIntToString(latt_m)<< " subgrid" <GlobalSum(tmp); + std::cout << GridLogMessage<< " Full nodes "<< tmp <GlobalSum(tmp); + std::cout << GridLogMessage<< " Split nodes "<< tmp <Barrier(); + + auto local_latt = GridN->LocalDimensions(); + + Full_cpy = zero; + std::vector seeds({1,2,3,4}); + GridParallelRNG RNG(GridN); RNG.SeedFixedIntegers(seeds); + + random(RNG,Full); + for(int t=0;t Date: Thu, 8 Nov 2018 08:58:09 +0000 Subject: [PATCH 4/4] Hadrons: sequential Aslash insertion and propagator on A2A vector --- Hadrons/Modules.hpp | 1 + Hadrons/Modules/MSolver/A2AAslashVector.cc | 35 ++++ Hadrons/Modules/MSolver/A2AAslashVector.hpp | 189 ++++++++++++++++++++ Hadrons/modules.inc | 2 + 4 files changed, 227 insertions(+) create mode 100644 Hadrons/Modules/MSolver/A2AAslashVector.cc create mode 100644 Hadrons/Modules/MSolver/A2AAslashVector.hpp diff --git a/Hadrons/Modules.hpp b/Hadrons/Modules.hpp index da57ab2b..27a889bf 100644 --- a/Hadrons/Modules.hpp +++ b/Hadrons/Modules.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/Hadrons/Modules/MSolver/A2AAslashVector.cc b/Hadrons/Modules/MSolver/A2AAslashVector.cc new file mode 100644 index 00000000..27344397 --- /dev/null +++ b/Hadrons/Modules/MSolver/A2AAslashVector.cc @@ -0,0 +1,35 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: Hadrons/Modules/MSolver/A2AAslashVector.cc + +Copyright (C) 2015-2018 + +Author: Vera Guelpers + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ +#include + +using namespace Grid; +using namespace Hadrons; +using namespace MSolver; + +template class Grid::Hadrons::MSolver::TA2AAslashVector; +template class Grid::Hadrons::MSolver::TA2AAslashVector; diff --git a/Hadrons/Modules/MSolver/A2AAslashVector.hpp b/Hadrons/Modules/MSolver/A2AAslashVector.hpp new file mode 100644 index 00000000..c99a6f6b --- /dev/null +++ b/Hadrons/Modules/MSolver/A2AAslashVector.hpp @@ -0,0 +1,189 @@ +/************************************************************************************* + +Grid physics library, www.github.com/paboyle/Grid + +Source file: Hadrons/Modules/MSolver/A2AAslashVector.hpp + +Copyright (C) 2015-2018 + +Author: Vera Guelpers + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +See the full license in the file "LICENSE" in the top level distribution directory +*************************************************************************************/ +/* END LEGAL */ +#ifndef Hadrons_MSolver_A2AAslashVector_hpp_ +#define Hadrons_MSolver_A2AAslashVector_hpp_ + +#include +#include +#include +#include + +BEGIN_HADRONS_NAMESPACE + +/****************************************************************************** + * Create all-to-all V & W vectors * + ******************************************************************************/ +BEGIN_MODULE_NAMESPACE(MSolver) + +/**************************************************************************** +* Calculate a sequential propagator on an insertion of i*g_mu*A_mu +* on an A2A vector +* +* vv_i(y) = S(y,x) * i * g_mu*A_mu(x) * v_i(x) +* +* with +* +* - v: A2A vector v_i(x) +* - photon: A_mu(x): electromagnetic photon field +* - solver: the solver for calculating the sequential propagator +* +*****************************************************************************/ + + +class A2AAslashVectorPar: Serializable +{ +public: + GRID_SERIALIZABLE_CLASS_MEMBERS(A2AAslashVectorPar, + std::string, v, + std::string, photon, + std::string, solver); +}; + +template +class TA2AAslashVector : public Module +{ +public: + FERM_TYPE_ALIASES(FImpl,); + SOLVER_TYPE_ALIASES(FImpl,); +public: + typedef PhotonR::GaugeField EmField; +public: + // constructor + TA2AAslashVector(const std::string name); + // destructor + virtual ~TA2AAslashVector(void) {}; + // dependency relation + virtual std::vector getInput(void); + virtual std::vector getOutput(void); + // setup + virtual void setup(void); + // execution + virtual void execute(void); +private: + unsigned int Ls_; +}; + +MODULE_REGISTER_TMP(A2AAslashVector,TA2AAslashVector, MSolver); +MODULE_REGISTER_TMP(ZA2AAslashVector,TA2AAslashVector, MSolver); + +/****************************************************************************** + * TA2AAslashVector implementation * + ******************************************************************************/ +// constructor ///////////////////////////////////////////////////////////////// +template +TA2AAslashVector::TA2AAslashVector(const std::string name) +: Module(name) +{} + +// dependencies/products /////////////////////////////////////////////////////// +template +std::vector TA2AAslashVector::getInput(void) +{ + std::vector in = {par().v, par().photon, par().solver}; + + return in; +} + +template +std::vector TA2AAslashVector::getOutput(void) +{ + std::vector out = {getName()}; + + return out; +} + +// setup /////////////////////////////////////////////////////////////////////// +template +void TA2AAslashVector::setup(void) +{ + Ls_ = env().getObjectLs(par().solver); + auto &vvector = envGet(std::vector, par().v); + unsigned int Nmodes = vvector.size(); + envCreate(std::vector, getName(), 1, + Nmodes, envGetGrid(FermionField)); + + envTmpLat(FermionField, "v4dtmp"); + envTmpLat(FermionField, "v5dtmp", Ls_); + envTmpLat(FermionField, "v5dtmp_sol", Ls_); +} + +// execution /////////////////////////////////////////////////////////////////// +template +void TA2AAslashVector::execute(void) +{ + auto &solver = envGet(Solver, par().solver); + auto &stoch_photon = envGet(EmField, par().photon); + auto &vvector = envGet(std::vector, par().v); + auto &Aslashv = envGet(std::vector, getName()); + unsigned int Nmodes = vvector.size(); + auto &mat = solver.getFMat(); + envGetTmp(FermionField, v4dtmp); + envGetTmp(FermionField, v5dtmp); + envGetTmp(FermionField, v5dtmp_sol); + + Complex ci(0.0,1.0); + + + startTimer("Seq Aslash"); + + LOG(Message) << "Calculate Sequential propagator on Aslash * v with the A2A vector " << par().v + << " and the photon field " << par().photon << std::endl; + + + for(unsigned int i=0; i(stoch_photon, mu) * (gmu * vvector[i]); + } + stopTimer("Multiply Aslash"); + + if (Ls_ == 1) + { + solver(Aslashv[i], v4dtmp); + } + else + { + mat.ImportPhysicalFermionSource(v4dtmp, v5dtmp); + solver(v5dtmp_sol, v5dtmp); + mat.ExportPhysicalFermionSolution(v5dtmp_sol, v4dtmp); + Aslashv[i] = v4dtmp; + } + } + + stopTimer("Seq Aslash"); +} + +END_MODULE_NAMESPACE + +END_HADRONS_NAMESPACE + +#endif // Hadrons_MSolver_A2AAslashVector_hpp_ diff --git a/Hadrons/modules.inc b/Hadrons/modules.inc index 157fe1a2..4ee09bb6 100644 --- a/Hadrons/modules.inc +++ b/Hadrons/modules.inc @@ -20,6 +20,7 @@ modules_cc =\ Modules/MSink/Point.cc \ Modules/MSink/Smear.cc \ Modules/MSolver/A2AVectors.cc \ + Modules/MSolver/A2AAslashVector.cc \ Modules/MSolver/RBPrecCG.cc \ Modules/MSolver/MixedPrecisionRBPrecCG.cc \ Modules/MSolver/LocalCoherenceLanczos.cc \ @@ -95,6 +96,7 @@ modules_hpp =\ Modules/MSolver/Guesser.hpp \ Modules/MSolver/RBPrecCG.hpp \ Modules/MSolver/A2AVectors.hpp \ + Modules/MSolver/A2AAslashVector.hpp \ Modules/MGauge/UnitEm.hpp \ Modules/MGauge/StoutSmearing.hpp \ Modules/MGauge/Unit.hpp \