mirror of
https://github.com/paboyle/Grid.git
synced 2024-11-10 07:55:35 +00:00
Merge branch 'feature/seqA2A' into develop
This commit is contained in:
Vera Guelpers
commit
d058b4e681
@ -50,8 +50,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv)
|
||||
assert(0);
|
||||
}
|
||||
|
||||
Grid_quiesce_nodes();
|
||||
|
||||
// Never clean up as done once.
|
||||
MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
|
||||
|
||||
@ -124,10 +122,8 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
|
||||
// split the communicator
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// int Nparent = parent._processors ;
|
||||
// std::cout << " splitting from communicator "<<parent.communicator <<std::endl;
|
||||
int Nparent;
|
||||
MPI_Comm_size(parent.communicator,&Nparent);
|
||||
// std::cout << " Parent size "<<Nparent <<std::endl;
|
||||
|
||||
int childsize=1;
|
||||
for(int d=0;d<processors.size();d++) {
|
||||
@ -136,8 +132,6 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
|
||||
int Nchild = Nparent/childsize;
|
||||
assert (childsize * Nchild == Nparent);
|
||||
|
||||
// std::cout << " child size "<<childsize <<std::endl;
|
||||
|
||||
std::vector<int> ccoor(_ndimension); // coor within subcommunicator
|
||||
std::vector<int> scoor(_ndimension); // coor of split within parent
|
||||
std::vector<int> ssize(_ndimension); // coor of split within parent
|
||||
|
||||
@ -413,7 +413,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
close(fd);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
// std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
}
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
@ -455,7 +455,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
close(fd);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
// std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
}
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
@ -499,7 +499,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
#endif
|
||||
void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0);
|
||||
|
||||
std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< size<< "bytes)"<<std::endl;
|
||||
// std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< size<< "bytes)"<<std::endl;
|
||||
if ( ptr == (void * )MAP_FAILED ) {
|
||||
perror("failed mmap");
|
||||
assert(0);
|
||||
|
||||
@ -464,8 +464,10 @@ void InsertSliceLocal(const Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int
|
||||
assert(orthog>=0);
|
||||
|
||||
for(int d=0;d<nh;d++){
|
||||
assert(lg->_processors[d] == hg->_processors[d]);
|
||||
assert(lg->_ldimensions[d] == hg->_ldimensions[d]);
|
||||
if ( d!=orthog ) {
|
||||
assert(lg->_processors[d] == hg->_processors[d]);
|
||||
assert(lg->_ldimensions[d] == hg->_ldimensions[d]);
|
||||
}
|
||||
}
|
||||
|
||||
// the above should guarantee that the operations are local
|
||||
@ -499,8 +501,10 @@ void ExtractSliceLocal(Lattice<vobj> &lowDim, Lattice<vobj> & higherDim,int slic
|
||||
assert(orthog>=0);
|
||||
|
||||
for(int d=0;d<nh;d++){
|
||||
assert(lg->_processors[d] == hg->_processors[d]);
|
||||
assert(lg->_ldimensions[d] == hg->_ldimensions[d]);
|
||||
if ( d!=orthog ) {
|
||||
assert(lg->_processors[d] == hg->_processors[d]);
|
||||
assert(lg->_ldimensions[d] == hg->_ldimensions[d]);
|
||||
}
|
||||
}
|
||||
|
||||
// the above should guarantee that the operations are local
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
#include <Hadrons/Modules/MSolver/Guesser.hpp>
|
||||
#include <Hadrons/Modules/MSolver/RBPrecCG.hpp>
|
||||
#include <Hadrons/Modules/MSolver/A2AVectors.hpp>
|
||||
#include <Hadrons/Modules/MSolver/A2AAslashVector.hpp>
|
||||
#include <Hadrons/Modules/MGauge/UnitEm.hpp>
|
||||
#include <Hadrons/Modules/MGauge/StoutSmearing.hpp>
|
||||
#include <Hadrons/Modules/MGauge/Unit.hpp>
|
||||
|
||||
35
Hadrons/Modules/MSolver/A2AAslashVector.cc
Normal file
35
Hadrons/Modules/MSolver/A2AAslashVector.cc
Normal file
@ -0,0 +1,35 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: Hadrons/Modules/MSolver/A2AAslashVector.cc
|
||||
|
||||
Copyright (C) 2015-2018
|
||||
|
||||
Author: Vera Guelpers <Vera.Guelpers@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Hadrons/Modules/MSolver/A2AAslashVector.hpp>
|
||||
|
||||
using namespace Grid;
|
||||
using namespace Hadrons;
|
||||
using namespace MSolver;
|
||||
|
||||
template class Grid::Hadrons::MSolver::TA2AAslashVector<FIMPL>;
|
||||
template class Grid::Hadrons::MSolver::TA2AAslashVector<ZFIMPL>;
|
||||
189
Hadrons/Modules/MSolver/A2AAslashVector.hpp
Normal file
189
Hadrons/Modules/MSolver/A2AAslashVector.hpp
Normal file
@ -0,0 +1,189 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: Hadrons/Modules/MSolver/A2AAslashVector.hpp
|
||||
|
||||
Copyright (C) 2015-2018
|
||||
|
||||
Author: Vera Guelpers <Vera.Guelpers@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef Hadrons_MSolver_A2AAslashVector_hpp_
|
||||
#define Hadrons_MSolver_A2AAslashVector_hpp_
|
||||
|
||||
#include <Hadrons/Global.hpp>
|
||||
#include <Hadrons/Module.hpp>
|
||||
#include <Hadrons/ModuleFactory.hpp>
|
||||
#include <Hadrons/Solver.hpp>
|
||||
|
||||
BEGIN_HADRONS_NAMESPACE
|
||||
|
||||
/******************************************************************************
|
||||
* Create all-to-all V & W vectors *
|
||||
******************************************************************************/
|
||||
BEGIN_MODULE_NAMESPACE(MSolver)
|
||||
|
||||
/****************************************************************************
|
||||
* Calculate a sequential propagator on an insertion of i*g_mu*A_mu
|
||||
* on an A2A vector
|
||||
*
|
||||
* vv_i(y) = S(y,x) * i * g_mu*A_mu(x) * v_i(x)
|
||||
*
|
||||
* with
|
||||
*
|
||||
* - v: A2A vector v_i(x)
|
||||
* - photon: A_mu(x): electromagnetic photon field
|
||||
* - solver: the solver for calculating the sequential propagator
|
||||
*
|
||||
*****************************************************************************/
|
||||
|
||||
|
||||
class A2AAslashVectorPar: Serializable
|
||||
{
|
||||
public:
|
||||
GRID_SERIALIZABLE_CLASS_MEMBERS(A2AAslashVectorPar,
|
||||
std::string, v,
|
||||
std::string, photon,
|
||||
std::string, solver);
|
||||
};
|
||||
|
||||
template <typename FImpl>
|
||||
class TA2AAslashVector : public Module<A2AAslashVectorPar>
|
||||
{
|
||||
public:
|
||||
FERM_TYPE_ALIASES(FImpl,);
|
||||
SOLVER_TYPE_ALIASES(FImpl,);
|
||||
public:
|
||||
typedef PhotonR::GaugeField EmField;
|
||||
public:
|
||||
// constructor
|
||||
TA2AAslashVector(const std::string name);
|
||||
// destructor
|
||||
virtual ~TA2AAslashVector(void) {};
|
||||
// dependency relation
|
||||
virtual std::vector<std::string> getInput(void);
|
||||
virtual std::vector<std::string> getOutput(void);
|
||||
// setup
|
||||
virtual void setup(void);
|
||||
// execution
|
||||
virtual void execute(void);
|
||||
private:
|
||||
unsigned int Ls_;
|
||||
};
|
||||
|
||||
MODULE_REGISTER_TMP(A2AAslashVector,TA2AAslashVector<FIMPL>, MSolver);
|
||||
MODULE_REGISTER_TMP(ZA2AAslashVector,TA2AAslashVector<ZFIMPL>, MSolver);
|
||||
|
||||
/******************************************************************************
|
||||
* TA2AAslashVector implementation *
|
||||
******************************************************************************/
|
||||
// constructor /////////////////////////////////////////////////////////////////
|
||||
template <typename FImpl>
|
||||
TA2AAslashVector<FImpl>::TA2AAslashVector(const std::string name)
|
||||
: Module<A2AAslashVectorPar>(name)
|
||||
{}
|
||||
|
||||
// dependencies/products ///////////////////////////////////////////////////////
|
||||
template <typename FImpl>
|
||||
std::vector<std::string> TA2AAslashVector<FImpl>::getInput(void)
|
||||
{
|
||||
std::vector<std::string> in = {par().v, par().photon, par().solver};
|
||||
|
||||
return in;
|
||||
}
|
||||
|
||||
template <typename FImpl>
|
||||
std::vector<std::string> TA2AAslashVector<FImpl>::getOutput(void)
|
||||
{
|
||||
std::vector<std::string> out = {getName()};
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// setup ///////////////////////////////////////////////////////////////////////
|
||||
template <typename FImpl>
|
||||
void TA2AAslashVector<FImpl>::setup(void)
|
||||
{
|
||||
Ls_ = env().getObjectLs(par().solver);
|
||||
auto &vvector = envGet(std::vector<FermionField>, par().v);
|
||||
unsigned int Nmodes = vvector.size();
|
||||
envCreate(std::vector<FermionField>, getName(), 1,
|
||||
Nmodes, envGetGrid(FermionField));
|
||||
|
||||
envTmpLat(FermionField, "v4dtmp");
|
||||
envTmpLat(FermionField, "v5dtmp", Ls_);
|
||||
envTmpLat(FermionField, "v5dtmp_sol", Ls_);
|
||||
}
|
||||
|
||||
// execution ///////////////////////////////////////////////////////////////////
|
||||
template <typename FImpl>
|
||||
void TA2AAslashVector<FImpl>::execute(void)
|
||||
{
|
||||
auto &solver = envGet(Solver, par().solver);
|
||||
auto &stoch_photon = envGet(EmField, par().photon);
|
||||
auto &vvector = envGet(std::vector<FermionField>, par().v);
|
||||
auto &Aslashv = envGet(std::vector<FermionField>, getName());
|
||||
unsigned int Nmodes = vvector.size();
|
||||
auto &mat = solver.getFMat();
|
||||
envGetTmp(FermionField, v4dtmp);
|
||||
envGetTmp(FermionField, v5dtmp);
|
||||
envGetTmp(FermionField, v5dtmp_sol);
|
||||
|
||||
Complex ci(0.0,1.0);
|
||||
|
||||
|
||||
startTimer("Seq Aslash");
|
||||
|
||||
LOG(Message) << "Calculate Sequential propagator on Aslash * v with the A2A vector " << par().v
|
||||
<< " and the photon field " << par().photon << std::endl;
|
||||
|
||||
|
||||
for(unsigned int i=0; i<Nmodes; i++)
|
||||
{
|
||||
v4dtmp = zero;
|
||||
startTimer("Multiply Aslash");
|
||||
for(unsigned int mu=0;mu<=3;mu++)
|
||||
{
|
||||
Gamma gmu(Gamma::gmu[mu]);
|
||||
v4dtmp += ci * PeekIndex<LorentzIndex>(stoch_photon, mu) * (gmu * vvector[i]);
|
||||
}
|
||||
stopTimer("Multiply Aslash");
|
||||
|
||||
if (Ls_ == 1)
|
||||
{
|
||||
solver(Aslashv[i], v4dtmp);
|
||||
}
|
||||
else
|
||||
{
|
||||
mat.ImportPhysicalFermionSource(v4dtmp, v5dtmp);
|
||||
solver(v5dtmp_sol, v5dtmp);
|
||||
mat.ExportPhysicalFermionSolution(v5dtmp_sol, v4dtmp);
|
||||
Aslashv[i] = v4dtmp;
|
||||
}
|
||||
}
|
||||
|
||||
stopTimer("Seq Aslash");
|
||||
}
|
||||
|
||||
END_MODULE_NAMESPACE
|
||||
|
||||
END_HADRONS_NAMESPACE
|
||||
|
||||
#endif // Hadrons_MSolver_A2AAslashVector_hpp_
|
||||
@ -20,6 +20,7 @@ modules_cc =\
|
||||
Modules/MSink/Point.cc \
|
||||
Modules/MSink/Smear.cc \
|
||||
Modules/MSolver/A2AVectors.cc \
|
||||
Modules/MSolver/A2AAslashVector.cc \
|
||||
Modules/MSolver/RBPrecCG.cc \
|
||||
Modules/MSolver/MixedPrecisionRBPrecCG.cc \
|
||||
Modules/MSolver/LocalCoherenceLanczos.cc \
|
||||
@ -96,6 +97,7 @@ modules_hpp =\
|
||||
Modules/MSolver/Guesser.hpp \
|
||||
Modules/MSolver/RBPrecCG.hpp \
|
||||
Modules/MSolver/A2AVectors.hpp \
|
||||
Modules/MSolver/A2AAslashVector.hpp \
|
||||
Modules/MGauge/UnitEm.hpp \
|
||||
Modules/MGauge/StoutSmearing.hpp \
|
||||
Modules/MGauge/Unit.hpp \
|
||||
|
||||
104
tests/debug/Test_split_laplacian.cc
Normal file
104
tests/debug/Test_split_laplacian.cc
Normal file
@ -0,0 +1,104 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./tests/Test_dwf_mrhs_cg.cc
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/Grid.h>
|
||||
|
||||
using namespace std;
|
||||
using namespace Grid;
|
||||
using namespace Grid::QCD;
|
||||
|
||||
|
||||
int main (int argc, char ** argv)
|
||||
{
|
||||
typedef LatticeComplex ComplexField;
|
||||
|
||||
Grid_init(&argc,&argv);
|
||||
|
||||
std::vector<int> latt_size = GridDefaultLatt();
|
||||
int nd = latt_size.size();
|
||||
int ndm1 = nd-1;
|
||||
|
||||
std::vector<int> simd_layout = GridDefaultSimd(nd,vComplex::Nsimd());
|
||||
std::vector<int> mpi_layout = GridDefaultMpi();
|
||||
std::vector<int> mpi_split (mpi_layout.size(),1);
|
||||
|
||||
std::cout << " Full " << GridCmdVectorIntToString(latt_size) << " subgrid" <<std::endl;
|
||||
std::cout << " Full " << GridCmdVectorIntToString(mpi_layout) << " sub communicator"<<std::endl;
|
||||
std::cout << " Full " << GridCmdVectorIntToString(simd_layout)<< " simd layout " <<std::endl;
|
||||
|
||||
GridCartesian * GridN = new GridCartesian(latt_size,
|
||||
simd_layout,
|
||||
mpi_layout);
|
||||
|
||||
std::vector<int> latt_m = latt_size; latt_m[nd-1] = 1;
|
||||
std::vector<int> mpi_m = mpi_layout; mpi_m [nd-1] = 1;
|
||||
std::vector<int> simd_m = GridDefaultSimd(ndm1,vComplex::Nsimd()); simd_m.push_back(1);
|
||||
|
||||
|
||||
std::cout << " Requesting " << GridCmdVectorIntToString(latt_m)<< " subgrid" <<std::endl;
|
||||
std::cout << " Requesting " << GridCmdVectorIntToString(mpi_m) << " sub communicator"<<std::endl;
|
||||
std::cout << " Requesting " << GridCmdVectorIntToString(simd_m)<< " simd layout " <<std::endl;
|
||||
GridCartesian * Grid_m = new GridCartesian(latt_m,
|
||||
simd_m,
|
||||
mpi_m,
|
||||
*GridN);
|
||||
|
||||
Complex C(1.0);
|
||||
Complex tmp;
|
||||
|
||||
ComplexField Full(GridN); Full = C;
|
||||
ComplexField Full_cpy(GridN);
|
||||
ComplexField Split(Grid_m);Split= C;
|
||||
|
||||
std::cout << GridLogMessage<< " Full volume "<< norm2(Full) <<std::endl;
|
||||
std::cout << GridLogMessage<< " Split volume "<< norm2(Split) <<std::endl;
|
||||
|
||||
tmp=C;
|
||||
GridN->GlobalSum(tmp);
|
||||
std::cout << GridLogMessage<< " Full nodes "<< tmp <<std::endl;
|
||||
|
||||
tmp=C;
|
||||
Grid_m->GlobalSum(tmp);
|
||||
std::cout << GridLogMessage<< " Split nodes "<< tmp <<std::endl;
|
||||
GridN->Barrier();
|
||||
|
||||
auto local_latt = GridN->LocalDimensions();
|
||||
|
||||
Full_cpy = zero;
|
||||
std::vector<int> seeds({1,2,3,4});
|
||||
GridParallelRNG RNG(GridN); RNG.SeedFixedIntegers(seeds);
|
||||
|
||||
random(RNG,Full);
|
||||
for(int t=0;t<local_latt[nd-1];t++){
|
||||
ExtractSliceLocal(Split,Full,0,t,Tp);
|
||||
InsertSliceLocal (Split,Full_cpy,0,t,Tp);
|
||||
}
|
||||
Full_cpy = Full_cpy - Full;
|
||||
std::cout << " NormFull " << norm2(Full)<<std::endl;
|
||||
std::cout << " NormDiff " << norm2(Full_cpy)<<std::endl;
|
||||
Grid_finalize();
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user