mirror of
https://github.com/paboyle/Grid.git
synced 2026-08-14 14:39:37 +01:00
Simplest starting point
This commit is contained in:
Peter Boyle
@@ -0,0 +1,803 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./tests/Test_padded_cell.cc
|
||||
|
||||
Copyright (C) 2023
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/Grid.h>
|
||||
#include <Grid/lattice/PaddedCell.h>
|
||||
#include <Grid/stencil/GeneralLocalStencil.h>
|
||||
|
||||
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidual.h>
|
||||
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
|
||||
#include <Grid/algorithms/iterative/BiCGSTAB.h>
|
||||
|
||||
using namespace std;
|
||||
using namespace Grid;
|
||||
RealD FineSmootherShift = 0.1;
|
||||
int FineSmootherOrder = 8;
|
||||
int FineSmootherTol = 0;
|
||||
//RealD CoarseSmootherShift = 0.1;
|
||||
//int CoarseSmootherOrder = 8;
|
||||
//int CoarseSmootherTol = 0;
|
||||
RealD CoarseSolverShift = 0.002;
|
||||
RealD CoarseSolverTol = 0.03;
|
||||
int CoarseSolverOrder = 200;
|
||||
int CoarseMmax = 20; // coarse GCR restart length (was hardcoded 20)
|
||||
RealD mass=0.00078;
|
||||
void ParseEnvironment(void)
|
||||
{
|
||||
|
||||
if(getenv("MASS") ) mass = atof(getenv("MASS"));
|
||||
if(getenv("FineSmootherShift")) FineSmootherShift = atof(getenv("FineSmootherShift"));
|
||||
if(getenv("FineSmootherOrder")) FineSmootherOrder = atoi(getenv("FineSmootherOrder"));
|
||||
if(getenv("CoarseSolverShift")) CoarseSolverShift = atof(getenv("CoarseSolverShift"));
|
||||
if(getenv("CoarseSolverTol")) CoarseSolverTol = atof(getenv("CoarseSolverTol"));
|
||||
if(getenv("CoarseSolverOrder")) CoarseSolverOrder = atoi(getenv("CoarseSolverOrder"));
|
||||
if(getenv("CoarseMmax")) CoarseMmax = atoi(getenv("CoarseMmax"));
|
||||
if(getenv("DiagInvPrec"))
|
||||
{
|
||||
std::cout << GridLogMessage << "WARNING: DiagInvPrec option REMOVED (diagonal-inverse preconditioning wrecks fine->coarse null-vector inheritance); IGNORED" << std::endl;
|
||||
}
|
||||
|
||||
// if(getenv("CoarseSmootherShift")) CoarseSmootherShift = atof(getenv("CoarseSmootherShift"));
|
||||
// if(getenv("CoarseSmootherOrder")) CoarseSmootherOrder = atoi(getenv("CoarseSmootherOrder"));
|
||||
|
||||
std::cout << GridLogMessage << "PARAM: FineSmootherShift "<<FineSmootherShift<<std::endl;
|
||||
std::cout << GridLogMessage << "PARAM: FineSmootherOrder "<<FineSmootherOrder<<std::endl;
|
||||
// std::cout << GridLogMessage << "PARAM: CoarseSmootherShift "<<CoarseSmootherShift<<std::endl;
|
||||
// std::cout << GridLogMessage << "PARAM: CoarseSmootherOrder "<<CoarseSmootherOrder<<std::endl;
|
||||
std::cout << GridLogMessage << "PARAM: CoarseSolverShift "<<CoarseSolverShift<<std::endl;
|
||||
std::cout << GridLogMessage << "PARAM: CoarseSolverTol "<<CoarseSolverTol<<std::endl;
|
||||
std::cout << GridLogMessage << "PARAM: CoarseSolverOrder "<<CoarseSolverOrder<<std::endl;
|
||||
std::cout << GridLogMessage << "PARAM: CoarseMmax "<<CoarseMmax<<std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "PARAM: MASS "<<mass<<std::endl;
|
||||
}
|
||||
|
||||
template <class T> void readFile(T& out, std::string const fname){
|
||||
#ifdef HAVE_LIME
|
||||
// Ref: https://github.com/paboyle/Grid/blob/feature/scidac-wp1/tests/debug/Test_general_coarse_hdcg_phys48.cc#L111
|
||||
std::cout << Grid::GridLogMessage << "Reads at: " << fname << std::endl;
|
||||
Grid::emptyUserRecord record;
|
||||
// Grid::ScidacReader SR(out.Grid()->IsBoss());
|
||||
Grid::ScidacReader SR;
|
||||
SR.open(fname);
|
||||
SR.readScidacFieldRecord(out, record);
|
||||
SR.close();
|
||||
#endif
|
||||
}
|
||||
template <class Field>
|
||||
void saveSubspace(std::vector<Field> &subspace, std::string const fname){
|
||||
#ifdef HAVE_LIME
|
||||
std::cout << Grid::GridLogMessage << "Saving subspace (" << subspace.size() << " vectors) to: " << fname << std::endl;
|
||||
Grid::emptyUserRecord record;
|
||||
Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
|
||||
SW.open(fname);
|
||||
for (int k = 0; k < (int)subspace.size(); k++)
|
||||
SW.writeScidacFieldRecord(subspace[k], record);
|
||||
SW.close();
|
||||
#endif
|
||||
}
|
||||
|
||||
template <class Field>
|
||||
void loadSubspace(std::vector<Field> &subspace, std::string const fname){
|
||||
#ifdef HAVE_LIME
|
||||
std::cout << Grid::GridLogMessage << "Loading subspace (" << subspace.size() << " vectors) from: " << fname << std::endl;
|
||||
Grid::emptyUserRecord record;
|
||||
Grid::ScidacReader SR;
|
||||
SR.open(fname);
|
||||
for (int k = 0; k < (int)subspace.size(); k++)
|
||||
SR.readScidacFieldRecord(subspace[k], record);
|
||||
SR.close();
|
||||
#endif
|
||||
}
|
||||
|
||||
template<class Matrix,class Field>
|
||||
class PVdagMLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
Matrix &_PV;
|
||||
int nApp;
|
||||
int nAppDag;
|
||||
public:
|
||||
PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV), nApp(0), nAppDag(0) {};
|
||||
|
||||
void OpDiag (const Field &in, Field &out) { assert(0); }
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) { assert(0); }
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); };
|
||||
void Op (const Field &in, Field &out){
|
||||
// std::cout << GridLogMessage<< "Op: PVdag M "<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
_Mat.M(in,tmp);
|
||||
_PV.Mdag(tmp,out);
|
||||
nApp++;
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
// std::cout << GridLogMessage<<"AdjOp: Mdag PV "<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
_PV.M(in,tmp);
|
||||
_Mat.Mdag(tmp,out);
|
||||
nAppDag++;
|
||||
}
|
||||
|
||||
void clear() {
|
||||
nApp = 0;
|
||||
nAppDag = 0;
|
||||
}
|
||||
|
||||
void getApplications() {
|
||||
std::cout << GridLogMessage << "# applications of PVdagM: " << nApp << std::endl;
|
||||
std::cout << GridLogMessage << "# applications of PVdagM^dag: " << nAppDag << std::endl;
|
||||
std::cout << GridLogMessage << "# applications total: " << nApp + nAppDag << std::endl;
|
||||
}
|
||||
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
// std::cout <<GridLogMessage<< "HermOp: Mdag PV PVdag M"<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
Op(in,tmp);
|
||||
AdjOp(tmp,out);
|
||||
// std::cout << "HermOp done "<<norm2(out)<<std::endl;
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class MdagPVLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
Matrix &_PV;
|
||||
public:
|
||||
MdagPVLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV){};
|
||||
|
||||
void OpDiag (const Field &in, Field &out) { assert(0); }
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) { assert(0); }
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); };
|
||||
void Op (const Field &in, Field &out){
|
||||
Field tmp(in.Grid());
|
||||
// std::cout <<GridLogMessage<< "Op: PVdag M "<<std::endl;
|
||||
_PV.M(in,tmp);
|
||||
_Mat.Mdag(tmp,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
// std::cout <<GridLogMessage<< "AdjOp: Mdag PV "<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
_Mat.M(in,tmp);
|
||||
_PV.Mdag(tmp,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
// std::cout << GridLogMessage<<"HermOp: PVdag M Mdag PV "<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
Op(in,tmp);
|
||||
AdjOp(tmp,out);
|
||||
// std::cout << "HermOp done "<<norm2(out)<<std::endl;
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class ShiftedPVdagMLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
Matrix &_PV;
|
||||
int nApp;
|
||||
int nAppDag;
|
||||
public:
|
||||
RealD shift;
|
||||
ShiftedPVdagMLinearOperator(RealD _shift,Matrix &Mat,Matrix &PV): shift(_shift),_Mat(Mat),_PV(PV) , nApp(0), nAppDag(0){};
|
||||
|
||||
void OpDiag (const Field &in, Field &out) { assert(0); }
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) { assert(0); }
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); };
|
||||
void Op (const Field &in, Field &out){
|
||||
// std::cout << "Op: PVdag M "<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
_Mat.M(in,tmp);
|
||||
_PV.Mdag(tmp,out);
|
||||
nApp++;
|
||||
out = out + shift * in;
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
// std::cout << "AdjOp: Mdag PV "<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
_PV.M(tmp,out);
|
||||
_Mat.Mdag(in,tmp);
|
||||
nAppDag++;
|
||||
out = out + shift * in;
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ assert(0); }
|
||||
void HermOp(const Field &in, Field &out){
|
||||
// std::cout << "HermOp: Mdag PV PVdag M"<<std::endl;
|
||||
Field tmp(in.Grid());
|
||||
Op(in,tmp);
|
||||
AdjOp(tmp,out);
|
||||
}
|
||||
void clear() {
|
||||
nApp = 0;
|
||||
nAppDag = 0;
|
||||
}
|
||||
void getApplications() {
|
||||
std::cout << GridLogMessage << "# applications of ShiftedPVdagM: " << nApp << std::endl;
|
||||
std::cout << GridLogMessage << "# applications of ShiftedPVdagM^dag: " << nAppDag << std::endl;
|
||||
std::cout << GridLogMessage << "# applications total: " << nApp + nAppDag << std::endl;
|
||||
}
|
||||
};
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class MGPreconditioner : public LinearFunction< Lattice<Fobj> > {
|
||||
public:
|
||||
|
||||
using LinearFunction<Lattice<Fobj> >::operator();
|
||||
|
||||
typedef Aggregation<Fobj,CComplex,nbasis> Aggregates;
|
||||
typedef typename Aggregation<Fobj,CComplex,nbasis>::FineField FineField;
|
||||
typedef typename Aggregation<Fobj,CComplex,nbasis>::CoarseVector CoarseVector;
|
||||
typedef typename Aggregation<Fobj,CComplex,nbasis>::CoarseMatrix CoarseMatrix;
|
||||
typedef LinearOperatorBase<FineField> FineOperator;
|
||||
typedef LinearFunction <FineField> FineSmoother;
|
||||
typedef LinearOperatorBase<CoarseVector> CoarseOperator;
|
||||
typedef LinearFunction <CoarseVector> CoarseSolver;
|
||||
Aggregates & _Aggregates;
|
||||
FineOperator & _FineOperator;
|
||||
FineSmoother & _PreSmoother;
|
||||
FineSmoother & _PostSmoother;
|
||||
CoarseOperator & _CoarseOperator;
|
||||
CoarseSolver & _CoarseSolve;
|
||||
std::string name;
|
||||
|
||||
int level; void Level(int lv) {level = lv; };
|
||||
|
||||
MGPreconditioner(Aggregates &Agg,
|
||||
FineOperator &Fine,
|
||||
FineSmoother &PreSmoother,
|
||||
FineSmoother &PostSmoother,
|
||||
CoarseOperator &CoarseOperator_,
|
||||
CoarseSolver &CoarseSolve_,
|
||||
std::string _name = std::string("unnamed"))
|
||||
: _Aggregates(Agg),
|
||||
_FineOperator(Fine),
|
||||
_PreSmoother(PreSmoother),
|
||||
_PostSmoother(PostSmoother),
|
||||
_CoarseOperator(CoarseOperator_),
|
||||
_CoarseSolve(CoarseSolve_),
|
||||
name(_name),
|
||||
level(1) { }
|
||||
|
||||
virtual void operator()(const FineField &in, FineField & out)
|
||||
{
|
||||
GridBase *CoarseGrid = _Aggregates.CoarseGrid;
|
||||
// auto CoarseGrid = _CoarseOperator.Grid();
|
||||
CoarseVector Csrc(CoarseGrid);
|
||||
CoarseVector Csol(CoarseGrid);
|
||||
FineField vec1(in.Grid());
|
||||
FineField vec2(in.Grid());
|
||||
|
||||
std::cout<<GridLogMessage << "Calling PreSmoother " <<std::endl;
|
||||
|
||||
// std::cout<<GridLogMessage << "Calling PreSmoother input residual "<<norm2(in) <<std::endl;
|
||||
double t;
|
||||
// Fine Smoother
|
||||
// out = in;
|
||||
out = Zero();
|
||||
t=-usecond();
|
||||
_PreSmoother(in,out);
|
||||
t+=usecond();
|
||||
|
||||
std::cout<<GridLogMessage << "PreSmoother took "<< t/1000.0<< "ms" <<std::endl;
|
||||
|
||||
// Update the residual
|
||||
_FineOperator.Op(out,vec1); sub(vec1, in ,vec1);
|
||||
// std::cout<<GridLogMessage <<"Residual-1 now " <<norm2(vec1)<<std::endl;
|
||||
|
||||
// Fine to Coarse
|
||||
t=-usecond();
|
||||
_Aggregates.ProjectToSubspace (Csrc,vec1);
|
||||
t+=usecond();
|
||||
std::cout<<GridLogMessage << "Project to coarse took "<< t/1000.0<< "ms" <<std::endl;
|
||||
|
||||
// Coarse correction
|
||||
t=-usecond();
|
||||
Csol = Zero();
|
||||
_CoarseSolve(Csrc,Csol);
|
||||
//Csol=Zero();
|
||||
t+=usecond();
|
||||
std::cout<<GridLogMessage << "Coarse solve took "<< t/1000.0<< "ms" <<std::endl;
|
||||
|
||||
// Coarse to Fine
|
||||
t=-usecond();
|
||||
// _CoarseOperator.PromoteFromSubspace(_Aggregates,Csol,vec1);
|
||||
_Aggregates.PromoteFromSubspace(Csol,vec1);
|
||||
add(out,out,vec1);
|
||||
t+=usecond();
|
||||
std::cout<<GridLogMessage << "Promote to this level took "<< t/1000.0<< "ms" <<std::endl;
|
||||
|
||||
// Residual
|
||||
_FineOperator.Op(out,vec1); sub(vec1 ,in , vec1);
|
||||
// std::cout<<GridLogMessage <<"Residual-2 now " <<norm2(vec1)<<std::endl;
|
||||
|
||||
// Fine Smoother
|
||||
t=-usecond();
|
||||
// vec2=vec1;
|
||||
vec2=Zero();
|
||||
_PostSmoother(vec1,vec2);
|
||||
t+=usecond();
|
||||
std::cout<<GridLogMessage << "PostSmoother took "<< t/1000.0<< "ms" <<std::endl;
|
||||
|
||||
add( out,out,vec2);
|
||||
std::cout<<GridLogMessage << "Done " <<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
template<class PVdagM_t, class ShiftedPVdagM_t, class Subspace, class LittleDiracOperator, class CoarseVector, class TwoLevelMG>
|
||||
void runMG(
|
||||
GridCartesian *FGrid,
|
||||
GridCartesian *Coarse5d,
|
||||
NextToNearestStencilGeometry5D geom,
|
||||
PVdagM_t PVdagM,
|
||||
ShiftedPVdagM_t ShiftedPVdagM,
|
||||
// std::vector<LatticeFermion> subspace
|
||||
Subspace AggregatesPD
|
||||
) {
|
||||
|
||||
// typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
|
||||
// typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
|
||||
// typedef LittleDiracOperator::CoarseVector CoarseVector;
|
||||
ParseEnvironment();
|
||||
|
||||
std::vector<LatticeFermion> subspace = AggregatesPD.subspace;
|
||||
int nbasis = subspace.size();
|
||||
const int cb = 0 ;
|
||||
|
||||
LatticeFermion err(FGrid);
|
||||
LatticeFermion prom(FGrid);
|
||||
LatticeFermion tmp(FGrid);
|
||||
|
||||
CoarseVector c_src (Coarse5d);
|
||||
CoarseVector c_res (Coarse5d);
|
||||
CoarseVector c_proj(Coarse5d);
|
||||
Complex one(1.0);
|
||||
|
||||
LatticeFermionD f_src(FGrid);
|
||||
LatticeFermionD f_res(FGrid);
|
||||
// typedef MGPreconditioner<vSpinColourVector, vTComplex,nbasis> TwoLevelMG;
|
||||
TrivialPrecon<CoarseVector> simple;
|
||||
TrivialPrecon<LatticeFermionD> simple_fine;
|
||||
|
||||
// Subspace AggregatesPD(Coarse5d,FGrid,cb);
|
||||
|
||||
// Orthonormalize subspace and compute nulliness
|
||||
|
||||
ShiftedPVdagM.shift = CoarseSolverShift;
|
||||
int nonherm = 0;
|
||||
LittleDiracOperator LittleDiracOpPV(geom,FGrid,Coarse5d,nonherm);
|
||||
LittleDiracOpPV.CoarsenOperator(ShiftedPVdagM, AggregatesPD);
|
||||
ShiftedPVdagM.shift = FineSmootherShift;
|
||||
|
||||
std::cout<<GridLogMessage<<std::endl;
|
||||
std::cout<<GridLogMessage<<"*******************************************"<<std::endl;
|
||||
std::cout<<GridLogMessage<<std::endl;
|
||||
std::cout<<GridLogMessage<<"Testing coarsened operator "<<std::endl;
|
||||
|
||||
c_src = one; // 1 in every element for vector 1.
|
||||
blockPromote(c_src,err,subspace);
|
||||
|
||||
prom=Zero();
|
||||
for(int b=0;b<nbasis;b++){
|
||||
prom=prom+subspace[b];
|
||||
}
|
||||
err=err-prom;
|
||||
std::cout<<GridLogMessage<<"Promoted back from subspace: err "<<norm2(err)<<std::endl;
|
||||
std::cout<<GridLogMessage<<"c_src "<<norm2(c_src)<<std::endl;
|
||||
std::cout<<GridLogMessage<<"prom "<<norm2(prom)<<std::endl;
|
||||
|
||||
// PVdagM.Op(prom,tmp);
|
||||
// blockProject(c_proj,tmp,subspace);
|
||||
// std::cout<<GridLogMessage<<" Called Big Dirac Op "<<norm2(tmp)<<std::endl;
|
||||
|
||||
// LittleDiracOpPV.M(c_src,c_res);
|
||||
// std::cout<<GridLogMessage<<" Called Little Dirac Op c_src "<< norm2(c_src) << " c_res "<< norm2(c_res) <<std::endl;
|
||||
|
||||
// std::cout<<GridLogMessage<<"Little dop : "<<norm2(c_res)<<std::endl;
|
||||
// // std::cout<<GridLogMessage<<" Little "<< c_res<<std::endl;
|
||||
// std::cout<<GridLogMessage<<"Big dop in subspace : "<<norm2(c_proj)<<std::endl;
|
||||
// // std::cout<<GridLogMessage<<" Big "<< c_proj<<std::endl;
|
||||
// c_proj = c_proj - c_res;
|
||||
// std::cout<<GridLogMessage<<" ldop error: "<<norm2(c_proj)<<std::endl;
|
||||
// // std::cout<<GridLogMessage<<" error "<< c_proj<<std::endl;
|
||||
|
||||
///////////////////////////////////////
|
||||
// Coarse grid solver test
|
||||
///////////////////////////////////////
|
||||
|
||||
std::cout<<GridLogMessage<<"******************* "<<std::endl;
|
||||
std::cout<<GridLogMessage<<" Coarse Grid Solve -- Level 2 "<<std::endl;
|
||||
std::cout<<GridLogMessage<<"******************* "<<std::endl;
|
||||
NonHermitianLinearOperator<LittleDiracOperator,CoarseVector> LinOpCoarse(LittleDiracOpPV);
|
||||
// DiagonalInverse preconditioning REMOVED (library support withdrawn: it
|
||||
// wrecks the collinearity that makes fine->coarse null-vector inheritance
|
||||
// free). TrivialPrecon reproduces the former DiagInvPrec=0 path exactly.
|
||||
PrecGeneralisedConjugateResidualNonHermitian<CoarseVector> L2PGCR(CoarseSolverTol, (CoarseSolverOrder+CoarseMmax-1)/CoarseMmax, LinOpCoarse,simple,CoarseMmax,CoarseMmax);
|
||||
L2PGCR.SetZeroGuess(1); // callers zero Csol / c_res
|
||||
L2PGCR.Level(2);
|
||||
L2PGCR.Name("Couter");
|
||||
c_res=Zero();
|
||||
L2PGCR(c_src,c_res);
|
||||
|
||||
|
||||
////////////////////////////////////////
|
||||
// Fine grid smoother
|
||||
////////////////////////////////////////
|
||||
// NonHermitianLinearOperator<PVdagM_t,LatticeFermionD> LinOpSmooth(PVdagM);
|
||||
|
||||
// PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD> SmootherGCR(0.05,1,ShiftedPVdagM,simple_fine,8,8);
|
||||
// Force 10 iters exactly, no early termination
|
||||
PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD> SmootherGCR(FineSmootherTol,1,
|
||||
ShiftedPVdagM,simple_fine,
|
||||
FineSmootherOrder,FineSmootherOrder);
|
||||
SmootherGCR.Level(1);
|
||||
SmootherGCR.Name("Fsmoother");
|
||||
SmootherGCR.SetZeroGuess(1); // pre/post slots + direct call all zero their guess
|
||||
|
||||
f_src = one; // 1 in every element for vector 1.
|
||||
f_res=Zero();
|
||||
SmootherGCR(f_src,f_res);
|
||||
|
||||
TwoLevelMG TwoLevelPrecon(AggregatesPD,
|
||||
PVdagM,
|
||||
simple_fine,
|
||||
SmootherGCR,
|
||||
LinOpCoarse,
|
||||
L2PGCR,
|
||||
"PVdagM");
|
||||
|
||||
PrecGeneralisedConjugateResidualNonHermitian<LatticeFermion> L1PGCR(1.0e-8,1000,PVdagM,TwoLevelPrecon,32,32);
|
||||
L1PGCR.SetZeroGuess(1); // f_res=Zero() before the solve
|
||||
L1PGCR.Level(1);
|
||||
L1PGCR.Name("Fouter");
|
||||
|
||||
std::cout<<GridLogMessage<<"******************* "<<std::endl;
|
||||
std::cout<<GridLogMessage<<" Running Multi Grid Solver "<<std::endl;
|
||||
std::cout<<GridLogMessage<<"******************* "<<std::endl;
|
||||
f_res=Zero();
|
||||
L1PGCR(f_src,f_res);
|
||||
|
||||
std::cout << GridLogMessage << "Fine Grid Smoother -- Level 2 operator uses: " << std::endl;
|
||||
PVdagM.getApplications();
|
||||
PVdagM.clear();
|
||||
ShiftedPVdagM.getApplications();
|
||||
ShiftedPVdagM.clear();
|
||||
|
||||
}
|
||||
|
||||
int main (int argc, char ** argv)
|
||||
{
|
||||
Grid_init(&argc,&argv);
|
||||
|
||||
// TODO read in more parameters: nbasis, GCR iters, smoother order, m
|
||||
// Might be impossible because nbasis needs to be a constant to be a template parameter
|
||||
// Usage : $ ./Example_pvdagm <nbasis> <smooth> <outerIters> <m>
|
||||
// std::string nbasisStr = argv[1];
|
||||
// std::string smoothStr = argv[2];
|
||||
// std::string outerStr = argv[3];
|
||||
// std::string mStr = argv[4];
|
||||
// int nbasis = std::stoi(nbasisStr);
|
||||
// int smooth = std::stoi(smoothStr);
|
||||
|
||||
const int Ls=24;
|
||||
RealD M5=1.8;
|
||||
|
||||
|
||||
// const int nbasis = 40;
|
||||
const int nbasis = 60;
|
||||
|
||||
std::cout << GridLogMessage << "Mass: " << mass << ", Ls: " << Ls << ", running Mobius kernel with b=1.5, c=0.5" << std::endl;
|
||||
std::cout << GridLogMessage << "nbasis: " << nbasis << std::endl;
|
||||
|
||||
std::vector<int> lat_size {48, 48, 48, 96};
|
||||
|
||||
GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(lat_size, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
|
||||
GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
|
||||
|
||||
GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
|
||||
GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
|
||||
|
||||
// Construct a coarsened grid
|
||||
// Coordinate clatt = GridDefaultLatt();
|
||||
Coordinate clatt = lat_size;
|
||||
Coordinate Block({4,4,4,4});
|
||||
std::cout << GridLogMessage << "Lattice size: " << lat_size << std::endl;
|
||||
for(int d=0;d<clatt.size();d++){
|
||||
clatt[d] = lat_size[d]/Block[d];
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "constructing coarse grid" << std::endl;
|
||||
GridCartesian *Coarse4d = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
|
||||
GridCartesian *Coarse5d = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
|
||||
|
||||
std::vector<int> seeds4({1,2,3,4});
|
||||
std::vector<int> seeds5({5,6,7,8});
|
||||
std::vector<int> cseeds({5,6,7,8});
|
||||
GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5);
|
||||
GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4);
|
||||
GridParallelRNG CRNG(Coarse5d);CRNG.SeedFixedIntegers(cseeds);
|
||||
|
||||
LatticeFermion src(FGrid); random(RNG5,src);
|
||||
LatticeFermion result(FGrid); result=Zero();
|
||||
LatticeFermion ref(FGrid); ref=Zero();
|
||||
LatticeFermion tmp(FGrid);
|
||||
LatticeFermion err(FGrid);
|
||||
LatticeGaugeField Umu(UGrid);
|
||||
|
||||
|
||||
std::cout << GridLogMessage << "Reading in gauge field" << std::endl;
|
||||
FieldMetaData header;
|
||||
// std::string file("/sdcc/u/poare/PETSc-Grid/ckpoint_lat.4000");
|
||||
std::string file("/ccs/home/poare/ckpoint_lat.1000");
|
||||
NerscIO::readConfiguration(Umu,header,file);
|
||||
|
||||
/*
|
||||
|
||||
// DWF, m=0.01
|
||||
// std::string eigenPath = "/hpcgpfs01/work/lqcd/staging/RBC/ckpoint_lat.4000/ks_evecs/PVdagM_Nm80_Nk40_Niter5000_337342/";
|
||||
|
||||
// DWF, m=0.001
|
||||
// std::string eigenPath = "/hpcgpfs01/work/lqcd/staging/RBC/ckpoint_lat.4000/ks_evecs/PVdagM_Nm80_Nk40_Niter5000_m0p001_339143/";
|
||||
|
||||
// Mobius, m=0.001
|
||||
// std::string eigenPath = "/hpcgpfs01/work/lqcd/staging/RBC/ckpoint_lat.4000/ks_evecs/PVdagM_Nm80_Nk40_Niter5000_346851/";
|
||||
|
||||
// Frontier path
|
||||
std::string eigenPath = "/ccs/home/poare/lqcd/multigrid/spectra/ckpoint_lat.1000/...";
|
||||
|
||||
|
||||
std::cout << GridLogMessage << "Loading eigenvalues" << std::endl;
|
||||
std::ifstream evalFile(eigenPath + "evals.txt");
|
||||
std::string str;
|
||||
std::vector<ComplexD> evals;
|
||||
while (std::getline(evalFile, str)) {
|
||||
std::cout << GridLogMessage << "Reading line: " << str << std::endl;
|
||||
int i1 = str.find("(") + 1;
|
||||
int i2 = str.find(",") + 1;
|
||||
int i3 = str.find(")");
|
||||
std::cout << "i1,i2,i3 = " << i1 << "," << i2 << "," << i3 << std::endl;
|
||||
std::string reStr = str.substr(i1, i2 - i1);
|
||||
std::string imStr = str.substr(i2, i3 - i2);
|
||||
std::cout << GridLogMessage << "Parsed re = " << reStr << " and im = " << imStr << std::endl;
|
||||
// ComplexD z (std::stof(reStr), std::stof(imStr));
|
||||
ComplexD z (std::stod(reStr), std::stod(imStr));
|
||||
evals.push_back(z);
|
||||
}
|
||||
std::cout << GridLogMessage << "Eigenvalues: " << evals << std::endl;
|
||||
|
||||
int Nevecs = 20;
|
||||
std::vector<LatticeFermion> evecs;
|
||||
LatticeFermion evec (FGrid);
|
||||
for (int i = 0; i < Nevecs; i++) {
|
||||
std::string evecPath = eigenPath + "evec" + std::to_string(i);
|
||||
readFile(evec, evecPath);
|
||||
evecs.push_back(evec);
|
||||
}
|
||||
std::cout << GridLogMessage << "Evecs loaded" << std::endl;
|
||||
|
||||
*/
|
||||
// TODO uncomment when evecs are computed!
|
||||
|
||||
// DomainWallFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
|
||||
// DomainWallFermionD Dpv(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0,M5);
|
||||
|
||||
// Mobius
|
||||
RealD b=1.5;// Scale factor b+c=2, b-c=1
|
||||
RealD c=0.5;
|
||||
MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
|
||||
MobiusFermionD Dpv(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0,M5,b,c);
|
||||
|
||||
const int cb = 0 ;
|
||||
LatticeFermion prom(FGrid);
|
||||
|
||||
// assert(nbasis <= Nevecs); // need to have enough evecs
|
||||
|
||||
typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
|
||||
typedef LittleDiracOperator::CoarseVector CoarseVector;
|
||||
|
||||
NextToNearestStencilGeometry5D geom(Coarse5d);
|
||||
|
||||
std::cout<<GridLogMessage<<std::endl;
|
||||
std::cout<<GridLogMessage<<"*******************************************"<<std::endl;
|
||||
std::cout<<GridLogMessage<<std::endl;
|
||||
|
||||
// typedef PVdagMLinearOperator<DomainWallFermionD,LatticeFermionD> PVdagM_t;
|
||||
// typedef MdagPVLinearOperator<DomainWallFermionD,LatticeFermionD> MdagPV_t;
|
||||
// typedef ShiftedPVdagMLinearOperator<DomainWallFermionD,LatticeFermionD> ShiftedPVdagM_t;
|
||||
typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD> PVdagM_t;
|
||||
typedef MdagPVLinearOperator<MobiusFermionD,LatticeFermionD> MdagPV_t;
|
||||
typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
|
||||
|
||||
PVdagM_t PVdagM(Ddwf,Dpv);
|
||||
MdagPV_t MdagPV(Ddwf,Dpv);
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(2.0,Ddwf,Dpv); // 355
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(1.0,Ddwf,Dpv); // 246
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.5,Ddwf,Dpv); // 183
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.25,Ddwf,Dpv); // 145
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 134
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 127 -- NULL space via inverse iteration
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 57 -- NULL space via inverse iteration; 3 iterations
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.25,Ddwf,Dpv); // 57 , tighter inversion
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.25,Ddwf,Dpv); // nbasis 20 -- 49 iters
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.25,Ddwf,Dpv); // nbasis 20 -- 70 iters; asymmetric
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.25,Ddwf,Dpv); // 58; Loosen coarse, tighten fine
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 56 ...
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 51 ... with 24 vecs
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 31 ... with 24 vecs and 2^4 blocking
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 43 ... with 16 vecs and 2^4 blocking, sloppier
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 35 ... with 20 vecs and 2^4 blocking
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 35 ... with 20 vecs and 2^4 blocking, looser coarse
|
||||
// ShiftedPVdagM_t ShiftedPVdagM(0.1,Ddwf,Dpv); // 64 ... with 20 vecs, Christoph setup, and 2^4 blocking, looser coarse
|
||||
|
||||
ShiftedPVdagM_t ShiftedPVdagM(FineSmootherShift,Ddwf,Dpv); //
|
||||
|
||||
// Run power method on HOA??
|
||||
PowerMethod<LatticeFermion> PM;
|
||||
|
||||
CoarseVector c_src (Coarse5d);
|
||||
CoarseVector c_res (Coarse5d);
|
||||
CoarseVector c_proj(Coarse5d);
|
||||
Complex one(1.0);
|
||||
|
||||
std::vector<LatticeFermion> subspace(nbasis,FGrid);
|
||||
|
||||
LatticeFermionD f_src(FGrid);
|
||||
LatticeFermionD f_res(FGrid);
|
||||
typedef MGPreconditioner<vSpinColourVector, vTComplex,nbasis> TwoLevelMG;
|
||||
TrivialPrecon<CoarseVector> simple;
|
||||
TrivialPrecon<LatticeFermionD> simple_fine;
|
||||
|
||||
// Warning: This routine calls PVdagM.Op, not PVdagM.HermOp
|
||||
typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
|
||||
|
||||
// Breeds right singular vectors with call to HermOp (V)
|
||||
// int chebyOrd = 500;
|
||||
// V.CreateSubspaceChebyshev(RNG5,PVdagM,
|
||||
// nbasis,
|
||||
// 4000.0,0.003,
|
||||
// chebyOrd);
|
||||
// AggregatesPD.CreateSubspaceChebyshev(RNG5,
|
||||
// PVdagM,
|
||||
// nbasis,
|
||||
// 4000.0,
|
||||
// 0.003,
|
||||
// chebyOrd);
|
||||
|
||||
// Subspace testing (uncomment blocks when needed)
|
||||
|
||||
// - nbasis = 20, m=0.01, 35 outer iterations
|
||||
// - nbasis = 40, m=0.01, 23 outer iterations
|
||||
std::cout << GridLogMessage << "*** GCR setup ***" << std::endl;
|
||||
|
||||
// Subspace cache: save after generation, reload on subsequent runs to skip expensive setup.
|
||||
// Set SUBSPACE_FILE to override the default path.
|
||||
std::string subspace_file = "/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb"
|
||||
+ std::to_string(nbasis) + ".scidac";
|
||||
if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
|
||||
|
||||
// Check if subspace file exists (boss rank checks, result broadcast via GlobalSum).
|
||||
uint64_t file_exists = 0;
|
||||
if ( UGrid->IsBoss() ) {
|
||||
std::ifstream f(subspace_file);
|
||||
file_exists = f.good() ? 1 : 0;
|
||||
}
|
||||
UGrid->GlobalSum(file_exists);
|
||||
|
||||
Subspace AggregatesGCR(Coarse5d,FGrid,cb);
|
||||
|
||||
if ( file_exists ) {
|
||||
std::cout << GridLogMessage << "*** Loading subspace from disk ***" << std::endl;
|
||||
loadSubspace(AggregatesGCR.subspace, subspace_file);
|
||||
// Insurance: GLOBAL (whole-lattice) orthonormalise, matching what
|
||||
// CreateSubspaceGCR applies to generated subspaces (Aggregates.h:196), so a
|
||||
// reloaded file ends in the same state. This replaces the block
|
||||
// Orthogonalise() previously called here -- that is redundant (CoarsenOperator
|
||||
// block-GS's the subspace internally) and would leave a loaded file block-
|
||||
// orthonormal while a generated one is globally orthonormal. Global GS is
|
||||
// span-preserving, so the coarse operator is unchanged.
|
||||
AggregatesGCR.GlobalOrthonormalise();
|
||||
std::cout << GridLogMessage << "Subspace loaded and globally orthonormalised." << std::endl;
|
||||
} else {
|
||||
std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
|
||||
AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
|
||||
std::cout << GridLogMessage << "Subspace generation: PVdagM operator uses:" << std::endl;
|
||||
PVdagM.getApplications();
|
||||
PVdagM.clear();
|
||||
saveSubspace(AggregatesGCR.subspace, subspace_file);
|
||||
std::cout << GridLogMessage << "Subspace saved to: " << subspace_file << std::endl;
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "Basis construction operator uses: " << std::endl;
|
||||
PVdagM.getApplications();
|
||||
PVdagM.clear();
|
||||
|
||||
std::cout << GridLogMessage << "Calling runMG " << std::endl;
|
||||
runMG<PVdagM_t, ShiftedPVdagM_t, Subspace, LittleDiracOperator, CoarseVector, TwoLevelMG>(
|
||||
FGrid,
|
||||
Coarse5d,
|
||||
geom,
|
||||
PVdagM,
|
||||
ShiftedPVdagM,
|
||||
AggregatesGCR
|
||||
);
|
||||
|
||||
//////////////////////////////////
|
||||
// Standard CG
|
||||
//////////////////////////////////
|
||||
#if 0
|
||||
{
|
||||
std::cout << "**************************************"<<std::endl;
|
||||
std::cout << "Calling red black CG"<<std::endl;
|
||||
std::cout << "**************************************"<<std::endl;
|
||||
ConjugateGradient<LatticeFermionD> CGfine(1.0e-8,30000,false);
|
||||
SchurDiagMooeeOperator<MobiusFermionD, LatticeFermion> HermOpEO(Ddwf);
|
||||
|
||||
LatticeFermion result(FrbGrid); result=Zero();
|
||||
LatticeFermion src(FrbGrid); random(RNG5,src);
|
||||
result=Zero();
|
||||
|
||||
CGfine(HermOpEO, src, result);
|
||||
}
|
||||
{
|
||||
std::cout << "**************************************"<<std::endl;
|
||||
std::cout << "Calling MdagM CG"<<std::endl;
|
||||
std::cout << "**************************************"<<std::endl;
|
||||
|
||||
LatticeFermion result(FGrid); result=Zero();
|
||||
LatticeFermion src(FGrid); random(RNG5,src);
|
||||
result=Zero();
|
||||
|
||||
MdagMLinearOperator<MobiusFermionD, LatticeFermionD> HermOp(Ddwf);
|
||||
ConjugateGradient<LatticeFermionD> CGfine(1.0e-8,100000,false);
|
||||
CGfine(HermOp, src, result);
|
||||
}
|
||||
{
|
||||
std::cout << "**************************************"<<std::endl;
|
||||
std::cout << "Calling PVdagM GCR"<<std::endl;
|
||||
std::cout << "**************************************"<<std::endl;
|
||||
|
||||
LatticeFermion result(FGrid); result=Zero();
|
||||
LatticeFermion src(FGrid); random(RNG5,src);
|
||||
result=Zero();
|
||||
PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD> GCR(1.0e-8,3000,PVdagM,simple_fine,50,50);
|
||||
GCR.Name("Fbaseline");
|
||||
GCR.SetZeroGuess(1); // result=Zero() above
|
||||
GCR(src,result);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
std::cout<<GridLogMessage<<std::endl;
|
||||
std::cout<<GridLogMessage << "Done "<< std::endl;
|
||||
|
||||
Grid_finalize();
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user