portelli/Grid
1
0 Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-10-10 09:48:06 +01:00
Code Releases Activity

Clean up and unification of PVdagM and HDCG, mixed precision support

Browse Source
This commit is contained in:
Peter Boyle committed 2026-09-24 01:38:39 -04:00
1 parent 2043072d9c
commit eeb56e6b39
46 files changed
+1404 -3815
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/Example_mdagm.cc
-43
View File
@@ -32,50 +32,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
using namespace std;
using namespace Grid;
// Routes Op/AdjOp -> HermOp so that CoarsenOperator and CreateSubspace
// both see the HPD operator M†M rather than bare M.
template<class Field>
class HermOpAdaptor : public LinearOperatorBase<Field>
{
LinearOperatorBase<Field> &wrapped;
public:
HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {};
void Op (const Field &in, Field &out) { wrapped.HermOp(in,out); }
void HermOp (const Field &in, Field &out) { wrapped.HermOp(in,out); }
void AdjOp (const Field &in, Field &out) { wrapped.HermOp(in,out); }
void OpDiag (const Field &in, Field &out) { GRID_ASSERT(0); }
void OpDir (const Field &in, Field &out,int dir,int disp) { GRID_ASSERT(0); }
void OpDirAll(const Field &in, std::vector<Field> &out) { GRID_ASSERT(0); }
void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
wrapped.HermOp(in, out);
ComplexD dot = innerProduct(in, out);
n1 = real(dot);
n2 = norm2(out);
}
};
// Fixed-iteration CG smoother: runs exactly `iters` steps of CG on the
// shifted operator. tolerance=0 so CG never exits early.
template<class Field>
class CGSmoother : public LinearFunction<Field>
{
public:
using LinearFunction<Field>::operator();
typedef LinearOperatorBase<Field> FineOperator;
FineOperator &_SmootherOperator;
int iters;
CGSmoother(int _iters, FineOperator &SmootherOperator)
: _SmootherOperator(SmootherOperator), iters(_iters)
{
std::cout << GridLogMessage << " CGSmoother order " << iters << std::endl;
}
void operator()(const Field &in, Field &out)
{
ConjugateGradient<Field> CG(0.0, iters, false);
out = Zero();
CG(_SmootherOperator, in, out);
}
};
int main (int argc, char ** argv)
{
examples/Example_mdagm_cg.cc
-39
View File
@@ -57,46 +57,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
using namespace std;
using namespace Grid;
// Wraps any LinearOperatorBase so that Op = AdjOp = HermOp.
// Required when coarsening an HPD operator whose Op != HermOp
// (e.g. MdagMLinearOperator where Op=M, HermOp=M†M).
template<class Field>
class HermOpAdaptor : public LinearOperatorBase<Field>
{
LinearOperatorBase<Field> &wrapped;
public:
HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {}
void Op (const Field &in, Field &out) { wrapped.HermOp(in, out); }
void HermOp (const Field &in, Field &out) { wrapped.HermOp(in, out); }
void AdjOp (const Field &in, Field &out) { wrapped.HermOp(in, out); }
void OpDiag (const Field &in, Field &out) { GRID_ASSERT(0); }
void OpDir (const Field &in, Field &out, int dir, int disp) { GRID_ASSERT(0); }
void OpDirAll(const Field &in, std::vector<Field> &out) { GRID_ASSERT(0); }
void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
wrapped.HermOp(in, out);
ComplexD dot = innerProduct(in, out);
n1 = real(dot); n2 = norm2(out);
}
};
// Fixed-iteration CG as a smoother (LinearFunction).
// Used as the IR-shifted smoother: solves (M†M + lo*I) x = b approximately.
template<class Field>
class CGSmoother : public LinearFunction<Field>
{
public:
using LinearFunction<Field>::operator();
LinearOperatorBase<Field> &_op;
int iters;
CGSmoother(int _iters, LinearOperatorBase<Field> &op) : _op(op), iters(_iters) {
std::cout << GridLogMessage << "CGSmoother order " << iters << std::endl;
}
void operator()(const Field &in, Field &out) {
ConjugateGradient<Field> CG(0.0, iters, false);
out = Zero();
CG(_op, in, out);
}
};
// Two-level V-cycle preconditioner (LinearFunction).
template<class Fobj, class CComplex, int nbasis>
examples/Example_pvdagm_mrhs.cc
+1 -280
View File
@@ -58,6 +58,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
using namespace std;
using namespace Grid;
@@ -183,194 +184,6 @@ public:
}
};
//////////////////////////////////////////////////////////////////////
// Minimal multi-RHS function interface (preconditioner slot)
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsLinearFunction {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
};
//////////////////////////////////////////////////////////////////////
// Single-polynomial multi-RHS PGCR (non-Hermitian).
//
// Verbatim adaptation of PrecGeneralisedConjugateResidualNonHermitian
// to std::vector<Field>: every innerProduct / norm2 is SUMMED over the
// RHS index, so one alpha/beta per step is shared by all RHS -- the
// single GCR on the enlarged block-diagonal system.
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsPGCRNonHermitian {
public:
RealD Tolerance;
Integer MaxIterations;
int mmax;
int nstep;
int steps;
int level;
int ZeroGuess = 0; int FirstCycle = 0; // caller contract: zero guess => first-cycle r0 = src
std::string name = "Level 1";
LinearOperatorBase<Field> &Linop;
MrhsLinearFunction<Field> &Preconditioner;
void Level(int lv) { name = "Level " + std::to_string(lv); level=lv; };
void Name(std::string n) { name = n; };
void SetZeroGuess(int z) { ZeroGuess=z; };
MrhsPGCRNonHermitian(RealD tol,Integer maxit,
LinearOperatorBase<Field> &_Linop,
MrhsLinearFunction<Field> &Prec,
int _mmax,int _nstep)
: Tolerance(tol), MaxIterations(maxit), Linop(_Linop), Preconditioner(Prec),
mmax(_mmax), nstep(_nstep) { level=1; }
///////////////////////////////////////////////////////////////
// vector-of-fields linear algebra, reductions summed over rhs
///////////////////////////////////////////////////////////////
static RealD vnorm2(std::vector<Field> &x){
RealD s=0.0; for(auto &f : x) s+=norm2(f); return s;
}
static ComplexD vinnerProduct(std::vector<Field> &x, std::vector<Field> &y){
ComplexD s(0.0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s;
}
static void vaxpy(std::vector<Field> &z, ComplexD a, std::vector<Field> &x, std::vector<Field> &y){
for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]);
}
void vOp(std::vector<Field> &in, std::vector<Field> &out){
for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]);
}
void operator() (std::vector<Field> &src, std::vector<Field> &psi){
RealD cp, ssq, rsq;
int nrhs = src.size();
GridBase *grid = src[0].Grid();
ssq=vnorm2(src);
rsq=Tolerance*Tolerance*ssq;
std::vector<Field> r(nrhs,grid);
GridStopWatch SolverTimer;
SolverTimer.Start();
steps=0;
FirstCycle=1;
for(int k=0;k<MaxIterations;k++){
cp=GCRnStep(src,psi,rsq);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
if(cp<rsq){
SolverTimer.Stop();
vOp(psi,r);
for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
RealD tr=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR: Converged on iteration "<<steps
<<" computed residual "<<std::sqrt(cp/ssq)
<<" true residual "<<std::sqrt(tr/ssq)
<<" target "<<Tolerance<<std::endl;
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR Time elapsed: Total "<<SolverTimer.Elapsed()<<std::endl;
// Per-RHS true residuals: the honest metric under the summed norm
for(int rr=0;rr<nrhs;rr++){
RealD rn = std::sqrt(norm2(r[rr])/norm2(src[rr]));
std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl;
}
return;
}
}
std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
}
RealD GCRnStep(std::vector<Field> &src, std::vector<Field> &psi, RealD rsq){
RealD cp;
ComplexD a, b, rq;
RealD zAAz;
int nrhs = src.size();
GridBase *grid = src[0].Grid();
std::vector<Field> r (nrhs,grid);
std::vector<Field> z (nrhs,grid);
std::vector<Field> Az(nrhs,grid);
////////////////////////////////
// history for flexible orthog: [mmax][nrhs]
////////////////////////////////
std::vector< std::vector<Field> > q(mmax, std::vector<Field>(nrhs,grid));
std::vector< std::vector<Field> > p(mmax, std::vector<Field>(nrhs,grid));
std::vector<RealD> qq(mmax);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
if (ZeroGuess && FirstCycle) {
for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; }
} else {
vOp(psi,Az);
for(int rr=0;rr<nrhs;rr++) r[rr] = src[rr]-Az[rr];
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR true residual r = src - A psi "<<vnorm2(r)<<std::endl;
}
FirstCycle=0;
Preconditioner(r,z);
vOp(z,Az);
zAAz=vnorm2(Az);
p[0]=z;
q[0]=Az;
qq[0]=zAAz;
cp=vnorm2(r);
for(int k=0;k<nstep;k++){
steps++;
int kp = k+1;
int peri_k = k %mmax;
int peri_kp= kp%mmax;
rq = vinnerProduct(q[peri_k],r);
a = rq/qq[peri_k];
vaxpy(psi,a,p[peri_k],psi);
vaxpy(r,-a,q[peri_k],r);
cp = vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR step["<<steps<<"] resid "<<cp<<" target "<<rsq<<std::endl;
if((k==nstep-1)||(cp<rsq)){
return cp;
}
Preconditioner(r,z);
vOp(z,Az);
zAAz=vnorm2(Az);
q[peri_kp]=Az;
p[peri_kp]=z;
int northog = ((kp)>(mmax-1))?(mmax-1):(kp);
for(int back=0;back<northog;back++){
int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
b = -real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]);
vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]);
}
qq[peri_kp]=vnorm2(q[peri_kp]);
}
GRID_ASSERT(0); // never reached
return cp;
}
};
//////////////////////////////////////////////////////////////////////
// Trivial multi-RHS preconditioner
//////////////////////////////////////////////////////////////////////
@@ -382,98 +195,6 @@ public:
}
};
//////////////////////////////////////////////////////////////////////
// MultiRHS two-level V-cycle.
//
// Mirrors MGPreconditioner in Example_pvdagm:
// out = in (trivial pre) [per rhs]
// r1 = in - A out [per rhs]
// batched blockProject -> pack -> ONE mrhs coarse PGCR -> unpack
// -> batched blockPromote; out += correction
// r2 = in - A out [per rhs]
// per-RHS fine post-smoother; out += smooth(r2)
//////////////////////////////////////////////////////////////////////
template<class FineField, class MrhsCoarseVector, class FineSmoother>
class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
public:
typedef MrhsCoarseVector CoarseVector; // same lattice type on Coarse5d and CoarseMrhs
LinearOperatorBase<FineField> &_FineOperator;
FineSmoother &_PostSmoother; // single-RHS smoother, looped
MultiRHSBlockProject<FineField> &_Projector;
LinearFunction<CoarseVector> &_CoarseSolve; // PGCR on the 6D mrhs field
GridBase *_CoarseGrid; // Coarse5d (single rhs)
GridBase *_CoarseGridMrhs; // 6D
MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp,
FineSmoother &Post,
MultiRHSBlockProject<FineField> &Projector,
LinearFunction<CoarseVector> &CoarseSolve,
GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
: _FineOperator(FineOp), _PostSmoother(Post), _Projector(Projector),
_CoarseSolve(CoarseSolve), _CoarseGrid(CoarseGrid), _CoarseGridMrhs(CoarseGridMrhs) {}
virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
int nrhs = in.size();
GridBase *fgrid = in[0].Grid();
double t;
std::vector<FineField> vec1(nrhs,fgrid);
std::vector<FineField> vec2(nrhs,fgrid);
// Trivial pre-smoother: out = in (as in Example_pvdagm with simple_fine)
for(int r=0;r<nrhs;r++) out[r]=in[r];
// Residual
for(int r=0;r<nrhs;r++){
_FineOperator.Op(out[r],vec1[r]);
sub(vec1[r],in[r],vec1[r]);
}
// Batched fine->coarse, pack rhs into 6D field
std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid);
std::vector<CoarseVector> Csol_split(nrhs,_CoarseGrid);
CoarseVector CsrcMrhs(_CoarseGridMrhs);
CoarseVector CsolMrhs(_CoarseGridMrhs);
t=-usecond();
_Projector.blockProject(vec1,Csrc_split);
for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
// ONE coarse solve for all rhs (GEMM coarse mults)
t=-usecond();
CsolMrhs=Zero();
_CoarseSolve(CsrcMrhs,CsolMrhs);
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
// Unpack, batched coarse->fine, add correction
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
_Projector.blockPromote(vec1,Csol_split);
for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
// Residual
for(int r=0;r<nrhs;r++){
_FineOperator.Op(out[r],vec1[r]);
sub(vec1[r],in[r],vec1[r]);
}
// Per-RHS post-smoother (fine level has no batching win; memory-light)
t=-usecond();
for(int r=0;r<nrhs;r++){
vec2[r]=Zero();
_PostSmoother(vec1[r],vec2[r]);
add(out[r],out[r],vec2[r]);
}
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
}
};
int main (int argc, char ** argv)
{
examples/Example_pvdagm_mrhs_3level.cc
+1 -200
View File
@@ -50,6 +50,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
using namespace std;
using namespace Grid;
@@ -167,206 +168,6 @@ public:
void HermOp (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
};
//////////////////////////////////////////////////////////////////////
// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from Example_pvdagm_mrhs.cc):
// reductions summed over rhs -> one alpha/beta per step for the enlarged system.
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsLinearFunction {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
};
template<class Field>
class MrhsPGCRNonHermitian {
public:
RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
int ZeroGuess = 0; int FirstCycle = 0; // caller contract: zero guess => first-cycle r0 = src
std::string name = "Level 1";
LinearOperatorBase<Field> &Linop;
MrhsLinearFunction<Field> &Preconditioner;
void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
void Name(std::string n){ name = n; }
void SetZeroGuess(int z){ ZeroGuess=z; }
MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
: Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
void vOp(std::vector<Field> &in,std::vector<Field> &out){ for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
void operator()(std::vector<Field> &src,std::vector<Field> &psi){
RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
std::vector<Field> r(nrhs,grid);
GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
for(int k=0;k<MaxIterations;k++){
cp=GCRnStep(src,psi,rsq);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
if(cp<rsq){
T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
RealD tr=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
<<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
return;
}
}
std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
}
RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
std::vector<RealD> qq(mmax);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
else { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
FirstCycle=0;
Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
for(int k=0;k<nstep;k++){
steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"] resid "<<cp<<" target "<<rsq<<std::endl;
if((k==nstep-1)||(cp<rsq)) return cp;
Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
q[peri_kp]=Az; p[peri_kp]=z;
int northog=((kp)>(mmax-1))?(mmax-1):(kp);
for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
qq[peri_kp]=vnorm2(q[peri_kp]);
}
GRID_ASSERT(0); return cp;
}
};
//////////////////////////////////////////////////////////////////////
// L2->L3 mrhs V-cycle: a LinearFunction on the 6D mrhs COARSE field.
// Mirrors Example_pvdagm_mrhs.cc's MrhsTwoLevelMG one level down, and the
// single-RHS MGPreconditioner of Example_pvdagm_3level_SVDdefl.cc:
// out = in (trivial pre)
// r = in - A_coarse out
// restrict (unpack 6D coarse -> blockProject -> pack 6D coarse-coarse)
// ONE coarse-coarse solve (L3, GEMM)
// prolong (unpack -> blockPromote -> pack); out += correction
// r = in - A_coarse out
// coarse smoother (shifted 6D coarse op); out += smooth(r)
//////////////////////////////////////////////////////////////////////
template<class CoarseField, class CoarseCoarseField>
class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
public:
LinearOperatorBase<CoarseField> &_CoarseOp; // mrhs coarse op (6D)
LinearFunction<CoarseField> &_CoarseSmoother; // shifted 6D coarse smoother
MultiRHSBlockProject<CoarseField> &_Projector; // L2->L3 (vector-based)
LinearFunction<CoarseCoarseField> &_CoarseCoarseSolve; // L3 solve (6D cc)
GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
int _nrhs;
MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
LinearFunction<CoarseField> &CoarseSmoother,
MultiRHSBlockProject<CoarseField> &Projector,
LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
: _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
_CoarseCoarseSolve(CoarseCoarseSolve),
_Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
using LinearFunction<CoarseField>::operator();
virtual void operator()(const CoarseField &in, CoarseField &out) {
int nrhs=_nrhs; double t;
CoarseField vec1(in.Grid());
CoarseField vec2(in.Grid());
// trivial pre-smoother
out = in;
// residual (6D coarse)
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
// restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
std::vector<CoarseField> csplit(nrhs,_Coarse5d);
std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
CoarseCoarseField CCsol(_CoarseCoarseMrhs);
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
_Projector.blockProject(csplit,ccsplit);
for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
t+=usecond();
std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
// L3 solve (6D coarse-coarse, GEMM)
t=-usecond();
CCsol=Zero();
_CoarseCoarseSolve(CCsrc,CCsol);
t+=usecond();
std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
// prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
_Projector.blockPromote(csplit,ccsplit);
for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
add(out,out,vec1);
t+=usecond();
std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
// residual + coarse smoother (6D coarse)
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
vec2=Zero();
_CoarseSmoother(vec1,vec2);
add(out,out,vec2);
}
};
//////////////////////////////////////////////////////////////////////
// L1->L2 mrhs V-cycle (verbatim from Example_pvdagm_mrhs.cc):
// per-rhs fine smoother + batched restriction + ONE coarse solve + batched prolong.
// The coarse solve passed in is now itself three-level (preconditioned by L2->L3).
//////////////////////////////////////////////////////////////////////
template<class FineField, class MrhsCoarseVector, class FineSmoother>
class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
public:
typedef MrhsCoarseVector CoarseVector;
LinearOperatorBase<FineField> &_FineOperator;
FineSmoother &_PostSmoother;
MultiRHSBlockProject<FineField> &_Projector;
LinearFunction<CoarseVector> &_CoarseSolve;
GridBase *_CoarseGrid, *_CoarseGridMrhs;
MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
: _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
_CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
for(int r=0;r<nrhs;r++) out[r]=in[r];
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
t=-usecond();
_Projector.blockProject(vec1,Csrc_split);
for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
t=-usecond(); CsolMrhs=Zero(); _CoarseSolve(CsrcMrhs,CsolMrhs); t+=usecond();
std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
_Projector.blockPromote(vec1,Csol_split);
for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
t=-usecond();
for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
}
};
int main (int argc, char ** argv)
{
examples/Example_pvdagm_mrhs_3level_DenseCoarseMatrix.cc
+2 -276
View File
@@ -50,6 +50,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
#include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
#include <memory>
@@ -173,281 +174,6 @@ public:
void HermOp (const Field &in, Field &out) { Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
};
//////////////////////////////////////////////////////////////////////
// Dense CC solve on the PACKED 6D mrhs coarse-coarse field: drop-in
// for the L3 PGCR, delegating to the library DenseCoarseMatrix.
//////////////////////////////////////////////////////////////////////
template<class DenseType, class CoarseCoarseField>
class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
public:
DenseType &_Dense;
GridBase *_CoarseCoarse5d;
int _nrhs;
MrhsDenseCCSolve(DenseType &D, GridBase *cc5d, int nrhs)
: _Dense(D), _CoarseCoarse5d(cc5d), _nrhs(nrhs) {}
using LinearFunction<CoarseCoarseField>::operator();
virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
if ( getenv("DENSE_CC_CHECK") ) {
// Audit path: per-rhs 5D unpack so ApplyBatch can run the _Op defect
// check per rhs. ~50ms/call of slice/split overhead -- audit only.
CoarseCoarseField tmp(in.Grid());
tmp = in;
std::vector<CoarseCoarseField> split_in (_nrhs,_CoarseCoarse5d);
std::vector<CoarseCoarseField> split_out(_nrhs,_CoarseCoarse5d);
for(int r=0;r<_nrhs;r++) ExtractSliceFast(split_in[r], tmp, r, 0);
_Dense.ApplyBatch(split_in, split_out);
for(int r=0;r<_nrhs;r++) InsertSliceFast(split_out[r], out, r, 0);
} else {
GRID_TRACE("MrhsDensCCSolve::ApplyBatch6D");
_Dense.ApplyBatch6D(in, out, _nrhs);
}
}
};
//////////////////////////////////////////////////////////////////////
// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from the
// frozen Example_pvdagm_mrhs_3level_dense.cc)
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsLinearFunction {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
};
template<class Field>
class MrhsPGCRNonHermitian {
public:
RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
int ZeroGuess = 0; int FirstCycle = 0; // caller contract: zero guess => first-cycle r0 = src
std::string name = "Level 1";
LinearOperatorBase<Field> &Linop;
MrhsLinearFunction<Field> &Preconditioner;
void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
void Name(std::string n){ name = n; }
void SetZeroGuess(int z){ ZeroGuess=z; }
MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
: Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
static RealD vnorm2(std::vector<Field> &x){ GRID_TRACE("GCR-vnorm2"); RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ GRID_TRACE("GCR-vinnerProduct"); ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ GRID_TRACE("GCR-vaxpy"); for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("GCR-vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
void operator()(std::vector<Field> &src,std::vector<Field> &psi){
GRID_TRACE((name+"MrhsPGCRNonHermitian").c_str());
RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
std::vector<Field> r(nrhs,grid);
GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
for(int k=0;k<MaxIterations;k++){
cp=GCRnStep(src,psi,rsq);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
if(cp<rsq){
T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
RealD tr=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
<<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
return;
}
}
std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
}
RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
std::vector<RealD> qq(mmax);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
else { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
FirstCycle=0;
Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
for(int k=0;k<nstep;k++){
steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
{
GRID_TRACE("GCR-project");
rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
}
// rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
// vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"] resid "<<cp<<" target "<<rsq<<std::endl;
if((k==nstep-1)||(cp<rsq)) return cp;
Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
{
GRID_TRACE("GCR-orthog");
q[peri_kp]=Az; p[peri_kp]=z;
int northog=((kp)>(mmax-1))?(mmax-1):(kp);
for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
qq[peri_kp]=vnorm2(q[peri_kp]);
}
// q[peri_kp]=Az; p[peri_kp]=z;
// int northog=((kp)>(mmax-1))?(mmax-1):(kp);
// for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
// b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
// vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
// qq[peri_kp]=vnorm2(q[peri_kp]);
}
GRID_ASSERT(0); return cp;
}
};
//////////////////////////////////////////////////////////////////////
// L2->L3 mrhs V-cycle: LinearFunction on the 6D mrhs COARSE field.
// The coarse-coarse solve slot takes EITHER the dense mrhs solve
// (DENSE_CC=1) or the L3 PGCR (DENSE_CC=0).
//////////////////////////////////////////////////////////////////////
template<class CoarseField, class CoarseCoarseField>
class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
public:
LinearOperatorBase<CoarseField> &_CoarseOp; // mrhs coarse op (6D)
LinearFunction<CoarseField> &_CoarseSmoother; // shifted 6D coarse smoother
MultiRHSBlockProject<CoarseField> &_Projector; // L2->L3 (vector-based)
LinearFunction<CoarseCoarseField> &_CoarseCoarseSolve; // L3 solve (6D cc)
GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
int _nrhs;
MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
LinearFunction<CoarseField> &CoarseSmoother,
MultiRHSBlockProject<CoarseField> &Projector,
LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
: _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
_CoarseCoarseSolve(CoarseCoarseSolve),
_Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
using LinearFunction<CoarseField>::operator();
virtual void operator()(const CoarseField &in, CoarseField &out) {
int nrhs=_nrhs; double t;
CoarseField vec1(in.Grid());
CoarseField vec2(in.Grid());
// trivial pre-smoother
out = in;
// residual (6D coarse)
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
// restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
std::vector<CoarseField> csplit(nrhs,_Coarse5d);
std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
CoarseCoarseField CCsol(_CoarseCoarseMrhs);
t=-usecond();
{
GRID_TRACE("L2L3-Vcycle - Extract/blockProject/Insert");
for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
_Projector.blockProject(csplit,ccsplit);
for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
}
t+=usecond();
std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
// L3 solve (dense mrhs GEMM, or PGCR)
t=-usecond();
{
GRID_TRACE("L2L3-Vcycle - CoarseCoarseSolve");
CCsol=Zero();
_CoarseCoarseSolve(CCsrc,CCsol);
}
t+=usecond();
std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
// prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
t=-usecond();
{
GRID_TRACE("L2L3-Vcycle - Ext/blockPromote/Ins");
for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
_Projector.blockPromote(csplit,ccsplit);
for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
add(out,out,vec1);
}
t+=usecond();
std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
// residual + coarse smoother (6D coarse)
{
GRID_TRACE("L2L3-Vcycle - Resid+CoarseSmooth");
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
vec2=Zero();
_CoarseSmoother(vec1,vec2);
add(out,out,vec2);
}
}
};
//////////////////////////////////////////////////////////////////////
// L1->L2 mrhs V-cycle (verbatim from the frozen example)
//////////////////////////////////////////////////////////////////////
template<class FineField, class MrhsCoarseVector, class FineSmoother>
class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
public:
typedef MrhsCoarseVector CoarseVector;
LinearOperatorBase<FineField> &_FineOperator;
FineSmoother &_PostSmoother;
MultiRHSBlockProject<FineField> &_Projector;
LinearFunction<CoarseVector> &_CoarseSolve;
GridBase *_CoarseGrid, *_CoarseGridMrhs;
MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
: _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
_CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
{
GRID_TRACE("L1L2-Vcycle - Resid");
for(int r=0;r<nrhs;r++) out[r]=in[r];
for(int r=0;r<nrhs;r++){
_FineOperator.Op(out[r],vec1[r]);
sub(vec1[r],in[r],vec1[r]);
}
}
std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
t=-usecond();
{
GRID_TRACE("L1L2-Vcycle - blockProject");
_Projector.blockProject(vec1,Csrc_split);
for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
}
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
t=-usecond();
{
GRID_TRACE("L1L2-Vcycle - CoarseSolve");
CsolMrhs=Zero();
_CoarseSolve(CsrcMrhs,CsolMrhs);
}
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
t=-usecond();
{
GRID_TRACE("L1L2-Vcycle - ExtractSlice/blockPromote/add delta");
for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
_Projector.blockPromote(vec1,Csol_split);
for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
}
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
{
GRID_TRACE("L1L2-Vcycle - Residuals");
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
}
t=-usecond();
{
GRID_TRACE("L1L2-Vcycle - Smoothers");
for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
}
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
}
};
int main (int argc, char ** argv)
{
@@ -629,7 +355,7 @@ int main (int argc, char ** argv)
std::cout << GridLogMessage << "**********************************************" << std::endl;
DenseCC.reset(new DenseCC_t(CoarseCoarse5d));
DenseCC->Import(LittleDiracOpL2);
MrhsDenseCC.reset(new MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector>(*DenseCC, CoarseCoarse5d, nrhs));
MrhsDenseCC.reset(new MrhsDenseCCSolve<DenseCC_t,CoarseCoarseVector>(*DenseCC, nrhs));
}
//////////////////////////////////////////////////////////////////////
examples/Example_pvdagm_mrhs_3level_dense.cc
+3 -222
View File
@@ -60,6 +60,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
#include <unordered_map>
#include <memory>
@@ -830,226 +831,6 @@ public:
}
};
//////////////////////////////////////////////////////////////////////
// Dense CC solve on the PACKED 6D mrhs coarse-coarse field: unpack per
// rhs, ONE batched dense apply, repack. Drop-in for the L3 PGCR.
//////////////////////////////////////////////////////////////////////
template<class CoarseCoarseField>
class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
public:
DistributedDenseInverse<CoarseCoarseField> &_Dense;
GridBase *_CoarseCoarse5d;
int _nrhs;
MrhsDenseCCSolve(DistributedDenseInverse<CoarseCoarseField> &D, GridBase *cc5d, int nrhs)
: _Dense(D), _CoarseCoarse5d(cc5d), _nrhs(nrhs) {}
using LinearFunction<CoarseCoarseField>::operator();
virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
if ( getenv("DENSE_CC_CHECK") ) {
// Audit path: per-rhs 5D unpack so ApplyBatch can run the _Op defect
// check per rhs. ~50ms/call of slice/split overhead -- audit only.
CoarseCoarseField tmp(in.Grid());
tmp = in;
std::vector<CoarseCoarseField> split_in (_nrhs,_CoarseCoarse5d);
std::vector<CoarseCoarseField> split_out(_nrhs,_CoarseCoarse5d);
for(int r=0;r<_nrhs;r++) ExtractSliceFast(split_in[r], tmp, r, 0);
_Dense.ApplyBatch(split_in, split_out);
for(int r=0;r<_nrhs;r++) InsertSliceFast(split_out[r], out, r, 0);
} else {
_Dense.ApplyBatch6D(in, out, _nrhs);
}
}
};
//////////////////////////////////////////////////////////////////////
// mrhs interfaces + single-polynomial mrhs PGCR (verbatim from Example_pvdagm_mrhs.cc)
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsLinearFunction {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
};
template<class Field>
class MrhsPGCRNonHermitian {
public:
RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
int ZeroGuess = 0; int FirstCycle = 0; // caller contract: zero guess => first-cycle r0 = src
std::string name = "Level 1";
LinearOperatorBase<Field> &Linop;
MrhsLinearFunction<Field> &Preconditioner;
void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
void Name(std::string n){ name = n; }
void SetZeroGuess(int z){ ZeroGuess=z; }
MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
: Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
void vOp(std::vector<Field> &in,std::vector<Field> &out){ for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
void operator()(std::vector<Field> &src,std::vector<Field> &psi){
RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
std::vector<Field> r(nrhs,grid);
GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
for(int k=0;k<MaxIterations;k++){
cp=GCRnStep(src,psi,rsq);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
if(cp<rsq){
T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
RealD tr=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
<<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
for(int rr=0;rr<nrhs;rr++){ RealD rn=std::sqrt(norm2(r[rr])/norm2(src[rr])); std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl; }
return;
}
}
std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
}
RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
RealD cp; ComplexD a,b,rq; RealD zAAz; int nrhs=src.size(); GridBase *grid=src[0].Grid();
std::vector<Field> r(nrhs,grid),z(nrhs,grid),Az(nrhs,grid);
std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
std::vector<RealD> qq(mmax);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
else { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
FirstCycle=0;
Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
p[0]=z; q[0]=Az; qq[0]=zAAz; cp=vnorm2(r);
for(int k=0;k<nstep;k++){
steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"] resid "<<cp<<" target "<<rsq<<std::endl;
if((k==nstep-1)||(cp<rsq)) return cp;
Preconditioner(r,z); vOp(z,Az); zAAz=vnorm2(Az);
q[peri_kp]=Az; p[peri_kp]=z;
int northog=((kp)>(mmax-1))?(mmax-1):(kp);
for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
b=-real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]); vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]); }
qq[peri_kp]=vnorm2(q[peri_kp]);
}
GRID_ASSERT(0); return cp;
}
};
//////////////////////////////////////////////////////////////////////
// L2->L3 mrhs V-cycle: LinearFunction on the 6D mrhs COARSE field.
// The coarse-coarse solve slot now takes EITHER the dense mrhs solve
// (DENSE_CC=1) or the L3 PGCR (DENSE_CC=0).
//////////////////////////////////////////////////////////////////////
template<class CoarseField, class CoarseCoarseField>
class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
public:
LinearOperatorBase<CoarseField> &_CoarseOp; // mrhs coarse op (6D)
LinearFunction<CoarseField> &_CoarseSmoother; // shifted 6D coarse smoother
MultiRHSBlockProject<CoarseField> &_Projector; // L2->L3 (vector-based)
LinearFunction<CoarseCoarseField> &_CoarseCoarseSolve; // L3 solve (6D cc)
GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
int _nrhs;
MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
LinearFunction<CoarseField> &CoarseSmoother,
MultiRHSBlockProject<CoarseField> &Projector,
LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
: _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
_CoarseCoarseSolve(CoarseCoarseSolve),
_Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
using LinearFunction<CoarseField>::operator();
virtual void operator()(const CoarseField &in, CoarseField &out) {
int nrhs=_nrhs; double t;
CoarseField vec1(in.Grid());
CoarseField vec2(in.Grid());
// trivial pre-smoother
out = in;
// residual (6D coarse)
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
// restrict: unpack 6D coarse -> vector<Coarse> -> blockProject -> vector<CoarseCoarse> -> pack 6D cc
std::vector<CoarseField> csplit(nrhs,_Coarse5d);
std::vector<CoarseCoarseField> ccsplit(nrhs,_CoarseCoarse5d);
CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
CoarseCoarseField CCsol(_CoarseCoarseMrhs);
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(csplit[r],vec1,r,0);
_Projector.blockProject(csplit,ccsplit);
for(int r=0;r<nrhs;r++) InsertSliceFast(ccsplit[r],CCsrc,r,0);
t+=usecond();
std::cout<<GridLogMessage<<"L2->L3 restrict took "<<t/1000.0<<"ms"<<std::endl;
// L3 solve (dense mrhs GEMM, or PGCR)
t=-usecond();
CCsol=Zero();
_CoarseCoarseSolve(CCsrc,CCsol);
t+=usecond();
std::cout<<GridLogMessage<<"L3 coarse-coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
// prolong: unpack 6D cc -> blockPromote -> pack 6D coarse; add correction
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(ccsplit[r],CCsol,r,0);
_Projector.blockPromote(csplit,ccsplit);
for(int r=0;r<nrhs;r++) InsertSliceFast(csplit[r],vec1,r,0);
add(out,out,vec1);
t+=usecond();
std::cout<<GridLogMessage<<"L2->L3 prolong took "<<t/1000.0<<"ms"<<std::endl;
// residual + coarse smoother (6D coarse)
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
vec2=Zero();
_CoarseSmoother(vec1,vec2);
add(out,out,vec2);
}
};
//////////////////////////////////////////////////////////////////////
// L1->L2 mrhs V-cycle (verbatim from Example_pvdagm_mrhs.cc)
//////////////////////////////////////////////////////////////////////
template<class FineField, class MrhsCoarseVector, class FineSmoother>
class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
public:
typedef MrhsCoarseVector CoarseVector;
LinearOperatorBase<FineField> &_FineOperator;
FineSmoother &_PostSmoother;
MultiRHSBlockProject<FineField> &_Projector;
LinearFunction<CoarseVector> &_CoarseSolve;
GridBase *_CoarseGrid, *_CoarseGridMrhs;
MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
: _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
_CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
int nrhs=in.size(); GridBase *fgrid=in[0].Grid(); double t;
std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
for(int r=0;r<nrhs;r++) out[r]=in[r];
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid), Csol_split(nrhs,_CoarseGrid);
CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
t=-usecond();
_Projector.blockProject(vec1,Csrc_split);
for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
t=-usecond(); CsolMrhs=Zero(); _CoarseSolve(CsrcMrhs,CsolMrhs); t+=usecond();
std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
_Projector.blockPromote(vec1,Csol_split);
for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
t=-usecond();
for(int r=0;r<nrhs;r++){ vec2[r]=Zero(); _PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]); }
t+=usecond(); std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
}
};
int main (int argc, char ** argv)
{
@@ -1221,13 +1002,13 @@ int main (int argc, char ** argv)
// its memory peak; needs only the hoisted single-RHS LinOpCC5d).
//////////////////////////////////////////////////////////////////////
std::unique_ptr<DistributedDenseInverse<CoarseCoarseVector>> DenseCC;
std::unique_ptr<MrhsDenseCCSolve<CoarseCoarseVector>> MrhsDenseCC;
std::unique_ptr<MrhsDenseCCSolve<DistributedDenseInverse<CoarseCoarseVector>,CoarseCoarseVector>> MrhsDenseCC;
if (UseDenseCC) {
std::cout << GridLogMessage << "**********************************************" << std::endl;
std::cout << GridLogMessage << " Dense CC inverse setup (mrhs bottom)" << std::endl;
std::cout << GridLogMessage << "**********************************************" << std::endl;
DenseCC.reset(new DistributedDenseInverse<CoarseCoarseVector>(LinOpCC5d, CoarseCoarse5d, nbasis));
MrhsDenseCC.reset(new MrhsDenseCCSolve<CoarseCoarseVector>(*DenseCC, CoarseCoarse5d, nrhs));
MrhsDenseCC.reset(new MrhsDenseCCSolve<DistributedDenseInverse<CoarseCoarseVector>,CoarseCoarseVector>(*DenseCC, nrhs));
}
//////////////////////////////////////////////////////////////////////
examples/Example_pvdagm_multigrid.cc
+42 -4
View File
@@ -33,7 +33,11 @@ Author: Peter Boyle <pboyle@bnl.gov>
// parameter. The effective parameters are always printed, so the log
// describes its own run.
//
// Compile-time: -DNBASIS=8 cuts the basis down for laptop runs.
// Compile-time: NBASIS defaults to 60, the production basis; a subspace file
// may hold more vectors, the load reads only the first NBASIS. -DNBASIS=8
// cuts it down for laptop runs.
// examples/Makefile.am builds the fp32-coarse and fp32-dense-inversion
// variants below as their own binaries.
//
#include <Grid/Grid.h>
#include <Grid/lattice/PaddedCell.h>
@@ -48,6 +52,16 @@ using namespace Grid;
#define NBASIS 60
#endif
// Precision of the coarse + coarse-coarse sector is a compile-time
// instantiation: -DCOARSE_SINGLE builds the fp32 coarse space (the dense
// bottom's apply slab is fp32 either way; its inversion is a configure
// option). Both levels carry the same site type iVector<CoarseScalar,NBASIS>.
#ifdef COARSE_SINGLE
typedef sTComplexF CoarseScalar_t;
#else
typedef sTComplexD CoarseScalar_t;
#endif
struct PVdagMDriverParams : Serializable {
GRID_SERIALIZABLE_CLASS_MEMBERS(PVdagMDriverParams,
int, Ls,
@@ -123,22 +137,46 @@ int main (int argc, char ** argv)
//////////////////////////////////////////////////////////////////////
// Grids -> coarsening -> dense bottom. Scope order is lifetime order.
//
// The fine operator applied during setup is always fp64, on the fp64
// basis. Everything the coarsening produces -- the Galerkin matrix
// elements and the transfer operator's store -- takes the coarse precision
// (CoarseScalar_t, a compile-time choice).
//
// There is exactly ONE fine transfer operator. Its STORE follows the
// coarse sector, since the coarse space is what it feeds, while its import
// and export accept either fine precision: they are already a layout
// transformation, and a scalar conversion inside one is free. So the fp64
// setup and an fp32 V-cycle share the same object and the same basis store.
// FinePrecision selects only the fine operator and smoother; the outer
// Krylov and its true-residual check stay fp64 throughout.
//////////////////////////////////////////////////////////////////////
typedef PVdagMMultiGridCoarsening<vSpinColourVector,sTComplexD,NBASIS> Coarsening_t;
typedef PVdagMMultiGridCoarsening<vSpinColourVector,CoarseScalar_t,NBASIS> Coarsening_t;
std::cout << GridLogMessage << "Coarse sector precision (compiled): "
<< (sizeof(typename GridTypeMapper<CoarseScalar_t>::scalar_type)==sizeof(ComplexF) ? "fp32" : "fp64")
<< ", nbasis " << NBASIS << std::endl;
MGCoarseGrids CGrids(FGrid, P.MultiGrid.Setup);
Coarsening_t Coarsening(CGrids, P.MultiGrid.Setup);
MGFineGridsF FGridsF(FGrid);
LatticeGaugeFieldF UmuF(FGridsF.UGridF);
precisionChange(UmuF,Umu);
MobiusFermionF DdwfF(UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,P.Mass,P.M5,P.MobiusB,P.MobiusC);
MobiusFermionF DpvF (UmuF,*FGridsF.FGridF,*FGridsF.FrbGridF,*FGridsF.UGridF,*FGridsF.UrbGridF,1.0, P.M5,P.MobiusB,P.MobiusC);
Coarsening_t Coarsening(CGrids, FGridsF, P.MultiGrid.Setup);
Coarsening.GetSubspace(RNG5, PVdagM);
Coarsening.Coarsen(PVdagM);
Coarsening.BuildDenseBottom();
Coarsening.CertifyCoarsening(PVdagM);
//////////////////////////////////////////////////////////////////////
// Solves: mrhs then (optionally) single RHS through the SAME objects.
//////////////////////////////////////////////////////////////////////
auto RunSolve = [&](int nr)
{
PVdagMMultiGridSolver<MobiusFermionD,Coarsening_t> Solver(Ddwf,Dpv,Coarsening,P.MultiGrid,nr);
PVdagMMultiGridSolver<MobiusFermionD,MobiusFermionF,Coarsening_t> Solver(Ddwf,Dpv,DdwfF,DpvF,Coarsening,P.MultiGrid,nr);
std::vector<LatticeFermionD> src(nr,FGrid), sol(nr,FGrid);
for(int r=0;r<nr;r++){ gaussian(RNG5,src[r]); sol[r]=Zero(); }
Solver.Solve(src,sol);
examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc
+3 -214
View File
@@ -62,6 +62,7 @@ Author: Peter Boyle <pboyle@bnl.gov>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <Grid/algorithms/multigrid/MrhsMultiGrid.h>
#include <Grid/algorithms/multigrid/DenseCoarseMatrix.h>
#include <memory>
@@ -396,220 +397,6 @@ void PowerIteration(const std::string &name, LinearOperatorBase<Field> &Op, Grid
<< std::endl;
}
//////////////////////////////////////////////////////////////////////
// Dense L3 solve on the packed D+1 coarse-coarse field
//////////////////////////////////////////////////////////////////////
template<class DenseType, class CoarseCoarseField>
class MrhsDenseCCSolve : public LinearFunction<CoarseCoarseField> {
public:
DenseType &_Dense;
int _nrhs;
MrhsDenseCCSolve(DenseType &D, int nrhs) : _Dense(D), _nrhs(nrhs) {}
using LinearFunction<CoarseCoarseField>::operator();
virtual void operator()(const CoarseCoarseField &in, CoarseCoarseField &out){
_Dense.ApplyBatch6D(in, out, _nrhs);
}
};
//////////////////////////////////////////////////////////////////////
// mrhs interfaces + single-polynomial mrhs PGCR
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsLinearFunction {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
};
template<class Field>
class MrhsPGCRNonHermitian {
public:
RealD Tolerance; Integer MaxIterations; int mmax,nstep,steps,level;
int ZeroGuess = 0; int FirstCycle = 0;
std::string name = "Level 1";
LinearOperatorBase<Field> &Linop;
MrhsLinearFunction<Field> &Preconditioner;
std::function<void(int)> OnStep; // called with the outer step count after every step (smoother switching)
void Level(int lv){ name = "Level " + std::to_string(lv); level=lv; }
void Name(std::string n){ name = n; }
void SetZeroGuess(int z){ ZeroGuess=z; }
MrhsPGCRNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,MrhsLinearFunction<Field> &Prec,int _mmax,int _nstep)
: Tolerance(tol),MaxIterations(maxit),Linop(_Linop),Preconditioner(Prec),mmax(_mmax),nstep(_nstep){ level=1; }
static RealD vnorm2(std::vector<Field> &x){ RealD s=0; for(auto &f:x) s+=norm2(f); return s; }
static ComplexD vinnerProduct(std::vector<Field> &x,std::vector<Field> &y){ ComplexD s(0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s; }
static void vaxpy(std::vector<Field> &z,ComplexD a,std::vector<Field> &x,std::vector<Field> &y){ for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]); }
void vOp(std::vector<Field> &in,std::vector<Field> &out){ GRID_TRACE("MrhsPGCR::vOp"); for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]); }
void operator()(std::vector<Field> &src,std::vector<Field> &psi){
RealD cp,ssq,rsq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
ssq=vnorm2(src); rsq=Tolerance*Tolerance*ssq;
std::vector<Field> r(nrhs,grid);
GridStopWatch T; T.Start(); steps=0; FirstCycle=1;
for(int k=0;k<MaxIterations;k++){
cp=GCRnStep(src,psi,rsq);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
if(cp<rsq){
T.Stop(); vOp(psi,r); for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
RealD tr=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR: Converged on iteration "<<steps
<<" computed residual "<<std::sqrt(cp/ssq)<<" true residual "<<std::sqrt(tr/ssq)<<" target "<<Tolerance<<std::endl;
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR Time elapsed: Total "<<T.Elapsed()<<std::endl;
return;
}
}
std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
}
RealD GCRnStep(std::vector<Field> &src,std::vector<Field> &psi,RealD rsq){
RealD cp; ComplexD a,b,rq; int nrhs=src.size(); GridBase *grid=src[0].Grid();
std::vector<Field> r(nrhs,grid),Az(nrhs,grid); // Az: restart residual scratch only
std::vector< std::vector<Field> > q(mmax,std::vector<Field>(nrhs,grid));
std::vector< std::vector<Field> > p(mmax,std::vector<Field>(nrhs,grid));
std::vector<RealD> qq(mmax);
if (ZeroGuess && FirstCycle) { for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; } }
else { vOp(psi,Az); for(int rr=0;rr<nrhs;rr++) r[rr]=src[rr]-Az[rr]; }
FirstCycle=0;
// p[0]=Prec(r), q[0]=A p[0], produced directly in the history slots (no copies)
Preconditioner(r,p[0]); vOp(p[0],q[0]); qq[0]=vnorm2(q[0]); cp=vnorm2(r);
for(int k=0;k<nstep;k++){
steps++; int kp=k+1, peri_k=k%mmax, peri_kp=kp%mmax;
if ( OnStep ) OnStep(steps);
rq=vinnerProduct(q[peri_k],r); a=rq/qq[peri_k];
vaxpy(psi,a,p[peri_k],psi); vaxpy(r,-a,q[peri_k],r); cp=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR step["<<steps<<"] resid "<<cp<<" target "<<rsq<<std::endl;
if((k==nstep-1)||(cp<rsq)) return cp;
// New direction straight into its history slot: p=Prec(r), q=A p.
Preconditioner(r,p[peri_kp]);
vOp(p[peri_kp],q[peri_kp]);
int northog=((kp)>(mmax-1))?(mmax-1):(kp);
MemoryManager::Snapshot(name+" orthog begin step "+std::to_string(steps));
{
GRID_TRACE("MrhsPGCR orthog");
// Classical Gram-Schmidt: all coefficients against the UN-updated new q
// (independent, batchable), then apply. Complex coefficient: the
// operator is non-Hermitian, real(<q_j,Aq>) alone left q's non-orthogonal.
// Batched per rhs (one fused kernel + one reduction each), the shared
// coefficient summed over rhs on the host, ONE GlobalSumVector.
std::vector<ComplexD> bcoef(northog,ComplexD(0.0)), part;
for(int rr=0;rr<nrhs;rr++){
std::vector<const Field*> qwin(northog);
for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0); qwin[back]=&q[peri_back][rr]; }
rankInnerProductMulti(part,qwin,q[peri_kp][rr]);
for(int back=0;back<northog;back++) bcoef[back]+=part[back];
}
if(northog) grid->GlobalSumVector(&bcoef[0],northog);
for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; bcoef[back]=-bcoef[back]/qq[peri_back]; }
for(int rr=0;rr<nrhs;rr++){
std::vector<const Field*> qwin(northog), pwin(northog);
for(int back=0;back<northog;back++){ int peri_back=(k-back)%mmax; qwin[back]=&q[peri_back][rr]; pwin[back]=&p[peri_back][rr]; }
axpyMulti(p[peri_kp][rr],bcoef,pwin);
axpyMulti(q[peri_kp][rr],bcoef,qwin);
}
}
qq[peri_kp]=vnorm2(q[peri_kp]);
MemoryManager::Snapshot(name+" orthog+vnorm2 end step "+std::to_string(steps));
}
GRID_ASSERT(0); return cp;
}
};
//////////////////////////////////////////////////////////////////////
// L2->L3 mrhs V-cycle on the D+1 coarse field
//////////////////////////////////////////////////////////////////////
template<class CoarseField, class CoarseCoarseField>
class MrhsCoarseThreeLevelPrec : public LinearFunction<CoarseField> {
public:
LinearOperatorBase<CoarseField> &_CoarseOp;
LinearFunction<CoarseField> &_CoarseSmoother;
MultiRHSBlockProject<CoarseField> &_Projector;
LinearFunction<CoarseCoarseField> &_CoarseCoarseSolve;
GridBase *_Coarse5d, *_CoarseCoarse5d, *_CoarseCoarseMrhs;
int _nrhs;
MrhsCoarseThreeLevelPrec(LinearOperatorBase<CoarseField> &CoarseOp,
LinearFunction<CoarseField> &CoarseSmoother,
MultiRHSBlockProject<CoarseField> &Projector,
LinearFunction<CoarseCoarseField> &CoarseCoarseSolve,
GridBase *Coarse5d, GridBase *CoarseCoarse5d, GridBase *CoarseCoarseMrhs, int nrhs)
: _CoarseOp(CoarseOp), _CoarseSmoother(CoarseSmoother), _Projector(Projector),
_CoarseCoarseSolve(CoarseCoarseSolve),
_Coarse5d(Coarse5d), _CoarseCoarse5d(CoarseCoarse5d), _CoarseCoarseMrhs(CoarseCoarseMrhs), _nrhs(nrhs) {}
using LinearFunction<CoarseField>::operator();
virtual void operator()(const CoarseField &in, CoarseField &out) {
int nrhs=_nrhs;
CoarseField vec1(in.Grid());
CoarseField vec2(in.Grid());
out = in;
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
// restrict, through the mixed blockProject: D+1 coarse in, D+1 cc out
CoarseCoarseField CCsrc(_CoarseCoarseMrhs);
CoarseCoarseField CCsol(_CoarseCoarseMrhs);
_Projector.blockProject(vec1,CCsrc);
// CCsol=Zero(); // Is this necessary?
_CoarseCoarseSolve(CCsrc,CCsol);
_Projector.blockPromote(vec1,CCsol);
add(out,out,vec1);
_CoarseOp.Op(out,vec1); sub(vec1,in,vec1);
// vec2=Zero(); // Zero guess
_CoarseSmoother(vec1,vec2);
add(out,out,vec2);
}
};
//////////////////////////////////////////////////////////////////////
// L1->L2 mrhs V-cycle
//////////////////////////////////////////////////////////////////////
template<class FineField, class MrhsCoarseVector, class FineSmoother>
class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
public:
typedef MrhsCoarseVector CoarseVector;
LinearOperatorBase<FineField> &_FineOperator;
FineSmoother &_PostSmoother;
MultiRHSBlockProject<FineField> &_Projector;
LinearFunction<CoarseVector> &_CoarseSolve;
GridBase *_CoarseGrid, *_CoarseGridMrhs;
MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp, FineSmoother &Post,
MultiRHSBlockProject<FineField> &Projector, LinearFunction<CoarseVector> &CoarseSolve,
GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
: _FineOperator(FineOp),_PostSmoother(Post),_Projector(Projector),_CoarseSolve(CoarseSolve),
_CoarseGrid(CoarseGrid),_CoarseGridMrhs(CoarseGridMrhs){}
virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
// The whole V-cycle is preconditioner: its fine residuals and the
// smoother run with sloppy halos; the caller (the outer Krylov) gets
// the exact operator back on exit.
GRID_TRACE("MGVcycle");
SetFineSloppy(FineSloppyComms);
int nrhs=in.size(); GridBase *fgrid=in[0].Grid();
std::vector<FineField> vec1(nrhs,fgrid),vec2(nrhs,fgrid);
for(int r=0;r<nrhs;r++) out[r]=in[r];
{ GRID_TRACE("MGFineResidual");
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
}
// fine vector -> D+1 coarse, via the mixed blockProject
CoarseVector CsrcMrhs(_CoarseGridMrhs), CsolMrhs(_CoarseGridMrhs);
{ GRID_TRACE("MGProject");
_Projector.blockProject(vec1,CsrcMrhs);
}
CsolMrhs=Zero();
{ GRID_TRACE("MGCoarseSolve");
_CoarseSolve(CsrcMrhs,CsolMrhs);
}
{ GRID_TRACE("MGPromote");
_Projector.blockPromote(vec1,CsolMrhs);
for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
}
{ GRID_TRACE("MGFineResidual2");
for(int r=0;r<nrhs;r++){ _FineOperator.Op(out[r],vec1[r]); sub(vec1[r],in[r],vec1[r]); }
}
{ GRID_TRACE("MGPostSmooth");
for(int r=0;r<nrhs;r++){
// vec2[r]=Zero();
_PostSmoother(vec1[r],vec2[r]); add(out[r],out[r],vec2[r]);
}
}
SetFineSloppy(0);
}
};
int main (int argc, char ** argv)
{
@@ -1074,6 +861,8 @@ int main (int argc, char ** argv)
SwitchableSmoother<LatticeFermionD> FineSmootherSlot(SmootherGCR,"Fsmoother GCR");
MrhsTwoLevelMG<LatticeFermionD,CoarseVector,SwitchableSmoother<LatticeFermionD> >
ThreeLevelPrecon(PVdagM, FineSmootherSlot, MrhsProjector, L2PGCR, Coarse5d, CMrhs);
ThreeLevelPrecon.SetSloppy = SetFineSloppy;
ThreeLevelPrecon.SloppyComms = FineSloppyComms;
MrhsPGCRNonHermitian<LatticeFermionD>
L1PGCR(OuterTol,1000,PVdagM,ThreeLevelPrecon,OuterMmax,OuterNstep);
examples/Makefile.am
+18
View File
@@ -2,5 +2,23 @@ SUBDIRS = .
include Make.inc
# Precision variants of the two multigrid drivers. The coarse-sector and
# dense-inversion precisions are compile-time instantiations, so each is its
# own binary built from the same source; the fine-level precision is a
# run-time parameter and needs no variant. Per-target flags give each
# variant its own object file.
bin_PROGRAMS += Example_pvdagm_multigrid_fp32coarse \
Example_pvdagm_multigrid_fp32coarse_fp32dense \
Example_hdcg_multigrid_fp32coarse
Example_pvdagm_multigrid_fp32coarse_SOURCES = Example_pvdagm_multigrid.cc
Example_pvdagm_multigrid_fp32coarse_CPPFLAGS = -DCOARSE_SINGLE
Example_pvdagm_multigrid_fp32coarse_LDADD = $(top_builddir)/Grid/libGrid.a
Example_pvdagm_multigrid_fp32coarse_fp32dense_SOURCES = Example_pvdagm_multigrid.cc
Example_pvdagm_multigrid_fp32coarse_fp32dense_CPPFLAGS = -DCOARSE_SINGLE -DGRID_DENSE_INVERSE_SINGLE
Example_pvdagm_multigrid_fp32coarse_fp32dense_LDADD = $(top_builddir)/Grid/libGrid.a
Example_hdcg_multigrid_fp32coarse_SOURCES = Example_hdcg_multigrid.cc
Example_hdcg_multigrid_fp32coarse_CPPFLAGS = -DCOARSE_SINGLE
Example_hdcg_multigrid_fp32coarse_LDADD = $(top_builddir)/Grid/libGrid.a