portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-21 01:49:35 +01:00
Code Releases Activity

New files, including v2 multiRHS coarse op

Browse Source
This commit is contained in:
Peter Boyle
2026-08-19 20:05:48 -04:00
parent ea5bf89955
commit 2900ce33b5
3 changed files with 1378 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/Example_mdagm.cc
+240
View File
@@ -0,0 +1,240 @@
/*************************************************************************************
Grid physics library, www.github.com/paboyle/Grid
Source file: ./examples/Example_mdagm.cc
Copyright (C) 2023
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
See the full license in the file "LICENSE" in the top level distribution directory
*************************************************************************************/
/* END LEGAL */
#include <Grid/Grid.h>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
using namespace std;
using namespace Grid;
// Routes Op/AdjOp -> HermOp so that CoarsenOperator and CreateSubspace
// both see the HPD operator M†M rather than bare M.
template<class Field>
class HermOpAdaptor : public LinearOperatorBase<Field>
{
LinearOperatorBase<Field> &wrapped;
public:
HermOpAdaptor(LinearOperatorBase<Field> &wrapme) : wrapped(wrapme) {};
void Op (const Field &in, Field &out) { wrapped.HermOp(in,out); }
void HermOp (const Field &in, Field &out) { wrapped.HermOp(in,out); }
void AdjOp (const Field &in, Field &out) { wrapped.HermOp(in,out); }
void OpDiag (const Field &in, Field &out) { GRID_ASSERT(0); }
void OpDir (const Field &in, Field &out,int dir,int disp) { GRID_ASSERT(0); }
void OpDirAll(const Field &in, std::vector<Field> &out) { GRID_ASSERT(0); }
void HermOpAndNorm(const Field &in, Field &out, RealD &n1, RealD &n2) {
wrapped.HermOp(in, out);
ComplexD dot = innerProduct(in, out);
n1 = real(dot);
n2 = norm2(out);
}
};
// Fixed-iteration CG smoother: runs exactly `iters` steps of CG on the
// shifted operator. tolerance=0 so CG never exits early.
template<class Field>
class CGSmoother : public LinearFunction<Field>
{
public:
using LinearFunction<Field>::operator();
typedef LinearOperatorBase<Field> FineOperator;
FineOperator &_SmootherOperator;
int iters;
CGSmoother(int _iters, FineOperator &SmootherOperator)
: _SmootherOperator(SmootherOperator), iters(_iters)
{
std::cout << GridLogMessage << " CGSmoother order " << iters << std::endl;
}
void operator()(const Field &in, Field &out)
{
ConjugateGradient<Field> CG(0.0, iters, false);
out = Zero();
CG(_SmootherOperator, in, out);
}
};
int main (int argc, char ** argv)
{
Grid_init(&argc,&argv);
const int Ls = 24;
const int nbasis = 60;
const int cb = 0;
RealD M5 = 1.8;
RealD b = 1.5;
RealD c = 0.5;
RealD mass = 0.00078;
{ const char *e = getenv("MASS"); if (e && *e) mass = atof(e); }
std::cout << GridLogMessage << "Mass: " << mass
<< " Ls: " << Ls << " b=" << b << " c=" << c << std::endl;
std::cout << GridLogMessage << "nbasis: " << nbasis << std::endl;
// ── Grids ──────────────────────────────────────────────────────────────
std::vector<int> lat_size{48,48,48,96};
GridCartesian *UGrid = SpaceTimeGrid::makeFourDimGrid(lat_size,
GridDefaultSimd(Nd,vComplex::Nsimd()),
GridDefaultMpi());
GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
GridCartesian *FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
Coordinate Block({4,4,3,4});
Coordinate clatt = lat_size;
for (int d = 0; d < (int)clatt.size(); d++) clatt[d] /= Block[d];
GridCartesian *Coarse4d = SpaceTimeGrid::makeFourDimGrid(clatt,
GridDefaultSimd(Nd,vComplex::Nsimd()),
GridDefaultMpi());
GridCartesian *Coarse5d = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
// ── RNGs ───────────────────────────────────────────────────────────────
GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
// ── Gauge field ────────────────────────────────────────────────────────
LatticeGaugeField Umu(UGrid);
FieldMetaData header;
NerscIO::readConfiguration(Umu, header, std::string("/ccs/home/poare/ckpoint_lat.1000"));
// ── Fermion operator ───────────────────────────────────────────────────
MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
MdagMLinearOperator<MobiusFermionD,LatticeFermionD> MdagMOp(Ddwf);
HermOpAdaptor<LatticeFermionD> HermFineOp(MdagMOp);
// ── Coarse geometry ────────────────────────────────────────────────────
typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
typedef LittleDiracOperator::CoarseVector CoarseVector;
typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
NextToNearestStencilGeometry5D geom(Coarse5d);
// ── Power method: estimate upper end of M†M spectrum ───────────────────
LatticeFermionD pm_src(FGrid); random(RNG5, pm_src);
PowerMethod<LatticeFermionD> PM;
RealD hi = PM(HermFineOp, pm_src);
std::cout << GridLogMessage << "Power method: hi = " << hi << std::endl;
// ── Smoother: fixed-iteration CG on (M†M + lo) ─────────────────────────
// lo/hi ~ 2/95 matches the HDCG ratio; tune empirically.
RealD lo = hi / 40.0;
int ord = 12;
std::cout << GridLogMessage << "Smoother shift lo = " << lo
<< " order = " << ord << std::endl;
ShiftedHermOpLinearOperator<LatticeFermionD> ShiftedFineOp(HermFineOp, lo);
CGSmoother<LatticeFermionD> Smoother(ord, ShiftedFineOp);
// ── Subspace via CG inverse iteration ──────────────────────────────────
Subspace Aggregates(Coarse5d, FGrid, cb);
Aggregates.CreateSubspace(RNG5, HermFineOp, nbasis);
// ── Cheap coarse deflation: diagonalise W BEFORE block-GS ──────────────
// Orthogonalise() applies block-local Gram-Schmidt which rotates subspace[i]
// into orthonormal block-local combinations, destroying the near-null
// property of individual vectors. We must extract the coarse zero-mode
// combinations χₖ = Σᵢ V[i,k] ψᵢ from the pre-GS near-null vectors first.
std::vector<LatticeFermionD> chi(nbasis, FGrid);
{
std::vector<LatticeFermionD> &psi = Aggregates.subspace;
LatticeFermionD tmp(FGrid);
Eigen::MatrixXcd W = Eigen::MatrixXcd::Zero(nbasis, nbasis);
for (int i = 0; i < nbasis; i++) {
HermFineOp.Op(psi[i], tmp);
for (int j = 0; j < nbasis; j++)
W(j, i) = TensorRemove(innerProduct(psi[j], tmp));
}
Eigen::SelfAdjointEigenSolver<Eigen::MatrixXcd> esolver(W);
for (int k = 0; k < nbasis; k++) {
chi[k] = Zero();
for (int i = 0; i < nbasis; i++)
chi[k] += ComplexD(esolver.eigenvectors()(i, k)) * psi[i];
}
}
Aggregates.Orthogonalise();
// ── Coarse operator ────────────────────────────────────────────────────
LittleDiracOperator LittleDiracOp(geom, FGrid, Coarse5d);
LittleDiracOp.CoarsenOperator(HermFineOp, Aggregates);
// ── Coarse linear operator ─────────────────────────────────────────────
HermitianLinearOperator<LittleDiracOperator,CoarseVector> LinOpCoarse(LittleDiracOp);
// ── Project χₖ to coarse grid; Rayleigh quotients give deflation evals ─
// ProjectToSubspace uses the post-GS basis, correctly mapping the pre-GS
// near-null combinations to coarse vectors via the block-local U†(x_c).
std::vector<CoarseVector> coarse_deflation_vecs(nbasis, Coarse5d);
std::vector<RealD> coarse_deflation_evals(nbasis);
{
CoarseVector Ac(Coarse5d);
for (int k = 0; k < nbasis; k++) {
Aggregates.ProjectToSubspace(coarse_deflation_vecs[k], chi[k]);
RealD n = norm2(coarse_deflation_vecs[k]);
coarse_deflation_vecs[k] *= 1.0 / std::sqrt(n);
LinOpCoarse.HermOp(coarse_deflation_vecs[k], Ac);
coarse_deflation_evals[k] = real(TensorRemove(innerProduct(coarse_deflation_vecs[k], Ac)));
std::cout << GridLogMessage << "Coarse deflation eval[" << k << "] = "
<< coarse_deflation_evals[k] << std::endl;
}
}
// ── Coarse solve: CG + deflation guesser ──────────────────────────────
ConjugateGradient<CoarseVector> coarseCG(5.0e-2, 10000, false);
DeflatedGuesser<CoarseVector> coarseGuess(coarse_deflation_vecs, coarse_deflation_evals);
HPDSolver<CoarseVector> CoarseSolve(LinOpCoarse, coarseCG, coarseGuess);
// ── ADEF2 outer solve ──────────────────────────────────────────────────
LatticeFermionD src(FGrid); random(RNG5, src);
LatticeFermionD result(FGrid); result = Zero();
TwoLevelADEF2<LatticeFermionD, CoarseVector, Subspace>
HDCG(1.0e-8, 1000,
HermFineOp,
Smoother,
CoarseSolve, // used in PcgM1
CoarseSolve, // used in Vstart
Aggregates);
HDCG(src, result);
// ── Reference RBCG ─────────────────────────────────────────────────────
#if 0
{
SchurDiagMooeeOperator<MobiusFermionD,LatticeFermion> HermOpEO(Ddwf);
LatticeFermionD rb_src(FrbGrid); random(RNG5, rb_src);
LatticeFermionD rb_res(FrbGrid); rb_res = Zero();
ConjugateGradient<LatticeFermionD> CG(1.0e-8, 30000, false);
CG(HermOpEO, rb_src, rb_res);
}
#endif
Grid_finalize();
return 0;
}
examples/Example_pvdagm_mrhs.cc
+662
View File
@@ -0,0 +1,662 @@
/*************************************************************************************
Grid physics library, www.github.com/paboyle/Grid
Source file: ./examples/Example_pvdagm_mrhs.cc
Copyright (C) 2026
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
See the full license in the file "LICENSE" in the top level distribution directory
*************************************************************************************/
/* END LEGAL */
// MultiRHS (valence) two-level multigrid for PVdagM.
//
// Philosophy: NO block Krylov, NO per-RHS lockstep coefficients. A single
// GCR polynomial for the enlarged block-diagonal system diag(A,...,A):
// all inner products are summed over the RHS index, giving one alpha/beta
// per step shared by every RHS.
//
// Level structure mirrors Example_pvdagm:
// outer: MrhsPGCRNonHermitian on PVdagM over std::vector<LatticeFermionD>
// precon: V-cycle -- per-RHS fine post-smoother (16-step shifted GCR),
// batched restriction (MultiRHSBlockProject / GEMM),
// ONE coarse PGCR on the 6D mrhs coarse operator
// (MultiGeneralCoarsenedMatrix, GEMM mults -- the ~10x win),
// batched prolongation.
//
// The coarse operator is coarsened once with the standard single-RHS
// machinery (subspace cache reused) and imported via CopyMatrix.
//
// Memory note: outer restart history is 2*mmax*nrhs fine fields
// (49GB/field global at 48^3x96,Ls=24). Default mmax=8, nrhs=12 needs
// ~10TB for history; use OuterMmax / NRHS to fit the partition.
//
// Env vars: MASS, SUBSPACE_FILE, BLOCK (dotted e.g. 4.4.3.4),
// NRHS (default 12, multiple of Nsimd),
// FineSmootherShift, FineSmootherOrder,
// CoarseSolverTol, CoarseSolverOrder,
// OuterMmax, OuterNstep, OuterTol
#include <Grid/Grid.h>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
using namespace std;
using namespace Grid;
RealD FineSmootherShift = 0.1;
int FineSmootherOrder = 16;
RealD CoarseSolverTol = 0.03;
int CoarseSolverOrder = 200;
RealD OuterTol = 1.0e-8;
int OuterMmax = 8;
int OuterNstep = 8;
int Nrhs = 12;
RealD mass = 0.00078;
void ParseEnvironment(void)
{
if(getenv("MASS")) mass = atof(getenv("MASS"));
if(getenv("FineSmootherShift")) FineSmootherShift = atof(getenv("FineSmootherShift"));
if(getenv("FineSmootherOrder")) FineSmootherOrder = atoi(getenv("FineSmootherOrder"));
if(getenv("CoarseSolverTol")) CoarseSolverTol = atof(getenv("CoarseSolverTol"));
if(getenv("CoarseSolverOrder")) CoarseSolverOrder = atoi(getenv("CoarseSolverOrder"));
if(getenv("OuterTol")) OuterTol = atof(getenv("OuterTol"));
if(getenv("OuterMmax")) OuterMmax = atoi(getenv("OuterMmax"));
if(getenv("OuterNstep")) OuterNstep = atoi(getenv("OuterNstep"));
if(getenv("NRHS")) Nrhs = atoi(getenv("NRHS"));
std::cout << GridLogMessage << "PARAM: MASS " << mass << std::endl;
std::cout << GridLogMessage << "PARAM: NRHS " << Nrhs << std::endl;
std::cout << GridLogMessage << "PARAM: FineSmootherShift " << FineSmootherShift << std::endl;
std::cout << GridLogMessage << "PARAM: FineSmootherOrder " << FineSmootherOrder << std::endl;
std::cout << GridLogMessage << "PARAM: CoarseSolverTol " << CoarseSolverTol << std::endl;
std::cout << GridLogMessage << "PARAM: CoarseSolverOrder " << CoarseSolverOrder << std::endl;
std::cout << GridLogMessage << "PARAM: OuterTol " << OuterTol << std::endl;
std::cout << GridLogMessage << "PARAM: OuterMmax " << OuterMmax << std::endl;
std::cout << GridLogMessage << "PARAM: OuterNstep " << OuterNstep << std::endl;
}
template <class Field>
void saveSubspace(std::vector<Field> &subspace, std::string const fname){
#ifdef HAVE_LIME
std::cout << Grid::GridLogMessage << "Saving subspace (" << subspace.size() << " vectors) to: " << fname << std::endl;
Grid::emptyUserRecord record;
Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
SW.open(fname);
for (int k = 0; k < (int)subspace.size(); k++)
SW.writeScidacFieldRecord(subspace[k], record);
SW.close();
#endif
}
template <class Field>
void loadSubspace(std::vector<Field> &subspace, std::string const fname){
#ifdef HAVE_LIME
std::cout << Grid::GridLogMessage << "Loading subspace (" << subspace.size() << " vectors) from: " << fname << std::endl;
Grid::emptyUserRecord record;
Grid::ScidacReader SR;
SR.open(fname);
for (int k = 0; k < (int)subspace.size(); k++)
SR.readScidacFieldRecord(subspace[k], record);
SR.close();
#endif
}
template<class Matrix,class Field>
class PVdagMLinearOperator : public LinearOperatorBase<Field> {
Matrix &_Mat;
Matrix &_PV;
public:
PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV) {};
void OpDiag (const Field &in, Field &out) { assert(0); }
void OpDir (const Field &in, Field &out,int dir,int disp) { assert(0); }
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); };
void Op (const Field &in, Field &out){
Field tmp(in.Grid());
_Mat.M(in,tmp);
_PV.Mdag(tmp,out);
}
void AdjOp (const Field &in, Field &out){
Field tmp(in.Grid());
_PV.M(in,tmp);
_Mat.Mdag(tmp,out);
}
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
HermOp(in,out);
ComplexD dot = innerProduct(in,out);
n1=real(dot);
n2=norm2(out);
}
void HermOp(const Field &in, Field &out){
Field tmp(in.Grid());
Op(in,tmp);
AdjOp(tmp,out);
}
};
template<class Matrix,class Field>
class ShiftedPVdagMLinearOperator : public LinearOperatorBase<Field> {
Matrix &_Mat;
Matrix &_PV;
public:
RealD shift;
ShiftedPVdagMLinearOperator(RealD _shift,Matrix &Mat,Matrix &PV): shift(_shift),_Mat(Mat),_PV(PV){};
void OpDiag (const Field &in, Field &out) { assert(0); }
void OpDir (const Field &in, Field &out,int dir,int disp) { assert(0); }
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); };
void Op (const Field &in, Field &out){
Field tmp(in.Grid());
_Mat.M(in,tmp);
_PV.Mdag(tmp,out);
out = out + shift * in;
}
void AdjOp (const Field &in, Field &out){
Field tmp(in.Grid());
_PV.M(tmp,out);
_Mat.Mdag(in,tmp);
out = out + shift * in;
}
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ assert(0); }
void HermOp(const Field &in, Field &out){
Field tmp(in.Grid());
Op(in,tmp);
AdjOp(tmp,out);
}
};
//////////////////////////////////////////////////////////////////////
// Minimal multi-RHS function interface (preconditioner slot)
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsLinearFunction {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out) = 0;
};
//////////////////////////////////////////////////////////////////////
// Single-polynomial multi-RHS PGCR (non-Hermitian).
//
// Verbatim adaptation of PrecGeneralisedConjugateResidualNonHermitian
// to std::vector<Field>: every innerProduct / norm2 is SUMMED over the
// RHS index, so one alpha/beta per step is shared by all RHS -- the
// single GCR on the enlarged block-diagonal system.
//////////////////////////////////////////////////////////////////////
template<class Field>
class MrhsPGCRNonHermitian {
public:
RealD Tolerance;
Integer MaxIterations;
int mmax;
int nstep;
int steps;
int level;
int ZeroGuess = 0; int FirstCycle = 0; // caller contract: zero guess => first-cycle r0 = src
std::string name = "Level 1";
LinearOperatorBase<Field> &Linop;
MrhsLinearFunction<Field> &Preconditioner;
void Level(int lv) { name = "Level " + std::to_string(lv); level=lv; };
void Name(std::string n) { name = n; };
void SetZeroGuess(int z) { ZeroGuess=z; };
MrhsPGCRNonHermitian(RealD tol,Integer maxit,
LinearOperatorBase<Field> &_Linop,
MrhsLinearFunction<Field> &Prec,
int _mmax,int _nstep)
: Tolerance(tol), MaxIterations(maxit), Linop(_Linop), Preconditioner(Prec),
mmax(_mmax), nstep(_nstep) { level=1; }
///////////////////////////////////////////////////////////////
// vector-of-fields linear algebra, reductions summed over rhs
///////////////////////////////////////////////////////////////
static RealD vnorm2(std::vector<Field> &x){
RealD s=0.0; for(auto &f : x) s+=norm2(f); return s;
}
static ComplexD vinnerProduct(std::vector<Field> &x, std::vector<Field> &y){
ComplexD s(0.0); for(int r=0;r<(int)x.size();r++) s+=innerProduct(x[r],y[r]); return s;
}
static void vaxpy(std::vector<Field> &z, ComplexD a, std::vector<Field> &x, std::vector<Field> &y){
for(int r=0;r<(int)z.size();r++) axpy(z[r],a,x[r],y[r]);
}
void vOp(std::vector<Field> &in, std::vector<Field> &out){
for(int r=0;r<(int)in.size();r++) Linop.Op(in[r],out[r]);
}
void operator() (std::vector<Field> &src, std::vector<Field> &psi){
RealD cp, ssq, rsq;
int nrhs = src.size();
GridBase *grid = src[0].Grid();
ssq=vnorm2(src);
rsq=Tolerance*Tolerance*ssq;
std::vector<Field> r(nrhs,grid);
GridStopWatch SolverTimer;
SolverTimer.Start();
steps=0;
FirstCycle=1;
for(int k=0;k<MaxIterations;k++){
cp=GCRnStep(src,psi,rsq);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR("<<mmax<<","<<nstep<<") "<<steps<<" steps cp = "<<cp<<" target "<<rsq<<std::endl;
if(cp<rsq){
SolverTimer.Stop();
vOp(psi,r);
for(int rr=0;rr<nrhs;rr++) axpy(r[rr],-1.0,src[rr],r[rr]);
RealD tr=vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR: Converged on iteration "<<steps
<<" computed residual "<<std::sqrt(cp/ssq)
<<" true residual "<<std::sqrt(tr/ssq)
<<" target "<<Tolerance<<std::endl;
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR Time elapsed: Total "<<SolverTimer.Elapsed()<<std::endl;
// Per-RHS true residuals: the honest metric under the summed norm
for(int rr=0;rr<nrhs;rr++){
RealD rn = std::sqrt(norm2(r[rr])/norm2(src[rr]));
std::cout<<GridLogMessage<<"MrhsPGCR per-rhs true residual["<<rr<<"] = "<<rn<<std::endl;
}
return;
}
}
std::cout<<GridLogMessage<<"MrhsPGCR: did not converge"<<std::endl;
}
RealD GCRnStep(std::vector<Field> &src, std::vector<Field> &psi, RealD rsq){
RealD cp;
ComplexD a, b, rq;
RealD zAAz;
int nrhs = src.size();
GridBase *grid = src[0].Grid();
std::vector<Field> r (nrhs,grid);
std::vector<Field> z (nrhs,grid);
std::vector<Field> Az(nrhs,grid);
////////////////////////////////
// history for flexible orthog: [mmax][nrhs]
////////////////////////////////
std::vector< std::vector<Field> > q(mmax, std::vector<Field>(nrhs,grid));
std::vector< std::vector<Field> > p(mmax, std::vector<Field>(nrhs,grid));
std::vector<RealD> qq(mmax);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name<<" MrhsPGCR nStep("<<nstep<<")"<<std::endl;
if (ZeroGuess && FirstCycle) {
for(int rr=0;rr<nrhs;rr++){ psi[rr]=Zero(); r[rr]=src[rr]; }
} else {
vOp(psi,Az);
for(int rr=0;rr<nrhs;rr++) r[rr] = src[rr]-Az[rr];
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR true residual r = src - A psi "<<vnorm2(r)<<std::endl;
}
FirstCycle=0;
Preconditioner(r,z);
vOp(z,Az);
zAAz=vnorm2(Az);
p[0]=z;
q[0]=Az;
qq[0]=zAAz;
cp=vnorm2(r);
for(int k=0;k<nstep;k++){
steps++;
int kp = k+1;
int peri_k = k %mmax;
int peri_kp= kp%mmax;
rq = vinnerProduct(q[peri_k],r);
a = rq/qq[peri_k];
vaxpy(psi,a,p[peri_k],psi);
vaxpy(r,-a,q[peri_k],r);
cp = vnorm2(r);
std::cout<<GridLogMessage<<std::string(level,'\t')<<" "<<name
<<" MrhsPGCR step["<<steps<<"] resid "<<cp<<" target "<<rsq<<std::endl;
if((k==nstep-1)||(cp<rsq)){
return cp;
}
Preconditioner(r,z);
vOp(z,Az);
zAAz=vnorm2(Az);
q[peri_kp]=Az;
p[peri_kp]=z;
int northog = ((kp)>(mmax-1))?(mmax-1):(kp);
for(int back=0;back<northog;back++){
int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
b = -real(vinnerProduct(q[peri_back],Az))/qq[peri_back];
vaxpy(p[peri_kp],b,p[peri_back],p[peri_kp]);
vaxpy(q[peri_kp],b,q[peri_back],q[peri_kp]);
}
qq[peri_kp]=vnorm2(q[peri_kp]);
}
GRID_ASSERT(0); // never reached
return cp;
}
};
//////////////////////////////////////////////////////////////////////
// Trivial multi-RHS preconditioner
//////////////////////////////////////////////////////////////////////
template<class Field>
class TrivialMrhsPrecon : public MrhsLinearFunction<Field> {
public:
virtual void operator()(std::vector<Field> &in, std::vector<Field> &out){
for(int r=0;r<(int)in.size();r++) out[r]=in[r];
}
};
//////////////////////////////////////////////////////////////////////
// MultiRHS two-level V-cycle.
//
// Mirrors MGPreconditioner in Example_pvdagm:
// out = in (trivial pre) [per rhs]
// r1 = in - A out [per rhs]
// batched blockProject -> pack -> ONE mrhs coarse PGCR -> unpack
// -> batched blockPromote; out += correction
// r2 = in - A out [per rhs]
// per-RHS fine post-smoother; out += smooth(r2)
//////////////////////////////////////////////////////////////////////
template<class FineField, class MrhsCoarseVector, class FineSmoother>
class MrhsTwoLevelMG : public MrhsLinearFunction<FineField> {
public:
typedef MrhsCoarseVector CoarseVector; // same lattice type on Coarse5d and CoarseMrhs
LinearOperatorBase<FineField> &_FineOperator;
FineSmoother &_PostSmoother; // single-RHS smoother, looped
MultiRHSBlockProject<FineField> &_Projector;
LinearFunction<CoarseVector> &_CoarseSolve; // PGCR on the 6D mrhs field
GridBase *_CoarseGrid; // Coarse5d (single rhs)
GridBase *_CoarseGridMrhs; // 6D
MrhsTwoLevelMG(LinearOperatorBase<FineField> &FineOp,
FineSmoother &Post,
MultiRHSBlockProject<FineField> &Projector,
LinearFunction<CoarseVector> &CoarseSolve,
GridBase *CoarseGrid, GridBase *CoarseGridMrhs)
: _FineOperator(FineOp), _PostSmoother(Post), _Projector(Projector),
_CoarseSolve(CoarseSolve), _CoarseGrid(CoarseGrid), _CoarseGridMrhs(CoarseGridMrhs) {}
virtual void operator()(std::vector<FineField> &in, std::vector<FineField> &out){
int nrhs = in.size();
GridBase *fgrid = in[0].Grid();
double t;
std::vector<FineField> vec1(nrhs,fgrid);
std::vector<FineField> vec2(nrhs,fgrid);
// Trivial pre-smoother: out = in (as in Example_pvdagm with simple_fine)
for(int r=0;r<nrhs;r++) out[r]=in[r];
// Residual
for(int r=0;r<nrhs;r++){
_FineOperator.Op(out[r],vec1[r]);
sub(vec1[r],in[r],vec1[r]);
}
// Batched fine->coarse, pack rhs into 6D field
std::vector<CoarseVector> Csrc_split(nrhs,_CoarseGrid);
std::vector<CoarseVector> Csol_split(nrhs,_CoarseGrid);
CoarseVector CsrcMrhs(_CoarseGridMrhs);
CoarseVector CsolMrhs(_CoarseGridMrhs);
t=-usecond();
_Projector.blockProject(vec1,Csrc_split);
for(int r=0;r<nrhs;r++) InsertSliceFast(Csrc_split[r],CsrcMrhs,r,0);
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs project+pack took "<<t/1000.0<<"ms"<<std::endl;
// ONE coarse solve for all rhs (GEMM coarse mults)
t=-usecond();
CsolMrhs=Zero();
_CoarseSolve(CsrcMrhs,CsolMrhs);
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs coarse solve took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
// Unpack, batched coarse->fine, add correction
t=-usecond();
for(int r=0;r<nrhs;r++) ExtractSliceFast(Csol_split[r],CsolMrhs,r,0);
_Projector.blockPromote(vec1,Csol_split);
for(int r=0;r<nrhs;r++) add(out[r],out[r],vec1[r]);
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs unpack+promote took "<<t/1000.0<<"ms"<<std::endl;
// Residual
for(int r=0;r<nrhs;r++){
_FineOperator.Op(out[r],vec1[r]);
sub(vec1[r],in[r],vec1[r]);
}
// Per-RHS post-smoother (fine level has no batching win; memory-light)
t=-usecond();
for(int r=0;r<nrhs;r++){
vec2[r]=Zero();
_PostSmoother(vec1[r],vec2[r]);
add(out[r],out[r],vec2[r]);
}
t+=usecond();
std::cout<<GridLogMessage<<"Mrhs post-smooth took "<<t/1000.0<<"ms ("<<t/1000.0/nrhs<<"ms/rhs)"<<std::endl;
}
};
int main (int argc, char ** argv)
{
Grid_init(&argc,&argv);
ParseEnvironment();
const int Ls=24;
RealD M5=1.8;
RealD b=1.5;
RealD c=0.5;
const int nbasis = 60;
const int nrhs = Nrhs;
GRID_ASSERT(nrhs % vComplex::Nsimd() == 0);
std::cout << GridLogMessage << "MultiRHS PVdagM MG: mass=" << mass << " Ls=" << Ls
<< " nbasis=" << nbasis << " nrhs=" << nrhs << std::endl;
std::vector<int> lat_size {48, 48, 48, 96};
GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(lat_size, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
// Blocking, env-overridable: BLOCK=2.2.2.2
Coordinate clatt = lat_size;
Coordinate Block({4,4,3,4});
if ( getenv("BLOCK") ) {
GridCmdOptionIntVector(std::string(getenv("BLOCK")),Block);
GRID_ASSERT(Block.size()==4);
}
for(int d=0;d<clatt.size();d++){
GRID_ASSERT(lat_size[d] % Block[d] == 0);
clatt[d] = lat_size[d]/Block[d];
}
std::cout << GridLogMessage << "Block " << Block << " coarse lattice " << clatt << std::endl;
GridCartesian *Coarse4d = SpaceTimeGrid::makeFourDimGrid(clatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
GridCartesian *Coarse5d = SpaceTimeGrid::makeFiveDimGrid(1,Coarse4d);
////////////////////////////////////////////////////////////
// 6D multi-RHS coarse grid: rhs is dim 0, SIMD across rhs
// (pattern: tests/debug/Test_general_coarse_hdcg_phys48.cc)
////////////////////////////////////////////////////////////
Coordinate mpi=GridDefaultMpi();
Coordinate rhMpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
Coordinate rhLatt({nrhs,1,clatt[0],clatt[1],clatt[2],clatt[3]});
Coordinate rhSimd({vComplex::Nsimd(),1, 1,1,1,1});
GridCartesian *CoarseMrhs = new GridCartesian(rhLatt,rhSimd,rhMpi);
GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
LatticeGaugeField Umu(UGrid);
std::cout << GridLogMessage << "Reading gauge field" << std::endl;
FieldMetaData header;
std::string file("/ccs/home/poare/ckpoint_lat.1000");
NerscIO::readConfiguration(Umu,header,file);
MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD> PVdagM_t;
typedef ShiftedPVdagMLinearOperator<MobiusFermionD,LatticeFermionD> ShiftedPVdagM_t;
typedef GeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
typedef LittleDiracOperator::CoarseVector CoarseVector;
typedef Aggregation<vSpinColourVector,vTComplex,nbasis> Subspace;
PVdagM_t PVdagM(Ddwf,Dpv);
ShiftedPVdagM_t ShiftedPVdagM(FineSmootherShift,Ddwf,Dpv);
NextToNearestStencilGeometry5D geom(Coarse5d);
////////////////////////////////////////////////////////////
// Subspace: load from cache or generate
////////////////////////////////////////////////////////////
std::string subspace_file = "/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle/subspace_nb"
+ std::to_string(nbasis) + ".scidac";
if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
uint64_t file_exists = 0;
if ( UGrid->IsBoss() ) { std::ifstream f(subspace_file); file_exists = f.good() ? 1 : 0; }
UGrid->GlobalSum(file_exists);
const int cb = 0;
Subspace AggregatesGCR(Coarse5d,FGrid,cb);
if ( file_exists ) {
std::cout << GridLogMessage << "*** Loading subspace from disk ***" << std::endl;
loadSubspace(AggregatesGCR.subspace, subspace_file);
} else {
std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
saveSubspace(AggregatesGCR.subspace, subspace_file);
}
////////////////////////////////////////////////////////////
// Coarsen once (single-RHS machinery), import into mrhs op.
// NB CoarsenOperator block-orthogonalises subspace in place;
// ImportBasis AFTER so the projector matches the coarse op.
////////////////////////////////////////////////////////////
MrhsLittleDiracOperator mrhsLittleDiracOpPV(geom,CoarseMrhs);
{
// Scope the single-RHS operator so its padded _A is freed after import.
// At small local volumes the depth-2 padded cell inflates ~8x
// (e.g. 2^4 blocking on 432 ranks: local 8x4x4x12 -> padded 12x8x8x16,
// ~23GB/GCD for _A alone -> OOM if kept alive).
LittleDiracOperator LittleDiracOpPV(geom,FGrid,Coarse5d);
LittleDiracOpPV.CoarsenOperator(PVdagM, AggregatesGCR);
mrhsLittleDiracOpPV.CopyMatrix(LittleDiracOpPV);
}
MultiRHSBlockProject<LatticeFermionD> MrhsProjector;
MrhsProjector.Allocate(nbasis,FGrid,Coarse5d);
MrhsProjector.ImportBasis(AggregatesGCR.subspace);
////////////////////////////////////////////////////////////
// Solvers
////////////////////////////////////////////////////////////
NonHermitianLinearOperator<MrhsLittleDiracOperator,CoarseVector> mrhsLinOpCoarse(mrhsLittleDiracOpPV);
TrivialPrecon<CoarseVector> simpleC;
PrecGeneralisedConjugateResidualNonHermitian<CoarseVector>
L2PGCRmrhs(CoarseSolverTol, CoarseSolverOrder/20, mrhsLinOpCoarse, simpleC, 20, 20);
L2PGCRmrhs.Level(2);
L2PGCRmrhs.Name("Couter");
L2PGCRmrhs.SetZeroGuess(1); // caller zeroes CsolMrhs
TrivialPrecon<LatticeFermionD> simple_fine;
PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD>
SmootherGCR(0.0, 1, ShiftedPVdagM, simple_fine, FineSmootherOrder, FineSmootherOrder);
SmootherGCR.Level(1);
SmootherGCR.Name("Fsmoother");
SmootherGCR.SetZeroGuess(1); // caller zeroes vec2[r]
typedef PrecGeneralisedConjugateResidualNonHermitian<LatticeFermionD> FineSmoother_t;
MrhsTwoLevelMG<LatticeFermionD,CoarseVector,FineSmoother_t>
TwoLevelPrecon(PVdagM, SmootherGCR, MrhsProjector, L2PGCRmrhs, Coarse5d, CoarseMrhs);
MrhsPGCRNonHermitian<LatticeFermionD>
L1PGCRmrhs(OuterTol, 1000, PVdagM, TwoLevelPrecon, OuterMmax, OuterNstep);
L1PGCRmrhs.Level(1);
L1PGCRmrhs.Name("Fouter");
L1PGCRmrhs.SetZeroGuess(1); // sol[r]=Zero() at source setup
////////////////////////////////////////////////////////////
// Sources and solve
////////////////////////////////////////////////////////////
std::vector<LatticeFermionD> src(nrhs,FGrid);
std::vector<LatticeFermionD> sol(nrhs,FGrid);
for(int r=0;r<nrhs;r++){
gaussian(RNG5,src[r]);
sol[r]=Zero();
}
std::cout << GridLogMessage << "**********************************************" << std::endl;
std::cout << GridLogMessage << " MultiRHS two-level solve: " << nrhs << " RHS " << std::endl;
std::cout << GridLogMessage << "**********************************************" << std::endl;
GridStopWatch w; w.Start();
L1PGCRmrhs(src,sol);
w.Stop();
std::cout << GridLogMessage << "MultiRHS solve total " << w.Elapsed()
<< " (per RHS: " << w.useconds()/1.0e6/nrhs << " s)" << std::endl;
// Independent final verification, per RHS
{
LatticeFermionD Ax(FGrid);
RealD worst=0.0;
for(int r=0;r<nrhs;r++){
PVdagM.Op(sol[r],Ax);
Ax = Ax - src[r];
RealD rn = std::sqrt(norm2(Ax)/norm2(src[r]));
std::cout << GridLogMessage << "FINAL: rhs["<<r<<"] true residual = " << rn << std::endl;
worst = std::max(worst,rn);
}
std::cout << GridLogMessage << "FINAL: worst-case residual = " << worst << std::endl;
}
std::cout << GridLogMessage << "Done" << std::endl;
Grid_finalize();
return 0;
}
examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc
+476
View File
@@ -0,0 +1,476 @@
/*************************************************************************************
Grid physics library, www.github.com/paboyle/Grid
Source file: ./examples/Example_pvdagm_v2_3level_DenseCoarseMatrix.cc
Copyright (C) 2026
Author: Peter Boyle <pboyle@bnl.gov>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
See the full license in the file "LICENSE" in the top level distribution directory
*************************************************************************************/
/* END LEGAL */
//
// PVdagM three level multigrid on the V2 coarse operator.
//
// STAGE ONE: grids, types, subspace, and the L1 coarsening only. The L2
// chain, the dense bottom and the solves are not here yet.
//
// Differences from Example_pvdagm_mrhs_3level_DenseCoarseMatrix.cc:
//
// * The coarse space is UNVECTORISED (sComplexD). The fine space stays
// vectorised. MultiRHSBlockProject carries the mixed layout.
//
// * One operator, not two. V1 needed GeneralCoarsenedMatrix to coarsen and
// MultiGeneralCoarsenedMatrix to apply, bridged by CopyMatrix. V2 does
// both, and single versus multiRHS is SetGrid on the same object with the
// matrix elements built once.
//
// * Nrhs is unconstrained. V1 required nrhs % vComplex::Nsimd() == 0 because
// its multiRHS grid carried the SIMD in the rhs direction.
//
// * CoarsenOperator takes the subspace vectors, not an Aggregation. It block
// orthonormalises them IN PLACE -- the vectors are far too large to copy
// defensively -- so rawNull is taken first and the RAW vectors are what
// define the L2 null space. Do not insert an Orthogonalise() anywhere:
// projecting a block-orthonormal vector onto its own block-orthonormalised
// aggregation gives e_k, and the near null content is silently gone. The
// ||<psi|psi> - I||_F guard below is what catches that.
//
// Env: LATT LS MASS NBASIS(compile time) NRHS BLOCK COARSEN_BATCH
// HOT_START CONFIG SUBSPACE_FILE V1_CHECK
//
#include <Grid/Grid.h>
#include <Grid/lattice/PaddedCell.h>
#include <Grid/stencil/GeneralLocalStencil.h>
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
#include <memory>
using namespace std;
using namespace Grid;
// Compile time so it can be cut down for laptop runs: -DNBASIS=8
#ifndef NBASIS
#define NBASIS 60
#endif
RealD mass = 0.00078;
int Nrhs = 12;
int Ls = 24;
int CoarsenBatch = 9;
std::vector<int> lat_size({48,48,48,96});
void ParseEnvironment(void)
{
if(getenv("MASS")) mass = atof(getenv("MASS"));
if(getenv("NRHS")) Nrhs = atoi(getenv("NRHS"));
if(getenv("LS")) Ls = atoi(getenv("LS"));
if(getenv("COARSEN_BATCH")) CoarsenBatch= atoi(getenv("COARSEN_BATCH"));
if(getenv("LATT")){
Coordinate l;
GridCmdOptionIntVector(std::string(getenv("LATT")),l);
GRID_ASSERT(l.size()==4);
for(int d=0;d<4;d++) lat_size[d]=l[d];
}
std::cout << GridLogMessage << "PARAM: LATT "
<< lat_size[0]<<"."<<lat_size[1]<<"."<<lat_size[2]<<"."<<lat_size[3] << std::endl;
std::cout << GridLogMessage << "PARAM: LS " << Ls << std::endl;
std::cout << GridLogMessage << "PARAM: MASS " << mass << std::endl;
std::cout << GridLogMessage << "PARAM: NBASIS " << NBASIS << std::endl;
std::cout << GridLogMessage << "PARAM: NRHS " << Nrhs << std::endl;
std::cout << GridLogMessage << "PARAM: COARSEN_BATCH " << CoarsenBatch << std::endl;
}
template <class Field>
void saveSubspace(std::vector<Field> &subspace, std::string const fname){
#ifdef HAVE_LIME
Grid::emptyUserRecord record;
Grid::ScidacWriter SW(subspace[0].Grid()->IsBoss());
SW.open(fname);
for (int k = 0; k < (int)subspace.size(); k++) SW.writeScidacFieldRecord(subspace[k], record);
SW.close();
#endif
}
template <class Field>
void loadSubspace(std::vector<Field> &subspace, std::string const fname){
#ifdef HAVE_LIME
Grid::emptyUserRecord record;
Grid::ScidacReader SR;
SR.open(fname);
for (int k = 0; k < (int)subspace.size(); k++) SR.readScidacFieldRecord(subspace[k], record);
SR.close();
#endif
}
//////////////////////////////////////////////////////////////////////
// A = PV^dag M (non-Hermitian)
//////////////////////////////////////////////////////////////////////
template<class Matrix,class Field>
class PVdagMLinearOperator : public LinearOperatorBase<Field> {
Matrix &_Mat; Matrix &_PV;
public:
PVdagMLinearOperator(Matrix &Mat,Matrix &PV): _Mat(Mat),_PV(PV) {};
void OpDiag (const Field &in, Field &out) { assert(0); }
void OpDir (const Field &in, Field &out,int dir,int disp) { assert(0); }
void OpDirAll (const Field &in, std::vector<Field> &out){ assert(0); };
void Op (const Field &in, Field &out){ Field tmp(in.Grid()); _Mat.M(in,tmp); _PV.Mdag(tmp,out); }
void AdjOp (const Field &in, Field &out){ Field tmp(in.Grid()); _PV.M(in,tmp); _Mat.Mdag(tmp,out); }
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){ HermOp(in,out); ComplexD d=innerProduct(in,out); n1=real(d); n2=norm2(out); }
void HermOp(const Field &in, Field &out){ Field tmp(in.Grid()); Op(in,tmp); AdjOp(tmp,out); }
};
//////////////////////////////////////////////////////////////////////
// ||<v|v> - I||_F over a set of coarse vectors. ~0.23 means the raw near
// null content survived the projection; ~sqrt(N_sites) means block
// orthonormal vectors leaked in and every image collapsed to e_k.
//////////////////////////////////////////////////////////////////////
template<class CoarseField>
RealD GramDefect(std::vector<CoarseField> &v)
{
RealD s2=0.0;
for(int i=0;i<(int)v.size();i++){
for(int j=0;j<(int)v.size();j++){
ComplexD sij=TensorRemove(innerProduct(v[i],v[j]));
ComplexD d=sij-(i==j?ComplexD(1.0):ComplexD(0.0));
s2+=real(d)*real(d)+imag(d)*imag(d);
}
}
return std::sqrt(s2);
}
int main (int argc, char ** argv)
{
Grid_init(&argc,&argv);
ParseEnvironment();
RealD M5=1.8, b=1.5, c=0.5;
const int nbasis=NBASIS;
const int nrhs=Nrhs;
const int batch=CoarsenBatch;
Coordinate mpi = GridDefaultMpi();
Coordinate fsimd= GridDefaultSimd(Nd,vComplex::Nsimd());
GridCartesian * UGrid = SpaceTimeGrid::makeFourDimGrid(lat_size,fsimd,mpi);
GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
GridCartesian * FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
// Level 1 blocking (default 2^4)
Coordinate clatt = lat_size;
Coordinate Block({2,2,2,2});
if ( getenv("BLOCK") ){ GridCmdOptionIntVector(std::string(getenv("BLOCK")),Block); GRID_ASSERT(Block.size()==4); }
for(int d=0;d<4;d++){ GRID_ASSERT(lat_size[d]%Block[d]==0); clatt[d]=lat_size[d]/Block[d]; }
std::cout << GridLogMessage << "Block " << Block << " coarse lattice " << clatt << std::endl;
//////////////////////////////////////////////////////////////////////
// The coarse space is unvectorised. The 5D coarse grid is built here
// rather than through SpaceTimeGrid so the SIMD layout is ours.
//////////////////////////////////////////////////////////////////////
Coordinate c5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
Coordinate c5simd({1,1,1,1,1});
Coordinate c5mpi ({1,mpi[0],mpi[1],mpi[2],mpi[3]});
GridCartesian *Coarse5d = new GridCartesian(c5latt,c5simd,c5mpi);
// 6D coarse multiRHS grid: rhs is dim 0, undistributed and unvectorised.
// No divisibility constraint on nrhs, unlike V1.
Coordinate cmlatt({nrhs,1,clatt[0],clatt[1],clatt[2],clatt[3]});
Coordinate cmsimd({1,1,1,1,1,1});
Coordinate cmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
GridCartesian *CoarseMrhs = new GridCartesian(cmlatt,cmsimd,cmmpi);
// 6D coarse grid at the coarsening batch, used only while CoarsenOperator
// runs. The matrix elements survive the change back to nrhs.
Coordinate cblatt({batch,1,clatt[0],clatt[1],clatt[2],clatt[3]});
GridCartesian *CoarseBatch = new GridCartesian(cblatt,cmsimd,cmmpi);
// 6D fine grid carrying the coarsening batch: fine SIMD layout preserved
Coordinate fmlatt({batch,Ls,lat_size[0],lat_size[1],lat_size[2],lat_size[3]});
Coordinate fmsimd({1,1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
Coordinate fmmpi ({1,1,mpi[0],mpi[1],mpi[2],mpi[3]});
GridCartesian *FineMrhs = new GridCartesian(fmlatt,fmsimd,fmmpi);
std::cout << GridLogMessage << "Nsimd fine " << FGrid->Nsimd()
<< " coarse " << Coarse5d->Nsimd() << std::endl;
GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers({1,2,3,4});
GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers({5,6,7,8});
//////////////////////////////////////////////////////////////////////
// Gauge field
//////////////////////////////////////////////////////////////////////
LatticeGaugeField Umu(UGrid);
if ( getenv("HOT_START") ) {
std::cout << GridLogMessage << "Hot start gauge field" << std::endl;
SU<Nc>::HotConfiguration(RNG4,Umu);
} else {
std::string file("/ccs/home/poare/ckpoint_lat.1000");
if ( getenv("CONFIG") ) file = std::string(getenv("CONFIG"));
std::cout << GridLogMessage << "Reading gauge field " << file << std::endl;
FieldMetaData header;
NerscIO::readConfiguration(Umu,header,file);
}
MobiusFermionD Ddwf(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5,b,c);
MobiusFermionD Dpv (Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,1.0, M5,b,c);
typedef PVdagMLinearOperator<MobiusFermionD,LatticeFermionD> PVdagM_t;
PVdagM_t PVdagM(Ddwf,Dpv);
//////////////////////////////////////////////////////////////////////
// Level 1 types: unvectorised coarse scalar
//////////////////////////////////////////////////////////////////////
typedef sTComplexD CComplexS;
typedef MultiGeneralCoarsenedOperatorV2<vSpinColourVector,CComplexS,nbasis> CoarseOperator;
typedef CoarseOperator::CoarseVector CoarseVector;
typedef Aggregation<vSpinColourVector,CComplexS,nbasis> Subspace;
NextToNearestStencilGeometry5D geom(Coarse5d);
//////////////////////////////////////////////////////////////////////
// Subspace: load RAW (no Orthogonalise!), or generate.
//
// The Aggregation is scaffolding for CreateSubspaceGCR only. That runs
// entirely on the fine grid and ends in GlobalOrthonormalise, which is a
// whole-lattice Gram-Schmidt, so the coarse grid it holds is never
// dereferenced and may be the unvectorised one.
//////////////////////////////////////////////////////////////////////
std::string subspace_file = "subspace_nb" + std::to_string(nbasis) + ".scidac";
if ( getenv("SUBSPACE_FILE") ) subspace_file = std::string(getenv("SUBSPACE_FILE"));
uint64_t file_exists=0;
if ( UGrid->IsBoss() ){ std::ifstream f(subspace_file); file_exists=f.good()?1:0; }
UGrid->GlobalSum(file_exists);
const int cb=0;
Subspace AggregatesGCR(Coarse5d,FGrid,cb);
if ( file_exists ){
std::cout << GridLogMessage << "*** Loading subspace from disk (kept RAW) ***" << std::endl;
loadSubspace(AggregatesGCR.subspace, subspace_file);
} else {
std::cout << GridLogMessage << "*** GCR subspace generation ***" << std::endl;
AggregatesGCR.CreateSubspaceGCR(RNG5,PVdagM,nbasis);
saveSubspace(AggregatesGCR.subspace, subspace_file);
}
// RAW copy BEFORE CoarsenOperator block-orthonormalises in place.
std::vector<LatticeFermionD> rawNull(nbasis,FGrid);
for(int k=0;k<nbasis;k++) rawNull[k]=AggregatesGCR.subspace[k];
//////////////////////////////////////////////////////////////////////
// L1 coarsening. The fine operator is single RHS, so it is promoted to
// the 6D batch grid; a natively multiRHS fine operator would substitute
// here with no other change.
//////////////////////////////////////////////////////////////////////
CoarseOperator CoarseOpPV(geom,Coarse5d);
CoarseOpPV.SetGrid(CoarseBatch);
std::cout << GridLogMessage << "*** L1 CoarsenOperator, batch "<<batch<<" ***" << std::endl;
if ( getenv("MRHS_COARSEN") ) {
// Promote the single RHS operator and pack the batch: costs an
// ExtractSlice/InsertSlice pair per rhs. Here for the A/B only; this is
// the path a natively multiRHS fine operator would take.
MrhsPromotedOperator<LatticeFermionD> MrhsPVdagM(PVdagM,FGrid,batch);
CoarseOpPV.CoarsenOperator(MrhsPVdagM,FineMrhs,AggregatesGCR.subspace,Coarse5d);
} else {
// PVdagM is single RHS: apply it directly, batch on the coarse side.
CoarseOpPV.CoarsenOperator(PVdagM,AggregatesGCR.subspace,Coarse5d,batch);
}
// Stay on the batch grid: the L2 coarsening drives this operator at the
// batch. It is switched to the solve Nrhs once L2 is built.
//////////////////////////////////////////////////////////////////////
// psi_coarse = P^dag (RAW fine null) -> Galerkin images that carry the
// near null content, and are free: A_c (P psi) = P A psi.
//////////////////////////////////////////////////////////////////////
MultiRHSBlockProject<LatticeFermionD> MrhsProjector;
MrhsProjector.Allocate(nbasis,FGrid,Coarse5d);
MrhsProjector.ImportBasis(AggregatesGCR.subspace); // block orthonormal basis
std::vector<CoarseVector> psi_coarse(nbasis,Coarse5d);
MrhsProjector.blockProject(rawNull,psi_coarse); // RAW vectors in
rawNull.clear(); rawNull.shrink_to_fit();
{
RealD defect = GramDefect(psi_coarse);
RealD leak = std::sqrt((double)Coarse5d->gSites());
std::cout << GridLogMessage << "GUARD: ||<psi_coarse|psi_coarse> - I||_F = " << defect
<< " (~0.23 good; ~sqrt(N_coarse)=" << leak << " = e_k leak)" << std::endl;
GRID_ASSERT( defect < leak );
}
//////////////////////////////////////////////////////////////////////
// Optional cross check of the coarse matrix elements against the V1
// path, which needs a vectorised coarse space. Block Gram-Schmidt is
// idempotent, so V1 may re-orthonormalise the same vectors in place
// without a second copy of the subspace.
//////////////////////////////////////////////////////////////////////
if ( getenv("V1_CHECK") ) {
typedef GeneralCoarsenedMatrix <vSpinColourVector,vTComplex,nbasis> LittleDiracOperator;
typedef MultiGeneralCoarsenedMatrix<vSpinColourVector,vTComplex,nbasis> MrhsLittleDiracOperator;
typedef Aggregation <vSpinColourVector,vTComplex,nbasis> SubspaceV;
Coordinate v5latt({1,clatt[0],clatt[1],clatt[2],clatt[3]});
Coordinate v5simd({1,fsimd[0],fsimd[1],fsimd[2],fsimd[3]});
GridCartesian *Coarse5dV = new GridCartesian(v5latt,v5simd,c5mpi);
int nrhs_v1 = vComplex::Nsimd();
Coordinate vmlatt({nrhs_v1,1,clatt[0],clatt[1],clatt[2],clatt[3]});
Coordinate vmsimd({vComplex::Nsimd(),1,1,1,1,1});
GridCartesian *CoarseMrhsV = new GridCartesian(vmlatt,vmsimd,cmmpi);
NextToNearestStencilGeometry5D geomV(Coarse5dV);
SubspaceV AggV(Coarse5dV,FGrid,cb);
for(int k=0;k<nbasis;k++) AggV.subspace[k]=AggregatesGCR.subspace[k];
LittleDiracOperator LittleDiracOpPV(geomV,FGrid,Coarse5dV);
std::cout << GridLogMessage << "*** V1 CoarsenOperator (cross check) ***" << std::endl;
LittleDiracOpPV.CoarsenOperator(PVdagM,AggV);
MrhsLittleDiracOperator mrhsV1(geomV,CoarseMrhsV);
mrhsV1.CopyMatrix(LittleDiracOpPV);
// BLAS_A is written by GridtoBLAS in lSite order and both sides carry
// the same scalar_object, so the two are directly comparable.
typedef MrhsLittleDiracOperator::calcMatrix calcMatrix;
int npoint = geom.npoint;
RealD num=0.0, den=0.0;
for(int p=0;p<npoint;p++){
int64_t sites = mrhsV1.BLAS_A[p].size();
GRID_ASSERT(sites == (int64_t)CoarseOpPV.BLAS_A[p].size());
std::vector<calcMatrix> h1(sites),h2(sites);
acceleratorCopyFromDevice(&mrhsV1.BLAS_A[p][0], &h1[0],sites*sizeof(calcMatrix));
acceleratorCopyFromDevice(&CoarseOpPV.BLAS_A[p][0],&h2[0],sites*sizeof(calcMatrix));
ComplexD *w1=(ComplexD *)&h1[0];
ComplexD *w2=(ComplexD *)&h2[0];
int64_t words = sites*sizeof(calcMatrix)/sizeof(ComplexD);
for(int64_t i=0;i<words;i++){
ComplexD d=w1[i]-w2[i];
num += real(d)*real(d)+imag(d)*imag(d);
den += real(w1[i])*real(w1[i])+imag(w1[i])*imag(w1[i]);
}
}
std::cout << GridLogMessage << "V1_CHECK: |A_V1|^2 = " << den << std::endl;
std::cout << GridLogMessage << "V1_CHECK: |A_V1 - A_V2|^2 / |A_V1|^2 = " << num/den << std::endl;
GRID_ASSERT( den > 0.0 );
GRID_ASSERT( num/den < 1.0e-18 );
}
//////////////////////////////////////////////////////////////////////
// STAGE TWO: L2 -> L3.
//
// The fine operator here is V2 at L1, which is natively multiRHS, so the
// multiRHS driver applies with no promotion adapter: its D+1 grid IS the
// batch grid the L1 operator is currently set to.
//////////////////////////////////////////////////////////////////////
Coordinate cclatt = clatt;
Coordinate Block2({8,4,3,6});
if ( getenv("BLOCK2") ){ GridCmdOptionIntVector(std::string(getenv("BLOCK2")),Block2); GRID_ASSERT(Block2.size()==4); }
for(int d=0;d<4;d++){ GRID_ASSERT(clatt[d]%Block2[d]==0); cclatt[d]=clatt[d]/Block2[d]; }
std::cout << GridLogMessage << "Block2 " << Block2 << " coarse-coarse lattice " << cclatt << std::endl;
Coordinate cc5latt({1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
GridCartesian *CoarseCoarse5d = new GridCartesian(cc5latt,c5simd,c5mpi);
Coordinate ccmlatt({nrhs,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
GridCartesian *CoarseCoarseMrhs = new GridCartesian(ccmlatt,cmsimd,cmmpi);
Coordinate ccblatt({batch,1,cclatt[0],cclatt[1],cclatt[2],cclatt[3]});
GridCartesian *CoarseCoarseBatch = new GridCartesian(ccblatt,cmsimd,cmmpi);
// Coarsening deepens the tensor nest by one iScalar
typedef CoarseVector::vector_object CoarseSiteObj;
typedef iScalar<CComplexS> CComplexS2;
typedef MultiGeneralCoarsenedOperatorV2<CoarseSiteObj,CComplexS2,nbasis> CoarseCoarseOperator;
typedef CoarseCoarseOperator::CoarseVector CoarseCoarseVector;
NextToNearestStencilGeometry5D geom2(CoarseCoarse5d);
// RAW copy of the coarse null vectors, for the same reason as rawNull:
// the L2 CoarsenOperator block-orthonormalises its subspace in place, and
// the L3 basis must be defined by the vectors that still carry content.
std::vector<CoarseVector> rawPsi(nbasis,Coarse5d);
for(int k=0;k<nbasis;k++) rawPsi[k]=psi_coarse[k];
CoarseCoarseOperator CoarseOpL2(geom2,CoarseCoarse5d);
CoarseOpL2.SetGrid(CoarseCoarseBatch);
NonHermitianLinearOperator<CoarseOperator,CoarseVector> LinOpCoarse(CoarseOpPV);
std::cout << GridLogMessage << "*** L2 CoarsenOperator, batch "<<batch<<" ***" << std::endl;
CoarseOpL2.CoarsenOperator(LinOpCoarse,CoarseBatch,psi_coarse,CoarseCoarse5d);
//////////////////////////////////////////////////////////////////////
// Both operators to the solve Nrhs. The matrix elements are Nrhs
// independent and survive the change.
//////////////////////////////////////////////////////////////////////
CoarseOpPV.SetGrid(CoarseMrhs);
CoarseOpL2.SetGrid(CoarseCoarseMrhs);
std::cout << GridLogMessage << "L1 operator at Nrhs " << CoarseOpPV.Nrhs()
<< ", L2 operator at Nrhs " << CoarseOpL2.Nrhs() << std::endl;
//////////////////////////////////////////////////////////////////////
// psi_cc from the RAW coarse null vectors, and the same guard
//////////////////////////////////////////////////////////////////////
MultiRHSBlockProject<CoarseVector> MrhsProjectorL2;
MrhsProjectorL2.Allocate(nbasis,Coarse5d,CoarseCoarse5d);
MrhsProjectorL2.ImportBasis(psi_coarse); // block orthonormal basis
{
std::vector<CoarseCoarseVector> psi_cc(nbasis,CoarseCoarse5d);
MrhsProjectorL2.blockProject(rawPsi,psi_cc); // RAW vectors in
RealD defect = GramDefect(psi_cc);
RealD leak = std::sqrt((double)CoarseCoarse5d->gSites());
std::cout << GridLogMessage << "GUARD: ||<psi_cc|psi_cc> - I||_F = " << defect
<< " (~0.23 good; ~sqrt(N_cc)=" << leak << " = e_k leak)" << std::endl;
GRID_ASSERT( defect < leak );
}
rawPsi.clear(); rawPsi.shrink_to_fit();
//////////////////////////////////////////////////////////////////////
// Both coarse operators apply on their solve grids
//////////////////////////////////////////////////////////////////////
{
GridParallelRNG cRNG(Coarse5d); cRNG.SeedFixedIntegers({3,4,5,6});
CoarseVector cin(CoarseMrhs), cout_(CoarseMrhs);
random(cRNG,cin);
CoarseOpPV.M(cin,cout_);
std::cout << GridLogMessage << "L1 apply |in|^2 = " << norm2(cin)
<< " |M in|^2 = " << norm2(cout_) << std::endl;
GRID_ASSERT( norm2(cout_) > 0.0 );
GridParallelRNG ccRNG(CoarseCoarse5d); ccRNG.SeedFixedIntegers({7,8,9,10});
CoarseCoarseVector ccin(CoarseCoarseMrhs), ccout(CoarseCoarseMrhs);
random(ccRNG,ccin);
CoarseOpL2.M(ccin,ccout);
std::cout << GridLogMessage << "L2 apply |in|^2 = " << norm2(ccin)
<< " |M in|^2 = " << norm2(ccout) << std::endl;
GRID_ASSERT( norm2(ccout) > 0.0 );
}
std::cout << GridLogMessage << "*** stage two complete: L1 and L2 coarse operators built ***" << std::endl;
Grid_finalize();
}