portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2025-04-24 12:45:56 +01:00
Code Releases Activity

Improve the coarse matrix calc

Browse Source
This commit is contained in:
Peter Boyle 2020-09-08 15:36:33 -04:00
parent baa668d3ac
commit 729882827c
1 changed files with 63 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
Grid/algorithms/CoarsenedMatrix.h
View File

@ -49,13 +49,11 @@ inline void blockMaskedInnerProduct(Lattice<CComplex> &CoarseInner,
Lattice<dotp> fine_inner_msk(fine); Lattice<dotp> fine_inner_msk(fine);
// Multiply could be fused with innerProduct // Multiply could be fused with innerProduct
// Single block sum kernel could do both masks.
fine_inner = localInnerProduct(fineX,fineY); fine_inner = localInnerProduct(fineX,fineY);
mult(fine_inner_msk, fine_inner,FineMask); mult(fine_inner_msk, fine_inner,FineMask);
blockSum(CoarseInner,fine_inner_msk); blockSum(CoarseInner,fine_inner_msk);
} }
class Geometry { class Geometry {
public: public:
int npoint; int npoint;
@ -80,8 +78,12 @@ public:
} }
directions [2*_d]=0; directions [2*_d]=0;
displacements[2*_d]=0; displacements[2*_d]=0;
}
std::cout <<GridLogMessage << "Geometry "<<std::endl;
for(int p=0;p<npoint;p++){
std::cout <<GridLogMessage << "point " <<p<<" dir "<<directions[p]<<" delta " <<displacements[p]<<std::endl;
}
}
}; };
template<class Fobj,class CComplex,int nbasis> template<class Fobj,class CComplex,int nbasis>
@ -285,6 +287,8 @@ public:
/////////////////////// ///////////////////////
GridBase * Grid(void) { return _grid; }; // this is all the linalg routines need to know GridBase * Grid(void) { return _grid; }; // this is all the linalg routines need to know
virtual std::vector<int> Directions(void) { return geom.directions; };
virtual std::vector<int> Displacements(void){ return geom.displacements; };
void M (const CoarseVector &in, CoarseVector &out) void M (const CoarseVector &in, CoarseVector &out)
{ {
conformable(_grid,in.Grid()); conformable(_grid,in.Grid());
@ -418,32 +422,17 @@ public:
int ndim = in.Grid()->Nd(); int ndim = in.Grid()->Nd();
////////////// int point=-1;
// 4D action like wilson for(int p=0;p<geom.npoint;p++){
// 0+ => 0 if( (dir==geom.directions[p])&&(disp==geom.displacements[p])) point=p;
// 0- => 1
// 1+ => 2
// 1- => 3
// etc..
//////////////
// 5D action like DWF
// 1+ => 0
// 1- => 1
// 2+ => 2
// 2- => 3
// etc..
auto point = [dir, disp, ndim](){
if(dir == 0 and disp == 0)
return 8;
else if ( ndim==4 ) {
return (4 * dir + 1 - disp) / 2;
} else {
return (4 * (dir-1) + 1 - disp) / 2;
} }
}(); assert(point!=-1);// Must find
std::cout <<GridLogMessage << "Mdir point "<<point<<" dir "<<dir<<" disp "<<disp <<std::endl;
for(int p=0;p<geom.npoint;p++){
std::cout <<GridLogMessage << "point " <<p<<" dir "<<geom.directions[p]<<" delta " <<geom.displacements[p]<<std::endl;
}
MdirCalc(in,out,point); MdirCalc(in,out,point);
}; };
void Mdiag(const CoarseVector &in, CoarseVector &out) void Mdiag(const CoarseVector &in, CoarseVector &out)
@ -486,29 +475,29 @@ public:
std::cout << GridLogMessage<< "************ "<<std::endl; std::cout << GridLogMessage<< "************ "<<std::endl;
// Coarse operator // Coarse operator
this->M(Cin,Cout); this->M(Cin,Cout);
std::cout << GridLogMessage<< " Cout "<<norm2(Cout)<<std::endl;
// Fine projected operator // Fine projected operator
_Aggregates.PromoteFromSubspace(Cin,Fin); _Aggregates.PromoteFromSubspace(Cin,Fin);
linop.Op(Fin,Fout); linop.Op(Fin,Fout);
_Aggregates.ProjectToSubspace(CFout,Fout); _Aggregates.ProjectToSubspace(CFout,Fout);
std::cout << GridLogMessage<< " CFout "<<norm2(CFout)<<std::endl;
CFout = CFout-Cout; CFout = CFout-Cout;
std::cout << GridLogMessage<< " diff "<<norm2(CFout)<<std::endl; RealD diff = norm2(CFout);
std::cout << GridLogMessage<< " diff "<<diff<<std::endl;
assert(diff<1.0e-5);
std::cout << GridLogMessage<< "************ "<<std::endl; std::cout << GridLogMessage<< "************ "<<std::endl;
std::cout << GridLogMessage<< " Testing Mdag "<<std::endl; std::cout << GridLogMessage<< " Testing Mdag "<<std::endl;
std::cout << GridLogMessage<< "************ "<<std::endl; std::cout << GridLogMessage<< "************ "<<std::endl;
// Coarse operator // Coarse operator
Mdag(Cin,Cout); Mdag(Cin,Cout);
std::cout << GridLogMessage<< " Cout "<<norm2(Cout)<<std::endl;
// Fine operator // Fine operator
linop.AdjOp(Fin,Fout); linop.AdjOp(Fin,Fout);
_Aggregates.ProjectToSubspace(CFout,Fout); _Aggregates.ProjectToSubspace(CFout,Fout);
std::cout << GridLogMessage<< " CFout "<<norm2(CFout)<<std::endl;
CFout = CFout-Cout; CFout = CFout-Cout;
std::cout << GridLogMessage<< " diff "<<norm2(CFout)<<std::endl; diff = norm2(CFout);
std::cout << GridLogMessage<< " diff "<<diff<<std::endl;
assert(diff<1.0e-5);
} }
void CoarsenOperator(GridBase *FineGrid,LinearOperatorBase<Lattice<Fobj> > &linop, void CoarsenOperator(GridBase *FineGrid,LinearOperatorBase<Lattice<Fobj> > &linop,
@ -547,13 +536,20 @@ public:
CoarseScalar InnerProd(Grid()); CoarseScalar InnerProd(Grid());
size_t free,total;
cudaMemGetInfo(&free,&total); std::cout << "ForceHermitian "<<free<<"/"<<total<<std::endl;
std::cout << GridLogMessage<< "CoarsenMatrix Orthog " << std::endl; std::cout << GridLogMessage<< "CoarsenMatrix Orthog " << std::endl;
// Orthogonalise the subblocks over the basis // Orthogonalise the subblocks over the basis
blockOrthogonalise(InnerProd,Subspace.subspace); blockOrthogonalise(InnerProd,Subspace.subspace);
std::cout << GridLogMessage<< "CoarsenMatrix Orthog done " << std::endl; std::cout << GridLogMessage<< "CoarsenMatrix Orthog done " << std::endl;
auto OpDirections = linop.Directions();
auto OpDisplacements = linop.Displacements();
std::cout<<" Coarsening an operator with "<< OpDirections.size()<<" terms "<<std::endl;
for(int p=0;p<OpDirections.size();p++) {
assert(OpDirections[p]==geom.directions[p]);
assert(OpDisplacements[p]==geom.displacements[p]);
}
// Compute the matrix elements of linop between this orthonormal // Compute the matrix elements of linop between this orthonormal
// set of vectors. // set of vectors.
int self_stencil=-1; int self_stencil=-1;
@ -587,9 +583,20 @@ public:
} }
evenmask = where(mod(bcb,2)==(Integer)0,one,zero); evenmask = where(mod(bcb,2)==(Integer)0,one,zero);
oddmask = one-evenmask; oddmask = one-evenmask;
std::cout << GridLogMessage<< "CoarsenMatrix Mask " << std::endl;
cudaMemGetInfo(&free,&total); std::cout << "ForceHermitian "<<free<<"/"<<total<<std::endl;
/*
{
phi=Subspace.subspace[0];
linop.OpDirAll(phi,Mphi_p);
for(int p=0;p<geom.npoint-1;p++){
int dir=geom.directions[p];
int disp=geom.displacements[p];
linop.OpDir(phi,Mphi,dir,disp);
Mphi=Mphi-Mphi_p[p];
std::cout << GridLogMessage <<" Direction mapping check " <<norm2(Mphi)<<std::endl;
}
}
*/
assert(self_stencil!=-1); assert(self_stencil!=-1);
int lhermitian=hermitian; int lhermitian=hermitian;
@ -603,7 +610,6 @@ public:
for(int p=0;p<geom.npoint;p++){ for(int p=0;p<geom.npoint;p++){
std::cout << GridLogMessage<< "CoarsenMatrix point "<<p << std::endl;
Mphi = Mphi_p[p]; Mphi = Mphi_p[p];
int dir = geom.directions[p]; int dir = geom.directions[p];
@ -627,7 +633,18 @@ public:
autoView( A_self , A[self_stencil], AcceleratorWrite); autoView( A_self , A[self_stencil], AcceleratorWrite);
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_p[ss](j,i),oZProj_v(ss)); }); accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_p[ss](j,i),oZProj_v(ss)); });
if ( lhermitian && (disp==-1) ) {
for(int pp=0;pp<geom.npoint;pp++){// Find the opposite link and set <j|A|i> = <i|A|j>*
int dirp = geom.directions[pp];
int dispp = geom.displacements[pp];
if ( (dirp==dir) && (dispp==1) ){
auto sft = conjugate(Cshift(oZProj,dir,1));
autoView( sft_v , sft , AcceleratorWrite);
autoView( A_pp , A[pp], AcceleratorWrite);
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_pp[ss](i,j),sft_v(ss)); });
}
}
}
} }
} }
} }
@ -664,17 +681,11 @@ public:
} }
} }
if(lhermitian) {
std::cout << GridLogMessage << " ForceHermitian, new code "<<std::endl;
cudaMemGetInfo(&free,&total);
std::cout << "ForceHermitian "<<free<<"/"<<total<<std::endl;
MemoryManager::PrintBytes(); MemoryManager::PrintBytes();
ForceHermitian();
std::cout << GridLogMessage << " ForceHermitian, done "<<std::endl; // Auto self test
cudaMemGetInfo(&free,&total); Test( Subspace,FineGrid,linop);
std::cout << "ForceHermitian "<<free<<"/"<<total<<std::endl;
MemoryManager::PrintBytes();
}
#if 0 #if 0
/////////////////////////// ///////////////////////////
@ -699,34 +710,6 @@ public:
} }
void ForceHermitian(void) {
for(int p=0;p<geom.npoint;p++){
int dir = geom.directions[p];
int disp = geom.displacements[p];
if(disp==-1) {
// Find the opposite link
for(int pp=0;pp<geom.npoint;pp++){
int dirp = geom.directions[pp];
int dispp = geom.displacements[pp];
if ( (dirp==dir) && (dispp==1) ){
std::cout << GridLogMessage<<" Replacing stencil leg "<<pp<<" with leg "<<p<<std::endl;
size_t free,total;
cudaMemGetInfo(&free,&total);
std::cout << "ForceHermitian before expr "<<free<<"/"<<total<<std::endl;
MemoryManager::PrintBytes();
A[pp] = Cshift(A[p],dir,1);
cudaMemGetInfo(&free,&total);
std::cout << "ForceHermitian during expr "<<free<<"/"<<total<<std::endl;
MemoryManager::PrintBytes();
A[pp] = adj(A[pp]);
cudaMemGetInfo(&free,&total);
std::cout << "ForceHermitian after expr "<<free<<"/"<<total<<std::endl;
MemoryManager::PrintBytes();
}
}
}
}
}
}; };
NAMESPACE_END(Grid); NAMESPACE_END(Grid);