mirror of
https://github.com/paboyle/Grid.git
synced 2024-11-09 23:45:36 +00:00
Blas based block project & deflate for multiRHS
This commit is contained in:
Peter Boyle
parent
cd15abe9d1
commit
ee0d460c8e
512
Grid/algorithms/deflation/MultiRHSBlockProject.h
Normal file
512
Grid/algorithms/deflation/MultiRHSBlockProject.h
Normal file
@ -0,0 +1,512 @@
|
|||||||
|
/*************************************************************************************
|
||||||
|
|
||||||
|
Grid physics library, www.github.com/paboyle/Grid
|
||||||
|
|
||||||
|
Source file: MultiRHSDeflation.h
|
||||||
|
|
||||||
|
Copyright (C) 2023
|
||||||
|
|
||||||
|
Author: Peter Boyle <pboyle@bnl.gov>
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation; either version 2 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License along
|
||||||
|
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||||
|
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||||
|
|
||||||
|
See the full license in the file "LICENSE" in the top level distribution directory
|
||||||
|
*************************************************************************************/
|
||||||
|
/* END LEGAL */
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
MultiRHS block projection
|
||||||
|
|
||||||
|
Import basis -> nblock x nbasis x (block x internal)
|
||||||
|
Import vector of fine lattice objects -> nblock x nrhs x (block x internal)
|
||||||
|
|
||||||
|
=> coarse_(nrhs x nbasis )^block = via batched GEMM
|
||||||
|
|
||||||
|
//template<class vobj,class CComplex,int nbasis,class VLattice>
|
||||||
|
//inline void blockProject(Lattice<iVector<CComplex,nbasis > > &coarseData,
|
||||||
|
// const VLattice &fineData,
|
||||||
|
// const VLattice &Basis)
|
||||||
|
*/
|
||||||
|
|
||||||
|
template<class Field>
|
||||||
|
class MultiRHSBlockProject
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
|
||||||
|
typedef typename Field::scalar_type scalar;
|
||||||
|
typedef typename Field::scalar_object scalar_object;
|
||||||
|
typedef Field Fermion;
|
||||||
|
|
||||||
|
int nbasis;
|
||||||
|
GridBase *coarse_grid;
|
||||||
|
GridBase *fine_grid;
|
||||||
|
uint64_t block_vol;
|
||||||
|
uint64_t fine_vol;
|
||||||
|
uint64_t coarse_vol;
|
||||||
|
uint64_t words;
|
||||||
|
|
||||||
|
// Row major layout "C" order:
|
||||||
|
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
||||||
|
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
||||||
|
// BLAS_C[coarse_vol][nrhs][nbasis]
|
||||||
|
/*
|
||||||
|
* in Fortran column major notation (cuBlas order)
|
||||||
|
*
|
||||||
|
* Vxb = [v1(x)][..][vn(x)] ... x coarse vol
|
||||||
|
*
|
||||||
|
* Fxr = [r1(x)][..][rm(x)] ... x coarse vol
|
||||||
|
*
|
||||||
|
* Block project:
|
||||||
|
* C_br = V^dag F x coarse vol
|
||||||
|
*
|
||||||
|
* Block promote:
|
||||||
|
* F_xr = Vxb Cbr x coarse_vol
|
||||||
|
*/
|
||||||
|
deviceVector<scalar> BLAS_V; // words * block_vol * nbasis x coarse_vol
|
||||||
|
deviceVector<scalar> BLAS_F; // nrhs x fine_vol * words -- the sources
|
||||||
|
deviceVector<scalar> BLAS_C; // nrhs x coarse_vol * nbasis -- the coarse coeffs
|
||||||
|
|
||||||
|
RealD blasNorm2(deviceVector<scalar> &blas)
|
||||||
|
{
|
||||||
|
scalar ss(0.0);
|
||||||
|
std::vector<scalar> tmp(blas.size());
|
||||||
|
acceleratorCopyFromDevice(&blas[0],&tmp[0],blas.size()*sizeof(scalar));
|
||||||
|
for(int64_t s=0;s<blas.size();s++){
|
||||||
|
ss=ss+tmp[s]*adj(tmp[s]);
|
||||||
|
}
|
||||||
|
coarse_grid->GlobalSum(ss);
|
||||||
|
return real(ss);
|
||||||
|
}
|
||||||
|
|
||||||
|
MultiRHSBlockProject(){};
|
||||||
|
~MultiRHSBlockProject(){ Deallocate(); };
|
||||||
|
|
||||||
|
void Deallocate(void)
|
||||||
|
{
|
||||||
|
nbasis=0;
|
||||||
|
coarse_grid=nullptr;
|
||||||
|
fine_grid=nullptr;
|
||||||
|
fine_vol=0;
|
||||||
|
block_vol=0;
|
||||||
|
coarse_vol=0;
|
||||||
|
words=0;
|
||||||
|
BLAS_V.resize(0);
|
||||||
|
BLAS_F.resize(0);
|
||||||
|
BLAS_C.resize(0);
|
||||||
|
}
|
||||||
|
void Allocate(int _nbasis,GridBase *_fgrid,GridBase *_cgrid)
|
||||||
|
{
|
||||||
|
nbasis=_nbasis;
|
||||||
|
|
||||||
|
fine_grid=_fgrid;
|
||||||
|
coarse_grid=_cgrid;
|
||||||
|
|
||||||
|
fine_vol = fine_grid->lSites();
|
||||||
|
coarse_vol = coarse_grid->lSites();
|
||||||
|
block_vol = fine_vol/coarse_vol;
|
||||||
|
|
||||||
|
words = sizeof(scalar_object)/sizeof(scalar);
|
||||||
|
|
||||||
|
BLAS_V.resize (fine_vol * words * nbasis );
|
||||||
|
}
|
||||||
|
void ImportFineGridVectors(std::vector <Field > &vecs, deviceVector<scalar> &blas)
|
||||||
|
{
|
||||||
|
int nvec = vecs.size();
|
||||||
|
typedef typename Field::vector_object vobj;
|
||||||
|
std::cout << " BlockProjector importing "<<nvec<< " vectors" <<std::endl;
|
||||||
|
|
||||||
|
assert(vecs[0].Grid()==fine_grid);
|
||||||
|
|
||||||
|
subdivides(coarse_grid,fine_grid); // require they map
|
||||||
|
|
||||||
|
int _ndimension = coarse_grid->_ndimension;
|
||||||
|
assert(block_vol == fine_grid->oSites() / coarse_grid->oSites());
|
||||||
|
|
||||||
|
Coordinate block_r (_ndimension);
|
||||||
|
for(int d=0 ; d<_ndimension;d++){
|
||||||
|
block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t sz = blas.size();
|
||||||
|
|
||||||
|
acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
|
||||||
|
|
||||||
|
Coordinate fine_rdimensions = fine_grid->_rdimensions;
|
||||||
|
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||||
|
int64_t bv= block_vol;
|
||||||
|
for(int v=0;v<vecs.size();v++){
|
||||||
|
|
||||||
|
// std::cout << " BlockProjector importing vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
|
||||||
|
autoView( fineData , vecs[v], AcceleratorRead);
|
||||||
|
|
||||||
|
auto blasData_p = &blas[0];
|
||||||
|
auto fineData_p = &fineData[0];
|
||||||
|
|
||||||
|
int64_t osites = fine_grid->oSites();
|
||||||
|
|
||||||
|
// loop over fine sites
|
||||||
|
const int Nsimd = vobj::Nsimd();
|
||||||
|
// std::cout << "sz "<<sz<<std::endl;
|
||||||
|
// std::cout << "prod "<<Nsimd * coarse_grid->oSites() * block_vol * nvec * words<<std::endl;
|
||||||
|
assert(sz == Nsimd * coarse_grid->oSites() * block_vol * nvec * words);
|
||||||
|
uint64_t lwords= words; // local variable for copy in to GPU
|
||||||
|
accelerator_for(sf,osites,Nsimd,{
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
{
|
||||||
|
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||||
|
#else
|
||||||
|
for(int lane=0;lane<Nsimd;lane++) {
|
||||||
|
#endif
|
||||||
|
// One thread per fine site
|
||||||
|
Coordinate coor_f(_ndimension);
|
||||||
|
Coordinate coor_b(_ndimension);
|
||||||
|
Coordinate coor_c(_ndimension);
|
||||||
|
|
||||||
|
// Fine site to fine coor
|
||||||
|
Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
|
||||||
|
|
||||||
|
for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
|
||||||
|
for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
|
||||||
|
|
||||||
|
int sc;// coarse site
|
||||||
|
int sb;// block site
|
||||||
|
Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
|
||||||
|
Lexicographic::IndexFromCoor(coor_b,sb,block_r);
|
||||||
|
|
||||||
|
scalar_object data = extractLane(lane,fineData[sf]);
|
||||||
|
|
||||||
|
// BLAS layout address calculation
|
||||||
|
// words * block_vol * nbasis x coarse_vol
|
||||||
|
// coarse oSite x block vole x lanes
|
||||||
|
int64_t site = (lane*osites + sc*bv)*nvec
|
||||||
|
+ v*bv
|
||||||
|
+ sb;
|
||||||
|
|
||||||
|
// assert(site*lwords<sz);
|
||||||
|
|
||||||
|
scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
|
||||||
|
|
||||||
|
*ptr = data;
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
});
|
||||||
|
// std::cout << " import fine Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||||
|
// std::cout << " BlockProjector imported vector"<<v<<std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
void ExportFineGridVectors(std::vector <Field> &vecs, deviceVector<scalar> &blas)
|
||||||
|
{
|
||||||
|
typedef typename Field::vector_object vobj;
|
||||||
|
|
||||||
|
int nvec = vecs.size();
|
||||||
|
|
||||||
|
assert(vecs[0].Grid()==fine_grid);
|
||||||
|
|
||||||
|
subdivides(coarse_grid,fine_grid); // require they map
|
||||||
|
|
||||||
|
int _ndimension = coarse_grid->_ndimension;
|
||||||
|
assert(block_vol == fine_grid->oSites() / coarse_grid->oSites());
|
||||||
|
|
||||||
|
Coordinate block_r (_ndimension);
|
||||||
|
for(int d=0 ; d<_ndimension;d++){
|
||||||
|
block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
|
||||||
|
}
|
||||||
|
Coordinate fine_rdimensions = fine_grid->_rdimensions;
|
||||||
|
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||||
|
|
||||||
|
// std::cout << " export fine Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||||
|
|
||||||
|
int64_t bv= block_vol;
|
||||||
|
for(int v=0;v<vecs.size();v++){
|
||||||
|
|
||||||
|
autoView( fineData , vecs[v], AcceleratorWrite);
|
||||||
|
|
||||||
|
auto blasData_p = &blas[0];
|
||||||
|
auto fineData_p = &fineData[0];
|
||||||
|
|
||||||
|
int64_t osites = fine_grid->oSites();
|
||||||
|
uint64_t lwords = words;
|
||||||
|
// std::cout << " Nsimd is "<<vobj::Nsimd() << std::endl;
|
||||||
|
// std::cout << " lwords is "<<lwords << std::endl;
|
||||||
|
// std::cout << " sizeof(scalar_object) is "<<sizeof(scalar_object) << std::endl;
|
||||||
|
// loop over fine sites
|
||||||
|
accelerator_for(sf,osites,vobj::Nsimd(),{
|
||||||
|
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
{
|
||||||
|
int lane=acceleratorSIMTlane(vobj::Nsimd()); // buffer lane
|
||||||
|
#else
|
||||||
|
for(int lane=0;lane<vobj::Nsimd();lane++) {
|
||||||
|
#endif
|
||||||
|
// One thread per fine site
|
||||||
|
Coordinate coor_f(_ndimension);
|
||||||
|
Coordinate coor_b(_ndimension);
|
||||||
|
Coordinate coor_c(_ndimension);
|
||||||
|
|
||||||
|
Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
|
||||||
|
|
||||||
|
for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
|
||||||
|
for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
|
||||||
|
|
||||||
|
int sc;
|
||||||
|
int sb;
|
||||||
|
Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
|
||||||
|
Lexicographic::IndexFromCoor(coor_b,sb,block_r);
|
||||||
|
|
||||||
|
// BLAS layout address calculation
|
||||||
|
// words * block_vol * nbasis x coarse_vol
|
||||||
|
int64_t site = (lane*osites + sc*bv)*nvec
|
||||||
|
+ v*bv
|
||||||
|
+ sb;
|
||||||
|
|
||||||
|
scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
|
||||||
|
|
||||||
|
scalar_object data = *ptr;
|
||||||
|
|
||||||
|
insertLane(lane,fineData[sf],data);
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
template<class vobj>
|
||||||
|
void ImportCoarseGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
|
||||||
|
{
|
||||||
|
int nvec = vecs.size();
|
||||||
|
typedef typename vobj::scalar_object coarse_scalar_object;
|
||||||
|
|
||||||
|
std::cout << " BlockProjector importing coarse grid "<<nvec<< " vectors" <<std::endl;
|
||||||
|
|
||||||
|
assert(vecs[0].Grid()==coarse_grid);
|
||||||
|
|
||||||
|
int _ndimension = coarse_grid->_ndimension;
|
||||||
|
|
||||||
|
uint64_t sz = blas.size();
|
||||||
|
|
||||||
|
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||||
|
|
||||||
|
for(int v=0;v<vecs.size();v++){
|
||||||
|
|
||||||
|
// std::cout << " BlockProjector importing coarse vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
|
||||||
|
autoView( coarseData , vecs[v], AcceleratorRead);
|
||||||
|
|
||||||
|
auto blasData_p = &blas[0];
|
||||||
|
auto coarseData_p = &coarseData[0];
|
||||||
|
|
||||||
|
int64_t osites = coarse_grid->oSites();
|
||||||
|
|
||||||
|
// loop over fine sites
|
||||||
|
const int Nsimd = vobj::Nsimd();
|
||||||
|
uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
|
||||||
|
assert(cwords==nbasis);
|
||||||
|
|
||||||
|
accelerator_for(sc,osites,Nsimd,{
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
{
|
||||||
|
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||||
|
#else
|
||||||
|
for(int lane=0;lane<Nsimd;lane++) {
|
||||||
|
#endif
|
||||||
|
// C_br per site
|
||||||
|
int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
|
||||||
|
|
||||||
|
coarse_scalar_object data = extractLane(lane,coarseData[sc]);
|
||||||
|
|
||||||
|
coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
|
||||||
|
|
||||||
|
*ptr = data;
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
});
|
||||||
|
// std::cout << " import coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
template<class vobj>
|
||||||
|
void ExportCoarseGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
|
||||||
|
{
|
||||||
|
int nvec = vecs.size();
|
||||||
|
typedef typename vobj::scalar_object coarse_scalar_object;
|
||||||
|
std::cout << " BlockProjector importing coarse grid "<<nvec<< " vectors" <<std::endl;
|
||||||
|
|
||||||
|
assert(vecs[0].Grid()==coarse_grid);
|
||||||
|
|
||||||
|
int _ndimension = coarse_grid->_ndimension;
|
||||||
|
|
||||||
|
uint64_t sz = blas.size();
|
||||||
|
|
||||||
|
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||||
|
|
||||||
|
// std::cout << " export coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||||
|
for(int v=0;v<vecs.size();v++){
|
||||||
|
|
||||||
|
// std::cout << " BlockProjector exporting coarse vector"<<v<<std::endl;
|
||||||
|
autoView( coarseData , vecs[v], AcceleratorWrite);
|
||||||
|
|
||||||
|
auto blasData_p = &blas[0];
|
||||||
|
auto coarseData_p = &coarseData[0];
|
||||||
|
|
||||||
|
int64_t osites = coarse_grid->oSites();
|
||||||
|
|
||||||
|
// loop over fine sites
|
||||||
|
const int Nsimd = vobj::Nsimd();
|
||||||
|
uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
|
||||||
|
assert(cwords==nbasis);
|
||||||
|
|
||||||
|
accelerator_for(sc,osites,Nsimd,{
|
||||||
|
// Wrap in a macro "FOR_ALL_LANES(lane,{ ... });
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
{
|
||||||
|
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||||
|
#else
|
||||||
|
for(int lane=0;lane<Nsimd;lane++) {
|
||||||
|
#endif
|
||||||
|
int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
|
||||||
|
coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
|
||||||
|
coarse_scalar_object data = *ptr;
|
||||||
|
insertLane(lane,coarseData[sc],data);
|
||||||
|
#ifdef GRID_SIMT
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
void ImportBasis(std::vector < Field > &vecs)
|
||||||
|
{
|
||||||
|
// std::cout << " BlockProjector Import basis size "<<vecs.size()<<std::endl;
|
||||||
|
ImportFineGridVectors(vecs,BLAS_V);
|
||||||
|
}
|
||||||
|
|
||||||
|
template<class cobj>
|
||||||
|
void blockProject(std::vector<Field> &fine,std::vector< Lattice<cobj> > & coarse)
|
||||||
|
{
|
||||||
|
int nrhs=fine.size();
|
||||||
|
int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
|
||||||
|
assert(nbasis==_nbasis);
|
||||||
|
|
||||||
|
BLAS_F.resize (fine_vol * words * nrhs );
|
||||||
|
BLAS_C.resize (coarse_vol * nbasis * nrhs );
|
||||||
|
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// Copy in the multi-rhs sources to same data layout
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// std::cout << "BlockProject import fine"<<std::endl;
|
||||||
|
ImportFineGridVectors(fine,BLAS_F);
|
||||||
|
|
||||||
|
deviceVector<scalar *> Vd(coarse_vol);
|
||||||
|
deviceVector<scalar *> Fd(coarse_vol);
|
||||||
|
deviceVector<scalar *> Cd(coarse_vol);
|
||||||
|
|
||||||
|
// std::cout << "BlockProject pointers"<<std::endl;
|
||||||
|
for(int c=0;c<coarse_vol;c++){
|
||||||
|
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
||||||
|
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
||||||
|
// BLAS_C[coarse_vol][nrhs][nbasis]
|
||||||
|
scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
|
||||||
|
scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
|
||||||
|
scalar * Ch = & BLAS_C[c*nrhs*nbasis];
|
||||||
|
|
||||||
|
acceleratorPut(Vd[c],Vh);
|
||||||
|
acceleratorPut(Fd[c],Fh);
|
||||||
|
acceleratorPut(Cd[c],Ch);
|
||||||
|
}
|
||||||
|
|
||||||
|
GridBLAS BLAS;
|
||||||
|
|
||||||
|
// std::cout << "BlockProject BLAS"<<std::endl;
|
||||||
|
int64_t vw = block_vol * words;
|
||||||
|
/////////////////////////////////////////
|
||||||
|
// C_br = V^dag R
|
||||||
|
/////////////////////////////////////////
|
||||||
|
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
||||||
|
nbasis,nrhs,vw,
|
||||||
|
ComplexD(1.0),
|
||||||
|
Vd,
|
||||||
|
Fd,
|
||||||
|
ComplexD(0.0), // wipe out C
|
||||||
|
Cd);
|
||||||
|
BLAS.synchronise();
|
||||||
|
// std::cout << "BlockProject done"<<std::endl;
|
||||||
|
ExportCoarseGridVectors(coarse, BLAS_C);
|
||||||
|
// std::cout << "BlockProject done"<<std::endl;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
template<class cobj>
|
||||||
|
void blockPromote(std::vector<Field> &fine,std::vector<Lattice<cobj> > & coarse)
|
||||||
|
{
|
||||||
|
int nrhs=fine.size();
|
||||||
|
int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
|
||||||
|
assert(nbasis==_nbasis);
|
||||||
|
|
||||||
|
BLAS_F.resize (fine_vol * words * nrhs );
|
||||||
|
BLAS_C.resize (coarse_vol * nbasis * nrhs );
|
||||||
|
|
||||||
|
ImportCoarseGridVectors(coarse, BLAS_C);
|
||||||
|
|
||||||
|
GridBLAS BLAS;
|
||||||
|
|
||||||
|
deviceVector<scalar *> Vd(coarse_vol);
|
||||||
|
deviceVector<scalar *> Fd(coarse_vol);
|
||||||
|
deviceVector<scalar *> Cd(coarse_vol);
|
||||||
|
|
||||||
|
for(int c=0;c<coarse_vol;c++){
|
||||||
|
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
||||||
|
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
||||||
|
// BLAS_C[coarse_vol][nrhs][nbasis]
|
||||||
|
scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
|
||||||
|
scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
|
||||||
|
scalar * Ch = & BLAS_C[c*nrhs*nbasis];
|
||||||
|
acceleratorPut(Vd[c],Vh);
|
||||||
|
acceleratorPut(Fd[c],Fh);
|
||||||
|
acceleratorPut(Cd[c],Ch);
|
||||||
|
}
|
||||||
|
|
||||||
|
/////////////////////////////////////////
|
||||||
|
// Block promote:
|
||||||
|
// F_xr = Vxb Cbr (x coarse_vol)
|
||||||
|
/////////////////////////////////////////
|
||||||
|
|
||||||
|
int64_t vw = block_vol * words;
|
||||||
|
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
||||||
|
vw,nrhs,nbasis,
|
||||||
|
ComplexD(1.0),
|
||||||
|
Vd,
|
||||||
|
Cd,
|
||||||
|
ComplexD(0.0), // wipe out C
|
||||||
|
Fd);
|
||||||
|
BLAS.synchronise();
|
||||||
|
// std::cout << " blas call done"<<std::endl;
|
||||||
|
|
||||||
|
ExportFineGridVectors(fine, BLAS_F);
|
||||||
|
// std::cout << " exported "<<std::endl;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
NAMESPACE_END(Grid);
|
||||||
234
Grid/algorithms/deflation/MultiRHSDeflation.h
Normal file
234
Grid/algorithms/deflation/MultiRHSDeflation.h
Normal file
@ -0,0 +1,234 @@
|
|||||||
|
/*************************************************************************************
|
||||||
|
|
||||||
|
Grid physics library, www.github.com/paboyle/Grid
|
||||||
|
|
||||||
|
Source file: MultiRHSDeflation.h
|
||||||
|
|
||||||
|
Copyright (C) 2023
|
||||||
|
|
||||||
|
Author: Peter Boyle <pboyle@bnl.gov>
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation; either version 2 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License along
|
||||||
|
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||||
|
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||||
|
|
||||||
|
See the full license in the file "LICENSE" in the top level distribution directory
|
||||||
|
*************************************************************************************/
|
||||||
|
/* END LEGAL */
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
NAMESPACE_BEGIN(Grid);
|
||||||
|
|
||||||
|
|
||||||
|
/* Need helper object for BLAS accelerated mrhs projection
|
||||||
|
|
||||||
|
i) MultiRHS Deflation
|
||||||
|
|
||||||
|
Import Evecs -> nev x vol x internal
|
||||||
|
Import vector of Lattice objects -> nrhs x vol x internal
|
||||||
|
=> Cij (nrhs x Nev) via GEMM.
|
||||||
|
=> Guess (nrhs x vol x internal) = C x evecs (via GEMM)
|
||||||
|
Export
|
||||||
|
|
||||||
|
|
||||||
|
ii) MultiRHS block projection
|
||||||
|
|
||||||
|
Import basis -> nblock x nbasis x (block x internal)
|
||||||
|
Import vector of fine lattice objects -> nblock x nrhs x (block x internal)
|
||||||
|
|
||||||
|
=> coarse_(nrhs x nbasis )^block = via batched GEMM
|
||||||
|
|
||||||
|
iii) Alternate interface:
|
||||||
|
Import higher dim Lattice object-> vol x nrhs layout
|
||||||
|
|
||||||
|
*/
|
||||||
|
template<class Field>
|
||||||
|
class MultiRHSDeflation
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
|
||||||
|
typedef typename Field::scalar_type scalar;
|
||||||
|
typedef typename Field::scalar_object scalar_object;
|
||||||
|
|
||||||
|
int nev;
|
||||||
|
std::vector<RealD> eval;
|
||||||
|
GridBase *grid;
|
||||||
|
uint64_t vol;
|
||||||
|
uint64_t words;
|
||||||
|
|
||||||
|
deviceVector<scalar> BLAS_E; // nev x vol -- the eigenbasis (up to a 1/sqrt(lambda))
|
||||||
|
deviceVector<scalar> BLAS_R; // nrhs x vol -- the sources
|
||||||
|
deviceVector<scalar> BLAS_G; // nrhs x vol -- the guess
|
||||||
|
deviceVector<scalar> BLAS_C; // nrhs x nev -- the coefficients
|
||||||
|
|
||||||
|
MultiRHSDeflation(){};
|
||||||
|
~MultiRHSDeflation(){ Deallocate(); };
|
||||||
|
|
||||||
|
void Deallocate(void)
|
||||||
|
{
|
||||||
|
nev=0;
|
||||||
|
grid=nullptr;
|
||||||
|
vol=0;
|
||||||
|
words=0;
|
||||||
|
BLAS_E.resize(0);
|
||||||
|
BLAS_R.resize(0);
|
||||||
|
BLAS_C.resize(0);
|
||||||
|
BLAS_G.resize(0);
|
||||||
|
}
|
||||||
|
void Allocate(int _nev,GridBase *_grid)
|
||||||
|
{
|
||||||
|
nev=_nev;
|
||||||
|
grid=_grid;
|
||||||
|
vol = grid->lSites();
|
||||||
|
words = sizeof(scalar_object)/sizeof(scalar);
|
||||||
|
eval.resize(nev);
|
||||||
|
BLAS_E.resize (vol * words * nev );
|
||||||
|
std::cout << GridLogMessage << " Allocate for "<<nev<<" eigenvectors and volume "<<vol<<std::endl;
|
||||||
|
}
|
||||||
|
void ImportEigenVector(Field &evec,RealD &_eval, int ev)
|
||||||
|
{
|
||||||
|
assert(ev<eval.size());
|
||||||
|
std::cout << " ev " <<ev<<" eval "<<_eval<< std::endl;
|
||||||
|
eval[ev] = _eval;
|
||||||
|
|
||||||
|
int64_t offset = ev*vol*words;
|
||||||
|
autoView(v,evec,AcceleratorRead);
|
||||||
|
acceleratorCopyDeviceToDevice(&v[0],&BLAS_E[offset],sizeof(scalar_object)*vol);
|
||||||
|
|
||||||
|
}
|
||||||
|
void ImportEigenBasis(std::vector<Field> &evec,std::vector<RealD> &_eval)
|
||||||
|
{
|
||||||
|
ImportEigenBasis(evec,_eval,0,evec.size());
|
||||||
|
}
|
||||||
|
// Could use to import a batch of eigenvectors
|
||||||
|
void ImportEigenBasis(std::vector<Field> &evec,std::vector<RealD> &_eval, int _ev0, int _nev)
|
||||||
|
{
|
||||||
|
assert(_ev0+_nev<=evec.size());
|
||||||
|
|
||||||
|
Allocate(_nev,evec[0].Grid());
|
||||||
|
|
||||||
|
// Imports a sub-batch of eigenvectors, _ev0, ..., _ev0+_nev-1
|
||||||
|
for(int e=0;e<nev;e++){
|
||||||
|
std::cout << "Importing eigenvector "<<e<<" evalue "<<_eval[_ev0+e]<<std::endl;
|
||||||
|
ImportEigenVector(evec[_ev0+e],_eval[_ev0+e],e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
void DeflateSources(std::vector<Field> &source,std::vector<Field> & guess)
|
||||||
|
{
|
||||||
|
int nrhs = source.size();
|
||||||
|
assert(source.size()==guess.size());
|
||||||
|
assert(grid == guess[0].Grid());
|
||||||
|
conformable(guess[0],source[0]);
|
||||||
|
|
||||||
|
int64_t vw = vol * words;
|
||||||
|
|
||||||
|
std::cout << GridLogMessage << "MultiRHSDelation for "<<nrhs<<" sources with "<<nev<<" eigenvectors "<<std::endl;
|
||||||
|
RealD t0 = usecond();
|
||||||
|
BLAS_R.resize(nrhs * vw); // cost free if size doesn't change
|
||||||
|
BLAS_G.resize(nrhs * vw); // cost free if size doesn't change
|
||||||
|
BLAS_C.resize(nev * nrhs);// cost free if size doesn't change
|
||||||
|
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// Copy in the multi-rhs sources
|
||||||
|
/////////////////////////////////////////////
|
||||||
|
// for(int r=0;r<nrhs;r++){
|
||||||
|
// std::cout << " source["<<r<<"] = "<<norm2(source[r])<<std::endl;
|
||||||
|
// }
|
||||||
|
for(int r=0;r<nrhs;r++){
|
||||||
|
int64_t offset = r*vw;
|
||||||
|
autoView(v,source[r],AcceleratorRead);
|
||||||
|
acceleratorCopyDeviceToDevice(&v[0],&BLAS_R[offset],sizeof(scalar_object)*vol);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* in Fortran column major notation (cuBlas order)
|
||||||
|
*
|
||||||
|
* Exe = [e1(x)][..][en(x)]
|
||||||
|
*
|
||||||
|
* Rxr = [r1(x)][..][rm(x)]
|
||||||
|
*
|
||||||
|
* C_er = E^dag R
|
||||||
|
* C_er = C_er / lambda_e
|
||||||
|
* G_xr = Exe Cer
|
||||||
|
*/
|
||||||
|
deviceVector<scalar *> Ed(1);
|
||||||
|
deviceVector<scalar *> Rd(1);
|
||||||
|
deviceVector<scalar *> Cd(1);
|
||||||
|
deviceVector<scalar *> Gd(1);
|
||||||
|
|
||||||
|
scalar * Eh = & BLAS_E[0];
|
||||||
|
scalar * Rh = & BLAS_R[0];
|
||||||
|
scalar * Ch = & BLAS_C[0];
|
||||||
|
scalar * Gh = & BLAS_G[0];
|
||||||
|
|
||||||
|
acceleratorPut(Ed[0],Eh);
|
||||||
|
acceleratorPut(Rd[0],Rh);
|
||||||
|
acceleratorPut(Cd[0],Ch);
|
||||||
|
acceleratorPut(Gd[0],Gh);
|
||||||
|
|
||||||
|
GridBLAS BLAS;
|
||||||
|
|
||||||
|
/////////////////////////////////////////
|
||||||
|
// C_er = E^dag R
|
||||||
|
/////////////////////////////////////////
|
||||||
|
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
||||||
|
nev,nrhs,vw,
|
||||||
|
ComplexD(1.0),
|
||||||
|
Ed,
|
||||||
|
Rd,
|
||||||
|
ComplexD(0.0), // wipe out C
|
||||||
|
Cd);
|
||||||
|
BLAS.synchronise();
|
||||||
|
|
||||||
|
assert(BLAS_C.size()==nev*nrhs);
|
||||||
|
|
||||||
|
std::vector<scalar> HOST_C(BLAS_C.size()); // nrhs . nev -- the coefficients
|
||||||
|
acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
|
||||||
|
grid->GlobalSumVector(&HOST_C[0],nev*nrhs);
|
||||||
|
for(int e=0;e<nev;e++){
|
||||||
|
RealD lam(1.0/eval[e]);
|
||||||
|
for(int r=0;r<nrhs;r++){
|
||||||
|
int off = e+nev*r;
|
||||||
|
HOST_C[off]=HOST_C[off] * lam;
|
||||||
|
// std::cout << "C["<<e<<"]["<<r<<"] ="<<HOST_C[off]<< " eval[e] "<<eval[e] <<std::endl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
acceleratorCopyToDevice(&HOST_C[0],&BLAS_C[0],BLAS_C.size()*sizeof(scalar));
|
||||||
|
|
||||||
|
|
||||||
|
/////////////////////////////////////////
|
||||||
|
// Guess G_xr = Exe Cer
|
||||||
|
/////////////////////////////////////////
|
||||||
|
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
||||||
|
vw,nrhs,nev,
|
||||||
|
ComplexD(1.0),
|
||||||
|
Ed, // x . nev
|
||||||
|
Cd, // nev . nrhs
|
||||||
|
ComplexD(0.0),
|
||||||
|
Gd);
|
||||||
|
BLAS.synchronise();
|
||||||
|
|
||||||
|
///////////////////////////////////////
|
||||||
|
// Copy out the multirhs
|
||||||
|
///////////////////////////////////////
|
||||||
|
for(int r=0;r<nrhs;r++){
|
||||||
|
int64_t offset = r*vw;
|
||||||
|
autoView(v,guess[r],AcceleratorWrite);
|
||||||
|
acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
|
||||||
|
}
|
||||||
|
RealD t1 = usecond();
|
||||||
|
std::cout << GridLogMessage << "MultiRHSDelation for "<<nrhs<<" sources with "<<nev<<" eigenvectors took " << (t1-t0)/1e3 <<" ms"<<std::endl;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
NAMESPACE_END(Grid);
|
||||||
Loading…
Reference in New Issue
Block a user