diff --git a/Grid/algorithms/deflation/MultiRHSBlockProject.h b/Grid/algorithms/deflation/MultiRHSBlockProject.h index 8212189c8..fbe871434 100644 --- a/Grid/algorithms/deflation/MultiRHSBlockProject.h +++ b/Grid/algorithms/deflation/MultiRHSBlockProject.h @@ -61,6 +61,17 @@ public: uint64_t coarse_vol; uint64_t words; + //////////////////////////////////////////////////////////////////////////// + // Blocking geometry in full local coordinates. Addressing in lSites rather + // than (lane,oSite) lets the fine and coarse spaces carry different SIMD + // layouts, so an unvectorised coarse space may block a vectorised fine one. + //////////////////////////////////////////////////////////////////////////// + Coordinate fine_ldimensions; + Coordinate coarse_ldimensions; + Coordinate block_ldimensions; + Coordinate fine_simd; + Coordinate coarse_simd; + // Row major layout "C" order: // BLAS_V[coarse_vol][nbasis][block_vol][words] // BLAS_F[coarse_vol][nrhs][block_vol][words] @@ -120,9 +131,26 @@ public: fine_vol = fine_grid->lSites(); coarse_vol = coarse_grid->lSites(); block_vol = fine_vol/coarse_vol; - + words = sizeof(scalar_object)/sizeof(scalar); + int nd = coarse_grid->_ndimension; + GRID_ASSERT(fine_grid->_ndimension == nd); + + fine_ldimensions.resize(nd); + coarse_ldimensions.resize(nd); + block_ldimensions.resize(nd); + fine_simd = fine_grid->_simd_layout; + coarse_simd = coarse_grid->_simd_layout; + for(int d=0;d_processors[d] == coarse_grid->_processors[d]); + fine_ldimensions [d] = fine_grid->_rdimensions [d]*fine_grid->_simd_layout [d]; + coarse_ldimensions[d] = coarse_grid->_rdimensions[d]*coarse_grid->_simd_layout[d]; + block_ldimensions [d] = fine_ldimensions[d]/coarse_ldimensions[d]; + GRID_ASSERT(block_ldimensions[d]*coarse_ldimensions[d] == fine_ldimensions[d]); + } + GRID_ASSERT(block_vol == fine_vol/coarse_vol); + BLAS_V.resize (fine_vol * words * nbasis ); } void ImportFineGridVectors(std::vector &vecs, deviceVector &blas) @@ -133,22 +161,16 @@ public: GRID_ASSERT(vecs[0].Grid()==fine_grid); - subdivides(coarse_grid,fine_grid); // require they map - int _ndimension = coarse_grid->_ndimension; - GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites()); - - Coordinate block_r (_ndimension); - for(int d=0 ; d<_ndimension;d++){ - block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d]; - } uint64_t sz = blas.size(); acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar)); Coordinate fine_rdimensions = fine_grid->_rdimensions; - Coordinate coarse_rdimensions = coarse_grid->_rdimensions; + Coordinate coarse_l = coarse_ldimensions; + Coordinate block_l = block_ldimensions; + Coordinate fsimd = fine_simd; int64_t bv= block_vol; for(int v=0;voSites() * block_vol * nvec * words<oSites() * block_vol * nvec * words); + GRID_ASSERT(sz == coarse_vol * block_vol * nvec * words); uint64_t lwords= words; // local variable for copy in to GPU accelerator_for(sf,osites,Nsimd,{ #ifdef GRID_SIMT @@ -175,27 +195,27 @@ public: #endif // One thread per fine site Coordinate coor_f(_ndimension); + Coordinate coor_l(_ndimension); Coordinate coor_b(_ndimension); Coordinate coor_c(_ndimension); - // Fine site to fine coor + // Fine (oSite,lane) to full local coor Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions); + Lexicographic::CoorFromIndex(coor_l,lane,fsimd); + for(int d=0;d<_ndimension;d++) coor_f[d] += fine_rdimensions[d]*coor_l[d]; + + for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_l[d]; + for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_l[d]; - for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d]; - for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d]; - int sc;// coarse site int sb;// block site - Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions); - Lexicographic::IndexFromCoor(coor_b,sb,block_r); + Lexicographic::IndexFromCoor(coor_c,sc,coarse_l); + Lexicographic::IndexFromCoor(coor_b,sb,block_l); scalar_object data = extractLane(lane,fineData[sf]); - // BLAS layout address calculation - // words * block_vol * nbasis x coarse_vol - // coarse oSite x block vole x lanes - int64_t site = (lane*osites + sc*bv)*nvec - + v*bv + // BLAS_F[coarse_vol][nvec][block_vol][words] + int64_t site = (sc*nvec + v)*bv + sb; // GRID_ASSERT(site*lwords &blas) + { + typedef typename Field::vector_object vobj; + + GridBase *fine_mrhs_grid = vec_mrhs.Grid(); + int _ndimension = coarse_grid->_ndimension; + + GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1); + GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1); + GRID_ASSERT(fine_mrhs_grid->_processors[0] == 1); + for(int d=0;d<_ndimension;d++){ + GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]); + GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]); + } + int nvec = fine_mrhs_grid->_rdimensions[0]; // nrhs + + uint64_t sz = blas.size(); + acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar)); + + Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions; + Coordinate fine_rdimensions = fine_grid->_rdimensions; + Coordinate coarse_l = coarse_ldimensions; + Coordinate block_l = block_ldimensions; + Coordinate fsimd = fine_simd; + int64_t bv= block_vol; + + autoView( fineData , vec_mrhs, AcceleratorRead); + auto blasData_p = &blas[0]; + auto fineData_p = &fineData[0]; + + int64_t osites = fine_grid->oSites(); // D dimensional + int64_t osites_hi = fine_mrhs_grid->oSites(); // nvec * osites + + const int Nsimd = vobj::Nsimd(); + GRID_ASSERT(sz == coarse_vol * block_vol * nvec * words); + uint64_t lwords= words; + int64_t lnvec = nvec; + + accelerator_for(sfr,osites_hi,Nsimd,{ +#ifdef GRID_SIMT + { + int lane=acceleratorSIMTlane(Nsimd); // buffer lane +#else + for(int lane=0;lane + void ExportCoarseGridMrhsVectors(Lattice &vec_mrhs, deviceVector &blas) + { + typedef typename vobj::scalar_object coarse_scalar_object; + + GridBase *coarse_mrhs_grid = vec_mrhs.Grid(); + int _ndimension = coarse_grid->_ndimension; + + GRID_ASSERT(coarse_mrhs_grid->_ndimension == _ndimension+1); + GRID_ASSERT(coarse_mrhs_grid->_simd_layout[0] == 1); + GRID_ASSERT(coarse_mrhs_grid->_processors[0] == 1); + for(int d=0;d<_ndimension;d++){ + GRID_ASSERT(coarse_mrhs_grid->_rdimensions[d+1] == coarse_grid->_rdimensions[d]); + GRID_ASSERT(coarse_mrhs_grid->_simd_layout[d+1] == coarse_grid->_simd_layout[d]); + } + int nvec = coarse_mrhs_grid->_rdimensions[0]; // nrhs + + Coordinate coarse_mrhs_rdimensions = coarse_mrhs_grid->_rdimensions; + Coordinate coarse_rdimensions = coarse_grid->_rdimensions; + Coordinate coarse_l = coarse_ldimensions; + Coordinate csimd = coarse_simd; + + autoView( coarseData , vec_mrhs, AcceleratorWrite); + auto blasData_p = &blas[0]; + auto coarseData_p = &coarseData[0]; + + int64_t osites = coarse_grid->oSites(); // D dimensional + int64_t osites_hi = coarse_mrhs_grid->oSites(); // nvec * osites + + const int Nsimd = vobj::Nsimd(); + uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(cwords==nbasis); + int64_t lnvec = nvec; + + accelerator_for(scr,osites_hi,Nsimd,{ +#ifdef GRID_SIMT + { + int lane=acceleratorSIMTlane(Nsimd); // buffer lane +#else + for(int lane=0;lane fine mrhs field, coarse mrhs field -> BLAS_C + //////////////////////////////////////////////////////////////////////////// + void ExportFineGridMrhsVectors(Field &vec_mrhs, deviceVector &blas) + { + typedef typename Field::vector_object vobj; + + GridBase *fine_mrhs_grid = vec_mrhs.Grid(); + int _ndimension = coarse_grid->_ndimension; + + GRID_ASSERT(fine_mrhs_grid->_ndimension == _ndimension+1); + GRID_ASSERT(fine_mrhs_grid->_simd_layout[0] == 1); + for(int d=0;d<_ndimension;d++){ + GRID_ASSERT(fine_mrhs_grid->_rdimensions[d+1] == fine_grid->_rdimensions[d]); + GRID_ASSERT(fine_mrhs_grid->_simd_layout[d+1] == fine_grid->_simd_layout[d]); + } + int nvec = fine_mrhs_grid->_rdimensions[0]; + + Coordinate fine_mrhs_rdimensions = fine_mrhs_grid->_rdimensions; + Coordinate fine_rdimensions = fine_grid->_rdimensions; + Coordinate coarse_l = coarse_ldimensions; + Coordinate block_l = block_ldimensions; + Coordinate fsimd = fine_simd; + int64_t bv= block_vol; + + autoView( fineData , vec_mrhs, AcceleratorWrite); + auto blasData_p = &blas[0]; + auto fineData_p = &fineData[0]; + + int64_t osites = fine_grid->oSites(); + int64_t osites_hi = fine_mrhs_grid->oSites(); + + const int Nsimd = vobj::Nsimd(); + uint64_t lwords= words; + int64_t lnvec = nvec; + + accelerator_for(sfr,osites_hi,Nsimd,{ +#ifdef GRID_SIMT + { + int lane=acceleratorSIMTlane(Nsimd); +#else + for(int lane=0;lane + void ImportCoarseGridMrhsVectors(Lattice &vec_mrhs, deviceVector &blas) + { + typedef typename vobj::scalar_object coarse_scalar_object; + + GridBase *coarse_mrhs_grid = vec_mrhs.Grid(); + int _ndimension = coarse_grid->_ndimension; + + GRID_ASSERT(coarse_mrhs_grid->_ndimension == _ndimension+1); + GRID_ASSERT(coarse_mrhs_grid->_simd_layout[0] == 1); + for(int d=0;d<_ndimension;d++){ + GRID_ASSERT(coarse_mrhs_grid->_rdimensions[d+1] == coarse_grid->_rdimensions[d]); + GRID_ASSERT(coarse_mrhs_grid->_simd_layout[d+1] == coarse_grid->_simd_layout[d]); + } + int nvec = coarse_mrhs_grid->_rdimensions[0]; + + Coordinate coarse_mrhs_rdimensions = coarse_mrhs_grid->_rdimensions; + Coordinate coarse_rdimensions = coarse_grid->_rdimensions; + Coordinate coarse_l = coarse_ldimensions; + Coordinate csimd = coarse_simd; + + autoView( coarseData , vec_mrhs, AcceleratorRead); + auto blasData_p = &blas[0]; + auto coarseData_p = &coarseData[0]; + + int64_t osites = coarse_grid->oSites(); + int64_t osites_hi = coarse_mrhs_grid->oSites(); + + const int Nsimd = vobj::Nsimd(); + uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(cwords==nbasis); + int64_t lnvec = nvec; + + accelerator_for(scr,osites_hi,Nsimd,{ +#ifdef GRID_SIMT + { + int lane=acceleratorSIMTlane(Nsimd); +#else + for(int lane=0;lane &vecs, deviceVector &blas) { typedef typename Field::vector_object vobj; @@ -221,17 +553,12 @@ public: GRID_ASSERT(vecs[0].Grid()==fine_grid); - subdivides(coarse_grid,fine_grid); // require they map - int _ndimension = coarse_grid->_ndimension; - GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites()); - - Coordinate block_r (_ndimension); - for(int d=0 ; d<_ndimension;d++){ - block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d]; - } + Coordinate fine_rdimensions = fine_grid->_rdimensions; - Coordinate coarse_rdimensions = coarse_grid->_rdimensions; + Coordinate coarse_l = coarse_ldimensions; + Coordinate block_l = block_ldimensions; + Coordinate fsimd = fine_simd; // std::cout << " export fine Blas norm "<_rdimensions; - + Coordinate coarse_l = coarse_ldimensions; + Coordinate csimd = coarse_simd; + for(int v=0;v_rdimensions; - + Coordinate coarse_l = coarse_ldimensions; + Coordinate csimd = coarse_simd; + // std::cout << " export coarsee Blas norm "< + void blockProject(Field &fine_mrhs,Lattice &coarse_mrhs) + { + int nrhs = fine_mrhs.Grid()->_rdimensions[0]; + int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(nbasis==_nbasis); + GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs); + + BLAS_F.resize (fine_vol * words * nrhs ); + BLAS_C.resize (coarse_vol * nbasis * nrhs ); + + ImportFineGridMrhsVectors(fine_mrhs,BLAS_F); + ProjectBLAS(nrhs); + ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C); + } + + template + void blockPromote(Field &fine_mrhs,Lattice &coarse_mrhs) + { + int nrhs = fine_mrhs.Grid()->_rdimensions[0]; + int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(nbasis==_nbasis); + GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs); + + BLAS_F.resize (fine_vol * words * nrhs ); + BLAS_C.resize (coarse_vol * nbasis * nrhs ); + + ImportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C); + PromoteBLAS(nrhs); + ExportFineGridMrhsVectors(fine_mrhs,BLAS_F); + } + + //////////////////////////////////////////////////////////////////////////// + // Mixed orderings. A single RHS fine operator produces a vector of fine + // fields with no packing; the coarse side is still wanted in mrhs order. + //////////////////////////////////////////////////////////////////////////// + template + void blockProject(std::vector &fine,Lattice &coarse_mrhs) + { + int nrhs = fine.size(); + int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(nbasis==_nbasis); + GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs); + + BLAS_F.resize (fine_vol * words * nrhs ); + BLAS_C.resize (coarse_vol * nbasis * nrhs ); + + ImportFineGridVectors(fine,BLAS_F); + ProjectBLAS(nrhs); + ExportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C); + } + + template + void blockProject(Field &fine_mrhs,std::vector< Lattice > &coarse) + { + int nrhs = fine_mrhs.Grid()->_rdimensions[0]; + int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(nbasis==_nbasis); + GRID_ASSERT(coarse.size()==nrhs); + + BLAS_F.resize (fine_vol * words * nrhs ); + BLAS_C.resize (coarse_vol * nbasis * nrhs ); + + ImportFineGridMrhsVectors(fine_mrhs,BLAS_F); + ProjectBLAS(nrhs); + ExportCoarseGridVectors(coarse,BLAS_C); + } + + template + void blockPromote(std::vector &fine,Lattice &coarse_mrhs) + { + int nrhs = fine.size(); + int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(nbasis==_nbasis); + GRID_ASSERT(coarse_mrhs.Grid()->_rdimensions[0]==nrhs); + + BLAS_F.resize (fine_vol * words * nrhs ); + BLAS_C.resize (coarse_vol * nbasis * nrhs ); + + ImportCoarseGridMrhsVectors(coarse_mrhs,BLAS_C); + PromoteBLAS(nrhs); + ExportFineGridVectors(fine,BLAS_F); + } + + template + void blockPromote(Field &fine_mrhs,std::vector< Lattice > &coarse) + { + int nrhs = fine_mrhs.Grid()->_rdimensions[0]; + int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar); + GRID_ASSERT(nbasis==_nbasis); + GRID_ASSERT(coarse.size()==nrhs); + + BLAS_F.resize (fine_vol * words * nrhs ); + BLAS_C.resize (coarse_vol * nbasis * nrhs ); + + ImportCoarseGridVectors(coarse,BLAS_C); + PromoteBLAS(nrhs); + ExportFineGridMrhsVectors(fine_mrhs,BLAS_F); + } + + //////////////////////////////////////////////////////////////////////////// + // Pointer tables and the batched GEMM, shared by both orderings + //////////////////////////////////////////////////////////////////////////// + void BLASPointers(int nrhs, + deviceVector &Vd, + deviceVector &Fd, + deviceVector &Cd) + { + for(int c=0;c Vd(coarse_vol); + deviceVector Fd(coarse_vol); + deviceVector Cd(coarse_vol); + BLASPointers(nrhs,Vd,Fd,Cd); + + GridBLAS BLAS; + int64_t vw = block_vol * words; + BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N, + nbasis,nrhs,vw, + scalar(1.0), + Vd, + Fd, + scalar(0.0), // wipe out C + Cd); + BLAS.synchronise(); + } + + // F_xr = Vxb Cbr + void PromoteBLAS(int nrhs) + { + deviceVector Vd(coarse_vol); + deviceVector Fd(coarse_vol); + deviceVector Cd(coarse_vol); + BLASPointers(nrhs,Vd,Fd,Cd); + + GridBLAS BLAS; + int64_t vw = block_vol * words; + BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N, + vw,nrhs,nbasis, + scalar(1.0), + Vd, + Cd, + scalar(0.0), // wipe out F + Fd); + BLAS.synchronise(); + } }; NAMESPACE_END(Grid);