portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-29 13:59:36 +01:00
Code Releases Activity

More probing why test harness vs. slate was a little faster running Grid's dense inverse than normal environment

Browse Source
This commit is contained in:
Peter Boyle
2026-08-27 13:42:18 -04:00
parent 68b9fa86d7
commit 6cea81bc2c
4 changed files with 171 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/debug/Test_schur2d_scale.cc
+32
View File
@@ -76,6 +76,20 @@ static ComplexD Fill(int64_t i, int64_t j, int64_t N)
int main(int argc, char **argv)
{
// Environment walk (2026-08-27): the SAME inverse runs 20.8 s in the SLATE
// harness and 26.5 s in the example, with the local GPU work (GEMM, leaf
// inverse) 25-40% slower in the example. Knobs to reproduce the example's
// environment here, one at a time (systems/Frontier/schur2d_env.job):
// GRID_MPI_THREAD_MULTIPLE=1 MPI_Init_thread(MULTIPLE) before Grid_init (SLATE harness does this)
// S2D_BALLAST_GB=x x GB of Lattice fields made device-resident before the invert
// (the example carries ~10.6 GB of fine-grid state in the MemoryManager)
// OMP_NUM_THREADS set in the job, read by nothing here but the runtime
if ( getenv("GRID_MPI_THREAD_MULTIPLE") ) {
int provided = 0;
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
std::cout << "GRID_MPI_THREAD_MULTIPLE: requested MPI_THREAD_MULTIPLE, provided " << provided
<< (provided==MPI_THREAD_MULTIPLE ? " (MULTIPLE)" : " (NOT multiple)") << std::endl;
}
Grid_init(&argc, &argv);
GridCartesian *grid = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(),
@@ -125,6 +139,24 @@ int main(int argc, char **argv)
BlockCyclicMatrix A0(grid,N,nb,Pr,Pc);
BlockCyclicSchurInverse RSI2;
// Device ballast: Lattice fields written on the accelerator so they sit in
// the MemoryManager's device LRU exactly as the example's fine-grid state does.
typedef Lattice<iVector<iVector<vComplexD,Nc>,Ns> > BallastField;
std::vector<BallastField> ballast;
if ( getenv("S2D_BALLAST_GB") ) {
double gb = atof(getenv("S2D_BALLAST_GB"));
uint64_t fbytes = (uint64_t)grid->oSites()*sizeof(BallastField::vector_object);
int nf = (int)(gb*1.0e9/(double)fbytes + 0.5);
ballast.reserve(nf);
for(int i=0;i<nf;i++){
ballast.emplace_back(grid);
autoView(v, ballast[i], AcceleratorWriteDiscard);
accelerator_for(ss, grid->oSites(), 1, { v[ss] = Zero(); });
}
std::cout << GridLogMessage << "Test_schur2d_scale: device ballast " << nf << " fields x " << fbytes/1.0e6
<< " MB = " << nf*fbytes/1.0e9 << " GB resident (S2D_BALLAST_GB=" << gb << ")" << std::endl;
}
BlockCyclicRedistribute::RowsToCyclic(grid,rowStart,&rows1d[0],myrows,A);
double t2 = usecond();
if ( A.data.size() )