portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-31 14:59:35 +01:00
Code Releases Activity

Buf fix on DenseCoarseMatrix.h, coefficient caching optimisation on BatchGEMM

Browse Source
This commit is contained in:
Peter Boyle
2026-08-13 13:26:42 -04:00
parent b039e659af
commit 4dfbd850ff
2 changed files with 58 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Grid/algorithms/blas/BatchedBlas.h
+53 -24
View File
@@ -68,6 +68,34 @@ NAMESPACE_BEGIN(Grid);
enum GridBLASOperation_t { GridBLAS_OP_N, GridBLAS_OP_T, GridBLAS_OP_C } ; enum GridBLASOperation_t { GridBLAS_OP_N, GridBLAS_OP_T, GridBLAS_OP_C } ;
enum GridBLASPrecision_t { GridBLAS_PRECISION_DEFAULT, GridBLAS_PRECISION_16F, GridBLAS_PRECISION_16BF, GridBLAS_PRECISION_TF32 }; enum GridBLASPrecision_t { GridBLAS_PRECISION_DEFAULT, GridBLAS_PRECISION_16F, GridBLAS_PRECISION_16BF, GridBLAS_PRECISION_TF32 };
///////////////////////////////////////////////////////////////////////////
// Device-resident scalar constant with VALUE CACHING: the host->device
// copy is issued ONLY when the requested value differs from what is
// already resident. Motivation (rocprof, Frontier, 2026-08-13): per-call
// alpha/beta staging in the gemmBatched family generated ~92k tiny staged
// hipMemcpys in a 12s solve window (~26% of host API time) at the
// latency-bound coarse level. With the single-slot cache the coarse-mult
// accumulation pattern beta = (p==0 ? 0 : 1) costs two copies per Mult
// instead of npoint. NB not thread safe -- matches the single-threaded
// host BLAS call pattern of the per-call staging it replaces.
///////////////////////////////////////////////////////////////////////////
template<class T>
class GridBLASDeviceConstant {
deviceVector<T> dev;
T host;
int valid;
public:
GridBLASDeviceConstant() : dev(1), valid(0) {};
T * put(T v) {
if ( (!valid) || (v != host) ) {
acceleratorCopyToDevice((void *)&v,(void *)&dev[0],sizeof(T));
host = v;
valid = 1;
}
return &dev[0];
}
};
class GridBLAS { class GridBLAS {
public: public:
@@ -240,11 +268,11 @@ public:
if(OpB!=GridBLAS_OP_N) if(OpB!=GridBLAS_OP_N)
ldb = n; ldb = n;
static deviceVector<ComplexD> alpha_p(1); // Cached device constants: copy only on value change (see GridBLASDeviceConstant)
static deviceVector<ComplexD> beta_p(1); static GridBLASDeviceConstant<ComplexD> alpha_c;
// can prestore the 1 and the zero on device static GridBLASDeviceConstant<ComplexD> beta_c;
acceleratorCopyToDevice((void *)&alpha,(void *)&alpha_p[0],sizeof(ComplexD)); ComplexD *alpha_p = alpha_c.put(alpha);
acceleratorCopyToDevice((void *)&beta ,(void *)&beta_p[0],sizeof(ComplexD)); ComplexD *beta_p = beta_c.put(beta);
RealD t0=usecond(); RealD t0=usecond();
// std::cout << "ZgemmBatched mnk "<<m<<","<<n<<","<<k<<" count "<<batchCount<<std::endl; // std::cout << "ZgemmBatched mnk "<<m<<","<<n<<","<<k<<" count "<<batchCount<<std::endl;
#ifdef GRID_HIP #ifdef GRID_HIP
@@ -498,11 +526,11 @@ public:
lda = k; lda = k;
if(OpB!=GridBLAS_OP_N) if(OpB!=GridBLAS_OP_N)
ldb = n; ldb = n;
static deviceVector<ComplexF> alpha_p(1); // Cached device constants: copy only on value change (see GridBLASDeviceConstant)
static deviceVector<ComplexF> beta_p(1); static GridBLASDeviceConstant<ComplexF> alpha_c;
// can prestore the 1 and the zero on device static GridBLASDeviceConstant<ComplexF> beta_c;
acceleratorCopyToDevice((void *)&alpha,(void *)&alpha_p[0],sizeof(ComplexF)); ComplexF *alpha_p = alpha_c.put(alpha);
acceleratorCopyToDevice((void *)&beta ,(void *)&beta_p[0],sizeof(ComplexF)); ComplexF *beta_p = beta_c.put(beta);
RealD t0=usecond(); RealD t0=usecond();
GRID_ASSERT(Bkn.size()==batchCount); GRID_ASSERT(Bkn.size()==batchCount);
@@ -723,10 +751,11 @@ public:
GRID_ASSERT( ldb >= ((OpB==GridBLAS_OP_N) ? k : n) ); GRID_ASSERT( ldb >= ((OpB==GridBLAS_OP_N) ? k : n) );
GRID_ASSERT( ldc >= m ); GRID_ASSERT( ldc >= m );
static deviceVector<ComplexF> alpha_p(1); // Cached device constants: copy only on value change (see GridBLASDeviceConstant)
static deviceVector<ComplexF> beta_p(1); static GridBLASDeviceConstant<ComplexF> alpha_c;
acceleratorCopyToDevice((void *)&alpha,(void *)&alpha_p[0],sizeof(ComplexF)); static GridBLASDeviceConstant<ComplexF> beta_c;
acceleratorCopyToDevice((void *)&beta ,(void *)&beta_p[0],sizeof(ComplexF)); ComplexF *alpha_p = alpha_c.put(alpha);
ComplexF *beta_p = beta_c.put(beta);
RealD t0=usecond(); RealD t0=usecond();
GRID_ASSERT(Bkn.size()==batchCount); GRID_ASSERT(Bkn.size()==batchCount);
@@ -943,11 +972,11 @@ public:
lda = k; lda = k;
if(OpB!=GridBLAS_OP_N) if(OpB!=GridBLAS_OP_N)
ldb = n; ldb = n;
static deviceVector<RealF> alpha_p(1); // Cached device constants: copy only on value change (see GridBLASDeviceConstant)
static deviceVector<RealF> beta_p(1); static GridBLASDeviceConstant<RealF> alpha_c;
// can prestore the 1 and the zero on device static GridBLASDeviceConstant<RealF> beta_c;
acceleratorCopyToDevice((void *)&alpha,(void *)&alpha_p[0],sizeof(RealF)); RealF *alpha_p = alpha_c.put(alpha);
acceleratorCopyToDevice((void *)&beta ,(void *)&beta_p[0],sizeof(RealF)); RealF *beta_p = beta_c.put(beta);
RealD t0=usecond(); RealD t0=usecond();
GRID_ASSERT(Bkn.size()==batchCount); GRID_ASSERT(Bkn.size()==batchCount);
@@ -1104,11 +1133,11 @@ public:
if(OpB!=GridBLAS_OP_N) if(OpB!=GridBLAS_OP_N)
ldb = n; ldb = n;
static deviceVector<RealD> alpha_p(1); // Cached device constants: copy only on value change (see GridBLASDeviceConstant)
static deviceVector<RealD> beta_p(1); static GridBLASDeviceConstant<RealD> alpha_c;
// can prestore the 1 and the zero on device static GridBLASDeviceConstant<RealD> beta_c;
acceleratorCopyToDevice((void *)&alpha,(void *)&alpha_p[0],sizeof(RealD)); RealD *alpha_p = alpha_c.put(alpha);
acceleratorCopyToDevice((void *)&beta ,(void *)&beta_p[0],sizeof(RealD)); RealD *beta_p = beta_c.put(beta);
RealD t0=usecond(); RealD t0=usecond();
GRID_ASSERT(Bkn.size()==batchCount); GRID_ASSERT(Bkn.size()==batchCount);
Grid/algorithms/multigrid/DenseCoarseMatrix.h
+5 -1
View File
@@ -272,7 +272,11 @@ public:
for(int p=0; p<_Op.geom.npoint; p++){ for(int p=0; p<_Op.geom.npoint; p++){
Coordinate shift = _Op.geom.shifts[p]; Coordinate shift = _Op.geom.shifts[p];
autoView(Av, _Op._A[p], CpuRead); // _A[p] is PADDED after ExchangeCoarseLinks (end of CoarsenOperator):
// extract the unpadded field before peeking with unpadded coordinates
// (exactly as MultiGeneralCoarsenedMatrix::CopyMatrix does).
CoarseMatrix Aun = _Op.Cell.Extract(_Op._A[p]);
autoView(Av, Aun, CpuRead);
thread_for(ss, lsites, { thread_for(ss, lsites, {
Coordinate ncoor(nd); Coordinate ncoor(nd);
for(int d=0; d<nd; d++){ for(int d=0; d<nd; d++){