portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-29 13:59:36 +01:00
Code Releases Activity

Probing environment effects on dense inversion time

Browse Source
This commit is contained in:
Peter Boyle
2026-08-27 13:56:02 -04:00
parent 6cea81bc2c
commit 074bddd48b
2 changed files with 32 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
systems/Frontier/schur2d_env.job
+7 -2
View File
@@ -4,7 +4,7 @@
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=7
#SBATCH --gpus-per-node=8
#SBATCH --time=0:40:00
#SBATCH --time=0:50:00
#SBATCH --account=phy157_dwf
#SBATCH --gpu-bind=none
#SBATCH --exclusive
@@ -70,7 +70,7 @@ run_cell () {
fname=log.env.$name
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 > $fname 2>&1
echo " exit $?"; sleep 30
grep -h "PROBE banner\|device ballast\|PROBE leaf\|BIG LEAVES\|gemm min/max\|invert\|certificate" $fname \
grep -h "PROBE banner\|device ballast\|pre-heat\|PROBE leaf\|BIG LEAVES\|gemm min/max\|invert\|certificate" $fname \
| sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-170
}
@@ -81,6 +81,11 @@ run_cell E3_omp7 OMP_NUM_THREADS=7
run_cell E4_omp7_ballast OMP_NUM_THREADS=7 S2D_BALLAST_GB=10.6 # + the example's device residency
run_cell E5_omp1_ballast OMP_NUM_THREADS=1 S2D_BALLAST_GB=10.6 # ballast alone
run_cell E6_omp1_again OMP_NUM_THREADS=1 # repeat E1: the noise floor
# E1-E4 (2026-08-27): 24.5 (first cell, cold) / 21.2 / 21.3 / 21.35 s -- thread level, OMP, residency all cleared.
# Remaining difference from the example (26.5-27.6 s, local kernels 20-40% slower, wires equal): the example
# inverts after ~100 s of full load on every GCD. Pre-heat reproduces that; "last zgemm" TF/s vs 23.7 is the tell.
run_cell E7_omp7_preheat OMP_NUM_THREADS=7 S2D_PREHEAT_S=100 # sustained load before the invert
run_cell E8_omp7_preheat_ballast OMP_NUM_THREADS=7 S2D_PREHEAT_S=100 S2D_BALLAST_GB=10.6 # the example's full state
echo "========================================================="
echo "summary"
tests/debug/Test_schur2d_scale.cc
+25
View File
@@ -139,6 +139,31 @@ int main(int argc, char **argv)
BlockCyclicMatrix A0(grid,N,nb,Pr,Pc);
BlockCyclicSchurInverse RSI2;
// Pre-heat: drive the GCD with back-to-back zgemm for S2D_PREHEAT_S seconds
// before the invert. The example calls the inverse after ~100 s of full
// load on all 288 GCDs and its LOCAL kernels run 20-40% slower than the
// idle-start harness (GEMM 3.1 vs 2.5 s, leaf 0.76 vs 0.24 s) with the
// wires unchanged; thread level / OMP / residency (E1-E5) did not reproduce
// that. If sustained load does, it is clock/power management, not code.
if ( getenv("S2D_PREHEAT_S") ) {
double secs = atof(getenv("S2D_PREHEAT_S"));
const int64_t W = 4320;
deviceVector<ComplexD> M((uint64_t)W*W), C((uint64_t)W*W);
{ ComplexD *m = &M[0]; accelerator_for(idx,(uint64_t)W*W,1,{ m[idx] = ComplexD(1.0e-3*(idx%97),1.0e-3*(idx%89)); }); accelerator_barrier(); }
deviceVector<ComplexD*> ap(1),bp(1),cp(1); std::vector<ComplexD*> ptr(1);
ptr[0]=&M[0]; acceleratorCopyToDevice(&ptr[0],&ap[0],sizeof(ComplexD*)); acceleratorCopyToDevice(&ptr[0],&bp[0],sizeof(ComplexD*));
ptr[0]=&C[0]; acceleratorCopyToDevice(&ptr[0],&cp[0],sizeof(ComplexD*));
double t0=usecond(); int n=0; double tlast=0;
while ( (usecond()-t0)/1.0e6 < secs ) {
double t1=usecond();
RSI2.SUMMA.BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,(int)W,(int)W,(int)W,ComplexD(1.0,0.0),ap,(int)W,bp,(int)W,ComplexD(0.0,0.0),cp,(int)W);
RSI2.SUMMA.BLAS.synchronise(); tlast=usecond()-t1; n++;
}
double tfirst = 0; (void)tfirst;
std::cout << GridLogMessage << "Test_schur2d_scale: pre-heat " << (usecond()-t0)/1.0e6 << " s, " << n << " zgemm W=" << W
<< ", last zgemm " << tlast/1.0e6 << " s (" << 8.0*W*W*W/tlast/1.0e6 << " TF/s; idle-start rate 23.7)" << std::endl;
}
// Device ballast: Lattice fields written on the accelerator so they sit in
// the MemoryManager's device LRU exactly as the example's fine-grid state does.
typedef Lattice<iVector<iVector<vComplexD,Nc>,Ns> > BallastField;