diff --git a/systems/Frontier/schur2d_env.job b/systems/Frontier/schur2d_env.job index fd974576a..7556ee544 100644 --- a/systems/Frontier/schur2d_env.job +++ b/systems/Frontier/schur2d_env.job @@ -4,7 +4,7 @@ #SBATCH --ntasks-per-node=8 #SBATCH --cpus-per-task=7 #SBATCH --gpus-per-node=8 -#SBATCH --time=0:40:00 +#SBATCH --time=0:50:00 #SBATCH --account=phy157_dwf #SBATCH --gpu-bind=none #SBATCH --exclusive @@ -70,7 +70,7 @@ run_cell () { fname=log.env.$name env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 > $fname 2>&1 echo " exit $?"; sleep 30 - grep -h "PROBE banner\|device ballast\|PROBE leaf\|BIG LEAVES\|gemm min/max\|invert\|certificate" $fname \ + grep -h "PROBE banner\|device ballast\|pre-heat\|PROBE leaf\|BIG LEAVES\|gemm min/max\|invert\|certificate" $fname \ | sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-170 } @@ -81,6 +81,11 @@ run_cell E3_omp7 OMP_NUM_THREADS=7 run_cell E4_omp7_ballast OMP_NUM_THREADS=7 S2D_BALLAST_GB=10.6 # + the example's device residency run_cell E5_omp1_ballast OMP_NUM_THREADS=1 S2D_BALLAST_GB=10.6 # ballast alone run_cell E6_omp1_again OMP_NUM_THREADS=1 # repeat E1: the noise floor +# E1-E4 (2026-08-27): 24.5 (first cell, cold) / 21.2 / 21.3 / 21.35 s -- thread level, OMP, residency all cleared. +# Remaining difference from the example (26.5-27.6 s, local kernels 20-40% slower, wires equal): the example +# inverts after ~100 s of full load on every GCD. Pre-heat reproduces that; "last zgemm" TF/s vs 23.7 is the tell. +run_cell E7_omp7_preheat OMP_NUM_THREADS=7 S2D_PREHEAT_S=100 # sustained load before the invert +run_cell E8_omp7_preheat_ballast OMP_NUM_THREADS=7 S2D_PREHEAT_S=100 S2D_BALLAST_GB=10.6 # the example's full state echo "=========================================================" echo "summary" diff --git a/tests/debug/Test_schur2d_scale.cc b/tests/debug/Test_schur2d_scale.cc index 609e76c88..bbe88bfb5 100644 --- a/tests/debug/Test_schur2d_scale.cc +++ b/tests/debug/Test_schur2d_scale.cc @@ -139,6 +139,31 @@ int main(int argc, char **argv) BlockCyclicMatrix A0(grid,N,nb,Pr,Pc); BlockCyclicSchurInverse RSI2; + // Pre-heat: drive the GCD with back-to-back zgemm for S2D_PREHEAT_S seconds + // before the invert. The example calls the inverse after ~100 s of full + // load on all 288 GCDs and its LOCAL kernels run 20-40% slower than the + // idle-start harness (GEMM 3.1 vs 2.5 s, leaf 0.76 vs 0.24 s) with the + // wires unchanged; thread level / OMP / residency (E1-E5) did not reproduce + // that. If sustained load does, it is clock/power management, not code. + if ( getenv("S2D_PREHEAT_S") ) { + double secs = atof(getenv("S2D_PREHEAT_S")); + const int64_t W = 4320; + deviceVector M((uint64_t)W*W), C((uint64_t)W*W); + { ComplexD *m = &M[0]; accelerator_for(idx,(uint64_t)W*W,1,{ m[idx] = ComplexD(1.0e-3*(idx%97),1.0e-3*(idx%89)); }); accelerator_barrier(); } + deviceVector ap(1),bp(1),cp(1); std::vector ptr(1); + ptr[0]=&M[0]; acceleratorCopyToDevice(&ptr[0],&ap[0],sizeof(ComplexD*)); acceleratorCopyToDevice(&ptr[0],&bp[0],sizeof(ComplexD*)); + ptr[0]=&C[0]; acceleratorCopyToDevice(&ptr[0],&cp[0],sizeof(ComplexD*)); + double t0=usecond(); int n=0; double tlast=0; + while ( (usecond()-t0)/1.0e6 < secs ) { + double t1=usecond(); + RSI2.SUMMA.BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,(int)W,(int)W,(int)W,ComplexD(1.0,0.0),ap,(int)W,bp,(int)W,ComplexD(0.0,0.0),cp,(int)W); + RSI2.SUMMA.BLAS.synchronise(); tlast=usecond()-t1; n++; + } + double tfirst = 0; (void)tfirst; + std::cout << GridLogMessage << "Test_schur2d_scale: pre-heat " << (usecond()-t0)/1.0e6 << " s, " << n << " zgemm W=" << W + << ", last zgemm " << tlast/1.0e6 << " s (" << 8.0*W*W*W/tlast/1.0e6 << " TF/s; idle-start rate 23.7)" << std::endl; + } + // Device ballast: Lattice fields written on the accelerator so they sit in // the MemoryManager's device LRU exactly as the example's fine-grid state does. typedef Lattice,Ns> > BallastField;