From cbe97cc40d413676333b5a893cc7ffb4578d13ca Mon Sep 17 00:00:00 2001 From: Peter Boyle Date: Wed, 26 Aug 2026 21:37:36 -0400 Subject: [PATCH] Smoother search improvement --- .../multigrid/BlockCyclicSchurInverse.h | 62 +++++++++++++++++++ systems/Frontier/smoother_modes.job | 26 ++++---- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h b/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h index 7ae154623..a530e54fe 100644 --- a/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h +++ b/Grid/algorithms/multigrid/BlockCyclicSchurInverse.h @@ -256,11 +256,73 @@ public: // PUBLIC ENTRY. In-place inverse of the whole matrix. Scratch (4x the // matrix footprint) is allocated here and released on return. /////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////// + // POWER - CLOCK - GROUND. Before the inverse runs, print the preconditions + // that have differed between fast (62 s) and slow (133-141 s) runs of the + // SAME inverse, and measure ONE SendToRecvFrom to the ACTUAL ring partners + // at three sizes, device and host buffers. Same code in the harness and in + // production, so the two processes are compared on the identical primitive + // before any explanation of the SUMMA rings is attempted. + // SCHUR2D_PROBE=0 disables (costs ~0.1-0.5 s). + /////////////////////////////////////////////////////////////////////////// + void Probe(BlockCyclicMatrix &A) + { + BlockCyclicLayout &L = A.layout; + GridBase *grid = A.grid; + int me = grid->ThisRank(); + // --- banner --- + int thr = -1; +#ifdef GRID_COMMS_MPI3 + MPI_Query_thread(&thr); +#endif + const char *omp = getenv("OMP_NUM_THREADS"); + MemoryStatus ms = MemoryManager::GetFootprint(); + std::cout << GridLogMessage << "Schur2D PROBE banner: MPI thread level " << thr + << " (0 single,1 funneled,2 serialized,3 multiple) OMP_NUM_THREADS=" << (omp?omp:"unset") + << " MemoryManager device bytes " << ms.DeviceBytes/1.0e9 << " GB (LRU " << ms.DeviceLRUBytes/1.0e9 + << " GB, cap " << ms.DeviceMaxBytes/1.0e9 << " GB)" + << " grid " << L.Pr << "x" << L.Pc << " nb " << L.nb << std::endl; +#ifdef GRID_HIP + if ( me==0 ) acceleratorMem(); +#endif + // --- ring partners exactly as SUMMA uses them --- + int prow=L.prow, pcol=L.pcol, Pr=L.Pr, Pc=L.Pc; + struct Ring { const char *name; int dest, src; }; + Ring rings[2] = { {"ringA(row, q+-1)", prow*Pc + (pcol+1)%Pc, prow*Pc + (pcol-1+Pc)%Pc}, + {"ringB(col, p+-1)", ((prow+1)%Pr)*Pc + pcol, ((prow-1+Pr)%Pr)*Pc + pcol} }; + uint64_t sizes[3] = { 64ull*1024, 1024ull*1024, 8ull*1024*1024 }; + uint64_t maxb = sizes[2]; + deviceVector dsend(maxb), drecv(maxb); + std::vector hsend(maxb), hrecv(maxb); + for(int r=0;r<2;r++){ + if ( (r==0 && Pc==1) || (r==1 && Pr==1) ) continue; + int off = grid->IsOffNode(rings[r].dest); + for(int si=0;si<3;si++){ + uint64_t bytes = sizes[si]; + // warm one, time five, both memory spaces + grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes); + double t0=usecond(); + for(int i=0;i<5;i++) grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes); + double td=(usecond()-t0)/5.0; + grid->SendToRecvFrom(&hsend[0], rings[r].dest, &hrecv[0], rings[r].src, bytes); + t0=usecond(); + for(int i=0;i<5;i++) grid->SendToRecvFrom(&hsend[0], rings[r].dest, &hrecv[0], rings[r].src, bytes); + double th=(usecond()-t0)/5.0; + // spread over ranks + RealD dmax=td, dmin=-td; grid->GlobalMax(dmax); grid->GlobalMax(dmin); dmin=-dmin; + std::cout << GridLogMessage << "Schur2D PROBE " << rings[r].name << (off?" OFF-node":" on-node") + << " " << bytes/1024 << " KB: device " << td << " us (" << bytes/td/1.0e3 << " GB/s) [min/max over ranks " << dmin << "/" << dmax << " us]" + << " host " << th << " us (" << bytes/th/1.0e3 << " GB/s)" << std::endl; + } + } + } + void Invert(BlockCyclicMatrix &A) { BlockCyclicLayout &L = A.layout; GRID_ASSERT( L.N >= 1 ); int64_t nblocks = (L.N + L.nb - 1)/L.nb; + if ( !(getenv("SCHUR2D_PROBE") && atoi(getenv("SCHUR2D_PROBE"))==0) ) Probe(A); BlockCyclicMatrix Bt(A.grid, L.N, L.nb, L.Pr, L.Pc); BlockCyclicMatrix Ct(A.grid, L.N, L.nb, L.Pr, L.Pc); diff --git a/systems/Frontier/smoother_modes.job b/systems/Frontier/smoother_modes.job index 4ed39a5a7..e0e4fc664 100644 --- a/systems/Frontier/smoother_modes.job +++ b/systems/Frontier/smoother_modes.job @@ -129,19 +129,21 @@ run_cell () { grep -h "SCHUR fp64 distributed invert took\|GB/s/rank" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-120 | head -2 } -# The (order x shift) table at Csn=2, one axis at a time from the overshoot, -# so a failing cell identifies WHICH knob it needed. Reference: 28.57 s. +# Ladder 2 (after L0-L3 of the first ladder): the stationary polynomial +# wants a LARGER shift than the adaptive one -- it must suppress the +# high-mode deviation on average, not per call. Measured (Csn=2): +# (12,1.0) 48 outer 31.4 s (8,1.0) 58 outer 30.9 s (8,0.5) 60 outer 34.3 s +# Reference (adaptive, Fso6/Fss0.1): 56 outer, 28.57 s. So walk the SHIFT UP +# at orders 8 and 6, trading shift against order. Nrhs=1 is the target. # name Fso Fss Csn fine coarse -run_cell L0_overshoot 12 1.0 6 replay gcr # M3's point with last-call selection + record 8..16 + refresh 5 -run_cell L1_12_10 12 1.0 2 replay gcr # coarse smoother back to 2 steps; the row/column anchor -run_cell L2_08_10 8 1.0 2 replay gcr # order axis -run_cell L3_06_10 6 1.0 2 replay gcr -run_cell L4_12_05 12 0.5 2 replay gcr # shift axis -run_cell L5_08_05 8 0.5 2 replay gcr -run_cell L6_06_05 6 0.5 2 replay gcr # nearest to the banked adaptive point (Fso6/Fss0.1) -# Coarse replay: record a DECENT polynomial first (6-step, shift 2.0) and back off. -run_cell L7_coarse6 8 1.0 6 replay replay -run_cell L8_coarse4 8 1.0 4 replay replay +run_cell S1_08_20 8 2.0 2 replay gcr # the promising cell: ~50 outer at 0.53 s/step would be ~26.5 s +run_cell S2_08_30 8 3.0 2 replay gcr +run_cell S3_06_20 6 2.0 2 replay gcr # banked order, larger shift +run_cell S4_06_10 6 1.0 2 replay gcr # the L3 of the second table (never ran) +run_cell S5_10_20 10 2.0 2 replay gcr # between 8 and 12 +# coarse replay, recorded at a decent polynomial and backed off, at the best fine point +run_cell S6_coarse6 8 2.0 6 replay replay +run_cell S7_coarse4 8 2.0 4 replay replay echo "=========================================================" echo "summary"