portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-28 21:39:35 +01:00
Code Releases Activity

Smoother search improvement

Browse Source
This commit is contained in:
Peter Boyle
2026-08-26 21:37:36 -04:00
parent dc1ae3185a
commit cbe97cc40d
2 changed files with 76 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Grid/algorithms/multigrid/BlockCyclicSchurInverse.h
+62
View File
@@ -256,11 +256,73 @@ public:
// PUBLIC ENTRY. In-place inverse of the whole matrix. Scratch (4x the
// matrix footprint) is allocated here and released on return.
///////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////
// POWER - CLOCK - GROUND. Before the inverse runs, print the preconditions
// that have differed between fast (62 s) and slow (133-141 s) runs of the
// SAME inverse, and measure ONE SendToRecvFrom to the ACTUAL ring partners
// at three sizes, device and host buffers. Same code in the harness and in
// production, so the two processes are compared on the identical primitive
// before any explanation of the SUMMA rings is attempted.
// SCHUR2D_PROBE=0 disables (costs ~0.1-0.5 s).
///////////////////////////////////////////////////////////////////////////
void Probe(BlockCyclicMatrix &A)
{
BlockCyclicLayout &L = A.layout;
GridBase *grid = A.grid;
int me = grid->ThisRank();
// --- banner ---
int thr = -1;
#ifdef GRID_COMMS_MPI3
MPI_Query_thread(&thr);
#endif
const char *omp = getenv("OMP_NUM_THREADS");
MemoryStatus ms = MemoryManager::GetFootprint();
std::cout << GridLogMessage << "Schur2D PROBE banner: MPI thread level " << thr
<< " (0 single,1 funneled,2 serialized,3 multiple) OMP_NUM_THREADS=" << (omp?omp:"unset")
<< " MemoryManager device bytes " << ms.DeviceBytes/1.0e9 << " GB (LRU " << ms.DeviceLRUBytes/1.0e9
<< " GB, cap " << ms.DeviceMaxBytes/1.0e9 << " GB)"
<< " grid " << L.Pr << "x" << L.Pc << " nb " << L.nb << std::endl;
#ifdef GRID_HIP
if ( me==0 ) acceleratorMem();
#endif
// --- ring partners exactly as SUMMA uses them ---
int prow=L.prow, pcol=L.pcol, Pr=L.Pr, Pc=L.Pc;
struct Ring { const char *name; int dest, src; };
Ring rings[2] = { {"ringA(row, q+-1)", prow*Pc + (pcol+1)%Pc, prow*Pc + (pcol-1+Pc)%Pc},
{"ringB(col, p+-1)", ((prow+1)%Pr)*Pc + pcol, ((prow-1+Pr)%Pr)*Pc + pcol} };
uint64_t sizes[3] = { 64ull*1024, 1024ull*1024, 8ull*1024*1024 };
uint64_t maxb = sizes[2];
deviceVector<char> dsend(maxb), drecv(maxb);
std::vector<char> hsend(maxb), hrecv(maxb);
for(int r=0;r<2;r++){
if ( (r==0 && Pc==1) || (r==1 && Pr==1) ) continue;
int off = grid->IsOffNode(rings[r].dest);
for(int si=0;si<3;si++){
uint64_t bytes = sizes[si];
// warm one, time five, both memory spaces
grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes);
double t0=usecond();
for(int i=0;i<5;i++) grid->SendToRecvFrom(&dsend[0], rings[r].dest, &drecv[0], rings[r].src, bytes);
double td=(usecond()-t0)/5.0;
grid->SendToRecvFrom(&hsend[0], rings[r].dest, &hrecv[0], rings[r].src, bytes);
t0=usecond();
for(int i=0;i<5;i++) grid->SendToRecvFrom(&hsend[0], rings[r].dest, &hrecv[0], rings[r].src, bytes);
double th=(usecond()-t0)/5.0;
// spread over ranks
RealD dmax=td, dmin=-td; grid->GlobalMax(dmax); grid->GlobalMax(dmin); dmin=-dmin;
std::cout << GridLogMessage << "Schur2D PROBE " << rings[r].name << (off?" OFF-node":" on-node")
<< " " << bytes/1024 << " KB: device " << td << " us (" << bytes/td/1.0e3 << " GB/s) [min/max over ranks " << dmin << "/" << dmax << " us]"
<< " host " << th << " us (" << bytes/th/1.0e3 << " GB/s)" << std::endl;
}
}
}
void Invert(BlockCyclicMatrix &A)
{
BlockCyclicLayout &L = A.layout;
GRID_ASSERT( L.N >= 1 );
int64_t nblocks = (L.N + L.nb - 1)/L.nb;
if ( !(getenv("SCHUR2D_PROBE") && atoi(getenv("SCHUR2D_PROBE"))==0) ) Probe(A);
BlockCyclicMatrix Bt(A.grid, L.N, L.nb, L.Pr, L.Pc);
BlockCyclicMatrix Ct(A.grid, L.N, L.nb, L.Pr, L.Pc);
systems/Frontier/smoother_modes.job
+14 -12
View File
@@ -129,19 +129,21 @@ run_cell () {
grep -h "SCHUR fp64 distributed invert took\|GB/s/rank" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-120 | head -2
}
# The (order x shift) table at Csn=2, one axis at a time from the overshoot,
# so a failing cell identifies WHICH knob it needed. Reference: 28.57 s.
# Ladder 2 (after L0-L3 of the first ladder): the stationary polynomial
# wants a LARGER shift than the adaptive one -- it must suppress the
# high-mode deviation on average, not per call. Measured (Csn=2):
# (12,1.0) 48 outer 31.4 s (8,1.0) 58 outer 30.9 s (8,0.5) 60 outer 34.3 s
# Reference (adaptive, Fso6/Fss0.1): 56 outer, 28.57 s. So walk the SHIFT UP
# at orders 8 and 6, trading shift against order. Nrhs=1 is the target.
# name Fso Fss Csn fine coarse
run_cell L0_overshoot 12 1.0 6 replay gcr # M3's point with last-call selection + record 8..16 + refresh 5
run_cell L1_12_10 12 1.0 2 replay gcr # coarse smoother back to 2 steps; the row/column anchor
run_cell L2_08_10 8 1.0 2 replay gcr # order axis
run_cell L3_06_10 6 1.0 2 replay gcr
run_cell L4_12_05 12 0.5 2 replay gcr # shift axis
run_cell L5_08_05 8 0.5 2 replay gcr
run_cell L6_06_05 6 0.5 2 replay gcr # nearest to the banked adaptive point (Fso6/Fss0.1)
# Coarse replay: record a DECENT polynomial first (6-step, shift 2.0) and back off.
run_cell L7_coarse6 8 1.0 6 replay replay
run_cell L8_coarse4 8 1.0 4 replay replay
run_cell S1_08_20 8 2.0 2 replay gcr # the promising cell: ~50 outer at 0.53 s/step would be ~26.5 s
run_cell S2_08_30 8 3.0 2 replay gcr
run_cell S3_06_20 6 2.0 2 replay gcr # banked order, larger shift
run_cell S4_06_10 6 1.0 2 replay gcr # the L3 of the second table (never ran)
run_cell S5_10_20 10 2.0 2 replay gcr # between 8 and 12
# coarse replay, recorded at a decent polynomial and backed off, at the best fine point
run_cell S6_coarse6 8 2.0 6 replay replay
run_cell S7_coarse4 8 2.0 4 replay replay
echo "========================================================="
echo "summary"