portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-29 13:59:36 +01:00
Code Releases Activity

Stop mucking about with THREAD_MULTIPLE

Browse Source
This commit is contained in:
Peter Boyle
2026-08-28 07:06:45 -04:00
parent 2264afda84
commit 27e32b1633
6 changed files with 72 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
systems/Frontier/allreduce_bench.job
+24 -4
View File
@@ -4,7 +4,7 @@
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=7
#SBATCH --gpus-per-node=8
#SBATCH --time=0:30:00
#SBATCH --time=0:20:00
#SBATCH --account=phy157_dwf
#SBATCH --gpu-bind=none
#SBATCH --exclusive
@@ -66,9 +66,29 @@ run_cell () {
}
# name nodes ntask geom vol
run_cell N1_8gcd 1 8 1.1.2.4 16.16.32.64
run_cell N4_32gcd 4 32 2.2.2.4 32.32.32.64
run_cell N36_288gcd 36 288 3.6.4.4 48.48.48.96
#run_cell N1_8gcd 1 8 1.1.2.4 16.16.32.64 # done 2026-08-27
#run_cell N4_32gcd 4 32 2.2.2.4 32.32.32.64 # done: MPI <1 MB, rings above, 2-2.5x at 512 MB
#run_cell N36_288gcd 36 288 3.6.4.4 48.48.48.96 # done: MPI <2 MB, cart 2-48 MB, flat above (3x at 512 MB);
# SCALAR: GlobalSumP2P 0.4 ms min / 12.8 ms MEAN vs MPI 45 us (!)
# SCALAR-ONLY sweep at 288 ranks (BENCH_MAX_MB=0 skips the vector tables; each cell ~30 s):
# S0 back-to-back as measured; S1 with ~2 ms of GPU work between sums (the solver's condition:
# fine-smoother steps are 16 ms apart incl. a ~10 ms matvec, so the solver cannot be paying
# 12 ms per reduction); S2-S3 one environment knob each. Readout: the five "min/mean us" lines.
export BENCH_MAX_MB=0
scalar_cell () { name=$1; shift; run_cell_env "$name" 36 288 3.6.4.4 48.48.48.96 "$@"; }
run_cell_env () {
name=$1; nodes=$2; ntask=$3; geom=$4; vol=$5; shift 5
echo "----- $name : $* -----"
fname=log.allreduce.$name
env "$@" srun -N$nodes -n$ntask --kill-on-bad-exit=1 ./select_gpu $BIN --mpi $geom --grid $vol $OPTS1 > $fname 2>&1
echo " exit $?"; sleep 20
grep -h "==== SCALAR\|gap kernel\| min .* us" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-140
}
scalar_cell S0_tight BENCH_SCALAR_GAP_US=0
scalar_cell S1_gap2ms BENCH_SCALAR_GAP_US=2000
scalar_cell S2_gap2ms_omp1 BENCH_SCALAR_GAP_US=2000 OMP_NUM_THREADS=1
scalar_cell S3_gap2ms_nogpu BENCH_SCALAR_GAP_US=2000 MPICH_GPU_SUPPORT_ENABLED=0
echo "========================================================="
echo "done: see log.allreduce.N1_8gcd, N4_32gcd, N36_288gcd"
systems/Frontier/schur2d_env.job
+1 -1
View File
@@ -76,7 +76,7 @@ run_cell () {
# name environment
run_cell E1_omp1 OMP_NUM_THREADS=1
run_cell E2_omp1_multiple OMP_NUM_THREADS=1 GRID_MPI_THREAD_MULTIPLE=1 MPICH_MAX_THREAD_SAFETY=multiple # = the SLATE harness environment
#run_cell E2_omp1_multiple (MPI_THREAD_MULTIPLE init: 21.2 s, NIL; the GRID_MPI_THREAD_MULTIPLE hack was removed 2026-08-28)
run_cell E3_omp7 OMP_NUM_THREADS=7 # the example's thread count
run_cell E4_omp7_ballast OMP_NUM_THREADS=7 S2D_BALLAST_GB=10.6 # + the example's device residency
run_cell E5_omp1_ballast OMP_NUM_THREADS=1 S2D_BALLAST_GB=10.6 # ballast alone
systems/Frontier/smoother_modes.job
+2 -4
View File
@@ -102,10 +102,8 @@ export PolyRecordIters=8 # outer steps recorded
export PolyRecordStart=8 # ...starting here: the early-step polynomials are unrepresentative (M3)
export PolyRecordSelect=last # replay ONE recorded call's polynomial (PB: every individual call beats the coefficient mean)
export PolyRefresh=5 # re-record every 5 outer steps: BFM BfmHDCG.C:2243, k%5==1 -> LdopM1MirsPolyRecord, single call, replayed 4 steps
# Inverse ring-rate hypotheses, ONE AT A TIME: (1) OMP_NUM_THREADS=1 (set above);
# (2) if (1) fails, uncomment the two lines below (harness ran 62 s with these).
#export MPICH_MAX_THREAD_SAFETY=multiple
#export GRID_MPI_THREAD_MULTIPLE=1
# Inverse ring-rate hypotheses (2026-08-27): OMP threads and MPI thread level both NIL
# (schur2d_env.job); the cause was partial ring participation -> SCHUR2D_LEAF_SPAN.
export PolyVerbose=1 # frozen smoothers print |r_m|/|r_0| per call: separates 'bad polynomial' from 'linear V-cycle stagnates the outer'
export FineChebLo=3.0 # harvested |R|<0.1 edge / PowerIteration edge x1.05
export FineChebHi=137.0