New script for ORNL to try

2026-09-10 03:33:16 +01:00 · 2026-09-09 15:13:12 -04:00
parent 03495bfcf8
commit a2ef8b39d3
1 changed files with 133 additions and 0 deletions
@@ -0,0 +1,133 @@
+#!/bin/bash -l
+#SBATCH --job-name=pvdagm-multigrid
+#SBATCH --nodes=36
+#SBATCH --ntasks-per-node=8
+#SBATCH --cpus-per-task=7
+#SBATCH --gpus-per-node=8
+#SBATCH --time=1:30:00
+#SBATCH --account=phy157_dwf
+#SBATCH --gpu-bind=none
+#SBATCH --exclusive
+#SBATCH --mem=0
+#SBATCH -S 0
+
+##############################################################################
+# Production run of the three-level mrhs PVdagM multigrid on the library
+# objects (examples/Example_pvdagm_multigrid.cc), 48^3x96 Ls=24, 288 GCDs.
+#
+# All algorithm/physics parameters are in ONE serialisable struct read from
+# XML (--pvdagm-params); there are NO Grid environment knobs any more.  The
+# library-internal choices the old nrhs_fault_36.job set by env are now the
+# only code path (2D block-cyclic dense inverse, ring-allgather apply,
+# inverseLU big leaves) and are gone from the environment.
+#
+# The binary must be built with -DNBASIS=64 to match the subspace file.
+##############################################################################
+
+LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
+RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
+mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
+
+cat << EOF > select_gpu
+#!/bin/bash
+export GPU_MAP=(0 1 2 3 7 6 5 4)
+export NUMA_MAP=(3 3 1 1 2 2 0 0)
+export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
+export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
+export HIP_VISIBLE_DEVICES=\$GPU
+unset ROCR_VISIBLE_DEVICES
+if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
+exec numactl -m \$NUMA -N \$NUMA \$*
+EOF
+chmod +x ./select_gpu
+
+root=$HOME/ParallelIO/systems/Frontier
+source $root/sourceme-rocm7.2.sh
+# sourceme sets FI_HMEM_ROCR_USE_DMABUF=0 -- the standing avoidance of the CXI
+# NO_TRANSLATION fault at NRHS>=12 (libfabric #12775).
+
+export OMP_NUM_THREADS=7
+ulimit -c 0                             # no 22 GB GPU core dumps
+export FI_MR_CACHE_MONITOR=kdreg2       # site default; device-buffer MPI on Slingshot
+export MPICH_GPU_SUPPORT_ENABLED=1
+export MPICH_SMP_SINGLE_COPY_MODE=CMA
+export MPICH_OFI_NIC_POLICY=GPU
+export GRID_ALLOC_NCACHE_LARGE=64       # allocator large-ring depth (Grid allocator, not a solver knob)
+module load libfabric
+
+BIN=$root/examples/Example_pvdagm_multigrid
+OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000 --comms-overlap"
+vol=48.48.48.96
+MPI_GEOM=3.6.4.4
+
+# Paths substituted into the XML below.
+SUBSPACE=$LUSTRE/subspace_nb64.scidac
+CONFIG=/ccs/home/poare/ckpoint_lat.1000     # gauge field (empty => hot start)
+
+##############################################################################
+# The parameter file.  Values are the banked adaptive operating point carried
+# over from nrhs_fault_36.job.  Tags mirror PVdagMDriverParams /
+# PVdagMMultiGridParams exactly; a mistyped tag deserialises silently to the
+# struct default, so keep this in step with the headers.
+##############################################################################
+cat << EOF > params.xml
+<?xml version="1.0"?>
+<grid>
+  <PVdagMDriver>
+    <Ls>24</Ls>
+    <Mass>0.00078</Mass>
+    <M5>1.8</M5>
+    <MobiusB>1.5</MobiusB>
+    <MobiusC>0.5</MobiusC>
+    <Config>$CONFIG</Config>
+    <Nrhs>12</Nrhs>
+    <SolveSingleRHS>1</SolveSingleRHS>
+    <MultiGrid>
+      <Setup>
+        <Block><elem>2</elem><elem>2</elem><elem>3</elem><elem>3</elem></Block>
+        <Block2><elem>4</elem><elem>4</elem><elem>2</elem><elem>4</elem></Block2>
+        <CoarsenBatch>9</CoarsenBatch>
+        <SubspaceFile>$SUBSPACE</SubspaceFile>
+        <FineSloppyComms>1</FineSloppyComms>
+      </Setup>
+      <FineSmoother>
+        <Shift>0.1</Shift><Nstep>6</Nstep><Mmax>4</Mmax>
+      </FineSmoother>
+      <CoarseSmoother>
+        <Shift>2.0</Shift><Nstep>2</Nstep><Mmax>2</Mmax>
+      </CoarseSmoother>
+      <CoarseSolver>
+        <Tol>0.05</Tol><Order>200</Order><Mmax>8</Mmax>
+      </CoarseSolver>
+      <Outer>
+        <Tol>1e-8</Tol><MaxIterations>1000</MaxIterations><Mmax>6</Mmax><Nstep>12</Nstep>
+      </Outer>
+      <Dense>
+        <LeafSpan>9</LeafSpan>
+      </Dense>
+    </MultiGrid>
+  </PVdagMDriver>
+</grid>
+EOF
+echo "=== params.xml ==="; cat params.xml; echo "=================="
+
+run_cell () {
+  name=$1; shift
+  echo "----- $name : $* -----"
+  export GRID_STDOUT_ROOT=$RUNDIR/mg_$name
+  env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS \
+       --pvdagm-params params.xml --debug-stdout --log Error,Warning,Message,Performance > log.mg.$name 2>&1
+  echo "  exit $?"; sleep 30
+  f=$(grep -l "Memory access fault\|NO_TRANSLATION" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
+  if [ -n "$f" ]; then
+    echo "  FAULT in $f  (rank $(basename $f | sed 's/Grid.stderr.//'))"; tail -20 $f | cut -c1-160
+  else
+    echo "  no fault; convergence + timing:"
+    grep -h "3-level solve\|FINAL Nrhs\|MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -8 | cut -c1-140
+  fi
+}
+
+# Two identical cells: the NRHS>=12 fault was intermittent (~1/3) before the
+# dmabuf avoidance, so run it twice to confirm a clean pair.
+run_cell  run1
+run_cell  run2