#!/bin/bash -l
#SBATCH --job-name=pvdagm-multigrid
#SBATCH --nodes=36
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=7
#SBATCH --gpus-per-node=8
#SBATCH --time=1:30:00
#SBATCH --account=phy157_dwf
#SBATCH --gpu-bind=none
#SBATCH --exclusive
#SBATCH --mem=0
#SBATCH -S 0

##############################################################################
# Production run of the three-level mrhs PVdagM multigrid on the library
# objects (examples/Example_pvdagm_multigrid.cc), 48^3x96 Ls=24, 288 GCDs.
#
# All algorithm/physics parameters are in ONE serialisable struct read from
# XML (--pvdagm-params); there are NO Grid environment knobs any more.  The
# library-internal choices the old nrhs_fault_36.job set by env are now the
# only code path (2D block-cyclic dense inverse, ring-allgather apply,
# inverseLU big leaves) and are gone from the environment.
#
# The binary must be built with -DNBASIS=64 to match the subspace file.
##############################################################################

root=$HOME/PVdagM/Grid/systems/Frontier
source $root/sourceme-rocm7.2.sh

# Paths substituted into the XML below.
LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
SUBSPACE=$LUSTRE/subspace_nb64.scidac
CONFIG=/ccs/home/paboyle/configs/ckpoint_lat.1000
RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"


cat << EOF > select_gpu
#!/bin/bash
export GPU_MAP=(0 1 2 3 7 6 5 4)
export NUMA_MAP=(3 3 1 1 2 2 0 0)
export GPU=\${GPU_MAP[\$SLURM_LOCALID]}
export NUMA=\${NUMA_MAP[\$SLURM_LOCALID]}
export HIP_VISIBLE_DEVICES=\$GPU
unset ROCR_VISIBLE_DEVICES
if [ \$SLURM_PROCID = "0" ]; then echo \$*; fi
exec numactl -m \$NUMA -N \$NUMA \$*
EOF
chmod +x ./select_gpu


export OMP_NUM_THREADS=7
ulimit -c 0                             # no 22 GB GPU core dumps
export FI_MR_CACHE_MONITOR=kdreg2       # site default; device-buffer MPI on Slingshot
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU

#### This raise the probability of failure ####
export GRID_ALLOC_NCACHE_LARGE=0       # allocator large-ring depth (Grid allocator, not a solver knob) -- default is 8
##############################################################################


module load libfabric

BIN=$root/examples/Example_pvdagm_multigrid
OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 0 --device-mem 32000 --comms-overlap"
vol=48.48.48.96
MPI_GEOM=3.6.4.4


##############################################################################
# The parameter file.  Values are the banked adaptive operating point carried
# over from nrhs_fault_36.job.  Tags mirror PVdagMDriverParams /
# PVdagMMultiGridParams exactly; a mistyped tag deserialises silently to the
# struct default, so keep this in step with the headers.
##############################################################################
cat << EOF > params.xml
<?xml version="1.0"?>
<grid>
  <PVdagMDriver>
    <Ls>24</Ls>
    <Mass>0.00078</Mass>
    <M5>1.8</M5>
    <MobiusB>1.5</MobiusB>
    <MobiusC>0.5</MobiusC>
    <Config>$CONFIG</Config>
    <Nrhs>12</Nrhs>
    <SolveSingleRHS>1</SolveSingleRHS>
    <MultiGrid>
      <Setup>
        <Block><elem>2</elem><elem>2</elem><elem>3</elem><elem>3</elem></Block>
        <Block2><elem>4</elem><elem>4</elem><elem>2</elem><elem>4</elem></Block2>
        <CoarsenBatch>9</CoarsenBatch>
        <SubspaceFile>$SUBSPACE</SubspaceFile>
        <FineSloppyComms>1</FineSloppyComms>
      </Setup>
      <FineSmoother>
        <Shift>0.1</Shift><Nstep>6</Nstep><Mmax>4</Mmax>
      </FineSmoother>
      <CoarseSmoother>
        <Shift>2.0</Shift><Nstep>2</Nstep><Mmax>2</Mmax>
      </CoarseSmoother>
      <CoarseSolver>
        <Tol>0.05</Tol><Order>200</Order><Mmax>8</Mmax>
      </CoarseSolver>
      <Outer>
        <Tol>1e-8</Tol><MaxIterations>1000</MaxIterations><Mmax>6</Mmax><Nstep>12</Nstep>
      </Outer>
      <Dense>
        <LeafSpan>9</LeafSpan>
      </Dense>
    </MultiGrid>
  </PVdagMDriver>
</grid>
EOF
echo "=== params.xml ==="; cat params.xml; echo "=================="

run_cell () {
  name=$1; shift
  echo "----- $name : $* -----"
  export GRID_STDOUT_ROOT=$RUNDIR/mg_$name
  env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS \
       --pvdagm-params params.xml --debug-stdout --log Error,Warning,Message,Performance > log.mg.$name 2>&1
  echo "  exit $?"; sleep 30
  f=$(grep -l "Memory access fault\|NO_TRANSLATION" $GRID_STDOUT_ROOT/*/Grid.stderr.* 2>/dev/null | head -1)
  if [ -n "$f" ]; then
    echo "  FAULT in $f  (rank $(basename $f | sed 's/Grid.stderr.//'))"; tail -20 $f | cut -c1-160
  else
    echo "  no fault; convergence + timing:"
    grep -h "3-level solve\|FINAL Nrhs\|MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -8 | cut -c1-140
  fi
}

# the NRHS>=12 fault was intermittent (~1/3) before the
# dmabuf avoidance, so run it twice to confirm a clean pair.

# sourceme sets FI_HMEM_ROCR_USE_DMABUF=0 -- the standing avoidance of the CXI
# NO_TRANSLATION fault at NRHS>=12 (libfabric #12775).
export FI_HMEM_ROCR_USE_DMABUF=0
run_cell  dmabuf0_run1 
run_cell  dmabuf0_run2 
run_cell  dmabuf0_run3 

export FI_HMEM_ROCR_USE_DMABUF=1
run_cell  dmabuf1_run1 
run_cell  dmabuf1_run2 
run_cell  dmabuf1_run3