Synch multigrid rewrite to repository and

broad unicode elimination effort

systems/Frontier/nrhs_fault.job

@@ -66,7 +66,7 @@ export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU
 module load libfabric
 
-BIN=$root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix
+BIN=$root/examples/Example_pvdagm_3level_DenseCoarseMatrix
 OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1"
 # LOCAL VOLUME MATCHED to production: 48.48.48.96 on 3.6.4.4 is local 16.8.12.24; on 8 GCDs
 # (1.1.2.4) that is global 16.8.24.96 with the production blockings 2.2.3.3 / 4.4.2.4:
@@ -112,7 +112,7 @@ run_cell () {
   logopt=""; [ -n "$GRID_LOG_EXTRA" ] && logopt="--log Error,Warning,Message,$GRID_LOG_EXTRA"
   env "$@" srun -N1 -n8 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --device-mem $devmem --comms-overlap $logopt > $fname 2>&1
   echo "  exit $?"; sleep 10
-  grep -h "V2 3-level solve\|Memory access fault\|GRID_ASSERT\|Converged on iteration" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | tail -4 | cut -c1-140
+  grep -h "3-level solve\|Memory access fault\|GRID_ASSERT\|Converged on iteration" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | tail -4 | cut -c1-140
   grep -n -B3 "Memory access fault" $fname | tail -5 | cut -c1-160
 }
 
@@ -124,5 +124,5 @@ run_cell   F3_nrhs6_nodense 16000 NRHS=6 DENSE_CC=0
 run_cell   F4_nrhs6_bigmem 48000 NRHS=6                                                    # no MemoryManager eviction
 
 echo "========================================================="
-for f in log.fault.F*; do echo "$f: $(grep -c 'Memory access fault' $f) faults; $(grep -h 'V2 3-level solve' $f | tail -1 | sed 's/.*V2/V2/' | cut -c1-60)"; done
+for f in log.fault.F*; do echo "$f: $(grep -c 'Memory access fault' $f) faults; $(grep -h '3-level solve' $f | tail -1 | sed 's/.*3-level/3-level/' | cut -c1-60)"; done
 echo "========================================================="

systems/Frontier/nrhs_fault_36.job

@@ -67,7 +67,7 @@ export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU
 module load libfabric
 
-BIN=$root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix
+BIN=$root/examples/Example_pvdagm_3level_DenseCoarseMatrix
 OPTS1="--accelerator-threads 8 --shm 4096 --shm-mpi 1 --device-mem 32000"
 OVERLAP="--comms-overlap"          # set empty for a no-overlap job
 vol=48.48.48.96
@@ -112,7 +112,7 @@ run_cell () {
     tail -15 $(dirname $f)/Grid.stdout.$rank | cut -c1-160
   else
     echo "  no 'Memory access fault' in any Grid.stderr.*"
-    grep -h "V2 3-level solve\|Fouter MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -3 | cut -c1-120
+    grep -h "3-level solve\|Fouter MrhsPGCR: Converged" $GRID_STDOUT_ROOT/0/Grid.stdout.0 | tail -3 | cut -c1-120
   fi
 }
 

systems/Frontier/pvdagm_multigrid.job

@@ -70,7 +70,7 @@ export GRID_ALLOC_NCACHE_LARGE=0       # allocator large-ring depth (Grid alloca
 module load libfabric
 
 BIN=$root/examples/Example_pvdagm_multigrid
-OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 0 --device-mem 32000 --comms-overlap"
+OPTS="--accelerator-threads 8 --shm 4096 --shm-mpi 0 --device-mem 40000 --comms-overlap"
 vol=48.48.48.96
 MPI_GEOM=3.6.4.4
 

systems/Frontier/schur2d_ladder.job

@@ -131,14 +131,14 @@ export NRHS=4
 
 echo "----- F4a: 1D baseline (DENSE_SCHUR2D unset) -----"
 unset DENSE_SCHUR2D
-srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \
+srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_3level_DenseCoarseMatrix \
      --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap
 
 if [ $F3RC -eq 0 ]
 then
 echo "----- F4b: 2D block-cyclic (DENSE_SCHUR2D=1) -----"
 export DENSE_SCHUR2D=1
-srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \
+srun -N36 -n288 ./select_gpu $root/examples/Example_pvdagm_3level_DenseCoarseMatrix \
      --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap
 else
 echo "----- F4b SKIPPED: F3 failed (rc=$F3RC), not spending the setup on it -----"

systems/Frontier/smoother_modes.job

@@ -121,10 +121,10 @@ run_cell () {
   export FineSmootherMode=$5; export CoarseSmootherMode=$6
   echo "----- $name : Fso=$FineSmootherOrder Fss=$FineSmootherShift Csn=$CoarseSmootherNstep fine=$FineSmootherMode coarse=$CoarseSmootherMode -----"
   fname=log.ladder.$name
-  srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $root/examples/Example_pvdagm_v2_3level_DenseCoarseMatrix \
+  srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $root/examples/Example_pvdagm_3level_DenseCoarseMatrix \
        --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap > $fname 2>&1
   echo "  exit $?"; sleep 60
-  echo "  $(grep -h 'V2 3-level solve Nrhs' $fname | sed 's/.*V2/V2/' | tr '\n' ' ')"
+  echo "  $(grep -h '3-level solve Nrhs' $fname | sed 's/.*3-level/3-level/' | tr '\n' ' ')"
   echo "  $(grep -h 'Fouter MrhsPGCR: Converged' $fname | sed 's/.*Converged/Converged/' | cut -c1-60 | tr '\n' ' ')"
   echo "  replay per-call |r|/|r0| (Nrhs=1 solve): $(awk '/THREE-level solve, Nrhs = 1/{s=1} s && /Fsmoother replay \|r\|/{v=$NF; n++; t+=v; if(v>mx)mx=v} END{if(n) printf "mean %.4f max %.4f over %d calls", t/n, mx, n}' $fname)"
   grep -h "SCHUR fp64 distributed invert took\|GB/s/rank\|BIG LEAVES\|ring histogram\|^Grid : Message : [0-9.]* s :    >=" $fname | sed 's/^Grid : Message : [0-9.]* s : //' | cut -c1-150 | head -12