Debug ulimit as core files driving me crazy

systems/Frontier/nrhs_fault_36.job

@@ -33,6 +33,13 @@
 #   (the last kernel name / MemoryManager line before the fault); same for fault36_B.
 ##############################################################################
 
+# Run in a per-job subdirectory on Lustre: logs, per-rank --debug-stdout trees and GPU
+# core files are large and must not land in $HOME (quota).  Everything below is
+# relative to RUNDIR; the job's own slurm-*.out stays where sbatch was run.
+LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
+RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
+mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
+
 cat << EOF > select_gpu
 #!/bin/bash
 export GPU_MAP=(0 1 2 3 7 6 5 4)
@@ -50,6 +57,10 @@ root=$HOME/ParallelIO/systems/Frontier
 source $root/sourceme-rocm7.2.sh
 
 export OMP_NUM_THREADS=7
+# No core dumps: the ROCm runtime's GPU core dump (gpucore.<pid>, ~22 GB per faulting GCD, the
+# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
+# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
+ulimit -c 0
 export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_SMP_SINGLE_COPY_MODE=CMA
 export MPICH_OFI_NIC_POLICY=GPU
@@ -86,7 +97,7 @@ export NRHS=6
 run_cell () {
   name=$1; shift
   echo "----- $name : $* -----"
-  export GRID_STDOUT_ROOT=$PWD/fault36_$name
+  export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name
   env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \
        --debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
   echo "  exit $?"; sleep 30