portelli/Grid
1
0
Fork 0
mirror of https://github.com/paboyle/Grid.git synced 2026-08-29 13:59:36 +01:00
Code Releases Activity

Debug ulimit as core files driving me crazy

Browse Source
This commit is contained in:
Peter Boyle
2026-08-28 11:20:43 -04:00
parent 9b4c388585
commit 63c2cdb712
4 changed files with 66 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
systems/Frontier/nrhs_fault.job
+12 -1
View File
@@ -32,6 +32,13 @@
# dense bottom or MemoryManager eviction is involved.
##############################################################################
# Run in a per-job subdirectory on Lustre: logs, per-rank --debug-stdout trees and GPU
# core files are large and must not land in $HOME (quota). Everything below is
# relative to RUNDIR; the job's own slurm-*.out stays where sbatch was run.
LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
cat << EOF > select_gpu
#!/bin/bash
export GPU_MAP=(0 1 2 3 7 6 5 4)
@@ -49,6 +56,10 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
# No core dumps: the ROCm runtime's GPU core dump (gpucore.<pid>, ~22 GB per faulting GCD, the
# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
ulimit -c 0
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
@@ -70,7 +81,7 @@ export LATT=$vol
export LS=24
# The subspace is generated on first use and saved to SUBSPACE_FILE (a few minutes).
export SUBSPACE_FILE=$PWD/subspace_16x8x24x96_nb64.scidac
export SUBSPACE_FILE=$LUSTRE/subspace_16x8x24x96_nb64.scidac # persistent across jobs (generated once)
unset SLAB_FILE
export HOT_START=1
export MASS=0.00078
systems/Frontier/nrhs_fault_36.job
+12 -1
View File
@@ -33,6 +33,13 @@
# (the last kernel name / MemoryManager line before the fault); same for fault36_B.
##############################################################################
# Run in a per-job subdirectory on Lustre: logs, per-rank --debug-stdout trees and GPU
# core files are large and must not land in $HOME (quota). Everything below is
# relative to RUNDIR; the job's own slurm-*.out stays where sbatch was run.
LUSTRE=/lustre/orion/phy157/proj-shared/phy157_dwf/paboyle
RUNDIR=$LUSTRE/runs/${SLURM_JOB_NAME}_${SLURM_JOB_ID}
mkdir -p $RUNDIR && cd $RUNDIR && echo "RUNDIR $RUNDIR"
cat << EOF > select_gpu
#!/bin/bash
export GPU_MAP=(0 1 2 3 7 6 5 4)
@@ -50,6 +57,10 @@ root=$HOME/ParallelIO/systems/Frontier
source $root/sourceme-rocm7.2.sh
export OMP_NUM_THREADS=7
# No core dumps: the ROCm runtime's GPU core dump (gpucore.<pid>, ~22 GB per faulting GCD, the
# .nfs quota disaster of 2026-08-25) obeys ulimit -c like the kernel's (ROCgdb docs, "AMD GPU
# core dumps"); Slurm propagates the batch shell's limits to srun tasks by default.
ulimit -c 0
export MPICH_GPU_SUPPORT_ENABLED=1
export MPICH_SMP_SINGLE_COPY_MODE=CMA
export MPICH_OFI_NIC_POLICY=GPU
@@ -86,7 +97,7 @@ export NRHS=6
run_cell () {
name=$1; shift
echo "----- $name : $* -----"
export GRID_STDOUT_ROOT=$PWD/fault36_$name
export GRID_STDOUT_ROOT=$RUNDIR/fault36_$name
env "$@" srun -N36 -n288 --kill-on-bad-exit=1 ./select_gpu $BIN --mpi ${MPI_GEOM} --grid $vol $OPTS1 --comms-overlap \
--debug-stdout --log Error,Warning,Message,Memory > log.fault36.$name 2>&1
echo " exit $?"; sleep 30