From af950e6e283ba432d57328504c08e8eb2e25f02d Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Wed, 1 Feb 2023 23:55:50 +0000
Subject: [PATCH] Grid Tursa 32 node batch script

---
 Grid/systems/tursa/files/run.gpu.32nodes.sh | 60 +++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 Grid/systems/tursa/files/run.gpu.32nodes.sh

diff --git a/Grid/systems/tursa/files/run.gpu.32nodes.sh b/Grid/systems/tursa/files/run.gpu.32nodes.sh
new file mode 100644
index 0000000..1cd3c40
--- /dev/null
+++ b/Grid/systems/tursa/files/run.gpu.32nodes.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1091,SC2050,SC2170
+
+#SBATCH -J benchmark-grid-16
+#SBATCH -t 1:00:00
+#SBATCH --nodes=32
+#SBATCH --ntasks=128
+#SBATCH --ntasks-per-node=4
+#SBATCH --cpus-per-task=8
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:4
+#SBATCH --output=%x.%j.out
+#SBATCH --error=%x.%j.err
+#SBATCH --qos=standard
+#SBATCH --no-requeue
+#SBATCH --gpu-freq=1410
+
+set -euo pipefail
+
+# load environment #############################################################
+env_cfg="${HOME}/.config/lattice-benchmarks/grid-env"
+if [ ! -f "${env_cfg}" ]; then
+	echo "error: ${env_cfg} does not exists, did you execute 'source env.sh' with your user account?"
+	exit 1
+fi
+env_dir="$(readlink -f "$(cat "${env_cfg}")")"
+source "${env_dir}/env.sh"      # load base Spack environment
+source "${env_dir}/env-gpu.sh"  # load GPU-sepcific packages
+source "${env_dir}/ompi-gpu.sh" # set GPU-specific OpenMPI variables
+
+# application and parameters ###################################################
+app="${env_dir}/prefix/gridbench_gpu/bin/Benchmark_Grid"
+
+# collect job information ######################################################
+job_info_dir=job/${SLURM_JOB_NAME}.${SLURM_JOB_ID}
+mkdir -p "${job_info_dir}"
+
+date                         > "${job_info_dir}/start-date"
+set                          > "${job_info_dir}/env"
+ldd "${app}"                 > "${job_info_dir}/ldd"
+md5sum "${app}"              > "${job_info_dir}/app-hash"
+readelf -a "${app}"          > "${job_info_dir}/elf"
+echo "${SLURM_JOB_NODELIST}" > "${job_info_dir}/nodes"
+cp "${BASH_SOURCE[0]}"       "${job_info_dir}/script"
+
+# run! #########################################################################
+mpirun -np "${SLURM_NTASKS}" -x LD_LIBRARY_PATH --bind-to none \
+	"${env_dir}/gpu-mpi-wrapper.sh" \
+  "${app}" \
+	--json-out "${job_info_dir}/result.json" \
+	--mpi 2.4.4.4 \
+  --accelerator-threads 8 \
+	--threads 8 \
+	--shm 2048 &> "${job_info_dir}/log"
+
+# if we reach that point the application exited successfully ###################
+touch "${job_info_dir}/success"
+date > "${job_info_dir}/end-date"
+
+################################################################################