working scripts for Tursa

Grid/systems/tursa/files/cpu-mpi-wrapper.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+lrank=$OMPI_COMM_WORLD_LOCAL_RANK
+numa=${lrank}
+cpus="$(( lrank*16 ))-$(( (lrank+1)*16-1 ))"
+places="$(( lrank*16 )):$(( (lrank+1)*16 ))"
+
+BINDING="taskset -c ${cpus} numactl -m ${numa}"
+export OMP_PLACES=${places}
+
+echo "$(hostname) - ${lrank} binding='${BINDING}'"
+
+${BINDING} "$@"

Grid/systems/tursa/files/env-cpu.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+spack env activate grid-cpu
+spack env status

Grid/systems/tursa/files/env-gpu.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+spack env activate grid-gpu
+spack env status

Grid/systems/tursa/files/gpu-mpi-wrapper.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+lrank=$OMPI_COMM_WORLD_LOCAL_RANK
+numa1=$(( 2 * lrank))
+numa2=$(( 2 * lrank + 1 ))
+netdev=mlx5_${lrank}:1
+
+export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
+export UCX_NET_DEVICES=${netdev}
+BINDING="--interleave=$numa1,$numa2"
+
+echo "$(hostname) - $lrank device=$CUDA_VISIBLE_DEVICES binding=$BINDING"
+
+numactl ${BINDING} "$@"