diff --git a/Grid/systems/tursa/files/gpu-mpi-wrapper.sh b/Grid/systems/tursa/files/gpu-mpi-wrapper.sh index 9e6871d..9031efa 100755 --- a/Grid/systems/tursa/files/gpu-mpi-wrapper.sh +++ b/Grid/systems/tursa/files/gpu-mpi-wrapper.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash lrank=$OMPI_COMM_WORLD_LOCAL_RANK -numa1=$(( 2 * lrank)) -numa2=$(( 2 * lrank + 1 )) +numa1=$((lrank)) netdev=mlx5_${lrank}:1 export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK export UCX_NET_DEVICES=${netdev} -BINDING="--interleave=$numa1,$numa2" +BINDING="--interleave=$numa1" echo "$(hostname) - $lrank device=$CUDA_VISIBLE_DEVICES binding=$BINDING" diff --git a/Grid/systems/tursa/spack-bootstrap.sh b/Grid/systems/tursa/spack-bootstrap.sh index 24c9d9c..6d928b4 100755 --- a/Grid/systems/tursa/spack-bootstrap.sh +++ b/Grid/systems/tursa/spack-bootstrap.sh @@ -80,7 +80,7 @@ mkdir -p build_gpu; cd build_gpu --enable-devel-headers --enable-examples --enable-optimizations \ --with-gdrcopy=${gdrcopy_path} --with-verbs --disable-logging \ --disable-debug --disable-assertions --enable-cma \ - --with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \ + --with-knem=/opt/knem-1.1.4.90mlnx2/ --with-rdmacm \ --without-rocm --without-ugni --without-java \ --enable-compiler-opt=3 --with-cuda="${cuda_path}" --without-cm \ --with-rc --with-ud --with-dc --with-mlx5-dv --with-dm \ @@ -96,7 +96,7 @@ mkdir -p build_cpu; cd build_cpu --enable-devel-headers --enable-examples --enable-optimizations \ --with-verbs --disable-logging --disable-debug \ --disable-assertions --enable-mt --enable-cma \ - --with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \ + --with-knem=/opt/knem-1.1.4.90mlnx2/--with-rdmacm \ --without-rocm --without-ugni --without-java \ --enable-compiler-opt=3 --without-cm --without-ugni --with-rc \ --with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --without-go @@ -122,7 +122,7 @@ mkdir build_gpu; cd build_gpu ../configure --prefix="${dir}"/prefix/ompi_gpu --without-xpmem \ --with-ucx="${dir}"/prefix/ucx_gpu \ --with-ucx-libdir="${dir}"/prefix/ucx_gpu/lib \ - --with-knem=/opt/knem-1.1.4.90mlnx1/ \ + --with-knem=/opt/knem-1.1.4.90mlnx2/ \ --enable-mca-no-build=btl-uct \ --with-cuda="${cuda_path}" --disable-getpwuid \ --with-verbs --with-slurm --enable-mpi-fortran=all \ @@ -136,7 +136,7 @@ mkdir build_cpu; cd build_cpu ../configure --prefix="${dir}"/prefix/ompi_cpu --without-xpmem \ --with-ucx="${dir}"/prefix/ucx_cpu \ --with-ucx-libdir="${dir}"/prefix/ucx_cpu/lib \ - --with-knem=/opt/knem-1.1.4.90mlnx1/ \ + --with-knem=/opt/knem-1.1.4.90mlnx2/ \ --enable-mca-no-build=btl-uct --disable-getpwuid \ --with-verbs --with-slurm --enable-mpi-fortran=all \ --with-pmix=internal --with-libevent=internal