portelli/lattice-benchmarks
3
1
Fork 3
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'Updates GPU launch wrapper and install script' (#8) from aturner/lattice-benchmarks:main into main

Browse Source 
Reviewed-on: #8
This commit is contained in:
Antonin Portelli 2025-03-31 16:08:45 +01:00
commit cb538bfbf1
2 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Grid/systems/tursa/files/gpu-mpi-wrapper.sh
View File

@ -1,13 +1,12 @@
#!/usr/bin/env bash #!/usr/bin/env bash
lrank=$OMPI_COMM_WORLD_LOCAL_RANK lrank=$OMPI_COMM_WORLD_LOCAL_RANK
numa1=$(( 2 * lrank)) numa1=$((lrank))
numa2=$(( 2 * lrank + 1 ))
netdev=mlx5_${lrank}:1 netdev=mlx5_${lrank}:1
export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK
export UCX_NET_DEVICES=${netdev} export UCX_NET_DEVICES=${netdev}
BINDING="--interleave=$numa1,$numa2" BINDING="--interleave=$numa1"
echo "$(hostname) - $lrank device=$CUDA_VISIBLE_DEVICES binding=$BINDING" echo "$(hostname) - $lrank device=$CUDA_VISIBLE_DEVICES binding=$BINDING"

8
Grid/systems/tursa/spack-bootstrap.sh
View File

@ -80,7 +80,7 @@ mkdir -p build_gpu; cd build_gpu
--enable-devel-headers --enable-examples --enable-optimizations \ --enable-devel-headers --enable-examples --enable-optimizations \
--with-gdrcopy=${gdrcopy_path} --with-verbs --disable-logging \ --with-gdrcopy=${gdrcopy_path} --with-verbs --disable-logging \
--disable-debug --disable-assertions --enable-cma \ --disable-debug --disable-assertions --enable-cma \
--with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \ --with-knem=/opt/knem-1.1.4.90mlnx2/ --with-rdmacm \
--without-rocm --without-ugni --without-java \ --without-rocm --without-ugni --without-java \
--enable-compiler-opt=3 --with-cuda="${cuda_path}" --without-cm \ --enable-compiler-opt=3 --with-cuda="${cuda_path}" --without-cm \
--with-rc --with-ud --with-dc --with-mlx5-dv --with-dm \ --with-rc --with-ud --with-dc --with-mlx5-dv --with-dm \
@ -96,7 +96,7 @@ mkdir -p build_cpu; cd build_cpu
--enable-devel-headers --enable-examples --enable-optimizations \ --enable-devel-headers --enable-examples --enable-optimizations \
--with-verbs --disable-logging --disable-debug \ --with-verbs --disable-logging --disable-debug \
--disable-assertions --enable-mt --enable-cma \ --disable-assertions --enable-mt --enable-cma \
--with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \ --with-knem=/opt/knem-1.1.4.90mlnx2/--with-rdmacm \
--without-rocm --without-ugni --without-java \ --without-rocm --without-ugni --without-java \
--enable-compiler-opt=3 --without-cm --without-ugni --with-rc \ --enable-compiler-opt=3 --without-cm --without-ugni --with-rc \
--with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --without-go --with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --without-go
@ -122,7 +122,7 @@ mkdir build_gpu; cd build_gpu
../configure --prefix="${dir}"/prefix/ompi_gpu --without-xpmem \ ../configure --prefix="${dir}"/prefix/ompi_gpu --without-xpmem \
--with-ucx="${dir}"/prefix/ucx_gpu \ --with-ucx="${dir}"/prefix/ucx_gpu \
--with-ucx-libdir="${dir}"/prefix/ucx_gpu/lib \ --with-ucx-libdir="${dir}"/prefix/ucx_gpu/lib \
--with-knem=/opt/knem-1.1.4.90mlnx1/ \ --with-knem=/opt/knem-1.1.4.90mlnx2/ \
--enable-mca-no-build=btl-uct \ --enable-mca-no-build=btl-uct \
--with-cuda="${cuda_path}" --disable-getpwuid \ --with-cuda="${cuda_path}" --disable-getpwuid \
--with-verbs --with-slurm --enable-mpi-fortran=all \ --with-verbs --with-slurm --enable-mpi-fortran=all \
@ -136,7 +136,7 @@ mkdir build_cpu; cd build_cpu
../configure --prefix="${dir}"/prefix/ompi_cpu --without-xpmem \ ../configure --prefix="${dir}"/prefix/ompi_cpu --without-xpmem \
--with-ucx="${dir}"/prefix/ucx_cpu \ --with-ucx="${dir}"/prefix/ucx_cpu \
--with-ucx-libdir="${dir}"/prefix/ucx_cpu/lib \ --with-ucx-libdir="${dir}"/prefix/ucx_cpu/lib \
--with-knem=/opt/knem-1.1.4.90mlnx1/ \ --with-knem=/opt/knem-1.1.4.90mlnx2/ \
--enable-mca-no-build=btl-uct --disable-getpwuid \ --enable-mca-no-build=btl-uct --disable-getpwuid \
--with-verbs --with-slurm --enable-mpi-fortran=all \ --with-verbs --with-slurm --enable-mpi-fortran=all \
--with-pmix=internal --with-libevent=internal --with-pmix=internal --with-libevent=internal