portelli/lattice-benchmarks
3
1
Fork 2
Code Issues Pull Requests 2 Packages Projects Releases Wiki Activity

working scripts for Tursa

Browse Source
This commit is contained in:
Antonin Portelli 2023-01-24 18:17:43 +00:00
parent 21c92f9fb7
commit 0f5733df97
12 changed files with 151 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Grid/systems/tursa/bootstrap-env.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -euo pipefail
if (( $# != 1 )); then
echo "usage: $(basename "$0") <environment directory>" 1>&2
exit 1
fi
dir=$1
call_dir=$(pwd -P)
script_dir="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
if [ -d "${dir}" ]; then
echo "error: directory '${dir}' exists"
exit 1
fi
mkdir -p "${dir}"
cd "${dir}"
git clone https://github.com/spack/spack.git
cd "${call_dir}"
cp "${script_dir}"/files/* "${dir}"
cp "${script_dir}/env.sh" "${script_dir}/grid-config.json" "${dir}"
source "${dir}"/spack/share/spack/setup-env.sh
"${script_dir}"/spack-bootstrap.sh "${dir}"

5
Grid/systems/tursa/env-cpu.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
# shellcheck disable=SC2046
script_dir="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
spack load $(cat "${script_dir}"/grid-cpu.spack)

5
Grid/systems/tursa/env-gpu.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
# shellcheck disable=SC2046
script_dir="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
spack load $(cat "${script_dir}"/grid-gpu.spack)

8
Grid/systems/tursa/env.sh Normal file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
# shellcheck disable=SC1091
GRIDENVDIR="$(dirname "$(readlink -f "${BASH_SOURCE:-$0}")")"
export GRIDENVDIR
export PATH="${GRIDENVDIR}/prefix/base/bin:${PATH}"
export ACLOCAL_PATH="${GRIDENVDIR}/prefix/base/share/aclocal:${ACLOCAL_PATH}"
source "${GRIDENVDIR}"/spack/share/spack/setup-env.sh

0
Grid/systems/tursa/cpu-mpi-wrapper.sh → Grid/systems/tursa/files/cpu-mpi-wrapper.sh
View File

4
Grid/systems/tursa/files/env-cpu.sh Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
spack env activate grid-cpu
spack env status

4
Grid/systems/tursa/files/env-gpu.sh Normal file
View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
spack env activate grid-gpu
spack env status

0
Grid/systems/tursa/gpu-mpi-wrapper.sh → Grid/systems/tursa/files/gpu-mpi-wrapper.sh
View File

25
Grid/systems/tursa/grid-config.json Normal file
View File

@ -0,0 +1,25 @@
{
"configs": [
{
"name": "gpu",
"env-script": "env-gpu.sh",
"commit": "796abfad80625d81bb16af7ff6ec612a836f17d8",
"config-options": "--enable-comms=mpi --enable-simd=GPU --enable-alloc-align=4k --enable-accelerator-cshift --enable-shm=nvlink --enable-gen-simd-width=64 --enable-accelerator=cuda --disable-unified --enable-gparity=no --enable-fermion-reps=no",
"env" : {
"CXX": "nvcc",
"LDFLAGS": "-cudart shared",
"CXXFLAGS": "-ccbin mpicxx -gencode arch=compute_80,code=sm_80 -std=c++14 -cudart shared"
}
},
{
"name": "cpu",
"env-script": "env-cpu.sh",
"commit": "796abfad80625d81bb16af7ff6ec612a836f17d8",
"config-options": "--enable-comms=mpi-auto --enable-simd=AVX2 --enable-alloc-align=4k --enable-shm=shmget --enable-gparity=no --enable-fermion-reps=no",
"env" : {
"CXX": "clang++",
"MPICXX": "mpicxx"
}
}
]
}

10
Grid/systems/tursa/grid-cpu.spack
View File

@ -1,10 +0,0 @@
gcc@9.4.0
llvm@12.0.1
ucx@1.12.0.CPU%gcc@9.4.0
openmpi@4.1.1.CPU%gcc@9.4.0
hdf5^openmpi@4.1.1.CPU%gcc@9.4.0
fftw^openmpi@4.1.1.CPU%gcc@9.4.0
openssl
gmp%gcc@9.4.0
mpfr%gcc@9.4.0
c-lime

10
Grid/systems/tursa/grid-gpu.spack
View File

@ -1,10 +0,0 @@
gcc@9.4.0
cuda@11.4.0
ucx@1.12.0.GPU%gcc@9.4.0
openmpi@4.1.1.GPU%gcc@9.4.0
hdf5^openmpi@4.1.1.GPU%gcc@9.4.0
fftw^openmpi@4.1.1.GPU%gcc@9.4.0
openssl
gmp%gcc@9.4.0
mpfr%gcc@9.4.0
c-lime

169
Grid/systems/tursa/spack-bootstrap.sh Normal file → Executable file
View File

@ -2,16 +2,19 @@
# shellcheck disable=SC2016 # shellcheck disable=SC2016
set -euo pipefail set -euo pipefail
GCC='gcc@9.4.0' gcc_spec='gcc@9.4.0'
CUDA='cuda@11.4.0' cuda_spec='cuda@11.4.0'
HDF5='hdf5@1.10.7' hdf5_spec='hdf5@1.10.7'
if (( $# != 1 )); then if (( $# != 1 )); then
echo "usage: $(basename "$0") <env dir>" 1>&2 echo "usage: $(basename "$0") <env dir>" 1>&2
exit 1 exit 1
fi fi
ENVDIR=$1 dir=$1
CWD=$(pwd -P) cwd=$(pwd -P)
cd "${dir}"
dir=$(pwd -P)
cd "${cwd}"
# General configuration ######################################################## # General configuration ########################################################
# build with 128 tasks # build with 128 tasks
@ -33,107 +36,92 @@ echo 'packages:
spack config --scope site add -f external.yaml spack config --scope site add -f external.yaml
rm external.yaml rm external.yaml
# Base compilers ###############################################################
# configure system base # configure system base
spack compiler find --scope site spack compiler find --scope site
# Base packages ################################################################ # install GCC, CUDA & LLVM
# install GCC spack install ${gcc_spec} ${cuda_spec} llvm
spack install ${GCC}
spack load ${GCC} spack load llvm
spack compiler find --scope site spack compiler find --scope site
spack unload ${GCC} spack unload llvm
# clean spack load ${gcc_spec}
spack clean
spack gc -y
# install CUDA
spack install ${CUDA}
# install development tools
dev_tools=("autoconf" "automake" "libtool" "git")
spack install "${dev_tools[@]}"
# create view for CLI & dev tools
spack view symlink -i "${ENVDIR}/prefix/base" "${dev_tools[@]}"
# install clang
spack install llvm@12.0.1
# locate new compilers
spack load llvm@12.0.1
spack compiler find --scope site spack compiler find --scope site
spack unload llvm@12.0.1 spack unload ${gcc_spec}
# Manual compilation of OpenMPI & UCX ########################################## # Manual compilation of OpenMPI & UCX ##########################################
# set build directories # set build directories
mkdir -p "${ENVDIR}"/build mkdir -p "${dir}"/build
cd "${ENVDIR}"/build cd "${dir}"/build
spack load ${GCC} ${CUDA} spack load ${gcc_spec} ${cuda_spec}
CUDA_PATH=$(which nvcc | sed "s/bin/@/g" | cut -d "@" -f1) cuda_path=$(spack find --format "{prefix}" cuda)
GDRCOPY_PATH=/mnt/lustre/tursafs1/apps/gdrcopy/2.3.1 gdrcopy_path=/mnt/lustre/tursafs1/apps/gdrcopy/2.3.1
# Install ucx 1.12.0 # Install ucx 1.12.0
UCX_URL=https://github.com/openucx/ucx/releases/download/v1.12.0/ucx-1.12.0.tar.gz ucx_url=https://github.com/openucx/ucx/releases/download/v1.12.0/ucx-1.12.0.tar.gz
echo "-- building UCX from source" echo "-- building UCX from source"
wget ${UCX_URL} wget ${ucx_url}
UCX_AR=$(basename ${UCX_URL}) ucx_ar=$(basename ${ucx_url})
tar -xvf "${UCX_AR}" tar -xvf "${ucx_ar}"
cd "${UCX_AR%.tar.gz}" cd "${ucx_ar%.tar.gz}"
# ucx gpu build # ucx gpu build
mkdir build_gpu; cd build_gpu mkdir -p build_gpu; cd build_gpu
../configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu \ ../configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu \
--disable-dependency-tracking --prefix="${ENVDIR}"/prefix/ucx_gpu \ --disable-dependency-tracking --prefix="${dir}"/prefix/ucx_gpu \
--enable-devel-headers --enable-examples --enable-optimizations \ --enable-devel-headers --enable-examples --enable-optimizations \
--with-gdrcopy=${GDRCOPY_PATH} --with-verbs --disable-logging \ --with-gdrcopy=${gdrcopy_path} --with-verbs --disable-logging \
--disable-debug --disable-assertions --enable-cma \ --disable-debug --disable-assertions --enable-cma \
--with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \ --with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \
--without-rocm --without-ugni --without-java \ --without-rocm --without-ugni --without-java \
--enable-compiler-opt=3 --with-cuda="${CUDA_PATH}" --without-cm \ --enable-compiler-opt=3 --with-cuda="${cuda_path}" --without-cm \
--with-rc --with-ud --with-dc --with-mlx5-dv --with-dm \ --with-rc --with-ud --with-dc --with-mlx5-dv --with-dm \
--enable-mt LDFLAGS=-L${GDRCOPY_PATH}/lib --enable-mt --without-go LDFLAGS=-L${gdrcopy_path}/lib
make -j 128 make -j 128
make install make install
cd .. cd ..
# ucx cpu build # ucx cpu build
mkdir build_cpu; cd build_cpu mkdir -p build_cpu; cd build_cpu
../configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu \ ../configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu \
--disable-dependency-tracking --prefix="${ENVDIR}"/prefix/ucx_cpu \ --disable-dependency-tracking --prefix="${dir}"/prefix/ucx_cpu \
--enable-devel-headers --enable-examples --enable-optimizations \ --enable-devel-headers --enable-examples --enable-optimizations \
--with-verbs --disable-logging --disable-debug \ --with-verbs --disable-logging --disable-debug \
--disable-assertions --enable-mt --enable-cma \ --disable-assertions --enable-mt --enable-cma \
--with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \ --with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \
--without-rocm --without-ugni --without-java \ --without-rocm --without-ugni --without-java \
--enable-compiler-opt=3 --without-cm --without-ugni --with-rc \ --enable-compiler-opt=3 --without-cm --without-ugni --with-rc \
--with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --without-go
make -j 128 make -j 128
make install make install
cd "${ENVDIR}"/build cd "${dir}"/build
# Install openmpi 4.1.1 (needs to be done on a gpu node) # Install openmpi 4.1.1
OMPI_URL=https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.1.tar.gz ompi_url=https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.1.tar.gz
echo "-- building OpenMPI from source" echo "-- building OpenMPI from source"
wget ${OMPI_URL} wget ${ompi_url}
OMPI_AR=$(basename ${OMPI_URL}) ompi_ar=$(basename ${ompi_url})
tar -xvf "${OMPI_AR}" tar -xvf "${ompi_ar}"
cd "${OMPI_AR%.tar.gz}" cd "${ompi_ar%.tar.gz}"
export AUTOMAKE_JOBS=128
./autogen.pl -f
# openmpi gpu build # openmpi gpu build
mkdir build_gpu; cd build_gpu mkdir build_gpu; cd build_gpu
../configure --prefix="${ENVDIR}"/prefix/ompi_gpu --without-xpmem \ ../configure --prefix="${dir}"/prefix/ompi_gpu --without-xpmem \
--with-ucx="${ENVDIR}"/prefix/ucx_gpu \ --with-ucx="${dir}"/prefix/ucx_gpu \
--with-ucx-libdir="${ENVDIR}"/prefix/ucx_gpu/lib \ --with-ucx-libdir="${dir}"/prefix/ucx_gpu/lib \
--with-knem=/opt/knem-1.1.4.90mlnx1/ \ --with-knem=/opt/knem-1.1.4.90mlnx1/ \
--enable-mca-no-build=btl-uct \ --enable-mca-no-build=btl-uct \
--with-cuda="${CUDA_PATH}" --disable-getpwuid \ --with-cuda="${cuda_path}" --disable-getpwuid \
--with-verbs --with-slurm --enable-mpi-fortran=all \ --with-verbs --with-slurm --enable-mpi-fortran=all \
--with-pmix=internal --with-libevent=internal --with-pmix=internal --with-libevent=internal
make -j 128 make -j 128
@ -142,61 +130,76 @@ cd ..
# openmpi cpu build # openmpi cpu build
mkdir build_cpu; cd build_cpu mkdir build_cpu; cd build_cpu
../configure --prefix="${ENVDIR}"/prefix/ompi_cpu --without-xpmem \ ../configure --prefix="${dir}"/prefix/ompi_cpu --without-xpmem \
--with-ucx="${ENVDIR}"/prefix/ucx_cpu \ --with-ucx="${dir}"/prefix/ucx_cpu \
--with-ucx-libdir="${ENVDIR}"/prefix/ucx_cpu/lib \ --with-ucx-libdir="${dir}"/prefix/ucx_cpu/lib \
--with-knem=/opt/knem-1.1.4.90mlnx1/ \ --with-knem=/opt/knem-1.1.4.90mlnx1/ \
--enable-mca-no-build=btl-uct --disable-getpwuid \ --enable-mca-no-build=btl-uct --disable-getpwuid \
--with-verbs --with-slurm --enable-mpi-fortran=all \ --with-verbs --with-slurm --enable-mpi-fortran=all \
--with-pmix=internal --with-libevent=internal --with-pmix=internal --with-libevent=internal
make -j 128 make -j 128
make install make install
cd "${ENVDIR}" cd "${dir}"
# Add externals to spack # Add externals to spack
echo "packages: echo "packages:
ucx: ucx:
externals: externals:
- spec: \"ucx@1.12.0.GPU%gcc@9.4.0\" - spec: \"ucx@1.12.0.GPU%gcc@9.4.0\"
prefix: ${ENVDIR}/prefix/ucx_gpu prefix: ${dir}/prefix/ucx_gpu
- spec: \"ucx@1.12.0.CPU%gcc@9.4.0\" - spec: \"ucx@1.12.0.CPU%gcc@9.4.0\"
prefix: ${ENVDIR}/prefix/ucx_cpu prefix: ${dir}/prefix/ucx_cpu
buildable: False buildable: False
openmpi: openmpi:
externals: externals:
- spec: \"openmpi@4.1.1.GPU%gcc@9.4.0\" - spec: \"openmpi@4.1.1.GPU%gcc@9.4.0\"
prefix: ${ENVDIR}/prefix/ompi_gpu prefix: ${dir}/prefix/ompi_gpu
- spec: \"openmpi@4.1.1.CPU%gcc@9.4.0\" - spec: \"openmpi@4.1.1.CPU%gcc@9.4.0\"
prefix: ${ENVDIR}/prefix/ompi_cpu prefix: ${dir}/prefix/ompi_cpu
buildable: False" > spack.yaml buildable: False" > spack.yaml
spack config --scope site add -f spack.yaml spack config --scope site add -f spack.yaml
rm spack.yaml rm spack.yaml
spack install ucx@1.12.0.GPU%gcc@9.4.0 spack install ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
spack install ucx@1.12.0.CPU%gcc@9.4.0 spack install ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
spack install openmpi@4.1.1.GPU%gcc@9.4.0
spack install openmpi@4.1.1.CPU%gcc@9.4.0
# Install Grid dependencies #################################################### cd "${cwd}"
cd "${CWD}"
OPENMPIGPUHASH=$(spack find --format "{hash}" openmpi@4.1.1.GPU) # environments #################################################################
OPENMPICPUHASH=$(spack find --format "{hash}" openmpi@4.1.1.CPU) dev_tools=("autoconf" "automake" "libtool" "jq")
ompi_gpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.GPU)
ompi_cpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.CPU)
spack install ${HDF5}+cxx+threadsafe ^/"${OPENMPIGPUHASH}" spack env create grid-gpu
spack install ${HDF5}+cxx+threadsafe ^/"${OPENMPICPUHASH}" spack env activate grid-gpu
spack install fftw ^/"${OPENMPIGPUHASH}" spack add ${gcc_spec} ${cuda_spec} "${dev_tools[@]}"
spack install fftw ^/"${OPENMPICPUHASH}" spack add ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
spack install openssl gmp mpfr c-lime spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_gpu_hash}"
spack add fftw ^/"${ompi_gpu_hash}"
spack add openssl gmp mpfr c-lime
spack install
spack env deactivate
spack env create grid-cpu
spack env activate grid-cpu
spack add llvm "${dev_tools[@]}"
spack add ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_cpu_hash}"
spack add fftw ^/"${ompi_cpu_hash}"
spack add openssl gmp mpfr c-lime
spack install
spack env deactivate
# Final setup ################################################################## # Final setup ##################################################################
spack clean spack clean
spack gc -y
# add more environment variables in module loading # add more environment variables in module loading
spack config --scope site add 'modules:prefix_inspections:lib:[LIBRARY_PATH]' spack config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH,LIBRARY_PATH]'
spack config --scope site add 'modules:prefix_inspections:lib64:[LD_LIBRARY_PATH,LIBRARY_PATH]'
spack config --scope site add 'modules:prefix_inspections:include:[C_INCLUDE_PATH,CPLUS_INCLUDE_PATH,INCLUDE]' spack config --scope site add 'modules:prefix_inspections:include:[C_INCLUDE_PATH,CPLUS_INCLUDE_PATH,INCLUDE]'
spack module tcl refresh -y spack module tcl refresh -y
# permission change for group access # permission change for group access
chmod -R g+rw "${ENVDIR}/spack/var/spack/cache" chmod -R g+rw "${dir}/spack/var/spack/cache"
setfacl -d -R -m g::rwX "${ENVDIR}/spack/var/spack/cache" setfacl -d -R -m g::rwX "${dir}/spack/var/spack/cache"