206 lines
6.9 KiB
Bash
Executable File
206 lines
6.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2016
|
|
set -euo pipefail
|
|
|
|
gcc_spec='gcc@9.4.0'
|
|
cuda_spec='cuda@11.4.0'
|
|
hdf5_spec='hdf5@1.10.7'
|
|
|
|
if (( $# != 1 )); then
|
|
echo "usage: $(basename "$0") <env dir>" 1>&2
|
|
exit 1
|
|
fi
|
|
dir=$1
|
|
cwd=$(pwd -P)
|
|
cd "${dir}"
|
|
dir=$(pwd -P)
|
|
cd "${cwd}"
|
|
|
|
# General configuration ########################################################
|
|
# build with 128 tasks
|
|
echo 'config:
|
|
build_jobs: 128
|
|
build_stage:
|
|
- $spack/var/spack/stage
|
|
test_stage: $spack/var/spack/test
|
|
misc_cache: $spack/var/spack/cache' > jobs.yaml
|
|
spack config --scope site add -f jobs.yaml
|
|
rm jobs.yaml
|
|
|
|
# add lustre as external package
|
|
echo 'packages:
|
|
lustre:
|
|
externals:
|
|
- spec: "lustre@2.12.6_ddn36"
|
|
prefix: /usr' > external.yaml
|
|
spack config --scope site add -f external.yaml
|
|
rm external.yaml
|
|
|
|
# Base compilers ###############################################################
|
|
# configure system base
|
|
spack compiler find --scope site
|
|
|
|
# install GCC, CUDA & LLVM
|
|
spack install ${gcc_spec} ${cuda_spec} llvm
|
|
|
|
spack load llvm
|
|
spack compiler find --scope site
|
|
spack unload llvm
|
|
|
|
spack load ${gcc_spec}
|
|
spack compiler find --scope site
|
|
spack unload ${gcc_spec}
|
|
|
|
# Manual compilation of OpenMPI & UCX ##########################################
|
|
# set build directories
|
|
mkdir -p "${dir}"/build
|
|
cd "${dir}"/build
|
|
|
|
spack load ${gcc_spec} ${cuda_spec}
|
|
|
|
cuda_path=$(spack find --format "{prefix}" cuda)
|
|
gdrcopy_path=/mnt/lustre/tursafs1/apps/gdrcopy/2.3.1
|
|
|
|
# Install ucx 1.12.0
|
|
ucx_url=https://github.com/openucx/ucx/releases/download/v1.12.0/ucx-1.12.0.tar.gz
|
|
|
|
echo "-- building UCX from source"
|
|
wget ${ucx_url}
|
|
ucx_ar=$(basename ${ucx_url})
|
|
tar -xvf "${ucx_ar}"
|
|
cd "${ucx_ar%.tar.gz}"
|
|
|
|
# ucx gpu build
|
|
mkdir -p build_gpu; cd build_gpu
|
|
../configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu \
|
|
--disable-dependency-tracking --prefix="${dir}"/prefix/ucx_gpu \
|
|
--enable-devel-headers --enable-examples --enable-optimizations \
|
|
--with-gdrcopy=${gdrcopy_path} --with-verbs --disable-logging \
|
|
--disable-debug --disable-assertions --enable-cma \
|
|
--with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \
|
|
--without-rocm --without-ugni --without-java \
|
|
--enable-compiler-opt=3 --with-cuda="${cuda_path}" --without-cm \
|
|
--with-rc --with-ud --with-dc --with-mlx5-dv --with-dm \
|
|
--enable-mt --without-go LDFLAGS=-L${gdrcopy_path}/lib
|
|
make -j 128
|
|
make install
|
|
cd ..
|
|
|
|
# ucx cpu build
|
|
mkdir -p build_cpu; cd build_cpu
|
|
../configure --build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu \
|
|
--disable-dependency-tracking --prefix="${dir}"/prefix/ucx_cpu \
|
|
--enable-devel-headers --enable-examples --enable-optimizations \
|
|
--with-verbs --disable-logging --disable-debug \
|
|
--disable-assertions --enable-mt --enable-cma \
|
|
--with-knem=/opt/knem-1.1.4.90mlnx1/ --with-rdmacm \
|
|
--without-rocm --without-ugni --without-java \
|
|
--enable-compiler-opt=3 --without-cm --without-ugni --with-rc \
|
|
--with-ud --with-dc --with-mlx5-dv --with-dm --enable-mt --without-go
|
|
make -j 128
|
|
make install
|
|
|
|
cd "${dir}"/build
|
|
|
|
# Install openmpi 4.1.1
|
|
ompi_url=https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.1.tar.gz
|
|
|
|
echo "-- building OpenMPI from source"
|
|
|
|
wget ${ompi_url}
|
|
ompi_ar=$(basename ${ompi_url})
|
|
tar -xvf "${ompi_ar}"
|
|
cd "${ompi_ar%.tar.gz}"
|
|
export AUTOMAKE_JOBS=128
|
|
./autogen.pl -f
|
|
|
|
# openmpi gpu build
|
|
mkdir build_gpu; cd build_gpu
|
|
../configure --prefix="${dir}"/prefix/ompi_gpu --without-xpmem \
|
|
--with-ucx="${dir}"/prefix/ucx_gpu \
|
|
--with-ucx-libdir="${dir}"/prefix/ucx_gpu/lib \
|
|
--with-knem=/opt/knem-1.1.4.90mlnx1/ \
|
|
--enable-mca-no-build=btl-uct \
|
|
--with-cuda="${cuda_path}" --disable-getpwuid \
|
|
--with-verbs --with-slurm --enable-mpi-fortran=all \
|
|
--with-pmix=internal --with-libevent=internal
|
|
make -j 128
|
|
make install
|
|
cd ..
|
|
|
|
# openmpi cpu build
|
|
mkdir build_cpu; cd build_cpu
|
|
../configure --prefix="${dir}"/prefix/ompi_cpu --without-xpmem \
|
|
--with-ucx="${dir}"/prefix/ucx_cpu \
|
|
--with-ucx-libdir="${dir}"/prefix/ucx_cpu/lib \
|
|
--with-knem=/opt/knem-1.1.4.90mlnx1/ \
|
|
--enable-mca-no-build=btl-uct --disable-getpwuid \
|
|
--with-verbs --with-slurm --enable-mpi-fortran=all \
|
|
--with-pmix=internal --with-libevent=internal
|
|
make -j 128
|
|
make install
|
|
cd "${dir}"
|
|
|
|
# Add externals to spack
|
|
echo "packages:
|
|
ucx:
|
|
externals:
|
|
- spec: \"ucx@1.12.0.GPU%gcc@9.4.0\"
|
|
prefix: ${dir}/prefix/ucx_gpu
|
|
- spec: \"ucx@1.12.0.CPU%gcc@9.4.0\"
|
|
prefix: ${dir}/prefix/ucx_cpu
|
|
buildable: False
|
|
openmpi:
|
|
externals:
|
|
- spec: \"openmpi@4.1.1.GPU%gcc@9.4.0\"
|
|
prefix: ${dir}/prefix/ompi_gpu
|
|
- spec: \"openmpi@4.1.1.CPU%gcc@9.4.0\"
|
|
prefix: ${dir}/prefix/ompi_cpu
|
|
buildable: False" > spack.yaml
|
|
|
|
spack config --scope site add -f spack.yaml
|
|
rm spack.yaml
|
|
spack install ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
|
|
spack install ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
|
|
|
|
cd "${cwd}"
|
|
|
|
# environments #################################################################
|
|
dev_tools=("autoconf" "automake" "libtool" "jq")
|
|
ompi_gpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.GPU)
|
|
ompi_cpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.CPU)
|
|
|
|
spack env create grid-gpu
|
|
spack env activate grid-gpu
|
|
spack add ${gcc_spec} ${cuda_spec} "${dev_tools[@]}"
|
|
spack add ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
|
|
spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_gpu_hash}"
|
|
spack add fftw ^/"${ompi_gpu_hash}"
|
|
spack add openssl gmp mpfr c-lime
|
|
spack install
|
|
spack env deactivate
|
|
|
|
spack env create grid-cpu
|
|
spack env activate grid-cpu
|
|
spack add llvm "${dev_tools[@]}"
|
|
spack add ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
|
|
spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_cpu_hash}"
|
|
spack add fftw ^/"${ompi_cpu_hash}"
|
|
spack add openssl gmp mpfr c-lime
|
|
spack install
|
|
spack env deactivate
|
|
|
|
# Final setup ##################################################################
|
|
spack clean
|
|
spack gc -y
|
|
|
|
# add more environment variables in module loading
|
|
spack config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH,LIBRARY_PATH]'
|
|
spack config --scope site add 'modules:prefix_inspections:lib64:[LD_LIBRARY_PATH,LIBRARY_PATH]'
|
|
spack config --scope site add 'modules:prefix_inspections:include:[C_INCLUDE_PATH,CPLUS_INCLUDE_PATH,INCLUDE]'
|
|
spack module tcl refresh -y
|
|
|
|
# permission change for group access
|
|
chmod -R g+rw "${dir}/spack/var/spack/cache"
|
|
setfacl -d -R -m g::rwX "${dir}/spack/var/spack/cache"
|