diff --git a/systems/Aurora-AOT/config-command b/systems/Aurora-AOT/config-command new file mode 100644 index 00000000..ead5f7c0 --- /dev/null +++ b/systems/Aurora-AOT/config-command @@ -0,0 +1,23 @@ +#Ahead of time compile for PVC +export LDFLAGS="-fiopenmp -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xs -device -Xs pvc -fsycl-device-lib=all -lze_loader -L${MKLROOT}/lib -qmkl=parallel -fsycl -lsycl " +export CXXFLAGS="-O3 -fiopenmp -fsycl-unnamed-lambda -fsycl -Wno-tautological-compare -qmkl=parallel -fsycl -fno-exceptions -fsycl-targets=spir64_gen -Xs -device -Xs pvc " + +#JIT compile +#export LDFLAGS="-fiopenmp -fsycl -fsycl-device-code-split=per_kernel -fsycl-device-lib=all -lze_loader -L${MKLROOT}/lib -qmkl=parallel -fsycl -lsycl " +#export CXXFLAGS="-O3 -fiopenmp -fsycl-unnamed-lambda -fsycl -Wno-tautological-compare -qmkl=parallel -fsycl -fno-exceptions " + +../../configure \ + --enable-simd=GPU \ + --enable-gen-simd-width=64 \ + --enable-comms=mpi-auto \ + --enable-debug \ + --disable-gparity \ + --disable-fermion-reps \ + --with-lime=$CLIME \ + --enable-shm=nvlink \ + --enable-accelerator=sycl \ + --enable-accelerator-aware-mpi=yes\ + --enable-unified=no \ + MPICXX=mpicxx \ + CXX=icpx + diff --git a/systems/Aurora-AOT/sourceme.sh b/systems/Aurora-AOT/sourceme.sh new file mode 100644 index 00000000..4cdc950f --- /dev/null +++ b/systems/Aurora-AOT/sourceme.sh @@ -0,0 +1,15 @@ +#module load oneapi/release/2023.12.15.001 +#module load mpich/icc-all-debug-pmix-gpu/52.2 +#module load mpich-config/mode/deterministic +#module load intel_compute_runtime/release/821.35 + +source ~/spack/share/spack/setup-env.sh +spack load c-lime +spack load openssl +export CLIME=`spack find --paths c-lime | grep ^c-lime | awk '{print $2}' ` +export HTTP_PROXY=http://proxy.alcf.anl.gov:3128 +export HTTPS_PROXY=http://proxy.alcf.anl.gov:3128 +export http_proxy=http://proxy.alcf.anl.gov:3128 +export https_proxy=http://proxy.alcf.anl.gov:3128 +git config --global http.proxy http://proxy.alcf.anl.gov:3128 +export SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file" diff --git a/systems/Aurora-AOT/tests/reproBigJob.pbs b/systems/Aurora-AOT/tests/reproBigJob.pbs new file mode 100644 index 00000000..10665df6 --- /dev/null +++ b/systems/Aurora-AOT/tests/reproBigJob.pbs @@ -0,0 +1,74 @@ +#!/bin/bash + +#PBS -l select=512 +#PBS -q EarlyAppAccess +#PBS -A LatticeQCD_aesp_CNDA +#PBS -l walltime=6:00:00 +#PBS -N reproBigJob +#PBS -k doe + +#export OMP_PROC_BIND=spread +#unset OMP_PLACES + +#module load oneapi/eng-compiler/2023.05.15.003 +#module load mpich/51.2/icc-all-deterministic-pmix-gpu + +# 56 cores / 6 threads ~9 +export OMP_NUM_THREADS=6 +export MPIR_CVAR_CH4_OFI_ENABLE_GPU_PIPELINE=1 +export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_D2H_ENGINE_TYPE=0 +export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_H2D_ENGINE_TYPE=0 +export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_BUFFER_SZ=10485760 +export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_THRESHOLD=131072 +export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_NUM_BUFFERS_PER_CHUNK=16 +export MPIR_CVAR_CH4_OFI_GPU_PIPELINE_MAX_NUM_BUFFERS=16 +#export MPIR_CVAR_GPU_USE_IMMEDIATE_COMMAND_LIST=1 + +#export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE=1 +export SYCL_PI_LEVEL_ZERO_USE_COPY_ENGINE_FOR_D2D_COPY=1 +export SYCL_PROGRAM_COMPILE_OPTIONS="-ze-opt-large-register-file" + +export GRID_PRINT_ENTIRE_LOG=0 +export GRID_CHECKSUM_RECV_BUF=0 +export GRID_CHECKSUM_SEND_BUF=0 + +export MPICH_OFI_NIC_POLICY=GPU + +#export MPIR_CVAR_ALLREDUCE_DEVICE_COLLECTIVE=0 +#export MPIR_CVAR_REDUCE_DEVICE_COLLECTIVE=0 +#export MPIR_CVAR_ALLREDUCE_INTRA_ALGORITHM=recursive_doubling +#unset MPIR_CVAR_CH4_COLL_SELECTION_TUNING_JSON_FILE +#unset MPIR_CVAR_COLL_SELECTION_TUNING_JSON_FILE +#unset MPIR_CVAR_CH4_POSIX_COLL_SELECTION_TUNING_JSON_FILE + +cd $PBS_O_WORKDIR + +cp $PBS_NODEFILE nodefile + +DIR=reproBigJob.$PBS_JOBID + +mkdir -p $DIR +cd $DIR + +cp $PBS_NODEFILE nodefile + +BINARY=../Test_dwf_mixedcg_prec + +echo > pingjob < command-line +env > environment +$CMD +grep Oops Grid.stderr.* > failures.$PBS_JOBID +rm core.*