Compare commits
No commits in common. "7235bfde4cb5d84a9e0c27edf4337e3e7069dd50" and "ce0d4d945720fe42a664092e2dbe150f71ecefd2" have entirely different histories.
7235bfde4c
...
ce0d4d9457
@ -73,8 +73,6 @@ class Benchmark
|
||||
{local[0] * mpi[0], local[1] * mpi[1], local[2] * mpi[2], local[3] * mpi[3]});
|
||||
GridCartesian *TmpGrid = SpaceTimeGrid::makeFourDimGrid(
|
||||
latt4, GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
|
||||
Grid::Coordinate shm;
|
||||
GlobalSharedMemory::GetShmDims(mpi, shm);
|
||||
|
||||
uint64_t NP = TmpGrid->RankCount();
|
||||
uint64_t NN = TmpGrid->NodeCount();
|
||||
@ -87,9 +85,7 @@ class Benchmark
|
||||
std::cout << GridLogMessage << "* OpenMP threads : " << GridThread::GetThreads()
|
||||
<< std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "* MPI layout : " << GridCmdVectorIntToString(mpi)
|
||||
<< std::endl;
|
||||
std::cout << GridLogMessage << "* Shm layout : " << GridCmdVectorIntToString(shm)
|
||||
std::cout << GridLogMessage << "* MPI tasks : " << GridCmdVectorIntToString(mpi)
|
||||
<< std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "* vReal : " << sizeof(vReal) * 8 << "bits ; "
|
||||
@ -122,7 +118,6 @@ class Benchmark
|
||||
for (unsigned int i = 0; i < mpi.size(); ++i)
|
||||
{
|
||||
tmp["mpi"].push_back(mpi[i]);
|
||||
tmp["shm"].push_back(shm[i]);
|
||||
}
|
||||
tmp["ranks"] = NP;
|
||||
tmp["nodes"] = NN;
|
||||
@ -137,8 +132,6 @@ class Benchmark
|
||||
|
||||
Coordinate simd_layout = GridDefaultSimd(Nd, vComplexD::Nsimd());
|
||||
Coordinate mpi_layout = GridDefaultMpi();
|
||||
Coordinate shm_layout;
|
||||
GlobalSharedMemory::GetShmDims(mpi_layout, shm_layout);
|
||||
|
||||
for (int mu = 0; mu < Nd; mu++)
|
||||
if (mpi_layout[mu] > 1)
|
||||
@ -150,8 +143,8 @@ class Benchmark
|
||||
std::cout << GridLogMessage << "Benchmarking threaded STENCIL halo exchange in "
|
||||
<< nmu << " dimensions" << std::endl;
|
||||
grid_small_sep();
|
||||
grid_printf("%5s %5s %7s %15s %15s %15s %15s %15s\n", "L", "dir", "shm",
|
||||
"payload (B)", "time (usec)", "rate (GB/s/node)", "std dev", "max");
|
||||
grid_printf("%5s %5s %15s %15s %15s %15s %15s\n", "L", "dir", "payload (B)",
|
||||
"time (usec)", "rate (GB/s/node)", "std dev", "max");
|
||||
|
||||
for (int lat = 16; lat <= maxlat; lat += 8)
|
||||
{
|
||||
@ -180,80 +173,74 @@ class Benchmark
|
||||
for (int dir = 0; dir < 8; dir++)
|
||||
{
|
||||
int mu = dir % 4;
|
||||
if (mpi_layout[mu] == 1) // skip directions that are not distributed
|
||||
continue;
|
||||
bool is_shm = mpi_layout[mu] == shm_layout[mu];
|
||||
bool is_partial_shm = !is_shm && shm_layout[mu] != 1;
|
||||
|
||||
std::vector<double> times(Nloop);
|
||||
for (int i = 0; i < NWARMUP; i++)
|
||||
{
|
||||
int xmit_to_rank;
|
||||
int recv_from_rank;
|
||||
|
||||
if (dir == mu)
|
||||
{
|
||||
int comm_proc = 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
else
|
||||
{
|
||||
int comm_proc = mpi_layout[mu] - 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, (void *)&rbuf[dir][0],
|
||||
recv_from_rank, bytes);
|
||||
}
|
||||
for (int i = 0; i < Nloop; i++)
|
||||
if (mpi_layout[mu] > 1)
|
||||
{
|
||||
|
||||
dbytes = 0;
|
||||
double start = usecond();
|
||||
int xmit_to_rank;
|
||||
int recv_from_rank;
|
||||
|
||||
if (dir == mu)
|
||||
std::vector<double> times(Nloop);
|
||||
for (int i = 0; i < NWARMUP; i++)
|
||||
{
|
||||
int comm_proc = 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
else
|
||||
{
|
||||
int comm_proc = mpi_layout[mu] - 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank, (void *)&rbuf[dir][0],
|
||||
recv_from_rank, bytes);
|
||||
dbytes += bytes;
|
||||
int xmit_to_rank;
|
||||
int recv_from_rank;
|
||||
|
||||
double stop = usecond();
|
||||
t_time[i] = stop - start; // microseconds
|
||||
if (dir == mu)
|
||||
{
|
||||
int comm_proc = 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
else
|
||||
{
|
||||
int comm_proc = mpi_layout[mu] - 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
|
||||
(void *)&rbuf[dir][0], recv_from_rank, bytes);
|
||||
}
|
||||
for (int i = 0; i < Nloop; i++)
|
||||
{
|
||||
|
||||
dbytes = 0;
|
||||
double start = usecond();
|
||||
int xmit_to_rank;
|
||||
int recv_from_rank;
|
||||
|
||||
if (dir == mu)
|
||||
{
|
||||
int comm_proc = 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
else
|
||||
{
|
||||
int comm_proc = mpi_layout[mu] - 1;
|
||||
Grid.ShiftedRanks(mu, comm_proc, xmit_to_rank, recv_from_rank);
|
||||
}
|
||||
Grid.SendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
|
||||
(void *)&rbuf[dir][0], recv_from_rank, bytes);
|
||||
dbytes += bytes;
|
||||
|
||||
double stop = usecond();
|
||||
t_time[i] = stop - start; // microseconds
|
||||
}
|
||||
timestat.statistics(t_time);
|
||||
|
||||
dbytes = dbytes * ppn;
|
||||
double bidibytes = 2. * dbytes;
|
||||
double rate = bidibytes / (timestat.mean / 1.e6) / 1024. / 1024. / 1024.;
|
||||
double rate_err = rate * timestat.err / timestat.mean;
|
||||
double rate_max = rate * timestat.mean / timestat.min;
|
||||
grid_printf("%5d %5d %15d %15.2f %15.2f %15.1f %15.2f\n", lat, dir, bytes,
|
||||
timestat.mean, rate, rate_err, rate_max);
|
||||
nlohmann::json tmp;
|
||||
nlohmann::json tmp_rate;
|
||||
tmp["L"] = lat;
|
||||
tmp["dir"] = dir;
|
||||
tmp["bytes"] = bytes;
|
||||
tmp["time_usec"] = timestat.mean;
|
||||
tmp_rate["mean"] = rate;
|
||||
tmp_rate["error"] = rate_err;
|
||||
tmp_rate["max"] = rate_max;
|
||||
tmp["rate_GBps"] = tmp_rate;
|
||||
json_results["comms"].push_back(tmp);
|
||||
}
|
||||
timestat.statistics(t_time);
|
||||
|
||||
dbytes = dbytes * ppn;
|
||||
double bidibytes = 2. * dbytes;
|
||||
double rate = bidibytes / (timestat.mean / 1.e6) / 1024. / 1024. / 1024.;
|
||||
double rate_err = rate * timestat.err / timestat.mean;
|
||||
double rate_max = rate * timestat.mean / timestat.min;
|
||||
grid_printf("%5d %5d %7s %15d %15.2f %15.2f %15.1f %15.2f\n", lat, dir,
|
||||
is_shm ? "yes"
|
||||
: is_partial_shm ? "partial"
|
||||
: "no",
|
||||
bytes, timestat.mean, rate, rate_err, rate_max);
|
||||
nlohmann::json tmp;
|
||||
nlohmann::json tmp_rate;
|
||||
tmp["L"] = lat;
|
||||
tmp["dir"] = dir;
|
||||
tmp["shared_mem"] = is_shm;
|
||||
tmp["partial_shared_mem"] = is_partial_shm;
|
||||
tmp["bytes"] = bytes;
|
||||
tmp["time_usec"] = timestat.mean;
|
||||
tmp_rate["mean"] = rate;
|
||||
tmp_rate["error"] = rate_err;
|
||||
tmp_rate["max"] = rate_max;
|
||||
tmp["rate_GBps"] = tmp_rate;
|
||||
json_results["comms"].push_back(tmp);
|
||||
}
|
||||
for (int d = 0; d < 8; d++)
|
||||
{
|
||||
|
||||
@ -4,13 +4,7 @@ set -euo pipefail
|
||||
|
||||
gcc_spec='gcc@9.4.0'
|
||||
cuda_spec='cuda@11.4.0'
|
||||
|
||||
# hdf5 and fftw depend on OpenMPI, which we install manually. To make sure this
|
||||
# dependency is picked by spack, we specify the compiler here explicitly. For
|
||||
# most other packages we dont really care about the compiler (i.e. system
|
||||
# compiler versus ${gcc_spec})
|
||||
hdf5_spec="hdf5@1.10.7+cxx+threadsafe%${gcc_spec}"
|
||||
fftw_spec="fftw%${gcc_spec}"
|
||||
hdf5_spec='hdf5@1.10.7'
|
||||
|
||||
if (( $# != 1 )); then
|
||||
echo "usage: $(basename "$0") <env dir>" 1>&2
|
||||
@ -24,7 +18,7 @@ cd "${cwd}"
|
||||
|
||||
# General configuration ########################################################
|
||||
# build with 128 tasks
|
||||
echo 'config:
|
||||
echo 'config:
|
||||
build_jobs: 128
|
||||
build_stage:
|
||||
- $spack/var/spack/stage
|
||||
@ -44,23 +38,26 @@ rm external.yaml
|
||||
|
||||
# Base compilers ###############################################################
|
||||
# configure system base
|
||||
|
||||
spack env create base
|
||||
spack env activate base
|
||||
spack compiler find --scope site
|
||||
|
||||
# install GCC, CUDA
|
||||
spack add ${gcc_spec} ${cuda_spec}
|
||||
spack concretize
|
||||
spack env depfile -o Makefile.tmp
|
||||
make -j128 -f Makefile.tmp
|
||||
# install GCC, CUDA & LLVM
|
||||
spack install ${gcc_spec} ${cuda_spec} llvm
|
||||
|
||||
spack load llvm
|
||||
spack compiler find --scope site
|
||||
spack unload llvm
|
||||
|
||||
spack load ${gcc_spec}
|
||||
spack compiler find --scope site
|
||||
spack unload ${gcc_spec}
|
||||
|
||||
# Manual compilation of OpenMPI & UCX ##########################################
|
||||
# set build directories
|
||||
mkdir -p "${dir}"/build
|
||||
cd "${dir}"/build
|
||||
|
||||
spack load ${gcc_spec} ${cuda_spec}
|
||||
|
||||
cuda_path=$(spack find --format "{prefix}" cuda)
|
||||
gdrcopy_path=/mnt/lustre/tursafs1/apps/gdrcopy/2.3.1
|
||||
|
||||
@ -127,8 +124,8 @@ mkdir build_gpu; cd build_gpu
|
||||
--with-cuda="${cuda_path}" --disable-getpwuid \
|
||||
--with-verbs --with-slurm --enable-mpi-fortran=all \
|
||||
--with-pmix=internal --with-libevent=internal
|
||||
make -j 128
|
||||
make install
|
||||
make -j 128
|
||||
make install
|
||||
cd ..
|
||||
|
||||
# openmpi cpu build
|
||||
@ -144,62 +141,60 @@ make -j 128
|
||||
make install
|
||||
cd "${dir}"
|
||||
|
||||
ucx_spec_gpu="ucx@1.12.0.GPU%${gcc_spec}"
|
||||
ucx_spec_cpu="ucx@1.12.0.CPU%${gcc_spec}"
|
||||
openmpi_spec_gpu="openmpi@4.1.1.GPU%${gcc_spec}"
|
||||
openmpi_spec_cpu="openmpi@4.1.1.CPU%${gcc_spec}"
|
||||
|
||||
# Add externals to spack
|
||||
echo "packages:
|
||||
ucx:
|
||||
externals:
|
||||
- spec: \"${ucx_spec_gpu}\"
|
||||
- spec: \"ucx@1.12.0.GPU%gcc@9.4.0\"
|
||||
prefix: ${dir}/prefix/ucx_gpu
|
||||
- spec: \"${ucx_spec_cpu}\"
|
||||
- spec: \"ucx@1.12.0.CPU%gcc@9.4.0\"
|
||||
prefix: ${dir}/prefix/ucx_cpu
|
||||
buildable: False
|
||||
openmpi:
|
||||
externals:
|
||||
- spec: \"${openmpi_spec_gpu}\"
|
||||
- spec: \"openmpi@4.1.1.GPU%gcc@9.4.0\"
|
||||
prefix: ${dir}/prefix/ompi_gpu
|
||||
- spec: \"${openmpi_spec_cpu}\"
|
||||
- spec: \"openmpi@4.1.1.CPU%gcc@9.4.0\"
|
||||
prefix: ${dir}/prefix/ompi_cpu
|
||||
buildable: False" > spack.yaml
|
||||
|
||||
spack config --scope site add -f spack.yaml
|
||||
rm spack.yaml
|
||||
spack env deactivate
|
||||
spack install ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
|
||||
spack install ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
|
||||
|
||||
cd "${cwd}"
|
||||
|
||||
# environments #################################################################
|
||||
dev_tools=("autoconf" "automake" "libtool" "jq" "git")
|
||||
ompi_gpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.GPU)
|
||||
ompi_cpu_hash=$(spack find --format "{hash}" openmpi@4.1.1.CPU)
|
||||
|
||||
spack env create grid-gpu
|
||||
spack env activate grid-gpu
|
||||
spack compiler find --scope site
|
||||
spack add ${gcc_spec} ${cuda_spec} ${ucx_spec_gpu} ${openmpi_spec_gpu}
|
||||
spack add ${hdf5_spec} ${fftw_spec}
|
||||
spack add openssl gmp mpfr c-lime "${dev_tools[@]}"
|
||||
spack concretize
|
||||
spack env depfile -o Makefile.tmp
|
||||
make -j128 -f Makefile.tmp
|
||||
spack add ${gcc_spec} ${cuda_spec} "${dev_tools[@]}"
|
||||
spack add ucx@1.12.0.GPU%gcc@9.4.0 openmpi@4.1.1.GPU%gcc@9.4.0
|
||||
spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_gpu_hash}"
|
||||
spack add fftw ^/"${ompi_gpu_hash}"
|
||||
spack add openssl gmp mpfr c-lime
|
||||
spack install
|
||||
spack env deactivate
|
||||
|
||||
spack env create grid-cpu
|
||||
spack env activate grid-cpu
|
||||
spack compiler find --scope site
|
||||
spack add ${gcc_spec} ${ucx_spec_cpu} ${openmpi_spec_cpu}
|
||||
spack add ${hdf5_spec} ${fftw_spec}
|
||||
spack add openssl gmp mpfr c-lime "${dev_tools[@]}"
|
||||
spack concretize
|
||||
spack env depfile -o Makefile.tmp
|
||||
make -j128 -f Makefile.tmp
|
||||
spack add llvm "${dev_tools[@]}"
|
||||
spack add ucx@1.12.0.CPU%gcc@9.4.0 openmpi@4.1.1.CPU%gcc@9.4.0
|
||||
spack add ${hdf5_spec}+cxx+threadsafe ^/"${ompi_cpu_hash}"
|
||||
spack add fftw ^/"${ompi_cpu_hash}"
|
||||
spack add openssl gmp mpfr c-lime
|
||||
spack install
|
||||
spack env deactivate
|
||||
|
||||
spack install jq git
|
||||
|
||||
# Final setup ##################################################################
|
||||
spack clean
|
||||
#spack gc -y # "spack gc" tends to get hung up for unknown reasons
|
||||
spack gc -y
|
||||
|
||||
# add more environment variables in module loading
|
||||
spack config --scope site add 'modules:prefix_inspections:lib:[LD_LIBRARY_PATH,LIBRARY_PATH]'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user