mirror of
https://github.com/paboyle/Grid.git
synced 2026-10-11 02:08:06 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 portelli
|
86a9cc8c27 |
No files matched your search
@@ -1,54 +0,0 @@
|
||||
name: Bug report
|
||||
description: Report a bug.
|
||||
title: "<insert title>"
|
||||
labels: [bug]
|
||||
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: >
|
||||
Thank you for taking the time to file a bug report.
|
||||
Please check that the code is pointing to the HEAD of develop
|
||||
or any commit in master which is tagged with a version number.
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: "Describe the issue:"
|
||||
description: >
|
||||
Describe the issue and any previous attempt to solve it.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: "Code example:"
|
||||
description: >
|
||||
If relevant, show how to reproduce the issue using a minimal working
|
||||
example.
|
||||
placeholder: |
|
||||
<< your code here >>
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: "Target platform:"
|
||||
description: >
|
||||
Give a description of the target platform (CPU, network, compiler).
|
||||
Please give the full CPU part description, using for example
|
||||
`cat /proc/cpuinfo | grep 'model name' | uniq` (Linux)
|
||||
or `sysctl machdep.cpu.brand_string` (macOS) and the full output
|
||||
the `--version` option of your compiler.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: "Configure options:"
|
||||
description: >
|
||||
Please give the exact configure command used and attach
|
||||
`config.log`, `grid.config.summary` and the output of `make V=1`.
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
@@ -1,7 +1,3 @@
|
||||
# Doxygen stuff
|
||||
html/*
|
||||
latex/*
|
||||
|
||||
# Compiled Object files #
|
||||
#########################
|
||||
*.slo
|
||||
@@ -92,7 +88,6 @@ Thumbs.db
|
||||
# build directory #
|
||||
###################
|
||||
build*/*
|
||||
Documentation/_build
|
||||
|
||||
# IDE related files #
|
||||
#####################
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
language: cpp
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- clang
|
||||
|
||||
matrix:
|
||||
include:
|
||||
- os: osx
|
||||
osx_image: xcode8.3
|
||||
compiler: clang
|
||||
env: PREC=single
|
||||
- os: osx
|
||||
osx_image: xcode8.3
|
||||
compiler: clang
|
||||
env: PREC=double
|
||||
|
||||
before_install:
|
||||
- export GRIDDIR=`pwd`
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]] && [ ! -e clang/bin ]; then wget $CLANG_LINK; tar -xf `basename $CLANG_LINK`; mkdir clang; mv clang+*/* clang/; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export PATH="${GRIDDIR}/clang/bin:${PATH}"; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc openssl; fi
|
||||
|
||||
install:
|
||||
- export CWD=`pwd`
|
||||
- echo $CWD
|
||||
- export CC=$CC$VERSION
|
||||
- export CXX=$CXX$VERSION
|
||||
- echo $PATH
|
||||
- which autoconf
|
||||
- autoconf --version
|
||||
- which automake
|
||||
- automake --version
|
||||
- which $CC
|
||||
- $CC --version
|
||||
- which $CXX
|
||||
- $CXX --version
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export LDFLAGS='-L/usr/local/lib'; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export EXTRACONF='--with-openssl=/usr/local/opt/openssl'; fi
|
||||
|
||||
script:
|
||||
- ./bootstrap.sh
|
||||
- mkdir build
|
||||
- cd build
|
||||
- mkdir lime
|
||||
- cd lime
|
||||
- mkdir build
|
||||
- cd build
|
||||
- wget http://usqcd-software.github.io/downloads/c-lime/lime-1.3.2.tar.gz
|
||||
- tar xf lime-1.3.2.tar.gz
|
||||
- cd lime-1.3.2
|
||||
- ./configure --prefix=$CWD/build/lime/install
|
||||
- make -j4
|
||||
- make install
|
||||
- cd $CWD/build
|
||||
- ../configure --enable-precision=$PREC --enable-simd=SSE4 --enable-comms=none --with-lime=$CWD/build/lime/install ${EXTRACONF}
|
||||
- make -j4
|
||||
- ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
|
||||
- make check
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,2 +0,0 @@
|
||||
|
||||
mpicxx -qmkl=parallel -fsycl BatchBlasBench.cc -o BatchBlasBench -DGRID_SYCL
|
||||
@@ -1,5 +0,0 @@
|
||||
CXX=hipcc
|
||||
MPICXX=mpicxx
|
||||
CXXFLAGS="-fPIC -I{$ROCM_PATH}/include/ -I${MPICH_DIR}/include -L/lib64 -I/opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3/include -DGRID_HIP"
|
||||
LDFLAGS="-L/lib64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -lamdhip64 -lhipblas -lrocblas -lmpi_gnu_123"
|
||||
hipcc $CXXFLAGS $LDFLAGS BatchBlasBench.cc -o BatchBlasBench
|
||||
@@ -1,2 +0,0 @@
|
||||
|
||||
mpicxx -qmkl=parallel -fsycl BatchBlasBench.cc -o BatchBlasBench -DGRID_SYCL
|
||||
@@ -1,98 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## What This Is
|
||||
|
||||
Grid is a data-parallel C++ library for lattice QCD. It provides SIMD-vectorised lattice containers, MPI-based domain decomposition, GPU acceleration (CUDA/HIP/SYCL), and a full suite of QCD algorithms including HMC.
|
||||
|
||||
## Build
|
||||
|
||||
Uses GNU Autotools. The bootstrap step only needs to run once (or after `configure.ac` changes).
|
||||
|
||||
```bash
|
||||
./bootstrap.sh # downloads Eigen 3.4.0, generates configure
|
||||
mkdir build && cd build
|
||||
../configure [options]
|
||||
make -j$(nproc)
|
||||
make check # run root-level tests
|
||||
make install
|
||||
```
|
||||
|
||||
Key configure options:
|
||||
|
||||
| Option | Common values |
|
||||
|--------|---------------|
|
||||
| `--enable-simd=` | `AVX2`, `AVX512`, `KNL`, `A64FX`, `NEONv8`, `GPU` |
|
||||
| `--enable-comms=` | `mpi-auto`, `mpi3-auto`, `none` |
|
||||
| `--enable-accelerator=` | `cuda`, `hip`, `sycl` |
|
||||
| `--enable-shm=` | `shmopen`, `hugetlbfs`, `nvlink` |
|
||||
| `--enable-Nc=` | `3` (default), `2`, `4`, `5` |
|
||||
| `--with-gmp=`, `--with-mpfr=`, `--with-fftw=`, `--with-lime=` | paths to libs |
|
||||
| `--enable-hdf5`, `--enable-mkl`, `--enable-lapack` | optional features |
|
||||
|
||||
Platform recipes from `README.md`:
|
||||
- **KNL**: `--enable-simd=KNL --enable-comms=mpi3-auto --enable-mkl`
|
||||
- **Skylake/Haswell**: `--enable-simd=AVX512` or `AVX2` + `--enable-comms=mpi3-auto`
|
||||
- **AMD EPYC**: `--enable-simd=AVX2 --enable-comms=mpi3`
|
||||
- **A64FX (Fugaku)**: `--enable-simd=A64FX --enable-comms=mpi3 --enable-shm=shmget` (see `SVE_README.txt`)
|
||||
|
||||
Required external libs: GMP, MPFR, OpenSSL, zlib.
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# From build directory
|
||||
make check # root-level tests (Test_simd, Test_cshift, etc.)
|
||||
make -C tests/<subdir> tests # build tests in a subdirectory
|
||||
./tests/core/Test_simd # run a single test binary directly
|
||||
```
|
||||
|
||||
Test subdirectories and their focus: `core` (SIMD, stencil, comms), `solver` (CG, GMRES, eigensolvers), `hmc` (MD integrators), `forces` (fermion forces), `lanczos`, `IO`, `smearing`, `sp2n`, `debug`.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Layer stack (bottom to top)
|
||||
|
||||
1. **SIMD layer** (`Grid/simd/`) — platform-specific intrinsics wrapped into `vRealF`, `vComplexD`, etc. The SIMD width and layout are compile-time constants controlled by `--enable-simd`.
|
||||
|
||||
2. **Tensor layer** (`Grid/tensors/`) — Lorentz/colour/spin tensor algebra built on top of SIMD types. `iMatrix`, `iVector`, `iScalar` templates compose into QCD types like `ColourMatrix`, `SpinColourVector`.
|
||||
|
||||
3. **Lattice layer** (`Grid/lattice/`) — `Lattice<T>` container: a site-local tensor replicated across a distributed Cartesian grid. All arithmetic is site-parallel and expression-template-fused.
|
||||
|
||||
4. **Cartesian/comms layer** (`Grid/cartesian/`, `Grid/communicator/`) — `GridCartesian` holds the MPI topology and local/global geometry. `Grid/cshift/` implements nearest-neighbour halo exchange; `Grid/stencil/` is the optimised multi-hop stencil used by Dirac operators.
|
||||
|
||||
5. **Algorithm layer** (`Grid/algorithms/`) — iterative solvers (CG, GMRES, BiCGSTAB, mixed-precision), eigensolvers (Lanczos, LAPACK), FFT, smearing.
|
||||
|
||||
6. **QCD layer** (`Grid/qcd/`) — gauge and fermion actions, HMC integrators, observables.
|
||||
|
||||
### QCD subsystem (`Grid/qcd/`)
|
||||
|
||||
- `action/fermion/` — Wilson, Clover, DWF (Mobius), Staggered, twisted-mass, G-parity variants
|
||||
- `action/gauge/` — Wilson gauge, Symanzik, Iwasaki, DBW2, plaquette+rect
|
||||
- `representations/` — Fundamental, Adjoint, Two-index, Sp(2n)
|
||||
- `hmc/` — Leapfrog, OMF2/OMF4 integrators; pseudofermion refreshment; Metropolis accept/reject
|
||||
- `smearing/` — APE, Stout, HEX, gradient flow
|
||||
- `observables/` — Polyakov loop, plaquette, topological charge
|
||||
|
||||
### GPU acceleration
|
||||
|
||||
GPU support is injected via macros (`accelerator_for`, `accelerator_for2dNB`). The `Grid/simd/` SIMD types map to scalar on GPU device code; host code paths remain vectorised. Unified virtual memory is on by default (`--enable-unified=yes`); device-aware MPI (`--enable-accelerator-aware-mpi`) avoids device→host copies on transfers.
|
||||
|
||||
### Memory and I/O
|
||||
|
||||
- `Grid/allocator/` — aligned/NUMA-aware allocators; caching allocator via `--enable-alloc-cache`
|
||||
- `Grid/parallelIO/` — distributed parallel reader/writer for ILDG (via LIME), SciDAC, and native binary formats
|
||||
- `Grid/serialisation/` — text, binary, HDF5, XML/JSON serialisation of arbitrary Grid objects
|
||||
|
||||
### HMC applications
|
||||
|
||||
`HMC/` contains production-ready HMC driver programmes (e.g. `Mobius2p1f.cc`, `DWF_plus_DSDR_nf2plus1_Shamir_Gparity.cc`). These are built separately from the library tests.
|
||||
|
||||
## Key Conventions
|
||||
|
||||
- **C++17** is required throughout.
|
||||
- Template structure: most classes are templated on `<_FImpl>` (fermion impl) or `<Gimpl>` (gauge impl), which encode the representation and precision. Instantiation is controlled by `--enable-fermion-instantiations`.
|
||||
- The `RealD`/`RealF`/`ComplexD`/`ComplexF` typedefs are used everywhere; avoid raw `double`/`float`.
|
||||
- Logging uses `Grid_log`, `Grid_error` macros (from `Grid/log/`); performance-critical paths use the `GRID_TRACE` / timer macros from `Grid/perfmon/`.
|
||||
- Reductions across MPI ranks go through `GridBase::GlobalSum` / `GlobalMax`; never reduce with bare MPI calls inside library code.
|
||||
+3
-15
@@ -37,31 +37,19 @@ directory
|
||||
#endif
|
||||
|
||||
//disables and intel compiler specific warning (in json.hpp)
|
||||
#ifdef __ICC
|
||||
#pragma warning disable 488
|
||||
#endif
|
||||
|
||||
#ifdef __NVCC__
|
||||
//disables nvcc specific warning in json.hpp
|
||||
#pragma clang diagnostic ignored "-Wdeprecated-register"
|
||||
|
||||
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__
|
||||
//disables nvcc specific warning in json.hpp
|
||||
#pragma nv_diag_suppress unsigned_compare_with_zero
|
||||
#pragma nv_diag_suppress cast_to_qualified_type
|
||||
//disables nvcc specific warning in many files
|
||||
#pragma nv_diag_suppress esa_on_defaulted_function_ignored
|
||||
#pragma nv_diag_suppress declared_but_not_referenced
|
||||
#pragma nv_diag_suppress extra_semicolon
|
||||
#else
|
||||
//disables nvcc specific warning in json.hpp
|
||||
#pragma diag_suppress unsigned_compare_with_zero
|
||||
#pragma diag_suppress cast_to_qualified_type
|
||||
#pragma diag_suppress declared_but_not_referenced
|
||||
|
||||
//disables nvcc specific warning in many files
|
||||
#pragma diag_suppress esa_on_defaulted_function_ignored
|
||||
#pragma diag_suppress extra_semicolon
|
||||
#endif
|
||||
|
||||
//Eigen only
|
||||
#endif
|
||||
|
||||
// Disable vectorisation in Eigen on the Power8/9 and PowerPC
|
||||
|
||||
+3
-5
@@ -44,13 +44,12 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
#include <Grid/GridStd.h>
|
||||
#include <Grid/threads/Pragmas.h>
|
||||
#include <Grid/perfmon/Timer.h>
|
||||
//#include <Grid/perfmon/PerfCount.h>
|
||||
#include <Grid/perfmon/PerfCount.h>
|
||||
#include <Grid/util/Util.h>
|
||||
#include <Grid/log/Log.h>
|
||||
#include <Grid/perfmon/Tracing.h>
|
||||
#include <Grid/allocator/Allocator.h>
|
||||
#include <Grid/allocator/AlignedAllocator.h>
|
||||
#include <Grid/simd/Simd.h>
|
||||
#include <Grid/threads/ThreadReduction.h>
|
||||
#include <Grid/threads/Threads.h>
|
||||
#include <Grid/serialisation/Serialisation.h>
|
||||
#include <Grid/util/Sha.h>
|
||||
#include <Grid/communicator/Communicator.h>
|
||||
@@ -59,7 +58,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
#include <Grid/lattice/Lattice.h>
|
||||
#include <Grid/cshift/Cshift.h>
|
||||
#include <Grid/stencil/Stencil.h>
|
||||
#include <Grid/stencil/GeneralLocalStencil.h>
|
||||
#include <Grid/parallelIO/BinaryIO.h>
|
||||
#include <Grid/algorithms/Algorithms.h>
|
||||
NAMESPACE_CHECK(GridCore)
|
||||
|
||||
@@ -36,7 +36,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
#include <Grid/GridCore.h>
|
||||
#include <Grid/qcd/QCD.h>
|
||||
#include <Grid/qcd/spin/Spin.h>
|
||||
#include <Grid/qcd/gparity/Gparity.h>
|
||||
#include <Grid/qcd/utils/Utils.h>
|
||||
#include <Grid/qcd/representations/Representations.h>
|
||||
NAMESPACE_CHECK(GridQCDCore);
|
||||
|
||||
+4
-44
@@ -1,20 +1,11 @@
|
||||
#ifndef GRID_STD_H
|
||||
#define GRID_STD_H
|
||||
|
||||
///////////////////
|
||||
// Grid config
|
||||
///////////////////
|
||||
#include "Config.h"
|
||||
|
||||
///////////////////
|
||||
// Std C++ dependencies
|
||||
///////////////////
|
||||
#define _NBACKTRACE (256)
|
||||
extern void * Grid_backtrace_buffer[_NBACKTRACE];
|
||||
|
||||
#include <cassert>
|
||||
#include <complex>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <string>
|
||||
@@ -23,48 +14,17 @@ extern void * Grid_backtrace_buffer[_NBACKTRACE];
|
||||
#include <random>
|
||||
#include <functional>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <strings.h>
|
||||
#include <stdio.h>
|
||||
#include <signal.h>
|
||||
#include <ctime>
|
||||
#include <sys/time.h>
|
||||
#include <chrono>
|
||||
#include <zlib.h>
|
||||
#ifdef HAVE_EXECINFO_H
|
||||
#include <execinfo.h>
|
||||
#endif
|
||||
|
||||
void GridAbort(void);
|
||||
///////////////////
|
||||
// Grid config
|
||||
///////////////////
|
||||
#include "Config.h"
|
||||
|
||||
#define ASSLOG(A) ::write(STDERR_FILENO,A,::strlen(A));
|
||||
#ifdef HAVE_EXECINFO_H
|
||||
#define GRID_ASSERT(b) if(!(b)) { \
|
||||
fflush(stdout); \
|
||||
ASSLOG(" GRID_ASSERT failure: "); \
|
||||
ASSLOG(__FILE__); \
|
||||
ASSLOG(" : "); \
|
||||
ASSLOG(#b); \
|
||||
ASSLOG(" : "); \
|
||||
int symbols = backtrace(Grid_backtrace_buffer,_NBACKTRACE); \
|
||||
backtrace_symbols_fd(Grid_backtrace_buffer,symbols,STDERR_FILENO); \
|
||||
GridAbort(); \
|
||||
};
|
||||
#else
|
||||
#define GRID_ASSERT(b) if(!(b)) { \
|
||||
ASSLOG(" GRID_ASSERT failure: "); \
|
||||
ASSLOG(__FILE__); \
|
||||
ASSLOG(" : "); \
|
||||
ASSLOG(#b); \
|
||||
ASSLOG(" : "); \
|
||||
GridAbort(); \
|
||||
};
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef TOFU
|
||||
#undef GRID_COMMS_THREADS
|
||||
#endif
|
||||
#endif /* GRID_STD_H */
|
||||
+3
-20
@@ -14,36 +14,25 @@
|
||||
/* NVCC save and restore compile environment*/
|
||||
#ifdef __NVCC__
|
||||
#pragma push
|
||||
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__
|
||||
#pragma nv_diag_suppress code_is_unreachable
|
||||
#else
|
||||
#pragma diag_suppress code_is_unreachable
|
||||
#endif
|
||||
#pragma push_macro("__CUDA_ARCH__")
|
||||
#pragma push_macro("__NVCC__")
|
||||
#pragma push_macro("__CUDACC__")
|
||||
#undef __CUDA_ARCH__
|
||||
#undef __NVCC__
|
||||
#undef __CUDACC__
|
||||
#undef __CUDA_ARCH__
|
||||
#define __NVCC__REDEFINE__
|
||||
#endif
|
||||
|
||||
/* SYCL save and restore compile environment*/
|
||||
#ifdef GRID_SYCL
|
||||
#ifdef __SYCL_DEVICE_ONLY__
|
||||
#pragma push
|
||||
#pragma push_macro("__SYCL_DEVICE_ONLY__")
|
||||
#undef __SYCL_DEVICE_ONLY__
|
||||
#define EIGEN_DONT_VECTORIZE
|
||||
#undef EIGEN_USE_SYCL
|
||||
#define __SYCL__REDEFINE__
|
||||
#define EIGEN_DONT_VECTORIZE
|
||||
#endif
|
||||
|
||||
/* HIP save and restore compile environment*/
|
||||
#ifdef GRID_HIP
|
||||
#pragma push
|
||||
#pragma push_macro("__HIP_DEVICE_COMPILE__")
|
||||
#endif
|
||||
#define EIGEN_NO_HIP
|
||||
|
||||
#include <Grid/Eigen/Dense>
|
||||
#include <Grid/Eigen/unsupported/CXX11/Tensor>
|
||||
@@ -62,12 +51,6 @@
|
||||
#pragma pop
|
||||
#endif
|
||||
|
||||
/*HIP restore*/
|
||||
#ifdef __HIP__REDEFINE__
|
||||
#pragma pop_macro("__HIP_DEVICE_COMPILE__")
|
||||
#pragma pop
|
||||
#endif
|
||||
|
||||
#if defined __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
+2
-24
@@ -21,8 +21,7 @@ if BUILD_HDF5
|
||||
extra_headers+=serialisation/Hdf5Type.h
|
||||
endif
|
||||
|
||||
|
||||
all: version-cache Version.h
|
||||
all: version-cache
|
||||
|
||||
version-cache:
|
||||
@if [ `git status --porcelain | grep -v '??' | wc -l` -gt 0 ]; then\
|
||||
@@ -43,7 +42,7 @@ version-cache:
|
||||
fi;\
|
||||
rm -f vertmp
|
||||
|
||||
Version.h: version-cache
|
||||
Version.h:
|
||||
cp version-cache Version.h
|
||||
|
||||
.PHONY: version-cache
|
||||
@@ -54,27 +53,6 @@ Version.h: version-cache
|
||||
include Make.inc
|
||||
include Eigen.inc
|
||||
|
||||
if BUILD_FERMION_INSTANTIATIONS
|
||||
extra_sources+=$(WILS_FERMION_FILES)
|
||||
extra_sources+=$(STAG_FERMION_FILES)
|
||||
if BUILD_ZMOBIUS
|
||||
extra_sources+=$(ZWILS_FERMION_FILES)
|
||||
endif
|
||||
if BUILD_GPARITY
|
||||
extra_sources+=$(GP_FERMION_FILES)
|
||||
endif
|
||||
if BUILD_FERMION_REPS
|
||||
extra_sources+=$(ADJ_FERMION_FILES)
|
||||
extra_sources+=$(TWOIND_FERMION_FILES)
|
||||
endif
|
||||
if BUILD_SP
|
||||
extra_sources+=$(SP_FERMION_FILES)
|
||||
if BUILD_FERMION_REPS
|
||||
extra_sources+=$(SP_TWOIND_FERMION_FILES)
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
lib_LIBRARIES = libGrid.a
|
||||
|
||||
CCFILES += $(extra_sources)
|
||||
|
||||
@@ -29,7 +29,6 @@ directory
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
#include <exception>
|
||||
#include <cassert>
|
||||
|
||||
#define NAMESPACE_BEGIN(A) namespace A {
|
||||
@@ -37,7 +36,3 @@ directory
|
||||
#define GRID_NAMESPACE_BEGIN NAMESPACE_BEGIN(Grid)
|
||||
#define GRID_NAMESPACE_END NAMESPACE_END(Grid)
|
||||
#define NAMESPACE_CHECK(x) struct namespaceTEST##x {}; static_assert(std::is_same<namespaceTEST##x, ::namespaceTEST##x>::value,"Not in :: at" );
|
||||
|
||||
#define EXCEPTION_CHECK_BEGIN(A) try {
|
||||
#define EXCEPTION_CHECK_END(A) } catch ( std::exception e ) { BACKTRACEFP(stderr); std::cerr << __PRETTY_FUNCTION__ << " : " <<__LINE__<< " Caught exception "<<e.what()<<std::endl; throw; }
|
||||
|
||||
@@ -29,14 +29,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
#ifndef GRID_ALGORITHMS_H
|
||||
#define GRID_ALGORITHMS_H
|
||||
|
||||
NAMESPACE_CHECK(blas);
|
||||
#include <Grid/algorithms/blas/BatchedBlas.h>
|
||||
|
||||
NAMESPACE_CHECK(algorithms);
|
||||
#include <Grid/algorithms/SparseMatrix.h>
|
||||
#include <Grid/algorithms/LinearOperator.h>
|
||||
#include <Grid/algorithms/Preconditioner.h>
|
||||
NAMESPACE_CHECK(SparseMatrix);
|
||||
|
||||
#include <Grid/algorithms/approx/Zolotarev.h>
|
||||
#include <Grid/algorithms/approx/Chebyshev.h>
|
||||
@@ -46,26 +41,15 @@ NAMESPACE_CHECK(SparseMatrix);
|
||||
#include <Grid/algorithms/approx/Forecast.h>
|
||||
#include <Grid/algorithms/approx/RemezGeneral.h>
|
||||
#include <Grid/algorithms/approx/ZMobius.h>
|
||||
NAMESPACE_CHECK(approx);
|
||||
#include <Grid/algorithms/deflation/Deflation.h>
|
||||
#include <Grid/algorithms/deflation/MultiRHSBlockProject.h>
|
||||
#include <Grid/algorithms/deflation/MultiRHSDeflation.h>
|
||||
#include <Grid/algorithms/deflation/MultiRHSBlockCGLinalg.h>
|
||||
// Not really deflation, but useful
|
||||
#include <Grid/algorithms/blas/MomentumProject.h>
|
||||
#include <Grid/algorithms/blas/A2ASpatialSum.h>
|
||||
NAMESPACE_CHECK(deflation);
|
||||
|
||||
#include <Grid/algorithms/iterative/Deflation.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradient.h>
|
||||
NAMESPACE_CHECK(ConjGrad);
|
||||
#include <Grid/algorithms/iterative/BiCGSTAB.h>
|
||||
NAMESPACE_CHECK(BiCGSTAB);
|
||||
#include <Grid/algorithms/iterative/ConjugateResidual.h>
|
||||
#include <Grid/algorithms/iterative/NormalEquations.h>
|
||||
#include <Grid/algorithms/iterative/SchurRedBlack.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMultiShift.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMixedPrec.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMultiShiftMixedPrec.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientMixedPrecBatched.h>
|
||||
#include <Grid/algorithms/iterative/BiCGSTABMixedPrec.h>
|
||||
#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
|
||||
#include <Grid/algorithms/iterative/ConjugateGradientReliableUpdate.h>
|
||||
@@ -77,11 +61,8 @@ NAMESPACE_CHECK(BiCGSTAB);
|
||||
#include <Grid/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h>
|
||||
#include <Grid/algorithms/iterative/ImplicitlyRestartedLanczos.h>
|
||||
#include <Grid/algorithms/iterative/PowerMethod.h>
|
||||
#include <Grid/algorithms/iterative/AdefGeneric.h>
|
||||
#include <Grid/algorithms/iterative/AdefMrhs.h>
|
||||
NAMESPACE_CHECK(PowerMethod);
|
||||
#include <Grid/algorithms/multigrid/MultiGrid.h>
|
||||
NAMESPACE_CHECK(multigrid);
|
||||
|
||||
#include <Grid/algorithms/CoarsenedMatrix.h>
|
||||
#include <Grid/algorithms/FFT.h>
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,960 @@
|
||||
// blockZaxpy in bockPromote - 3s, 5%
|
||||
// noncoalesced linalg in Preconditionoer ~ 3s 5%
|
||||
// Lancos tuning or replace 10-20s ~ 25%, open ended
|
||||
// setup tuning 5s ~ 8%
|
||||
// -- e.g. ordermin, orderstep tunables.
|
||||
// MdagM path without norm in LinOp code. few seconds
|
||||
|
||||
// Mdir calc blocking kernels
|
||||
// Fuse kernels in blockMaskedInnerProduct
|
||||
// preallocate Vectors in Cayley 5D ~ few percent few seconds
|
||||
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/CoarsenedMatrix.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_ALGORITHM_COARSENED_MATRIX_H
|
||||
#define GRID_ALGORITHM_COARSENED_MATRIX_H
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class vobj,class CComplex>
|
||||
inline void blockMaskedInnerProduct(Lattice<CComplex> &CoarseInner,
|
||||
const Lattice<decltype(innerProduct(vobj(),vobj()))> &FineMask,
|
||||
const Lattice<vobj> &fineX,
|
||||
const Lattice<vobj> &fineY)
|
||||
{
|
||||
typedef decltype(innerProduct(vobj(),vobj())) dotp;
|
||||
|
||||
GridBase *coarse(CoarseInner.Grid());
|
||||
GridBase *fine (fineX.Grid());
|
||||
|
||||
Lattice<dotp> fine_inner(fine); fine_inner.Checkerboard() = fineX.Checkerboard();
|
||||
Lattice<dotp> fine_inner_msk(fine);
|
||||
|
||||
// Multiply could be fused with innerProduct
|
||||
// Single block sum kernel could do both masks.
|
||||
fine_inner = localInnerProduct(fineX,fineY);
|
||||
mult(fine_inner_msk, fine_inner,FineMask);
|
||||
blockSum(CoarseInner,fine_inner_msk);
|
||||
}
|
||||
|
||||
|
||||
class Geometry {
|
||||
public:
|
||||
int npoint;
|
||||
std::vector<int> directions ;
|
||||
std::vector<int> displacements;
|
||||
|
||||
Geometry(int _d) {
|
||||
|
||||
int base = (_d==5) ? 1:0;
|
||||
|
||||
// make coarse grid stencil for 4d , not 5d
|
||||
if ( _d==5 ) _d=4;
|
||||
|
||||
npoint = 2*_d+1;
|
||||
directions.resize(npoint);
|
||||
displacements.resize(npoint);
|
||||
for(int d=0;d<_d;d++){
|
||||
directions[d ] = d+base;
|
||||
directions[d+_d] = d+base;
|
||||
displacements[d ] = +1;
|
||||
displacements[d+_d]= -1;
|
||||
}
|
||||
directions [2*_d]=0;
|
||||
displacements[2*_d]=0;
|
||||
|
||||
//// report back
|
||||
std::cout<<GridLogMessage<<"directions :";
|
||||
for(int d=0;d<npoint;d++) std::cout<< directions[d]<< " ";
|
||||
std::cout<<std::endl;
|
||||
std::cout<<GridLogMessage<<"displacements :";
|
||||
for(int d=0;d<npoint;d++) std::cout<< displacements[d]<< " ";
|
||||
std::cout<<std::endl;
|
||||
}
|
||||
|
||||
/*
|
||||
// Original cleaner code
|
||||
Geometry(int _d) : dimension(_d), npoint(2*_d+1), directions(npoint), displacements(npoint) {
|
||||
for(int d=0;d<dimension;d++){
|
||||
directions[2*d ] = d;
|
||||
directions[2*d+1] = d;
|
||||
displacements[2*d ] = +1;
|
||||
displacements[2*d+1] = -1;
|
||||
}
|
||||
directions [2*dimension]=0;
|
||||
displacements[2*dimension]=0;
|
||||
}
|
||||
std::vector<int> GetDelta(int point) {
|
||||
std::vector<int> delta(dimension,0);
|
||||
delta[directions[point]] = displacements[point];
|
||||
return delta;
|
||||
};
|
||||
*/
|
||||
|
||||
};
|
||||
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class Aggregation {
|
||||
public:
|
||||
typedef iVector<CComplex,nbasis > siteVector;
|
||||
typedef Lattice<siteVector> CoarseVector;
|
||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
||||
|
||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj > FineField;
|
||||
|
||||
GridBase *CoarseGrid;
|
||||
GridBase *FineGrid;
|
||||
std::vector<Lattice<Fobj> > subspace;
|
||||
int checkerboard;
|
||||
int Checkerboard(void){return checkerboard;}
|
||||
Aggregation(GridBase *_CoarseGrid,GridBase *_FineGrid,int _checkerboard) :
|
||||
CoarseGrid(_CoarseGrid),
|
||||
FineGrid(_FineGrid),
|
||||
subspace(nbasis,_FineGrid),
|
||||
checkerboard(_checkerboard)
|
||||
{
|
||||
};
|
||||
|
||||
void Orthogonalise(void){
|
||||
CoarseScalar InnerProd(CoarseGrid);
|
||||
std::cout << GridLogMessage <<" Block Gramm-Schmidt pass 1"<<std::endl;
|
||||
blockOrthogonalise(InnerProd,subspace);
|
||||
// std::cout << GridLogMessage <<" Block Gramm-Schmidt pass 2"<<std::endl; // Really have to do twice? Yuck
|
||||
// blockOrthogonalise(InnerProd,subspace);
|
||||
// std::cout << GridLogMessage <<" Gramm-Schmidt checking orthogonality"<<std::endl;
|
||||
// CheckOrthogonal();
|
||||
}
|
||||
void CheckOrthogonal(void){
|
||||
CoarseVector iProj(CoarseGrid);
|
||||
CoarseVector eProj(CoarseGrid);
|
||||
for(int i=0;i<nbasis;i++){
|
||||
blockProject(iProj,subspace[i],subspace);
|
||||
eProj=Zero();
|
||||
accelerator_for(ss, CoarseGrid->oSites(),1,{
|
||||
eProj[ss](i)=CComplex(1.0);
|
||||
});
|
||||
eProj=eProj - iProj;
|
||||
std::cout<<GridLogMessage<<"Orthog check error "<<i<<" " << norm2(eProj)<<std::endl;
|
||||
}
|
||||
std::cout<<GridLogMessage <<"CheckOrthog done"<<std::endl;
|
||||
}
|
||||
void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
|
||||
blockProject(CoarseVec,FineVec,subspace);
|
||||
}
|
||||
void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
|
||||
FineVec.Checkerboard() = subspace[0].Checkerboard();
|
||||
blockPromote(CoarseVec,FineVec,subspace);
|
||||
}
|
||||
void CreateSubspaceRandom(GridParallelRNG &RNG){
|
||||
for(int i=0;i<nbasis;i++){
|
||||
random(RNG,subspace[i]);
|
||||
}
|
||||
}
|
||||
|
||||
virtual void CreateSubspace(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,int nn=nbasis) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
ConjugateGradient<FineField> CG(1.0e-2,100,false);
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
|
||||
for(int b=0;b<nn;b++){
|
||||
|
||||
subspace[b] = Zero();
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise ["<<b<<"] <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
for(int i=0;i<1;i++){
|
||||
|
||||
CG(hermop,noise,subspace[b]);
|
||||
|
||||
noise = subspace[b];
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
}
|
||||
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "filtered["<<b<<"] <f|MdagM|f> "<<norm2(Mn)<<std::endl;
|
||||
subspace[b] = noise;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// World of possibilities here. But have tried quite a lot of experiments (250+ jobs run on Summit)
|
||||
// and this is the best I found
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if 1
|
||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
double lo,
|
||||
int orderfilter,
|
||||
int ordermin,
|
||||
int orderstep,
|
||||
double filterlo
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
int b =0;
|
||||
{
|
||||
// Filter
|
||||
Chebyshev<FineField> Cheb(lo,hi,orderfilter);
|
||||
Cheb(hermop,noise,Mn);
|
||||
// normalise
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
b++;
|
||||
}
|
||||
|
||||
// Generate a full sequence of Chebyshevs
|
||||
{
|
||||
lo=filterlo;
|
||||
noise=Mn;
|
||||
|
||||
FineField T0(FineGrid); T0 = noise;
|
||||
FineField T1(FineGrid);
|
||||
FineField T2(FineGrid);
|
||||
FineField y(FineGrid);
|
||||
|
||||
FineField *Tnm = &T0;
|
||||
FineField *Tn = &T1;
|
||||
FineField *Tnp = &T2;
|
||||
|
||||
// Tn=T1 = (xscale M + mscale)in
|
||||
RealD xscale = 2.0/(hi-lo);
|
||||
RealD mscale = -(hi+lo)/(hi-lo);
|
||||
hermop.HermOp(T0,y);
|
||||
T1=y*xscale+noise*mscale;
|
||||
|
||||
for(int n=2;n<=ordermin+orderstep*(nn-2);n++){
|
||||
|
||||
hermop.HermOp(*Tn,y);
|
||||
|
||||
auto y_v = y.View();
|
||||
auto Tn_v = Tn->View();
|
||||
auto Tnp_v = Tnp->View();
|
||||
auto Tnm_v = Tnm->View();
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
accelerator_forNB(ss, FineGrid->oSites(), Nsimd, {
|
||||
coalescedWrite(y_v[ss],xscale*y_v(ss)+mscale*Tn_v(ss));
|
||||
coalescedWrite(Tnp_v[ss],2.0*y_v(ss)-Tnm_v(ss));
|
||||
});
|
||||
|
||||
// Possible more fine grained control is needed than a linear sweep,
|
||||
// but huge productivity gain if this is simple algorithm and not a tunable
|
||||
int m =1;
|
||||
if ( n>=ordermin ) m=n-ordermin;
|
||||
if ( (m%orderstep)==0 ) {
|
||||
Mn=*Tnp;
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << n<<" filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
b++;
|
||||
}
|
||||
|
||||
// Cycle pointers to avoid copies
|
||||
FineField *swizzle = Tnm;
|
||||
Tnm =Tn;
|
||||
Tn =Tnp;
|
||||
Tnp =swizzle;
|
||||
|
||||
}
|
||||
}
|
||||
assert(b==nn);
|
||||
}
|
||||
#endif
|
||||
#if 0
|
||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
double lo,
|
||||
int orderfilter,
|
||||
int ordermin,
|
||||
int orderstep,
|
||||
double filterlo
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
FineField combined(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
int b =0;
|
||||
#define FILTERb(llo,hhi,oorder) \
|
||||
{ \
|
||||
Chebyshev<FineField> Cheb(llo,hhi,oorder); \
|
||||
Cheb(hermop,noise,Mn); \
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale; \
|
||||
subspace[b] = Mn; \
|
||||
hermop.Op(Mn,tmp); \
|
||||
std::cout<<GridLogMessage << oorder<< " Cheb filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl; \
|
||||
b++; \
|
||||
}
|
||||
|
||||
// JacobiPolynomial<FineField> Cheb(0.002,60.0,1500,-0.5,3.5); \
|
||||
|
||||
RealD alpha=-0.8;
|
||||
RealD beta =-0.8;
|
||||
#define FILTER(llo,hhi,oorder) \
|
||||
{ \
|
||||
Chebyshev<FineField> Cheb(llo,hhi,oorder); \
|
||||
/* JacobiPolynomial<FineField> Cheb(0.0,60.0,oorder,alpha,beta);*/\
|
||||
Cheb(hermop,noise,Mn); \
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale; \
|
||||
subspace[b] = Mn; \
|
||||
hermop.Op(Mn,tmp); \
|
||||
std::cout<<GridLogMessage << oorder<< "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl; \
|
||||
b++; \
|
||||
}
|
||||
|
||||
#define FILTERc(llo,hhi,oorder) \
|
||||
{ \
|
||||
Chebyshev<FineField> Cheb(llo,hhi,oorder); \
|
||||
Cheb(hermop,noise,combined); \
|
||||
}
|
||||
|
||||
double node = 0.000;
|
||||
FILTERb(lo,hi,orderfilter);// 0
|
||||
// FILTERc(node,hi,51);// 0
|
||||
noise = Mn;
|
||||
int base = 0;
|
||||
int mult = 100;
|
||||
FILTER(node,hi,base+1*mult);
|
||||
FILTER(node,hi,base+2*mult);
|
||||
FILTER(node,hi,base+3*mult);
|
||||
FILTER(node,hi,base+4*mult);
|
||||
FILTER(node,hi,base+5*mult);
|
||||
FILTER(node,hi,base+6*mult);
|
||||
FILTER(node,hi,base+7*mult);
|
||||
FILTER(node,hi,base+8*mult);
|
||||
FILTER(node,hi,base+9*mult);
|
||||
FILTER(node,hi,base+10*mult);
|
||||
FILTER(node,hi,base+11*mult);
|
||||
FILTER(node,hi,base+12*mult);
|
||||
FILTER(node,hi,base+13*mult);
|
||||
FILTER(node,hi,base+14*mult);
|
||||
FILTER(node,hi,base+15*mult);
|
||||
assert(b==nn);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if 0
|
||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
double lo,
|
||||
int orderfilter,
|
||||
int ordermin,
|
||||
int orderstep,
|
||||
double filterlo
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
FineField combined(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
int b =0;
|
||||
{
|
||||
Chebyshev<FineField> JacobiPoly(0.005,60.,1500);
|
||||
// JacobiPolynomial<FineField> JacobiPoly(0.002,60.0,1500,-0.5,3.5);
|
||||
//JacobiPolynomial<FineField> JacobiPoly(0.03,60.0,500,-0.5,3.5);
|
||||
// JacobiPolynomial<FineField> JacobiPoly(0.00,60.0,1000,-0.5,3.5);
|
||||
JacobiPoly(hermop,noise,Mn);
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
b++;
|
||||
// scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
||||
// subspace[b] = tmp; b++;
|
||||
// }
|
||||
}
|
||||
|
||||
#define FILTER(lambda) \
|
||||
{ \
|
||||
hermop.HermOp(subspace[0],tmp); \
|
||||
tmp = tmp - lambda *subspace[0]; \
|
||||
scale = std::pow(norm2(tmp),-0.5); \
|
||||
tmp=tmp*scale; \
|
||||
subspace[b] = tmp; \
|
||||
hermop.Op(subspace[b],tmp); \
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl; \
|
||||
b++; \
|
||||
}
|
||||
// scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
||||
// subspace[b] = tmp; b++;
|
||||
// }
|
||||
|
||||
FILTER(2.0e-5);
|
||||
FILTER(2.0e-4);
|
||||
FILTER(4.0e-4);
|
||||
FILTER(8.0e-4);
|
||||
FILTER(8.0e-4);
|
||||
|
||||
FILTER(2.0e-3);
|
||||
FILTER(3.0e-3);
|
||||
FILTER(4.0e-3);
|
||||
FILTER(5.0e-3);
|
||||
FILTER(6.0e-3);
|
||||
|
||||
FILTER(2.5e-3);
|
||||
FILTER(3.5e-3);
|
||||
FILTER(4.5e-3);
|
||||
FILTER(5.5e-3);
|
||||
FILTER(6.5e-3);
|
||||
|
||||
// FILTER(6.0e-5);//6
|
||||
// FILTER(7.0e-5);//8
|
||||
// FILTER(8.0e-5);//9
|
||||
// FILTER(9.0e-5);//3
|
||||
|
||||
/*
|
||||
// FILTER(1.0e-4);//10
|
||||
FILTER(2.0e-4);//11
|
||||
// FILTER(3.0e-4);//12
|
||||
// FILTER(4.0e-4);//13
|
||||
FILTER(5.0e-4);//14
|
||||
|
||||
FILTER(6.0e-3);//4
|
||||
FILTER(7.0e-4);//1
|
||||
FILTER(8.0e-4);//7
|
||||
FILTER(9.0e-4);//15
|
||||
FILTER(1.0e-3);//2
|
||||
|
||||
FILTER(2.0e-3);//2
|
||||
FILTER(3.0e-3);//2
|
||||
FILTER(4.0e-3);//2
|
||||
FILTER(5.0e-3);//2
|
||||
FILTER(6.0e-3);//2
|
||||
|
||||
FILTER(7.0e-3);//2
|
||||
FILTER(8.0e-3);//2
|
||||
FILTER(1.0e-2);//2
|
||||
*/
|
||||
std::cout << GridLogMessage <<"Jacobi filtering done" <<std::endl;
|
||||
assert(b==nn);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
};
|
||||
|
||||
// Fine Object == (per site) type of fine field
|
||||
// nbasis == number of deflation vectors
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class CoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
|
||||
typedef iVector<CComplex,nbasis > siteVector;
|
||||
typedef Lattice<CComplex > CoarseComplexField;
|
||||
typedef Lattice<siteVector> CoarseVector;
|
||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj > FineField;
|
||||
|
||||
////////////////////
|
||||
// Data members
|
||||
////////////////////
|
||||
Geometry geom;
|
||||
GridBase * _grid;
|
||||
int hermitian;
|
||||
|
||||
CartesianStencil<siteVector,siteVector,int> Stencil;
|
||||
|
||||
std::vector<CoarseMatrix> A;
|
||||
|
||||
///////////////////////
|
||||
// Interface
|
||||
///////////////////////
|
||||
GridBase * Grid(void) { return _grid; }; // this is all the linalg routines need to know
|
||||
|
||||
void M (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
conformable(_grid,in.Grid());
|
||||
conformable(in.Grid(),out.Grid());
|
||||
|
||||
SimpleCompressor<siteVector> compressor;
|
||||
|
||||
Stencil.HaloExchange(in,compressor);
|
||||
|
||||
auto in_v = in.View();
|
||||
auto out_v = out.View();
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
|
||||
Vector<Aview> AcceleratorViewContainer;
|
||||
for(int p=0;p<geom.npoint;p++) AcceleratorViewContainer.push_back(A[p].View());
|
||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
int osites=Grid()->oSites();
|
||||
|
||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
int lane=SIMTlane(Nsimd);
|
||||
for(int point=0;point<geom.npoint;point++){
|
||||
|
||||
SE=Stencil.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute,lane);
|
||||
} else {
|
||||
nbr = coalescedRead(Stencil.CommBuf()[SE->_offset],lane);
|
||||
}
|
||||
synchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res,lane);
|
||||
});
|
||||
};
|
||||
|
||||
void Mdag (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
if(hermitian) {
|
||||
// corresponds to Petrov-Galerkin coarsening
|
||||
return M(in,out);
|
||||
} else {
|
||||
// corresponds to Galerkin coarsening
|
||||
CoarseVector tmp(Grid());
|
||||
G5C(tmp, in);
|
||||
M(tmp, out);
|
||||
G5C(out, out);
|
||||
}
|
||||
};
|
||||
void MdirComms(const CoarseVector &in)
|
||||
{
|
||||
SimpleCompressor<siteVector> compressor;
|
||||
Stencil.HaloExchange(in,compressor);
|
||||
}
|
||||
void MdirCalc(const CoarseVector &in, CoarseVector &out, int point)
|
||||
{
|
||||
conformable(_grid,in.Grid());
|
||||
conformable(_grid,out.Grid());
|
||||
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
Vector<Aview> AcceleratorViewContainer;
|
||||
for(int p=0;p<geom.npoint;p++) AcceleratorViewContainer.push_back(A[p].View());
|
||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||
|
||||
auto out_v = out.View();
|
||||
auto in_v = in.View();
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
int lane=SIMTlane(Nsimd);
|
||||
SE=Stencil.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute,lane);
|
||||
} else {
|
||||
nbr = coalescedRead(Stencil.CommBuf()[SE->_offset],lane);
|
||||
}
|
||||
synchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res,lane);
|
||||
});
|
||||
#if 0
|
||||
accelerator_for(ss,Grid()->oSites(),1,{
|
||||
|
||||
siteVector res = Zero();
|
||||
siteVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
SE=Stencil.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local&&SE->_permute) {
|
||||
permute(nbr,in_v[SE->_offset],ptype);
|
||||
} else if(SE->_is_local) {
|
||||
nbr = in_v[SE->_offset];
|
||||
} else {
|
||||
nbr = Stencil.CommBuf()[SE->_offset];
|
||||
}
|
||||
synchronise();
|
||||
|
||||
res = res + Aview_p[point][ss]*nbr;
|
||||
|
||||
out_v[ss]=res;
|
||||
});
|
||||
#endif
|
||||
}
|
||||
void MdirAll(const CoarseVector &in,std::vector<CoarseVector> &out)
|
||||
{
|
||||
this->MdirComms(in);
|
||||
int ndir=geom.npoint-1;
|
||||
if ((out.size()!=ndir)&&(out.size()!=ndir+1)) {
|
||||
std::cout <<"MdirAll out size "<< out.size()<<std::endl;
|
||||
std::cout <<"MdirAll ndir "<< ndir<<std::endl;
|
||||
assert(0);
|
||||
}
|
||||
for(int p=0;p<ndir;p++){
|
||||
MdirCalc(in,out[p],p);
|
||||
}
|
||||
};
|
||||
void Mdir(const CoarseVector &in, CoarseVector &out, int dir, int disp){
|
||||
|
||||
this->MdirComms(in);
|
||||
|
||||
int ndim = in.Grid()->Nd();
|
||||
|
||||
//////////////
|
||||
// 4D action like wilson
|
||||
// 0+ => 0
|
||||
// 0- => 1
|
||||
// 1+ => 2
|
||||
// 1- => 3
|
||||
// etc..
|
||||
//////////////
|
||||
// 5D action like DWF
|
||||
// 1+ => 0
|
||||
// 1- => 1
|
||||
// 2+ => 2
|
||||
// 2- => 3
|
||||
// etc..
|
||||
auto point = [dir, disp, ndim](){
|
||||
if(dir == 0 and disp == 0)
|
||||
return 8;
|
||||
else if ( ndim==4 ) {
|
||||
return (4 * dir + 1 - disp) / 2;
|
||||
} else {
|
||||
return (4 * (dir-1) + 1 - disp) / 2;
|
||||
}
|
||||
}();
|
||||
|
||||
MdirCalc(in,out,point);
|
||||
|
||||
};
|
||||
|
||||
void Mdiag(const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
int point=geom.npoint-1;
|
||||
MdirCalc(in, out, point); // No comms
|
||||
};
|
||||
|
||||
|
||||
CoarsenedMatrix(GridCartesian &CoarseGrid, int hermitian_=0) :
|
||||
|
||||
_grid(&CoarseGrid),
|
||||
geom(CoarseGrid._ndimension),
|
||||
hermitian(hermitian_),
|
||||
Stencil(&CoarseGrid,geom.npoint,Even,geom.directions,geom.displacements,0),
|
||||
A(geom.npoint,&CoarseGrid)
|
||||
{
|
||||
};
|
||||
|
||||
void CoarsenOperator(GridBase *FineGrid,LinearOperatorBase<Lattice<Fobj> > &linop,
|
||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
||||
{
|
||||
typedef Lattice<typename Fobj::tensor_reduced> FineComplexField;
|
||||
typedef typename Fobj::scalar_type scalar_type;
|
||||
|
||||
FineComplexField one(FineGrid); one=scalar_type(1.0,0.0);
|
||||
FineComplexField zero(FineGrid); zero=scalar_type(0.0,0.0);
|
||||
|
||||
std::vector<FineComplexField> masks(geom.npoint,FineGrid);
|
||||
FineComplexField imask(FineGrid); // contributions from within this block
|
||||
FineComplexField omask(FineGrid); // contributions from outwith this block
|
||||
|
||||
FineComplexField evenmask(FineGrid);
|
||||
FineComplexField oddmask(FineGrid);
|
||||
|
||||
FineField phi(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
FineField zz(FineGrid); zz=Zero();
|
||||
FineField Mphi(FineGrid);
|
||||
FineField Mphie(FineGrid);
|
||||
FineField Mphio(FineGrid);
|
||||
std::vector<FineField> Mphi_p(geom.npoint,FineGrid);
|
||||
|
||||
Lattice<iScalar<vInteger> > coor (FineGrid);
|
||||
Lattice<iScalar<vInteger> > bcoor(FineGrid);
|
||||
Lattice<iScalar<vInteger> > bcb (FineGrid); bcb = Zero();
|
||||
|
||||
CoarseVector iProj(Grid());
|
||||
CoarseVector oProj(Grid());
|
||||
CoarseVector SelfProj(Grid());
|
||||
CoarseComplexField iZProj(Grid());
|
||||
CoarseComplexField oZProj(Grid());
|
||||
|
||||
CoarseScalar InnerProd(Grid());
|
||||
|
||||
// Orthogonalise the subblocks over the basis
|
||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
||||
|
||||
// Compute the matrix elements of linop between this orthonormal
|
||||
// set of vectors.
|
||||
int self_stencil=-1;
|
||||
for(int p=0;p<geom.npoint;p++)
|
||||
{
|
||||
int dir = geom.directions[p];
|
||||
int disp = geom.displacements[p];
|
||||
A[p]=Zero();
|
||||
if( geom.displacements[p]==0){
|
||||
self_stencil=p;
|
||||
}
|
||||
|
||||
Integer block=(FineGrid->_rdimensions[dir])/(Grid()->_rdimensions[dir]);
|
||||
|
||||
LatticeCoordinate(coor,dir);
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// Work out even and odd block checkerboarding for fast diagonal term
|
||||
///////////////////////////////////////////////////////
|
||||
if ( disp==1 ) {
|
||||
bcb = bcb + div(coor,block);
|
||||
}
|
||||
|
||||
if ( disp==0 ) {
|
||||
masks[p]= Zero();
|
||||
} else if ( disp==1 ) {
|
||||
masks[p] = where(mod(coor,block)==(block-1),one,zero);
|
||||
} else if ( disp==-1 ) {
|
||||
masks[p] = where(mod(coor,block)==(Integer)0,one,zero);
|
||||
}
|
||||
}
|
||||
evenmask = where(mod(bcb,2)==(Integer)0,one,zero);
|
||||
oddmask = one-evenmask;
|
||||
|
||||
assert(self_stencil!=-1);
|
||||
|
||||
for(int i=0;i<nbasis;i++){
|
||||
|
||||
phi=Subspace.subspace[i];
|
||||
|
||||
// std::cout << GridLogMessage<< "CoarsenMatrix vector "<<i << std::endl;
|
||||
linop.OpDirAll(phi,Mphi_p);
|
||||
linop.OpDiag (phi,Mphi_p[geom.npoint-1]);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
|
||||
Mphi = Mphi_p[p];
|
||||
|
||||
int dir = geom.directions[p];
|
||||
int disp = geom.displacements[p];
|
||||
|
||||
if ( (disp==-1) || (!hermitian ) ) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Pick out contributions coming from this cell and neighbour cell
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
omask = masks[p];
|
||||
imask = one-omask;
|
||||
|
||||
for(int j=0;j<nbasis;j++){
|
||||
|
||||
blockMaskedInnerProduct(oZProj,omask,Subspace.subspace[j],Mphi);
|
||||
|
||||
auto iZProj_v = iZProj.View() ;
|
||||
auto oZProj_v = oZProj.View() ;
|
||||
auto A_p = A[p].View();
|
||||
auto A_self = A[self_stencil].View();
|
||||
|
||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_p[ss](j,i),oZProj_v(ss)); });
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////
|
||||
// Faster alternate self coupling.. use hermiticity to save 2x
|
||||
///////////////////////////////////////////
|
||||
{
|
||||
mult(tmp,phi,evenmask); linop.Op(tmp,Mphie);
|
||||
mult(tmp,phi,oddmask ); linop.Op(tmp,Mphio);
|
||||
|
||||
{
|
||||
auto tmp_ = tmp.View();
|
||||
auto evenmask_ = evenmask.View();
|
||||
auto oddmask_ = oddmask.View();
|
||||
auto Mphie_ = Mphie.View();
|
||||
auto Mphio_ = Mphio.View();
|
||||
accelerator_for(ss, FineGrid->oSites(), Fobj::Nsimd(),{
|
||||
coalescedWrite(tmp_[ss],evenmask_(ss)*Mphie_(ss) + oddmask_(ss)*Mphio_(ss));
|
||||
});
|
||||
}
|
||||
|
||||
blockProject(SelfProj,tmp,Subspace.subspace);
|
||||
|
||||
auto SelfProj_ = SelfProj.View();
|
||||
auto A_self = A[self_stencil].View();
|
||||
|
||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{
|
||||
for(int j=0;j<nbasis;j++){
|
||||
coalescedWrite(A_self[ss](j,i), SelfProj_(ss)(j));
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
if(hermitian) {
|
||||
std::cout << GridLogMessage << " ForceHermitian, new code "<<std::endl;
|
||||
ForceHermitian();
|
||||
}
|
||||
// AssertHermitian();
|
||||
// ForceDiagonal();
|
||||
}
|
||||
|
||||
#if 0
|
||||
///////////////////////////
|
||||
// test code worth preserving in if block
|
||||
///////////////////////////
|
||||
std::cout<<GridLogMessage<< " Computed matrix elements "<< self_stencil <<std::endl;
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
std::cout<<GridLogMessage<< "A["<<p<<"]" << std::endl;
|
||||
std::cout<<GridLogMessage<< A[p] << std::endl;
|
||||
}
|
||||
std::cout<<GridLogMessage<< " picking by block0 "<< self_stencil <<std::endl;
|
||||
|
||||
phi=Subspace.subspace[0];
|
||||
std::vector<int> bc(FineGrid->_ndimension,0);
|
||||
|
||||
blockPick(Grid(),phi,tmp,bc); // Pick out a block
|
||||
linop.Op(tmp,Mphi); // Apply big dop
|
||||
blockProject(iProj,Mphi,Subspace.subspace); // project it and print it
|
||||
std::cout<<GridLogMessage<< " Computed matrix elements from block zero only "<<std::endl;
|
||||
std::cout<<GridLogMessage<< iProj <<std::endl;
|
||||
std::cout<<GridLogMessage<<"Computed Coarse Operator"<<std::endl;
|
||||
#endif
|
||||
|
||||
|
||||
void ForceHermitian(void) {
|
||||
CoarseMatrix Diff (Grid());
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
int dir = geom.directions[p];
|
||||
int disp = geom.displacements[p];
|
||||
if(disp==-1) {
|
||||
// Find the opposite link
|
||||
for(int pp=0;pp<geom.npoint;pp++){
|
||||
int dirp = geom.directions[pp];
|
||||
int dispp = geom.displacements[pp];
|
||||
if ( (dirp==dir) && (dispp==1) ){
|
||||
// Diff = adj(Cshift(A[p],dir,1)) - A[pp];
|
||||
// std::cout << GridLogMessage<<" Replacing stencil leg "<<pp<<" with leg "<<p<< " diff "<<norm2(Diff) <<std::endl;
|
||||
A[pp] = adj(Cshift(A[p],dir,1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
void AssertHermitian(void) {
|
||||
CoarseMatrix AA (Grid());
|
||||
CoarseMatrix AAc (Grid());
|
||||
CoarseMatrix Diff (Grid());
|
||||
for(int d=0;d<4;d++){
|
||||
|
||||
int dd=d+1;
|
||||
AAc = Cshift(A[2*d+1],dd,1);
|
||||
AA = A[2*d];
|
||||
|
||||
Diff = AA - adj(AAc);
|
||||
|
||||
std::cout<<GridLogMessage<<"Norm diff dim "<<d<<" "<< norm2(Diff)<<std::endl;
|
||||
std::cout<<GridLogMessage<<"Norm dim "<<d<<" "<< norm2(AA)<<std::endl;
|
||||
|
||||
}
|
||||
Diff = A[8] - adj(A[8]);
|
||||
std::cout<<GridLogMessage<<"Norm diff local "<< norm2(Diff)<<std::endl;
|
||||
std::cout<<GridLogMessage<<"Norm local "<< norm2(A[8])<<std::endl;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
+221
-425
@@ -1,6 +1,7 @@
|
||||
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/Cshift.h
|
||||
|
||||
@@ -28,465 +29,260 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
#ifndef _GRID_FFT_H_
|
||||
#define _GRID_FFT_H_
|
||||
|
||||
#ifdef GRID_CUDA
|
||||
#include <cufft.h>
|
||||
#endif
|
||||
|
||||
#ifdef GRID_HIP
|
||||
#include <hipfft/hipfft.h>
|
||||
#endif
|
||||
|
||||
#if !defined(GRID_CUDA) && !defined(GRID_HIP)
|
||||
#ifdef HAVE_FFTW
|
||||
#if defined(USE_MKL) || defined(GRID_SYCL)
|
||||
#ifdef USE_MKL
|
||||
#include <fftw/fftw3.h>
|
||||
#else
|
||||
#include <fftw3.h>
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class scalar> struct FFTW { };
|
||||
|
||||
#ifdef HAVE_FFTW
|
||||
template<> struct FFTW<ComplexD> {
|
||||
public:
|
||||
|
||||
typedef fftw_complex FFTW_scalar;
|
||||
typedef fftw_plan FFTW_plan;
|
||||
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, const int *n,int howmany,
|
||||
FFTW_scalar *in, const int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, const int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
return ::fftw_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
||||
}
|
||||
|
||||
static void fftw_flops(const FFTW_plan p,double *add, double *mul, double *fmas){
|
||||
::fftw_flops(p,add,mul,fmas);
|
||||
}
|
||||
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out) {
|
||||
::fftw_execute_dft(p,in,out);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
||||
::fftw_destroy_plan(p);
|
||||
}
|
||||
};
|
||||
|
||||
template<> struct FFTW<ComplexF> {
|
||||
public:
|
||||
|
||||
typedef fftwf_complex FFTW_scalar;
|
||||
typedef fftwf_plan FFTW_plan;
|
||||
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, const int *n,int howmany,
|
||||
FFTW_scalar *in, const int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, const int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
return ::fftwf_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
||||
}
|
||||
|
||||
static void fftw_flops(const FFTW_plan p,double *add, double *mul, double *fmas){
|
||||
::fftwf_flops(p,add,mul,fmas);
|
||||
}
|
||||
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out) {
|
||||
::fftwf_execute_dft(p,in,out);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
||||
::fftwf_destroy_plan(p);
|
||||
}
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef FFTW_FORWARD
|
||||
#define FFTW_FORWARD (-1)
|
||||
#define FFTW_BACKWARD (+1)
|
||||
#define FFTW_ESTIMATE (0)
|
||||
#endif
|
||||
|
||||
template<class scalar> struct FFTW {
|
||||
};
|
||||
|
||||
#ifdef GRID_HIP
|
||||
template<> struct FFTW<ComplexD> {
|
||||
class FFT {
|
||||
private:
|
||||
|
||||
GridCartesian *vgrid;
|
||||
GridCartesian *sgrid;
|
||||
|
||||
int Nd;
|
||||
double flops;
|
||||
double flops_call;
|
||||
uint64_t usec;
|
||||
|
||||
Coordinate dimensions;
|
||||
Coordinate processors;
|
||||
Coordinate processor_coor;
|
||||
|
||||
public:
|
||||
|
||||
static const int forward=FFTW_FORWARD;
|
||||
static const int backward=FFTW_BACKWARD;
|
||||
typedef hipfftDoubleComplex FFTW_scalar;
|
||||
typedef hipfftHandle FFTW_plan;
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
||||
FFTW_scalar *in, int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
FFTW_plan p;
|
||||
auto rv = hipfftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,HIPFFT_Z2Z,howmany);
|
||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
||||
return p;
|
||||
}
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
||||
hipfftResult rv;
|
||||
if ( sign == forward ) rv =hipfftExecZ2Z(p,in,out,HIPFFT_FORWARD);
|
||||
else rv =hipfftExecZ2Z(p,in,out,HIPFFT_BACKWARD);
|
||||
accelerator_barrier();
|
||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) { hipfftDestroy(p); }
|
||||
};
|
||||
template<> struct FFTW<ComplexF> {
|
||||
public:
|
||||
static const int forward=FFTW_FORWARD;
|
||||
static const int backward=FFTW_BACKWARD;
|
||||
typedef hipfftComplex FFTW_scalar;
|
||||
typedef hipfftHandle FFTW_plan;
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
||||
FFTW_scalar *in, int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
FFTW_plan p;
|
||||
auto rv = hipfftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,HIPFFT_C2C,howmany);
|
||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
||||
return p;
|
||||
}
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
||||
hipfftResult rv;
|
||||
if ( sign == forward ) rv =hipfftExecC2C(p,in,out,HIPFFT_FORWARD);
|
||||
else rv =hipfftExecC2C(p,in,out,HIPFFT_BACKWARD);
|
||||
accelerator_barrier();
|
||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) { hipfftDestroy(p); }
|
||||
};
|
||||
#endif
|
||||
|
||||
double Flops(void) {return flops;}
|
||||
double MFlops(void) {return flops/usec;}
|
||||
double USec(void) {return (double)usec;}
|
||||
|
||||
#ifdef GRID_CUDA
|
||||
template<> struct FFTW<ComplexD> {
|
||||
public:
|
||||
static const int forward=FFTW_FORWARD;
|
||||
static const int backward=FFTW_BACKWARD;
|
||||
typedef cufftDoubleComplex FFTW_scalar;
|
||||
typedef cufftHandle FFTW_plan;
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
||||
FFTW_scalar *in, int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
FFTW_plan p;
|
||||
cufftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,CUFFT_Z2Z,howmany);
|
||||
return p;
|
||||
}
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
||||
if ( sign == forward ) cufftExecZ2Z(p,in,out,CUFFT_FORWARD);
|
||||
else cufftExecZ2Z(p,in,out,CUFFT_INVERSE);
|
||||
accelerator_barrier();
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) { cufftDestroy(p); }
|
||||
};
|
||||
template<> struct FFTW<ComplexF> {
|
||||
public:
|
||||
static const int forward=FFTW_FORWARD;
|
||||
static const int backward=FFTW_BACKWARD;
|
||||
typedef cufftComplex FFTW_scalar;
|
||||
typedef cufftHandle FFTW_plan;
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
||||
FFTW_scalar *in, int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
FFTW_plan p;
|
||||
cufftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,CUFFT_C2C,howmany);
|
||||
return p;
|
||||
}
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
||||
if ( sign == forward ) cufftExecC2C(p,in,out,CUFFT_FORWARD);
|
||||
else cufftExecC2C(p,in,out,CUFFT_INVERSE);
|
||||
accelerator_barrier();
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) { cufftDestroy(p); }
|
||||
};
|
||||
#endif
|
||||
|
||||
#if !defined(GRID_CUDA) && !defined(GRID_HIP)
|
||||
#ifdef HAVE_FFTW
|
||||
template<> struct FFTW<ComplexD> {
|
||||
public:
|
||||
typedef fftw_complex FFTW_scalar;
|
||||
typedef fftw_plan FFTW_plan;
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
||||
FFTW_scalar *in, int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
return ::fftw_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
||||
}
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
||||
::fftw_execute_dft(p,in,out);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) { ::fftw_destroy_plan(p); }
|
||||
};
|
||||
template<> struct FFTW<ComplexF> {
|
||||
public:
|
||||
typedef fftwf_complex FFTW_scalar;
|
||||
typedef fftwf_plan FFTW_plan;
|
||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
||||
FFTW_scalar *in, int *inembed,
|
||||
int istride, int idist,
|
||||
FFTW_scalar *out, int *onembed,
|
||||
int ostride, int odist,
|
||||
int sign, unsigned flags) {
|
||||
return ::fftwf_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
||||
}
|
||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
||||
::fftwf_execute_dft(p,in,out);
|
||||
}
|
||||
inline static void fftw_destroy_plan(const FFTW_plan p) { ::fftwf_destroy_plan(p); }
|
||||
};
|
||||
#endif
|
||||
#endif
|
||||
|
||||
struct FFTbase {
|
||||
double flops;
|
||||
double flops_call;
|
||||
uint64_t usec;
|
||||
GridCartesian *_grid;
|
||||
|
||||
static const int forward = FFTW_FORWARD;
|
||||
static const int backward = FFTW_BACKWARD;
|
||||
|
||||
double Flops(void) { return flops; }
|
||||
double MFlops(void) { return flops / usec; }
|
||||
double USec(void) { return (double)usec; }
|
||||
|
||||
FFTbase(GridCartesian *grid) : _grid(grid), flops(0), flops_call(0), usec(0) {}
|
||||
};
|
||||
|
||||
// Barrel-shift gather, FFT execute, and insert. Called by both FFT and PlannedFFT.
|
||||
// The caller is responsible for plan acquisition and destruction.
|
||||
template<class vobj>
|
||||
static void FFT_dim_execute(
|
||||
Lattice<vobj> &result,
|
||||
const Lattice<vobj> &source,
|
||||
int dim, int sign,
|
||||
typename FFTW<typename vobj::scalar_type>::FFTW_plan p,
|
||||
GridCartesian *grid,
|
||||
double &flops, double &flops_call, uint64_t &usec)
|
||||
{
|
||||
typedef typename vobj::scalar_type scalar;
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
|
||||
|
||||
const int Ndim = grid->Nd();
|
||||
int L = grid->_ldimensions[dim];
|
||||
int G = grid->_fdimensions[dim];
|
||||
int Ncomp = sizeof(sobj) / sizeof(scalar);
|
||||
int64_t Nlow = 1, Nhigh = 1;
|
||||
for (int d = 0; d < dim; d++) Nlow *= grid->_ldimensions[d];
|
||||
for (int d = dim+1; d < Ndim; d++) Nhigh *= grid->_ldimensions[d];
|
||||
int64_t Nperp = Nlow * Nhigh;
|
||||
|
||||
deviceVector<scalar> pgbuf(Nperp * Ncomp * G);
|
||||
scalar *pgbuf_v = &pgbuf[0];
|
||||
int howmany = Ncomp * Nperp;
|
||||
|
||||
scalar div;
|
||||
if (sign == FFTW_BACKWARD) div = 1.0 / G;
|
||||
else if (sign == FFTW_FORWARD) div = 1.0;
|
||||
else GRID_ASSERT(0);
|
||||
|
||||
double t_pencil = 0, t_fft = 0, t_copy = 0, t_shift = 0;
|
||||
double t_total = -usecond();
|
||||
|
||||
result = source;
|
||||
int pc = grid->_processor_coor[dim];
|
||||
|
||||
const Coordinate ldims = grid->_ldimensions;
|
||||
const Coordinate rdims = grid->_rdimensions;
|
||||
const Coordinate sdims = grid->_simd_layout;
|
||||
const Coordinate processors = grid->_processors;
|
||||
|
||||
Coordinate pgdims(Ndim);
|
||||
pgdims[0] = G;
|
||||
for (int d = 0, dd = 1; d < Ndim; d++)
|
||||
if (d != dim) pgdims[dd++] = ldims[d];
|
||||
int64_t pgvol = 1;
|
||||
for (int d = 0; d < Ndim; d++) pgvol *= pgdims[d];
|
||||
|
||||
const int Nsimd = vobj::Nsimd();
|
||||
t_pencil = -usecond();
|
||||
for (int p_idx = 0; p_idx < processors[dim]; p_idx++) {
|
||||
t_copy -= usecond();
|
||||
autoView(r_v, result, AcceleratorRead);
|
||||
accelerator_for(idx, grid->oSites(), vobj::Nsimd(), {
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane = acceleratorSIMTlane(Nsimd);
|
||||
#else
|
||||
for (int lane = 0; lane < Nsimd; lane++) {
|
||||
#endif
|
||||
Coordinate icoor, ocoor, pgcoor;
|
||||
Lexicographic::CoorFromIndex(icoor, lane, sdims);
|
||||
Lexicographic::CoorFromIndex(ocoor, idx, rdims);
|
||||
pgcoor[0] = ocoor[dim] + icoor[dim]*rdims[dim] + ((pc+p_idx)%processors[dim])*L;
|
||||
for (int d = 0, dd = 1; d < Ndim; d++)
|
||||
if (d != dim) { pgcoor[dd] = ocoor[d] + icoor[d]*rdims[d]; dd++; }
|
||||
int64_t pgidx;
|
||||
Lexicographic::IndexFromCoor(pgcoor, pgidx, pgdims);
|
||||
vector_type *from = (vector_type *)&r_v[idx];
|
||||
scalar_type stmp;
|
||||
for (int w = 0; w < Ncomp; w++) {
|
||||
stmp = getlane(from[w], lane);
|
||||
pgbuf_v[pgidx + w*pgvol] = stmp;
|
||||
}
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
t_copy += usecond();
|
||||
if (p_idx != processors[dim] - 1) {
|
||||
Lattice<vobj> temp(grid);
|
||||
t_shift -= usecond();
|
||||
temp = Cshift(result, dim, L); result = temp;
|
||||
t_shift += usecond();
|
||||
}
|
||||
}
|
||||
t_pencil += usecond();
|
||||
|
||||
FFTW_scalar *in = (FFTW_scalar *)pgbuf_v;
|
||||
FFTW_scalar *out = (FFTW_scalar *)pgbuf_v;
|
||||
t_fft = -usecond();
|
||||
FFTW<scalar>::fftw_execute_dft(p, in, out, sign);
|
||||
t_fft += usecond();
|
||||
|
||||
flops_call = 5.0 * howmany * G * log2(G);
|
||||
usec = t_fft;
|
||||
flops = flops_call;
|
||||
|
||||
result = Zero();
|
||||
double t_insert = -usecond();
|
||||
FFT ( GridCartesian * grid ) :
|
||||
vgrid(grid),
|
||||
Nd(grid->_ndimension),
|
||||
dimensions(grid->_fdimensions),
|
||||
processors(grid->_processors),
|
||||
processor_coor(grid->_processor_coor)
|
||||
{
|
||||
autoView(r_v, result, AcceleratorWrite);
|
||||
accelerator_for(idx, grid->oSites(), Nsimd, {
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane = acceleratorSIMTlane(Nsimd);
|
||||
#else
|
||||
for (int lane = 0; lane < Nsimd; lane++) {
|
||||
#endif
|
||||
Coordinate icoor(Ndim), ocoor(Ndim), pgcoor(Ndim);
|
||||
Lexicographic::CoorFromIndex(icoor, lane, sdims);
|
||||
Lexicographic::CoorFromIndex(ocoor, idx, rdims);
|
||||
pgcoor[0] = ocoor[dim] + icoor[dim]*rdims[dim] + pc*L;
|
||||
for (int d = 0, dd = 1; d < Ndim; d++)
|
||||
if (d != dim) { pgcoor[dd] = ocoor[d] + icoor[d]*rdims[d]; dd++; }
|
||||
int64_t pgidx;
|
||||
Lexicographic::IndexFromCoor(pgcoor, pgidx, pgdims);
|
||||
vector_type *to = (vector_type *)&r_v[idx];
|
||||
scalar_type stmp;
|
||||
for (int w = 0; w < Ncomp; w++) {
|
||||
stmp = pgbuf_v[pgidx + w*pgvol];
|
||||
putlane(to[w], stmp, lane);
|
||||
}
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
flops=0;
|
||||
usec =0;
|
||||
Coordinate layout(Nd,1);
|
||||
sgrid = new GridCartesian(dimensions,layout,processors);
|
||||
};
|
||||
|
||||
~FFT ( void) {
|
||||
delete sgrid;
|
||||
}
|
||||
result = result * div;
|
||||
t_insert += usecond();
|
||||
t_total += usecond();
|
||||
|
||||
std::cout << GridLogPerformance << " FFT took " << t_total/1.0e6 << " s" << std::endl;
|
||||
std::cout << GridLogPerformance << " FFT pencil " << t_pencil/1.0e6 << " s" << std::endl;
|
||||
std::cout << GridLogPerformance << " of which copy " << t_copy/1.0e6 << " s" << std::endl;
|
||||
std::cout << GridLogPerformance << " of which shift" << t_shift/1.0e6 << " s" << std::endl;
|
||||
std::cout << GridLogPerformance << " FFT kernels " << t_fft/1.0e6 << " s" << std::endl;
|
||||
std::cout << GridLogPerformance << " FFT insert " << t_insert/1.0e6 << " s" << std::endl;
|
||||
}
|
||||
|
||||
class FFT : public FFTbase {
|
||||
public:
|
||||
FFT(GridCartesian *grid) : FFTbase(grid) {}
|
||||
~FFT() {}
|
||||
|
||||
|
||||
template<class vobj>
|
||||
void FFT_dim_mask(Lattice<vobj> &result, const Lattice<vobj> &source, Coordinate mask, int sign) {
|
||||
const int Ndim = _grid->Nd();
|
||||
Lattice<vobj> tmp = source;
|
||||
for (int d = 0; d < Ndim; d++) {
|
||||
if (mask[d]) {
|
||||
FFT_dim(result, tmp, d, sign);
|
||||
tmp = result;
|
||||
void FFT_dim_mask(Lattice<vobj> &result,const Lattice<vobj> &source,Coordinate mask,int sign){
|
||||
|
||||
conformable(result.Grid(),vgrid);
|
||||
conformable(source.Grid(),vgrid);
|
||||
Lattice<vobj> tmp(vgrid);
|
||||
tmp = source;
|
||||
for(int d=0;d<Nd;d++){
|
||||
if( mask[d] ) {
|
||||
FFT_dim(result,tmp,d,sign);
|
||||
tmp=result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj>
|
||||
void FFT_all_dim(Lattice<vobj> &result, const Lattice<vobj> &source, int sign) {
|
||||
Coordinate mask(_grid->Nd(), 1);
|
||||
FFT_dim_mask(result, source, mask, sign);
|
||||
void FFT_all_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int sign){
|
||||
Coordinate mask(Nd,1);
|
||||
FFT_dim_mask(result,source,mask,sign);
|
||||
}
|
||||
|
||||
template<class vobj>
|
||||
void FFT_dim(Lattice<vobj> &result, const Lattice<vobj> &source, int dim, int sign) {
|
||||
GRID_ASSERT(source.Grid() == _grid);
|
||||
GRID_ASSERT(result.Grid() == _grid);
|
||||
conformable(result.Grid(), source.Grid());
|
||||
|
||||
typedef typename vobj::scalar_type scalar;
|
||||
template<class vobj>
|
||||
void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int sign){
|
||||
#ifndef HAVE_FFTW
|
||||
assert(0);
|
||||
#else
|
||||
conformable(result.Grid(),vgrid);
|
||||
conformable(source.Grid(),vgrid);
|
||||
|
||||
int L = vgrid->_ldimensions[dim];
|
||||
int G = vgrid->_fdimensions[dim];
|
||||
|
||||
Coordinate layout(Nd,1);
|
||||
Coordinate pencil_gd(vgrid->_fdimensions);
|
||||
|
||||
pencil_gd[dim] = G*processors[dim];
|
||||
|
||||
// Pencil global vol LxLxGxLxL per node
|
||||
GridCartesian pencil_g(pencil_gd,layout,processors);
|
||||
|
||||
// Construct pencils
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
typedef typename sobj::scalar_type scalar;
|
||||
|
||||
Lattice<sobj> pgbuf(&pencil_g);
|
||||
auto pgbuf_v = pgbuf.View();
|
||||
|
||||
typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
|
||||
typedef typename FFTW<scalar>::FFTW_plan FFTW_plan;
|
||||
|
||||
const int Ndim = _grid->Nd();
|
||||
int G = _grid->_fdimensions[dim];
|
||||
int Ncomp = sizeof(sobj) / sizeof(scalar);
|
||||
int64_t Nperp = 1;
|
||||
for (int d = 0; d < Ndim; d++)
|
||||
if (d != dim) Nperp *= _grid->_ldimensions[d];
|
||||
int n[] = {G};
|
||||
int howmany = Ncomp * Nperp;
|
||||
|
||||
deviceVector<scalar> dummy(2);
|
||||
FFTW_scalar *buf = (FFTW_scalar *)&dummy[0];
|
||||
FFTW_plan p = FFTW<scalar>::fftw_plan_many_dft(1, n, howmany,
|
||||
buf, n, 1, G,
|
||||
buf, n, 1, G,
|
||||
sign, FFTW_ESTIMATE);
|
||||
FFT_dim_execute(result, source, dim, sign, p, _grid, flops, flops_call, usec);
|
||||
FFTW<scalar>::fftw_destroy_plan(p);
|
||||
}
|
||||
};
|
||||
|
||||
template<class vobj>
|
||||
class PlannedFFT : public FFTbase {
|
||||
private:
|
||||
typedef typename vobj::scalar_type scalar;
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
|
||||
typedef typename FFTW<scalar>::FFTW_plan FFTW_plan;
|
||||
|
||||
std::vector<FFTW_plan> forward_plans;
|
||||
std::vector<FFTW_plan> backward_plans;
|
||||
|
||||
void PlanCreate() {
|
||||
const int Ndim = _grid->Nd();
|
||||
forward_plans.resize(Ndim);
|
||||
backward_plans.resize(Ndim);
|
||||
|
||||
for (int d = 0; d < Ndim; d++) {
|
||||
int G = _grid->_fdimensions[d];
|
||||
int Ncomp = sizeof(sobj) / sizeof(scalar);
|
||||
int64_t Nperp = 1;
|
||||
for (int dd = 0; dd < Ndim; dd++)
|
||||
if (dd != d) Nperp *= _grid->_ldimensions[dd];
|
||||
int howmany = Ncomp * (int)Nperp;
|
||||
int n[] = {G};
|
||||
|
||||
deviceVector<scalar> dummy(2);
|
||||
FFTW_scalar *buf = (FFTW_scalar *)&dummy[0];
|
||||
|
||||
forward_plans[d] = FFTW<scalar>::fftw_plan_many_dft(1, n, howmany, buf, n, 1, G, buf, n, 1, G, FFTW_FORWARD, FFTW_ESTIMATE);
|
||||
backward_plans[d] = FFTW<scalar>::fftw_plan_many_dft(1, n, howmany, buf, n, 1, G, buf, n, 1, G, FFTW_BACKWARD, FFTW_ESTIMATE);
|
||||
|
||||
int Ncomp = sizeof(sobj)/sizeof(scalar);
|
||||
int Nlow = 1;
|
||||
for(int d=0;d<dim;d++){
|
||||
Nlow*=vgrid->_ldimensions[d];
|
||||
}
|
||||
}
|
||||
|
||||
void PlanDestroy() {
|
||||
for (auto p : forward_plans) FFTW<scalar>::fftw_destroy_plan(p);
|
||||
for (auto p : backward_plans) FFTW<scalar>::fftw_destroy_plan(p);
|
||||
forward_plans.clear();
|
||||
backward_plans.clear();
|
||||
}
|
||||
|
||||
public:
|
||||
PlannedFFT(GridCartesian *grid) : FFTbase(grid) { PlanCreate(); }
|
||||
~PlannedFFT() { PlanDestroy(); }
|
||||
|
||||
void FFT_dim_mask(Lattice<vobj> &result, const Lattice<vobj> &source, Coordinate mask, int sign) {
|
||||
const int Ndim = _grid->Nd();
|
||||
Lattice<vobj> tmp = source;
|
||||
for (int d = 0; d < Ndim; d++) {
|
||||
if (mask[d]) {
|
||||
FFT_dim(result, tmp, d, sign);
|
||||
tmp = result;
|
||||
|
||||
int rank = 1; /* 1d transforms */
|
||||
int n[] = {G}; /* 1d transforms of length G */
|
||||
int howmany = Ncomp;
|
||||
int odist,idist,istride,ostride;
|
||||
idist = odist = 1; /* Distance between consecutive FT's */
|
||||
istride = ostride = Ncomp*Nlow; /* distance between two elements in the same FT */
|
||||
int *inembed = n, *onembed = n;
|
||||
|
||||
scalar div;
|
||||
if ( sign == backward ) div = 1.0/G;
|
||||
else if ( sign == forward ) div = 1.0;
|
||||
else assert(0);
|
||||
|
||||
FFTW_plan p;
|
||||
{
|
||||
FFTW_scalar *in = (FFTW_scalar *)&pgbuf_v[0];
|
||||
FFTW_scalar *out= (FFTW_scalar *)&pgbuf_v[0];
|
||||
p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
|
||||
in,inembed,
|
||||
istride,idist,
|
||||
out,onembed,
|
||||
ostride, odist,
|
||||
sign,FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
// Barrel shift and collect global pencil
|
||||
Coordinate lcoor(Nd), gcoor(Nd);
|
||||
result = source;
|
||||
int pc = processor_coor[dim];
|
||||
for(int p=0;p<processors[dim];p++) {
|
||||
thread_for(idx, sgrid->lSites(),{
|
||||
Coordinate cbuf(Nd);
|
||||
sobj s;
|
||||
sgrid->LocalIndexToLocalCoor(idx,cbuf);
|
||||
peekLocalSite(s,result,cbuf);
|
||||
cbuf[dim]+=((pc+p) % processors[dim])*L;
|
||||
// cbuf[dim]+=p*L;
|
||||
pokeLocalSite(s,pgbuf,cbuf);
|
||||
});
|
||||
if (p != processors[dim] - 1) {
|
||||
result = Cshift(result,dim,L);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void FFT_all_dim(Lattice<vobj> &result, const Lattice<vobj> &source, int sign) {
|
||||
Coordinate mask(_grid->Nd(), 1);
|
||||
FFT_dim_mask(result, source, mask, sign);
|
||||
}
|
||||
|
||||
void FFT_dim(Lattice<vobj> &result, const Lattice<vobj> &source, int dim, int sign) {
|
||||
GRID_ASSERT(source.Grid() == _grid);
|
||||
GRID_ASSERT(result.Grid() == _grid);
|
||||
GRID_ASSERT((int)forward_plans.size() == _grid->Nd());
|
||||
conformable(result.Grid(), source.Grid());
|
||||
FFTW_plan p = (sign == forward ? forward_plans : backward_plans)[dim];
|
||||
FFT_dim_execute(result, source, dim, sign, p, _grid, flops, flops_call, usec);
|
||||
|
||||
// Loop over orthog coords
|
||||
int NN=pencil_g.lSites();
|
||||
GridStopWatch timer;
|
||||
timer.Start();
|
||||
thread_for( idx,NN,{
|
||||
Coordinate cbuf(Nd);
|
||||
pencil_g.LocalIndexToLocalCoor(idx, cbuf);
|
||||
if ( cbuf[dim] == 0 ) { // restricts loop to plane at lcoor[dim]==0
|
||||
FFTW_scalar *in = (FFTW_scalar *)&pgbuf_v[idx];
|
||||
FFTW_scalar *out= (FFTW_scalar *)&pgbuf_v[idx];
|
||||
FFTW<scalar>::fftw_execute_dft(p,in,out);
|
||||
}
|
||||
});
|
||||
timer.Stop();
|
||||
|
||||
// performance counting
|
||||
double add,mul,fma;
|
||||
FFTW<scalar>::fftw_flops(p,&add,&mul,&fma);
|
||||
flops_call = add+mul+2.0*fma;
|
||||
usec += timer.useconds();
|
||||
flops+= flops_call*NN;
|
||||
|
||||
// writing out result
|
||||
thread_for(idx,sgrid->lSites(),{
|
||||
Coordinate clbuf(Nd), cgbuf(Nd);
|
||||
sobj s;
|
||||
sgrid->LocalIndexToLocalCoor(idx,clbuf);
|
||||
cgbuf = clbuf;
|
||||
cgbuf[dim] = clbuf[dim]+L*pc;
|
||||
peekLocalSite(s,pgbuf,cgbuf);
|
||||
pokeLocalSite(s,result,clbuf);
|
||||
});
|
||||
result = result*div;
|
||||
|
||||
// destroying plan
|
||||
FFTW<scalar>::fftw_destroy_plan(p);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -52,7 +52,6 @@ public:
|
||||
virtual void AdjOp (const Field &in, Field &out) = 0; // Abstract base
|
||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
|
||||
virtual void HermOp(const Field &in, Field &out)=0;
|
||||
virtual ~LinearOperatorBase(){};
|
||||
};
|
||||
|
||||
|
||||
@@ -64,7 +63,7 @@ public:
|
||||
//
|
||||
// I'm not entirely happy with implementation; to share the Schur code between herm and non-herm
|
||||
// while still having a "OpAndNorm" in the abstract base I had to implement it in both cases
|
||||
// with an GRID_ASSERT trap in the non-herm. This isn't right; there must be a better C++ way to
|
||||
// with an assert trap in the non-herm. This isn't right; there must be a better C++ way to
|
||||
// do it, but I fear it required multiple inheritance and mixed in abstract base classes
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -103,38 +102,6 @@ public:
|
||||
_Mat.MdagM(in,out);
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class MMdagLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
public:
|
||||
MMdagLinearOperator(Matrix &Mat): _Mat(Mat){};
|
||||
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
_Mat.MdirAll(in,out);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.Mdag(in,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
_Mat.MMdag(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
_Mat.MMdag(in,out);
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Construct herm op and shift it for mgrid smoother
|
||||
@@ -148,22 +115,22 @@ public:
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.Mdag(in,out);
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
@@ -177,44 +144,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Create a shifted HermOp
|
||||
////////////////////////////////////////////////////////////////////
|
||||
template<class Field>
|
||||
class ShiftedHermOpLinearOperator : public LinearOperatorBase<Field> {
|
||||
LinearOperatorBase<Field> &_Mat;
|
||||
RealD _shift;
|
||||
public:
|
||||
ShiftedHermOpLinearOperator(LinearOperatorBase<Field> &Mat,RealD shift): _Mat(Mat), _shift(shift){};
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
GRID_ASSERT(0);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
HermOp(in,out);
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
HermOp(in,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
_Mat.HermOp(in,out);
|
||||
out = out + _shift*in;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// Wrap an already herm matrix
|
||||
////////////////////////////////////////////////////////////////////
|
||||
@@ -271,42 +200,10 @@ public:
|
||||
_Mat.Mdag(in,out);
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
class ShiftedNonHermitianLinearOperator : public LinearOperatorBase<Field> {
|
||||
Matrix &_Mat;
|
||||
RealD shift;
|
||||
public:
|
||||
ShiftedNonHermitianLinearOperator(Matrix &Mat,RealD shft): _Mat(Mat),shift(shft){};
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
_Mat.Mdiag(in,out);
|
||||
out = out + shift*in;
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
_Mat.Mdir(in,out,dir,disp);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
_Mat.MdirAll(in,out);
|
||||
};
|
||||
void Op (const Field &in, Field &out){
|
||||
_Mat.M(in,out);
|
||||
out = out + shift * in;
|
||||
}
|
||||
void AdjOp (const Field &in, Field &out){
|
||||
_Mat.Mdag(in,out);
|
||||
out = out + shift * in;
|
||||
}
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
void HermOp(const Field &in, Field &out){
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -345,13 +242,13 @@ class SchurOperatorBase : public LinearOperatorBase<Field> {
|
||||
}
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag (const Field &in, Field &out) {
|
||||
GRID_ASSERT(0); // must coarsen the unpreconditioned system
|
||||
assert(0); // must coarsen the unpreconditioned system
|
||||
}
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
};
|
||||
};
|
||||
template<class Matrix,class Field>
|
||||
@@ -447,10 +344,10 @@ class NonHermitianSchurOperatorBase : public LinearOperatorBase<Field>
|
||||
MpcDag(tmp,out);
|
||||
}
|
||||
virtual void HermOpAndNorm(const Field& in, Field& out, RealD& n1, RealD& n2) {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
virtual void HermOp(const Field& in, Field& out) {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void Op(const Field& in, Field& out) {
|
||||
Mpc(in, out);
|
||||
@@ -460,13 +357,13 @@ class NonHermitianSchurOperatorBase : public LinearOperatorBase<Field>
|
||||
}
|
||||
// Support for coarsening to a multigrid
|
||||
void OpDiag(const Field& in, Field& out) {
|
||||
GRID_ASSERT(0); // must coarsen the unpreconditioned system
|
||||
assert(0); // must coarsen the unpreconditioned system
|
||||
}
|
||||
void OpDir(const Field& in, Field& out, int dir, int disp) {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void OpDirAll(const Field& in, std::vector<Field>& out){
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
};
|
||||
};
|
||||
|
||||
@@ -580,7 +477,7 @@ class SchurStaggeredOperator : public SchurOperatorBase<Field> {
|
||||
public:
|
||||
SchurStaggeredOperator (Matrix &Mat): _Mat(Mat), tmp(_Mat.RedBlackGrid())
|
||||
{
|
||||
GRID_ASSERT( _Mat.isTrivialEE() );
|
||||
assert( _Mat.isTrivialEE() );
|
||||
mass = _Mat.Mass();
|
||||
}
|
||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
@@ -610,8 +507,8 @@ class SchurStaggeredOperator : public SchurOperatorBase<Field> {
|
||||
virtual void MpcDag (const Field &in, Field &out){
|
||||
Mpc(in,out);
|
||||
}
|
||||
virtual void MpcDagMpc(const Field &in, Field &out) {
|
||||
GRID_ASSERT(0);// Never need with staggered
|
||||
virtual void MpcDagMpc(const Field &in, Field &out,RealD &ni,RealD &no) {
|
||||
assert(0);// Never need with staggered
|
||||
}
|
||||
};
|
||||
template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
|
||||
@@ -623,28 +520,16 @@ template<class Field> class OperatorFunction {
|
||||
public:
|
||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) = 0;
|
||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const std::vector<Field> &in,std::vector<Field> &out) {
|
||||
GRID_ASSERT(in.size()==out.size());
|
||||
assert(in.size()==out.size());
|
||||
for(int k=0;k<in.size();k++){
|
||||
(*this)(Linop,in[k],out[k]);
|
||||
}
|
||||
};
|
||||
virtual ~OperatorFunction(){};
|
||||
};
|
||||
|
||||
template<class Field> class LinearFunction {
|
||||
public:
|
||||
virtual void operator() (const Field &in, Field &out) = 0;
|
||||
|
||||
virtual void operator() (const std::vector<Field> &in, std::vector<Field> &out)
|
||||
{
|
||||
GRID_ASSERT(in.size() == out.size());
|
||||
|
||||
for (unsigned int i = 0; i < in.size(); ++i)
|
||||
{
|
||||
(*this)(in[i], out[i]);
|
||||
}
|
||||
}
|
||||
virtual ~LinearFunction(){};
|
||||
};
|
||||
|
||||
template<class Field> class IdentityLinearFunction : public LinearFunction<Field> {
|
||||
@@ -690,7 +575,6 @@ class HermOpOperatorFunction : public OperatorFunction<Field> {
|
||||
template<typename Field>
|
||||
class PlainHermOp : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
LinearOperatorBase<Field> &_Linop;
|
||||
|
||||
PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop)
|
||||
@@ -704,7 +588,6 @@ public:
|
||||
template<typename Field>
|
||||
class FunctionHermOp : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
OperatorFunction<Field> & _poly;
|
||||
LinearOperatorBase<Field> &_Linop;
|
||||
|
||||
|
||||
@@ -30,19 +30,13 @@ Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Field> using Preconditioner = LinearFunction<Field> ;
|
||||
|
||||
/*
|
||||
template<class Field> class Preconditioner : public LinearFunction<Field> {
|
||||
using LinearFunction<Field>::operator();
|
||||
template<class Field> class Preconditioner : public LinearFunction<Field> {
|
||||
virtual void operator()(const Field &src, Field & psi)=0;
|
||||
};
|
||||
*/
|
||||
|
||||
template<class Field> class TrivialPrecon : public Preconditioner<Field> {
|
||||
public:
|
||||
using Preconditioner<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field & psi){
|
||||
void operator()(const Field &src, Field & psi){
|
||||
psi = src;
|
||||
}
|
||||
TrivialPrecon(void){};
|
||||
|
||||
@@ -45,15 +45,9 @@ public:
|
||||
M(in,tmp);
|
||||
Mdag(tmp,out);
|
||||
}
|
||||
virtual void MMdag(const Field &in, Field &out) {
|
||||
Field tmp (in.Grid());
|
||||
Mdag(in,tmp);
|
||||
M(tmp,out);
|
||||
}
|
||||
virtual void Mdiag (const Field &in, Field &out)=0;
|
||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp)=0;
|
||||
virtual void MdirAll (const Field &in, std::vector<Field> &out)=0;
|
||||
virtual ~SparseMatrixBase() {};
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -78,7 +72,7 @@ public:
|
||||
virtual void MeooeDag (const Field &in, Field &out)=0;
|
||||
virtual void MooeeDag (const Field &in, Field &out)=0;
|
||||
virtual void MooeeInvDag (const Field &in, Field &out)=0;
|
||||
virtual ~CheckerBoardedSparseMatrixBase() {};
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -59,7 +59,7 @@ public:
|
||||
RealD diff = hi-lo;
|
||||
RealD delta = diff*1.0e-9;
|
||||
for (RealD x=lo; x<hi; x+=delta) {
|
||||
delta*=1.02;
|
||||
delta*=1.1;
|
||||
RealD f = approx(x);
|
||||
out<< x<<" "<<f<<std::endl;
|
||||
}
|
||||
@@ -90,8 +90,9 @@ public:
|
||||
order=_order;
|
||||
|
||||
if(order < 2) exit(-1);
|
||||
Coeffs.resize(order,0.0);
|
||||
Coeffs[order-1] = 1.0;
|
||||
Coeffs.resize(order);
|
||||
Coeffs.assign(0.,order);
|
||||
Coeffs[order-1] = 1.;
|
||||
};
|
||||
|
||||
// PB - more efficient low pass drops high modes above the low as 1/x uses all Chebyshev's.
|
||||
@@ -131,26 +132,6 @@ public:
|
||||
Coeffs[j] = s * 2.0/order;
|
||||
}
|
||||
};
|
||||
template<class functor>
|
||||
void Init(RealD _lo,RealD _hi,int _order, functor & func)
|
||||
{
|
||||
lo=_lo;
|
||||
hi=_hi;
|
||||
order=_order;
|
||||
|
||||
if(order < 2) exit(-1);
|
||||
Coeffs.resize(order);
|
||||
for(int j=0;j<order;j++){
|
||||
RealD s=0;
|
||||
for(int k=0;k<order;k++){
|
||||
RealD y=std::cos(M_PI*(k+0.5)/order);
|
||||
RealD x=0.5*(y*(hi-lo)+(hi+lo));
|
||||
RealD f=func(x);
|
||||
s=s+f*std::cos( j*M_PI*(k+0.5)/order );
|
||||
}
|
||||
Coeffs[j] = s * 2.0/order;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
void JacksonSmooth(void){
|
||||
@@ -269,9 +250,7 @@ public:
|
||||
RealD xscale = 2.0/(hi-lo);
|
||||
RealD mscale = -(hi+lo)/(hi-lo);
|
||||
Linop.HermOp(T0,y);
|
||||
grid->Barrier();
|
||||
axpby(T1,xscale,mscale,y,in);
|
||||
grid->Barrier();
|
||||
|
||||
// sum = .5 c[0] T0 + c[1] T1
|
||||
// out = ()*T0 + Coeffs[1]*T1;
|
||||
@@ -279,12 +258,26 @@ public:
|
||||
for(int n=2;n<order;n++){
|
||||
|
||||
Linop.HermOp(*Tn,y);
|
||||
#if 0
|
||||
auto y_v = y.View();
|
||||
auto Tn_v = Tn->View();
|
||||
auto Tnp_v = Tnp->View();
|
||||
auto Tnm_v = Tnm->View();
|
||||
constexpr int Nsimd = vector_type::Nsimd();
|
||||
accelerator_forNB(ss, in.Grid()->oSites(), Nsimd, {
|
||||
coalescedWrite(y_v[ss],xscale*y_v(ss)+mscale*Tn_v(ss));
|
||||
coalescedWrite(Tnp_v[ss],2.0*y_v(ss)-Tnm_v(ss));
|
||||
});
|
||||
if ( Coeffs[n] != 0.0) {
|
||||
axpy(out,Coeffs[n],*Tnp,out);
|
||||
}
|
||||
#else
|
||||
axpby(y,xscale,mscale,y,(*Tn));
|
||||
axpby(*Tnp,2.0,-1.0,y,(*Tnm));
|
||||
if ( Coeffs[n] != 0.0) {
|
||||
axpy(out,Coeffs[n],*Tnp,out);
|
||||
}
|
||||
|
||||
#endif
|
||||
// Cycle pointers to avoid copies
|
||||
Field *swizzle = Tnm;
|
||||
Tnm =Tn;
|
||||
|
||||
@@ -40,7 +40,7 @@ public:
|
||||
RealD norm;
|
||||
RealD lo,hi;
|
||||
|
||||
MultiShiftFunction(int n,RealD _lo,RealD _hi): poles(n), residues(n), tolerances(n), lo(_lo), hi(_hi) {;};
|
||||
MultiShiftFunction(int n,RealD _lo,RealD _hi): poles(n), residues(n), lo(_lo), hi(_hi) {;};
|
||||
RealD approx(RealD x);
|
||||
void csv(std::ostream &out);
|
||||
void gnuplot(std::ostream &out);
|
||||
|
||||
@@ -121,7 +121,7 @@ double AlgRemez::generateApprox(int num_degree, int den_degree,
|
||||
// Reallocate arrays, since degree has changed
|
||||
if (num_degree != n || den_degree != d) allocate(num_degree,den_degree);
|
||||
|
||||
GRID_ASSERT(a_len<=SUM_MAX);
|
||||
assert(a_len<=SUM_MAX);
|
||||
|
||||
step = new bigfloat[num_degree+den_degree+2];
|
||||
|
||||
@@ -151,9 +151,9 @@ double AlgRemez::generateApprox(int num_degree, int den_degree,
|
||||
equations();
|
||||
if (delta < tolerance) {
|
||||
std::cout<<"Delta too small, try increasing precision\n";
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
};
|
||||
GRID_ASSERT( delta>= tolerance);
|
||||
assert( delta>= tolerance);
|
||||
|
||||
search(step);
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ class AlgRemez
|
||||
virtual ~AlgRemez();
|
||||
|
||||
int getDegree(void){
|
||||
GRID_ASSERT(n==d);
|
||||
assert(n==d);
|
||||
return n;
|
||||
}
|
||||
// Reset the bounds of the approximation
|
||||
|
||||
@@ -28,11 +28,11 @@ void AlgRemezGeneral::setupPolyProperties(int num_degree, int den_degree, PolyTy
|
||||
pow_n = num_degree;
|
||||
pow_d = den_degree;
|
||||
|
||||
if(pow_n % 2 == 0 && num_type_in == PolyType::Odd) GRID_ASSERT(0);
|
||||
if(pow_n % 2 == 1 && num_type_in == PolyType::Even) GRID_ASSERT(0);
|
||||
if(pow_n % 2 == 0 && num_type_in == PolyType::Odd) assert(0);
|
||||
if(pow_n % 2 == 1 && num_type_in == PolyType::Even) assert(0);
|
||||
|
||||
if(pow_d % 2 == 0 && den_type_in == PolyType::Odd) GRID_ASSERT(0);
|
||||
if(pow_d % 2 == 1 && den_type_in == PolyType::Even) GRID_ASSERT(0);
|
||||
if(pow_d % 2 == 0 && den_type_in == PolyType::Odd) assert(0);
|
||||
if(pow_d % 2 == 1 && den_type_in == PolyType::Even) assert(0);
|
||||
|
||||
num_type = num_type_in;
|
||||
den_type = den_type_in;
|
||||
@@ -112,9 +112,9 @@ double AlgRemezGeneral::generateApprox(const int num_degree, const int den_degre
|
||||
equations();
|
||||
if (delta < tolerance) {
|
||||
std::cout<<"Iteration " << iter-1 << " delta too small (" << delta << "<" << tolerance << "), try increasing precision\n";
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
};
|
||||
GRID_ASSERT( delta>= tolerance );
|
||||
assert( delta>= tolerance );
|
||||
|
||||
search();
|
||||
}
|
||||
@@ -278,7 +278,7 @@ void AlgRemezGeneral::equations(){
|
||||
if(num_pows[j] != -1){ *aa++ = z; t++; }
|
||||
z *= x;
|
||||
}
|
||||
GRID_ASSERT(t == n+1);
|
||||
assert(t == n+1);
|
||||
|
||||
z = (bigfloat)1l;
|
||||
t = 0;
|
||||
@@ -286,7 +286,7 @@ void AlgRemezGeneral::equations(){
|
||||
if(den_pows[j] != -1){ *aa++ = -y * z; t++; }
|
||||
z *= x;
|
||||
}
|
||||
GRID_ASSERT(t == d);
|
||||
assert(t == d);
|
||||
|
||||
B[i] = y * z; // Right hand side vector
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ class AlgRemezGeneral{
|
||||
bigfloat (*f)(bigfloat x, void *data), void *data);
|
||||
|
||||
inline int getDegree(void) const{
|
||||
GRID_ASSERT(n==d);
|
||||
assert(n==d);
|
||||
return n;
|
||||
}
|
||||
// Reset the bounds of the approximation
|
||||
|
||||
@@ -74,7 +74,7 @@ bigfloat epsilonMobius(bigfloat x, void* data){
|
||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out,
|
||||
const std::vector<RealD> &omega_in, const int Ls_in,
|
||||
const RealD lambda_bound){
|
||||
GRID_ASSERT(omega_in.size() == Ls_in);
|
||||
assert(omega_in.size() == Ls_in);
|
||||
omega_out.resize(Ls_out);
|
||||
|
||||
//Use the Remez algorithm to generate the appropriate rational polynomial
|
||||
|
||||
@@ -293,7 +293,7 @@ static void sncndnFK(INTERNAL_PRECISION u, INTERNAL_PRECISION k,
|
||||
* Set type = 0 for the Zolotarev approximation, which is zero at x = 0, and
|
||||
* type = 1 for the approximation which is infinite at x = 0. */
|
||||
|
||||
zolotarev_data* zolotarev(ZOLO_PRECISION epsilon, int n, int type) {
|
||||
zolotarev_data* zolotarev(PRECISION epsilon, int n, int type) {
|
||||
INTERNAL_PRECISION A, c, cp, kp, ksq, sn, cn, dn, Kp, Kj, z, z0, t, M, F,
|
||||
l, invlambda, xi, xisq, *tv, s, opl;
|
||||
int m, czero, ts;
|
||||
@@ -375,12 +375,12 @@ zolotarev_data* zolotarev(ZOLO_PRECISION epsilon, int n, int type) {
|
||||
construct_partfrac(d);
|
||||
construct_contfrac(d);
|
||||
|
||||
/* Converting everything to ZOLO_PRECISION for external use only */
|
||||
/* Converting everything to PRECISION for external use only */
|
||||
|
||||
zd = (zolotarev_data*) malloc(sizeof(zolotarev_data));
|
||||
zd -> A = (ZOLO_PRECISION) d -> A;
|
||||
zd -> Delta = (ZOLO_PRECISION) d -> Delta;
|
||||
zd -> epsilon = (ZOLO_PRECISION) d -> epsilon;
|
||||
zd -> A = (PRECISION) d -> A;
|
||||
zd -> Delta = (PRECISION) d -> Delta;
|
||||
zd -> epsilon = (PRECISION) d -> epsilon;
|
||||
zd -> n = d -> n;
|
||||
zd -> type = d -> type;
|
||||
zd -> dn = d -> dn;
|
||||
@@ -390,24 +390,24 @@ zolotarev_data* zolotarev(ZOLO_PRECISION epsilon, int n, int type) {
|
||||
zd -> deg_num = d -> deg_num;
|
||||
zd -> deg_denom = d -> deg_denom;
|
||||
|
||||
zd -> a = (ZOLO_PRECISION*) malloc(zd -> dn * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> dn; m++) zd -> a[m] = (ZOLO_PRECISION) d -> a[m];
|
||||
zd -> a = (PRECISION*) malloc(zd -> dn * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dn; m++) zd -> a[m] = (PRECISION) d -> a[m];
|
||||
free(d -> a);
|
||||
|
||||
zd -> ap = (ZOLO_PRECISION*) malloc(zd -> dd * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> dd; m++) zd -> ap[m] = (ZOLO_PRECISION) d -> ap[m];
|
||||
zd -> ap = (PRECISION*) malloc(zd -> dd * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dd; m++) zd -> ap[m] = (PRECISION) d -> ap[m];
|
||||
free(d -> ap);
|
||||
|
||||
zd -> alpha = (ZOLO_PRECISION*) malloc(zd -> da * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> da; m++) zd -> alpha[m] = (ZOLO_PRECISION) d -> alpha[m];
|
||||
zd -> alpha = (PRECISION*) malloc(zd -> da * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> da; m++) zd -> alpha[m] = (PRECISION) d -> alpha[m];
|
||||
free(d -> alpha);
|
||||
|
||||
zd -> beta = (ZOLO_PRECISION*) malloc(zd -> db * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> db; m++) zd -> beta[m] = (ZOLO_PRECISION) d -> beta[m];
|
||||
zd -> beta = (PRECISION*) malloc(zd -> db * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> db; m++) zd -> beta[m] = (PRECISION) d -> beta[m];
|
||||
free(d -> beta);
|
||||
|
||||
zd -> gamma = (ZOLO_PRECISION*) malloc(zd -> n * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> n; m++) zd -> gamma[m] = (ZOLO_PRECISION) d -> gamma[m];
|
||||
zd -> gamma = (PRECISION*) malloc(zd -> n * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> n; m++) zd -> gamma[m] = (PRECISION) d -> gamma[m];
|
||||
free(d -> gamma);
|
||||
|
||||
free(d);
|
||||
@@ -426,7 +426,7 @@ void zolotarev_free(zolotarev_data *zdata)
|
||||
}
|
||||
|
||||
|
||||
zolotarev_data* higham(ZOLO_PRECISION epsilon, int n) {
|
||||
zolotarev_data* higham(PRECISION epsilon, int n) {
|
||||
INTERNAL_PRECISION A, M, c, cp, z, z0, t, epssq;
|
||||
int m, czero;
|
||||
zolotarev_data *zd;
|
||||
@@ -481,9 +481,9 @@ zolotarev_data* higham(ZOLO_PRECISION epsilon, int n) {
|
||||
/* Converting everything to PRECISION for external use only */
|
||||
|
||||
zd = (zolotarev_data*) malloc(sizeof(zolotarev_data));
|
||||
zd -> A = (ZOLO_PRECISION) d -> A;
|
||||
zd -> Delta = (ZOLO_PRECISION) d -> Delta;
|
||||
zd -> epsilon = (ZOLO_PRECISION) d -> epsilon;
|
||||
zd -> A = (PRECISION) d -> A;
|
||||
zd -> Delta = (PRECISION) d -> Delta;
|
||||
zd -> epsilon = (PRECISION) d -> epsilon;
|
||||
zd -> n = d -> n;
|
||||
zd -> type = d -> type;
|
||||
zd -> dn = d -> dn;
|
||||
@@ -493,24 +493,24 @@ zolotarev_data* higham(ZOLO_PRECISION epsilon, int n) {
|
||||
zd -> deg_num = d -> deg_num;
|
||||
zd -> deg_denom = d -> deg_denom;
|
||||
|
||||
zd -> a = (ZOLO_PRECISION*) malloc(zd -> dn * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> dn; m++) zd -> a[m] = (ZOLO_PRECISION) d -> a[m];
|
||||
zd -> a = (PRECISION*) malloc(zd -> dn * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dn; m++) zd -> a[m] = (PRECISION) d -> a[m];
|
||||
free(d -> a);
|
||||
|
||||
zd -> ap = (ZOLO_PRECISION*) malloc(zd -> dd * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> dd; m++) zd -> ap[m] = (ZOLO_PRECISION) d -> ap[m];
|
||||
zd -> ap = (PRECISION*) malloc(zd -> dd * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> dd; m++) zd -> ap[m] = (PRECISION) d -> ap[m];
|
||||
free(d -> ap);
|
||||
|
||||
zd -> alpha = (ZOLO_PRECISION*) malloc(zd -> da * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> da; m++) zd -> alpha[m] = (ZOLO_PRECISION) d -> alpha[m];
|
||||
zd -> alpha = (PRECISION*) malloc(zd -> da * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> da; m++) zd -> alpha[m] = (PRECISION) d -> alpha[m];
|
||||
free(d -> alpha);
|
||||
|
||||
zd -> beta = (ZOLO_PRECISION*) malloc(zd -> db * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> db; m++) zd -> beta[m] = (ZOLO_PRECISION) d -> beta[m];
|
||||
zd -> beta = (PRECISION*) malloc(zd -> db * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> db; m++) zd -> beta[m] = (PRECISION) d -> beta[m];
|
||||
free(d -> beta);
|
||||
|
||||
zd -> gamma = (ZOLO_PRECISION*) malloc(zd -> n * sizeof(ZOLO_PRECISION));
|
||||
for (m = 0; m < zd -> n; m++) zd -> gamma[m] = (ZOLO_PRECISION) d -> gamma[m];
|
||||
zd -> gamma = (PRECISION*) malloc(zd -> n * sizeof(PRECISION));
|
||||
for (m = 0; m < zd -> n; m++) zd -> gamma[m] = (PRECISION) d -> gamma[m];
|
||||
free(d -> gamma);
|
||||
|
||||
free(d);
|
||||
@@ -523,17 +523,17 @@ NAMESPACE_END(Grid);
|
||||
#ifdef TEST
|
||||
|
||||
#undef ZERO
|
||||
#define ZERO ((ZOLO_PRECISION) 0)
|
||||
#define ZERO ((PRECISION) 0)
|
||||
#undef ONE
|
||||
#define ONE ((ZOLO_PRECISION) 1)
|
||||
#define ONE ((PRECISION) 1)
|
||||
#undef TWO
|
||||
#define TWO ((ZOLO_PRECISION) 2)
|
||||
#define TWO ((PRECISION) 2)
|
||||
|
||||
/* Evaluate the rational approximation R(x) using the factored form */
|
||||
|
||||
static ZOLO_PRECISION zolotarev_eval(ZOLO_PRECISION x, zolotarev_data* rdata) {
|
||||
static PRECISION zolotarev_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
ZOLO_PRECISION R;
|
||||
PRECISION R;
|
||||
|
||||
if (rdata -> type == 0) {
|
||||
R = rdata -> A * x;
|
||||
@@ -551,9 +551,9 @@ static ZOLO_PRECISION zolotarev_eval(ZOLO_PRECISION x, zolotarev_data* rdata) {
|
||||
|
||||
/* Evaluate the rational approximation R(x) using the partial fraction form */
|
||||
|
||||
static ZOLO_PRECISION zolotarev_partfrac_eval(ZOLO_PRECISION x, zolotarev_data* rdata) {
|
||||
static PRECISION zolotarev_partfrac_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
ZOLO_PRECISION R = rdata -> alpha[rdata -> da - 1];
|
||||
PRECISION R = rdata -> alpha[rdata -> da - 1];
|
||||
for (m = 0; m < rdata -> dd; m++)
|
||||
R += rdata -> alpha[m] / (x * x - rdata -> ap[m]);
|
||||
if (rdata -> type == 1) R += rdata -> alpha[rdata -> dd] / (x * x);
|
||||
@@ -568,18 +568,18 @@ static ZOLO_PRECISION zolotarev_partfrac_eval(ZOLO_PRECISION x, zolotarev_data*
|
||||
* non-signalling overflow this will work correctly since 1/(1/0) = 1/INF = 0,
|
||||
* but with signalling overflow you will get an error message. */
|
||||
|
||||
static ZOLO_PRECISION zolotarev_contfrac_eval(ZOLO_PRECISION x, zolotarev_data* rdata) {
|
||||
static PRECISION zolotarev_contfrac_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
ZOLO_PRECISION R = rdata -> beta[0] * x;
|
||||
PRECISION R = rdata -> beta[0] * x;
|
||||
for (m = 1; m < rdata -> db; m++) R = rdata -> beta[m] * x + ONE / R;
|
||||
return R;
|
||||
}
|
||||
|
||||
/* Evaluate the rational approximation R(x) using Cayley form */
|
||||
|
||||
static ZOLO_PRECISION zolotarev_cayley_eval(ZOLO_PRECISION x, zolotarev_data* rdata) {
|
||||
static PRECISION zolotarev_cayley_eval(PRECISION x, zolotarev_data* rdata) {
|
||||
int m;
|
||||
ZOLO_PRECISION T;
|
||||
PRECISION T;
|
||||
|
||||
T = rdata -> type == 0 ? ONE : -ONE;
|
||||
for (m = 0; m < rdata -> n; m++)
|
||||
@@ -607,7 +607,7 @@ int main(int argc, char** argv) {
|
||||
int m, n, plotpts = 5000, type = 0;
|
||||
float eps, x, ypferr, ycferr, ycaylerr, maxypferr, maxycferr, maxycaylerr;
|
||||
zolotarev_data *rdata;
|
||||
ZOLO_PRECISION y;
|
||||
PRECISION y;
|
||||
FILE *plot_function, *plot_error,
|
||||
*plot_partfrac, *plot_contfrac, *plot_cayley;
|
||||
|
||||
@@ -626,13 +626,13 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
|
||||
rdata = type == 2
|
||||
? higham((ZOLO_PRECISION) eps, n)
|
||||
: zolotarev((ZOLO_PRECISION) eps, n, type);
|
||||
? higham((PRECISION) eps, n)
|
||||
: zolotarev((PRECISION) eps, n, type);
|
||||
|
||||
printf("Zolotarev Test: R(epsilon = %g, n = %d, type = %d)\n\t"
|
||||
STRINGIFY(VERSION) "\n\t" STRINGIFY(HVERSION)
|
||||
"\n\tINTERNAL_PRECISION = " STRINGIFY(INTERNAL_PRECISION)
|
||||
"\tZOLO_PRECISION = " STRINGIFY(ZOLO_PRECISION)
|
||||
"\tPRECISION = " STRINGIFY(PRECISION)
|
||||
"\n\n\tRational approximation of degree (%d,%d), %s at x = 0\n"
|
||||
"\tDelta = %g (maximum error)\n\n"
|
||||
"\tA = %g (overall factor)\n",
|
||||
@@ -681,15 +681,15 @@ int main(int argc, char** argv) {
|
||||
x = 2.4 * (float) m / plotpts - 1.2;
|
||||
if (rdata -> type == 0 || fabs(x) * (float) plotpts > 1.0) {
|
||||
/* skip x = 0 for type 1, as R(0) is singular */
|
||||
y = zolotarev_eval((ZOLO_PRECISION) x, rdata);
|
||||
y = zolotarev_eval((PRECISION) x, rdata);
|
||||
fprintf(plot_function, "%g %g\n", x, (float) y);
|
||||
fprintf(plot_error, "%g %g\n",
|
||||
x, (float)((y - ((x > 0.0 ? ONE : -ONE))) / rdata -> Delta));
|
||||
ypferr = (float)((zolotarev_partfrac_eval((ZOLO_PRECISION) x, rdata) - y)
|
||||
ypferr = (float)((zolotarev_partfrac_eval((PRECISION) x, rdata) - y)
|
||||
/ rdata -> Delta);
|
||||
ycferr = (float)((zolotarev_contfrac_eval((ZOLO_PRECISION) x, rdata) - y)
|
||||
ycferr = (float)((zolotarev_contfrac_eval((PRECISION) x, rdata) - y)
|
||||
/ rdata -> Delta);
|
||||
ycaylerr = (float)((zolotarev_cayley_eval((ZOLO_PRECISION) x, rdata) - y)
|
||||
ycaylerr = (float)((zolotarev_cayley_eval((PRECISION) x, rdata) - y)
|
||||
/ rdata -> Delta);
|
||||
if (fabs(x) < 1.0 && fabs(x) > rdata -> epsilon) {
|
||||
maxypferr = MAX(maxypferr, fabs(ypferr));
|
||||
|
||||
@@ -9,10 +9,10 @@ NAMESPACE_BEGIN(Approx);
|
||||
#define HVERSION Header Time-stamp: <14-OCT-2004 09:26:51.00 adk@MISSCONTRARY>
|
||||
|
||||
#ifndef ZOLOTAREV_INTERNAL
|
||||
#ifndef ZOLO_PRECISION
|
||||
#define ZOLO_PRECISION double
|
||||
#ifndef PRECISION
|
||||
#define PRECISION double
|
||||
#endif
|
||||
#define ZPRECISION ZOLO_PRECISION
|
||||
#define ZPRECISION PRECISION
|
||||
#define ZOLOTAREV_DATA zolotarev_data
|
||||
#endif
|
||||
|
||||
@@ -77,8 +77,8 @@ typedef struct {
|
||||
* zolotarev_data structure. The arguments must satisfy the constraints that
|
||||
* epsilon > 0, n > 0, and type = 0 or 1. */
|
||||
|
||||
ZOLOTAREV_DATA* higham(ZOLO_PRECISION epsilon, int n) ;
|
||||
ZOLOTAREV_DATA* zolotarev(ZOLO_PRECISION epsilon, int n, int type);
|
||||
ZOLOTAREV_DATA* higham(PRECISION epsilon, int n) ;
|
||||
ZOLOTAREV_DATA* zolotarev(PRECISION epsilon, int n, int type);
|
||||
void zolotarev_free(zolotarev_data *zdata);
|
||||
#endif
|
||||
|
||||
@@ -86,4 +86,3 @@ void zolotarev_free(zolotarev_data *zdata);
|
||||
NAMESPACE_END(Approx);
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: Grid/algorithms/blas/A2ASpatialSum.h
|
||||
|
||||
Copyright (C) 2025
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/*
|
||||
A2ASpatialSum
|
||||
|
||||
Replaces the scalar spatial accumulation loop in A2A extended meson field
|
||||
contractions with a batched GEMM over local time slices, enabling GPU offload.
|
||||
|
||||
Given:
|
||||
leftv[N_i][osite] — conjugated left SpinColourVectors (SIMD-packed)
|
||||
loopRight[N_j][osite]— type-contracted right SpinColourVectors (SIMD-packed)
|
||||
|
||||
Computes:
|
||||
EMF[i,j,t] = sum_{x,s,c} leftv[i][x,t,s,c] * loopRight[j][x,t,s,c]
|
||||
|
||||
via batched GEMM over nt local time slices, then GlobalSumVector across MPI.
|
||||
|
||||
Memory layout (all C row-major):
|
||||
W_buf [nt][N_i][nxyz*Nsc] — W[t][i][x*Nsc+sc] = leftv[i] at (x,t)
|
||||
LR_buf [nt][N_j][nxyz*Nsc] — LR[t][j][x*Nsc+sc] = loopRight[j] at (x,t)
|
||||
EMF_buf[nt][N_j][N_i] — column-major result; EMF[i,j,t] = EMF_buf[t][j][i]
|
||||
|
||||
BLAS call (column-major, OP_T on A so A is read as W[i][k]):
|
||||
C = A^T * B where A=W[N_i×K C-row], B=LR[N_j×K C-row], C=[N_j×N_i C-row]
|
||||
→ C[i,j] = sum_k W[i][k] * LR[j][k] = EMF[i,j] ✓
|
||||
*/
|
||||
template<class vobj>
|
||||
class A2ASpatialSum
|
||||
{
|
||||
public:
|
||||
typedef typename vobj::scalar_type scalar;
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
|
||||
GridBase *grid;
|
||||
int N_i, N_j;
|
||||
int nt, nxyz, Nsc;
|
||||
|
||||
deviceVector<scalar> W_buf;
|
||||
deviceVector<scalar> LR_buf;
|
||||
deviceVector<scalar> EMF_buf;
|
||||
deviceVector<scalar *> W_ptrs;
|
||||
deviceVector<scalar *> LR_ptrs;
|
||||
deviceVector<scalar *> EMF_ptrs;
|
||||
|
||||
A2ASpatialSum() : grid(nullptr), N_i(0), N_j(0), nt(0), nxyz(0), Nsc(0) {}
|
||||
|
||||
void Allocate(int _N_i, int _N_j, GridBase *_grid)
|
||||
{
|
||||
grid = _grid;
|
||||
N_i = _N_i;
|
||||
N_j = _N_j;
|
||||
Coordinate ldims = grid->LocalDimensions();
|
||||
nt = ldims[grid->Nd() - 1];
|
||||
nxyz = grid->lSites() / nt;
|
||||
Nsc = sizeof(sobj) / sizeof(scalar);
|
||||
|
||||
W_buf.resize(nt * N_i * nxyz * Nsc);
|
||||
LR_buf.resize(nt * N_j * nxyz * Nsc);
|
||||
EMF_buf.resize(nt * N_j * N_i);
|
||||
|
||||
// Build persistent batch pointer arrays
|
||||
W_ptrs.resize(nt);
|
||||
LR_ptrs.resize(nt);
|
||||
EMF_ptrs.resize(nt);
|
||||
scalar *Wh = &W_buf[0];
|
||||
scalar *LRh = &LR_buf[0];
|
||||
scalar *EMFh = &EMF_buf[0];
|
||||
int lN_i = N_i, lN_j = N_j, lnxyz = nxyz, lNsc = Nsc;
|
||||
for (int t = 0; t < nt; t++) {
|
||||
acceleratorPut(W_ptrs[t], Wh + t * lN_i * lnxyz * lNsc);
|
||||
acceleratorPut(LR_ptrs[t], LRh + t * lN_j * lnxyz * lNsc);
|
||||
acceleratorPut(EMF_ptrs[t], EMFh + t * lN_j * lN_i);
|
||||
}
|
||||
}
|
||||
|
||||
void PackLeft(const std::vector<Lattice<vobj>> &leftv)
|
||||
{
|
||||
GRID_ASSERT((int)leftv.size() == N_i);
|
||||
PackVectors(leftv, &W_buf[0], N_i);
|
||||
}
|
||||
|
||||
void PackRight(const std::vector<Lattice<vobj>> &loopRight)
|
||||
{
|
||||
GRID_ASSERT((int)loopRight.size() == N_j);
|
||||
PackVectors(loopRight, &LR_buf[0], N_j);
|
||||
}
|
||||
|
||||
private:
|
||||
// Pack vecs[N] lattice fields into buf[nt][N][nxyz*Nsc], extracting all SIMD lanes.
|
||||
void PackVectors(const std::vector<Lattice<vobj>> &vecs, scalar *buf, int N)
|
||||
{
|
||||
int nd = grid->_ndimension;
|
||||
int osites = grid->oSites();
|
||||
int Nsimd = vobj::Nsimd();
|
||||
int lN = N;
|
||||
int lNsc = Nsc;
|
||||
int lnxyz = nxyz;
|
||||
Coordinate rdimensions = grid->_rdimensions;
|
||||
Coordinate ldims = grid->LocalDimensions();
|
||||
Coordinate simd = grid->_simd_layout;
|
||||
|
||||
for (int n = 0; n < N; n++) {
|
||||
autoView(src_v, vecs[n], AcceleratorRead);
|
||||
accelerator_for(sf, osites, Nsimd, {
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane = acceleratorSIMTlane(Nsimd);
|
||||
#else
|
||||
for (int lane = 0; lane < Nsimd; lane++) {
|
||||
#endif
|
||||
Coordinate icoor(nd), ocoor(nd), lcoor(nd);
|
||||
Lexicographic::CoorFromIndex(icoor, lane, simd);
|
||||
Lexicographic::CoorFromIndex(ocoor, sf, rdimensions);
|
||||
for (int d = 0; d < nd; d++)
|
||||
lcoor[d] = rdimensions[d] * icoor[d] + ocoor[d];
|
||||
|
||||
int l_t = lcoor[nd - 1];
|
||||
Coordinate xyz_coor = lcoor;
|
||||
xyz_coor[nd - 1] = 0;
|
||||
int64_t l_xyz;
|
||||
Lexicographic::IndexFromCoor(xyz_coor, l_xyz, ldims);
|
||||
|
||||
sobj data = extractLane(lane, src_v[sf]);
|
||||
scalar *data_s = (scalar *)&data;
|
||||
|
||||
int64_t base = (int64_t)l_t * lN * lnxyz * lNsc
|
||||
+ (int64_t)n * lnxyz * lNsc
|
||||
+ l_xyz * lNsc;
|
||||
for (int sc = 0; sc < lNsc; sc++)
|
||||
buf[base + sc] = data_s[sc];
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
// Batched GEMM + MPI reduction → result[nt_global][N_i][N_j]
|
||||
//
|
||||
// BLAS (column-major, OP_T on A):
|
||||
// C[N_j×N_i] = A^T[N_i×K] * B[N_j×K] with K=nxyz*Nsc
|
||||
// reading A as C row-major [N_i][K] and B as C row-major [N_j][K]
|
||||
// → C[i,j] = sum_k W[i,k] * LR[j,k] = EMF[i,j] ✓
|
||||
void Sum(Eigen::Tensor<ComplexD, 3> &result)
|
||||
{
|
||||
GridBLAS BLAS;
|
||||
|
||||
int K = nxyz * Nsc;
|
||||
BLAS.gemmBatched(GridBLAS_OP_T, GridBLAS_OP_N,
|
||||
N_i, N_j, K,
|
||||
scalar(1.0),
|
||||
W_ptrs,
|
||||
LR_ptrs,
|
||||
scalar(0.0),
|
||||
EMF_ptrs);
|
||||
BLAS.synchronise();
|
||||
|
||||
// Copy from device and distribute into global-t layout
|
||||
int nt_global = result.dimension(0);
|
||||
int nd = grid->Nd();
|
||||
int lt_start = grid->LocalStarts()[nd - 1];
|
||||
|
||||
std::vector<scalar> host_emf(nt * N_j * N_i);
|
||||
acceleratorCopyFromDevice(&EMF_buf[0], host_emf.data(),
|
||||
nt * N_j * N_i * sizeof(scalar));
|
||||
|
||||
// EMF_buf[t][j*N_i + i] = EMF[i,j] for local t
|
||||
std::vector<scalar> global_emf(nt_global * N_i * N_j, scalar(0.0));
|
||||
for (int lt = 0; lt < nt; lt++) {
|
||||
int gt = lt + lt_start;
|
||||
for (int i = 0; i < N_i; i++)
|
||||
for (int j = 0; j < N_j; j++)
|
||||
global_emf[gt * N_i * N_j + i * N_j + j] = host_emf[lt * N_j * N_i + j * N_i + i];
|
||||
}
|
||||
grid->GlobalSumVector(global_emf.data(), nt_global * N_i * N_j);
|
||||
|
||||
for (int gt = 0; gt < nt_global; gt++)
|
||||
for (int i = 0; i < N_i; i++)
|
||||
for (int j = 0; j < N_j; j++)
|
||||
result(gt, i, j) = global_emf[gt * N_i * N_j + i * N_j + j];
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,34 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: BatchedBlas.h
|
||||
|
||||
Copyright (C) 2023
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#include <Grid/GridCore.h>
|
||||
#include <Grid/algorithms/blas/BatchedBlas.h>
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
gridblasHandle_t GridBLAS::gridblasHandle;
|
||||
int GridBLAS::gridblasInit;
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,300 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: MomentumProject.h
|
||||
|
||||
Copyright (C) 2025
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
/*
|
||||
MultiMomProject
|
||||
|
||||
Import vectors -> nxyz x (ncomponent x nt)
|
||||
Import complex phases -> nmom x nxy
|
||||
|
||||
apply = via (possibly batched) GEMM
|
||||
*/
|
||||
template<class Field, class ComplexField>
|
||||
class MomentumProject
|
||||
{
|
||||
public:
|
||||
|
||||
typedef typename Field::scalar_type scalar;
|
||||
typedef typename Field::scalar_object scalar_object;
|
||||
|
||||
GridBase *grid;
|
||||
uint64_t nmom;
|
||||
uint64_t nxyz;
|
||||
uint64_t nt;
|
||||
uint64_t nbtw;
|
||||
uint64_t words;
|
||||
|
||||
deviceVector<scalar> BLAS_V; //
|
||||
deviceVector<scalar> BLAS_M; //
|
||||
deviceVector<scalar> BLAS_P; //
|
||||
|
||||
MomentumProject(){};
|
||||
~MomentumProject(){ Deallocate(); };
|
||||
|
||||
void Deallocate(void)
|
||||
{
|
||||
grid=nullptr;
|
||||
nmom=0;
|
||||
nxyz=0;
|
||||
nt=0;
|
||||
nbtw=0;
|
||||
words=0;
|
||||
BLAS_V.resize(0);
|
||||
BLAS_M.resize(0);
|
||||
BLAS_P.resize(0);
|
||||
}
|
||||
void Allocate(int _nmom,GridBase *_grid)
|
||||
{
|
||||
grid=_grid;
|
||||
Coordinate ldims = grid->LocalDimensions();
|
||||
|
||||
nmom=_nmom;
|
||||
nt = ldims[grid->Nd()-1];
|
||||
nxyz = grid->lSites()/nt;
|
||||
words = sizeof(scalar_object)/sizeof(scalar);
|
||||
nbtw = nt * words;
|
||||
|
||||
BLAS_V.resize (nxyz * nt * words );
|
||||
BLAS_M.resize (nmom * nxyz );
|
||||
BLAS_P.resize (nmom * nt * words );
|
||||
}
|
||||
void ImportMomenta(const std::vector <ComplexField> &momenta)
|
||||
{
|
||||
GRID_ASSERT(momenta.size()==nmom);
|
||||
// might as well just make the momenta here
|
||||
typedef typename Field::vector_object vobj;
|
||||
|
||||
int nd = grid->_ndimension;
|
||||
|
||||
uint64_t sz = BLAS_M.size();
|
||||
|
||||
GRID_ASSERT(momenta.size()==nmom)
|
||||
GRID_ASSERT(momenta[0].Grid()==grid);
|
||||
GRID_ASSERT(sz = nxyz * nmom);
|
||||
|
||||
Coordinate rdimensions = grid->_rdimensions;
|
||||
Coordinate ldims = grid->LocalDimensions();
|
||||
int64_t osites = grid->oSites();
|
||||
Coordinate simd = grid->_simd_layout;
|
||||
const int Nsimd = vobj::Nsimd();
|
||||
uint64_t lwords = words; // local variable for copy in to GPU
|
||||
int64_t Nxyz = nxyz;
|
||||
auto blasData_p = &BLAS_M[0];
|
||||
for(int m=0;m<momenta.size();m++){
|
||||
|
||||
autoView( Data , momenta[m], AcceleratorRead);
|
||||
auto Data_p = &Data[0];
|
||||
|
||||
accelerator_for(xyz,nxyz,1,{
|
||||
//////////////////////////////////////////
|
||||
// isite -- map lane within buffer to lane within lattice
|
||||
////////////////////////////////////////////
|
||||
Coordinate lcoor(nd,0);
|
||||
Lexicographic::CoorFromIndex(lcoor,xyz,ldims);
|
||||
|
||||
Coordinate icoor(nd);
|
||||
Coordinate ocoor(nd);
|
||||
for (int d = 0; d < nd; d++) {
|
||||
icoor[d] = lcoor[d]/rdimensions[d];
|
||||
ocoor[d] = lcoor[d]%rdimensions[d];
|
||||
}
|
||||
int64_t osite;
|
||||
int64_t isite;
|
||||
Lexicographic::IndexFromCoor(ocoor,osite,rdimensions);
|
||||
Lexicographic::IndexFromCoor(icoor,isite,simd);
|
||||
|
||||
// BLAS_M[nmom][slice_vol]
|
||||
// Fortran Column major BLAS layout is M_xyz,mom
|
||||
scalar data = extractLane(isite,Data[osite]);
|
||||
uint64_t idx = xyz+m*Nxyz;
|
||||
blasData_p[idx] = data;
|
||||
});
|
||||
}
|
||||
}
|
||||
void ImportVector(Field &vec)
|
||||
{
|
||||
typedef typename Field::vector_object vobj;
|
||||
|
||||
int nd = grid->_ndimension;
|
||||
|
||||
uint64_t sz = BLAS_V.size();
|
||||
|
||||
GRID_ASSERT(sz = nxyz * words * nt);
|
||||
|
||||
Coordinate rdimensions = grid->_rdimensions;
|
||||
Coordinate ldims= grid->LocalDimensions();
|
||||
int64_t osites = grid->oSites();
|
||||
Coordinate simd = grid->_simd_layout;
|
||||
const int Nsimd = vobj::Nsimd();
|
||||
uint64_t lwords= words; // local variable for copy in to GPU
|
||||
|
||||
auto blasData_p = &BLAS_V[0];
|
||||
autoView( Data , vec, AcceleratorRead);
|
||||
auto Data_p = &Data[0];
|
||||
|
||||
int64_t nwords = words;// for capture
|
||||
int64_t Nt = nt;// for capture
|
||||
|
||||
accelerator_for(sf,osites,Nsimd,{
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||
#else
|
||||
for(int lane=0;lane<Nsimd;lane++) {
|
||||
#endif
|
||||
//////////////////////////////////////////
|
||||
// isite -- map lane within buffer to lane within lattice
|
||||
////////////////////////////////////////////
|
||||
Coordinate lcoor(nd,0);
|
||||
Coordinate icoor(nd);
|
||||
Coordinate ocoor(nd);
|
||||
|
||||
Lexicographic::CoorFromIndex(icoor,lane,simd);
|
||||
Lexicographic::CoorFromIndex(ocoor,sf,rdimensions);
|
||||
|
||||
|
||||
int64_t l_xyz = 0;
|
||||
for (int d = 0; d < nd; d++) {
|
||||
lcoor[d] = rdimensions[d]*icoor[d] + ocoor[d];
|
||||
}
|
||||
uint64_t l_t = lcoor[nd-1];
|
||||
|
||||
Coordinate xyz_coor = lcoor;
|
||||
xyz_coor[nd-1] =0;
|
||||
Lexicographic::IndexFromCoor(xyz_coor,l_xyz,ldims);
|
||||
|
||||
|
||||
scalar_object data = extractLane(lane,Data[sf]);
|
||||
scalar *data_words = (scalar *) &data;
|
||||
for(int w = 0 ; w < nwords; w++) {
|
||||
// BLAS_V[slice_vol][nt][words]
|
||||
// Fortran Column major BLAS layout is V_(t,w)_xyz
|
||||
uint64_t idx = w+l_t*nwords + l_xyz * nwords * Nt;
|
||||
blasData_p[idx] = data_words[w];
|
||||
}
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
}
|
||||
void ExportMomentumProjection(std::vector<typename Field::scalar_object> &projection)
|
||||
{
|
||||
projection.resize(nmom*nt);
|
||||
acceleratorCopyFromDevice(&BLAS_P[0],(scalar *)&projection[0],BLAS_P.size()*sizeof(scalar));
|
||||
// Could decide on a layout late?
|
||||
}
|
||||
|
||||
// Row major layout "C" order:
|
||||
// BLAS_V[slice_vol][nt][words]
|
||||
// BLAS_M[nmom][slice_vol]
|
||||
// BLAS_P[nmom][nt][words]
|
||||
//
|
||||
// Fortran Column major BLAS layout is V_(w,t)_xyz
|
||||
// Fortran Column major BLAS layout is M_xyz,mom
|
||||
// Fortran Column major BLAS layout is P_(w,t),mom
|
||||
//
|
||||
// Projected
|
||||
//
|
||||
// P = (V * M)_(w,t),mom
|
||||
//
|
||||
void Project(Field &data,std::vector< typename Field::scalar_object > & projected_gdata)
|
||||
{
|
||||
double t_import=0;
|
||||
double t_export=0;
|
||||
double t_gemm =0;
|
||||
double t_allreduce=0;
|
||||
t_import-=usecond();
|
||||
this->ImportVector(data);
|
||||
|
||||
std::vector< typename Field::scalar_object > projected_planes;
|
||||
|
||||
deviceVector<scalar *> Vd(1);
|
||||
deviceVector<scalar *> Md(1);
|
||||
deviceVector<scalar *> Pd(1);
|
||||
|
||||
scalar * Vh = & BLAS_V[0];
|
||||
scalar * Mh = & BLAS_M[0];
|
||||
scalar * Ph = & BLAS_P[0];
|
||||
|
||||
acceleratorPut(Vd[0],Vh);
|
||||
acceleratorPut(Md[0],Mh);
|
||||
acceleratorPut(Pd[0],Ph);
|
||||
t_import+=usecond();
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
/////////////////////////////////////////
|
||||
// P_im = VMmx . Vxi
|
||||
/////////////////////////////////////////
|
||||
t_gemm-=usecond();
|
||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
||||
words*nt,nmom,nxyz,
|
||||
scalar(1.0),
|
||||
Vd,
|
||||
Md,
|
||||
scalar(0.0), // wipe out result
|
||||
Pd);
|
||||
BLAS.synchronise();
|
||||
t_gemm+=usecond();
|
||||
|
||||
t_export-=usecond();
|
||||
ExportMomentumProjection(projected_planes); // resizes
|
||||
t_export+=usecond();
|
||||
|
||||
/////////////////////////////////
|
||||
// Reduce across MPI ranks
|
||||
/////////////////////////////////
|
||||
int nd = grid->Nd();
|
||||
int gt = grid->GlobalDimensions()[nd-1];
|
||||
int lt = grid->LocalDimensions()[nd-1];
|
||||
projected_gdata.resize(gt*nmom);
|
||||
for(int t=0;t<gt*nmom;t++){ // global Nt array with zeroes for stuff not on this node
|
||||
projected_gdata[t]=Zero();
|
||||
}
|
||||
for(int t=0;t<lt;t++){
|
||||
for(int m=0;m<nmom;m++){
|
||||
int st = grid->LocalStarts()[nd-1];
|
||||
projected_gdata[t+st + gt*m] = projected_planes[t+lt*m];
|
||||
}}
|
||||
t_allreduce-=usecond();
|
||||
grid->GlobalSumVector((scalar *)&projected_gdata[0],gt*nmom*words);
|
||||
t_allreduce+=usecond();
|
||||
|
||||
std::cout << GridLogPerformance<<" MomentumProject t_import "<<t_import<<"us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<" MomentumProject t_export "<<t_export<<"us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<" MomentumProject t_gemm "<<t_gemm<<"us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<" MomentumProject t_reduce "<<t_allreduce<<"us"<<std::endl;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,376 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: MultiRHSBlockCGLinalg.h
|
||||
|
||||
Copyright (C) 2024
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
/* Need helper object for BLAS accelerated mrhs blockCG */
|
||||
template<class Field>
|
||||
class MultiRHSBlockCGLinalg
|
||||
{
|
||||
public:
|
||||
|
||||
typedef typename Field::scalar_type scalar;
|
||||
typedef typename Field::scalar_object scalar_object;
|
||||
typedef typename Field::vector_object vector_object;
|
||||
|
||||
deviceVector<scalar> BLAS_X; // nrhs x vol -- the sources
|
||||
deviceVector<scalar> BLAS_Y; // nrhs x vol -- the result
|
||||
deviceVector<scalar> BLAS_C; // nrhs x nrhs -- the coefficients
|
||||
deviceVector<scalar> BLAS_Cred; // nrhs x nrhs x oSites -- reduction buffer
|
||||
deviceVector<scalar *> Xdip;
|
||||
deviceVector<scalar *> Ydip;
|
||||
deviceVector<scalar *> Cdip;
|
||||
|
||||
MultiRHSBlockCGLinalg() {};
|
||||
~MultiRHSBlockCGLinalg(){ Deallocate(); };
|
||||
|
||||
void Deallocate(void)
|
||||
{
|
||||
Xdip.resize(0);
|
||||
Ydip.resize(0);
|
||||
Cdip.resize(0);
|
||||
BLAS_Cred.resize(0);
|
||||
BLAS_C.resize(0);
|
||||
BLAS_X.resize(0);
|
||||
BLAS_Y.resize(0);
|
||||
}
|
||||
void MaddMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X,const std::vector<Field> &Y,RealD scale=1.0)
|
||||
{
|
||||
std::vector<Field> Y_copy(AP.size(),AP[0].Grid());
|
||||
for(int r=0;r<AP.size();r++){
|
||||
Y_copy[r] = Y[r];
|
||||
}
|
||||
MulMatrix(AP,m,X);
|
||||
for(int r=0;r<AP.size();r++){
|
||||
AP[r] = scale*AP[r]+Y_copy[r];
|
||||
}
|
||||
}
|
||||
void MulMatrix(std::vector<Field> &Y, Eigen::MatrixXcd &m , const std::vector<Field> &X)
|
||||
{
|
||||
typedef typename Field::scalar_type scomplex;
|
||||
GridBase *grid;
|
||||
uint64_t vol;
|
||||
uint64_t words;
|
||||
|
||||
int nrhs = Y.size();
|
||||
grid = X[0].Grid();
|
||||
vol = grid->lSites();
|
||||
words = sizeof(scalar_object)/sizeof(scalar);
|
||||
int64_t vw = vol * words;
|
||||
|
||||
RealD t0 = usecond();
|
||||
BLAS_X.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_Y.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_C.resize(nrhs * nrhs);// cost free if size doesn't change
|
||||
RealD t1 = usecond();
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// Copy in the multi-rhs sources
|
||||
/////////////////////////////////////////////
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int64_t offset = r*vw;
|
||||
autoView(x_v,X[r],AcceleratorRead);
|
||||
acceleratorCopyDeviceToDevice(&x_v[0],&BLAS_X[offset],sizeof(scalar_object)*vol);
|
||||
}
|
||||
|
||||
// Assumes Eigen storage contiguous
|
||||
acceleratorCopyToDevice(&m(0,0),&BLAS_C[0],BLAS_C.size()*sizeof(scalar));
|
||||
|
||||
/*
|
||||
* in Fortran column major notation (cuBlas order)
|
||||
*
|
||||
* Xxr = [X1(x)][..][Xn(x)]
|
||||
* Yxr = [Y1(x)][..][Ym(x)]
|
||||
* Y = X . C
|
||||
*/
|
||||
deviceVector<scalar *> Xd(1);
|
||||
deviceVector<scalar *> Yd(1);
|
||||
deviceVector<scalar *> Cd(1);
|
||||
|
||||
scalar * Xh = & BLAS_X[0];
|
||||
scalar * Yh = & BLAS_Y[0];
|
||||
scalar * Ch = & BLAS_C[0];
|
||||
|
||||
acceleratorPut(Xd[0],Xh);
|
||||
acceleratorPut(Yd[0],Yh);
|
||||
acceleratorPut(Cd[0],Ch);
|
||||
|
||||
RealD t2 = usecond();
|
||||
GridBLAS BLAS;
|
||||
/////////////////////////////////////////
|
||||
// Y = X*C (transpose?)
|
||||
/////////////////////////////////////////
|
||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
||||
vw,nrhs,nrhs,
|
||||
scalar(1.0),
|
||||
Xd,
|
||||
Cd,
|
||||
scalar(0.0), // wipe out Y
|
||||
Yd);
|
||||
BLAS.synchronise();
|
||||
RealD t3 = usecond();
|
||||
|
||||
// Copy back Y = m X
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int64_t offset = r*vw;
|
||||
autoView(y_v,Y[r],AcceleratorWrite);
|
||||
acceleratorCopyDeviceToDevice(&BLAS_Y[offset],&y_v[0],sizeof(scalar_object)*vol);
|
||||
}
|
||||
RealD t4 = usecond();
|
||||
std::cout <<GridLogPerformance << "MulMatrix alloc took "<< t1-t0<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "MulMatrix preamble took "<< t2-t1<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "MulMatrix blas took "<< t3-t2<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "MulMatrix copy took "<< t4-t3<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "MulMatrix total "<< t4-t0<<" us"<<std::endl;
|
||||
}
|
||||
|
||||
void InnerProductMatrix(Eigen::MatrixXcd &m , const std::vector<Field> &X, const std::vector<Field> &Y)
|
||||
{
|
||||
#if 0
|
||||
int nrhs;
|
||||
GridBase *grid;
|
||||
uint64_t vol;
|
||||
uint64_t words;
|
||||
|
||||
nrhs = X.size();
|
||||
GRID_ASSERT(X.size()==Y.size());
|
||||
conformable(X[0],Y[0]);
|
||||
|
||||
grid = X[0].Grid();
|
||||
vol = grid->lSites();
|
||||
words = sizeof(scalar_object)/sizeof(scalar);
|
||||
int64_t vw = vol * words;
|
||||
|
||||
RealD t0 = usecond();
|
||||
BLAS_X.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_Y.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_C.resize(nrhs * nrhs);// cost free if size doesn't change
|
||||
RealD t1 = usecond();
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// Copy in the multi-rhs sources
|
||||
/////////////////////////////////////////////
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int64_t offset = r*vw;
|
||||
autoView(x_v,X[r],AcceleratorRead);
|
||||
acceleratorCopyDeviceToDevice(&x_v[0],&BLAS_X[offset],sizeof(scalar_object)*vol);
|
||||
autoView(y_v,Y[r],AcceleratorRead);
|
||||
acceleratorCopyDeviceToDevice(&y_v[0],&BLAS_Y[offset],sizeof(scalar_object)*vol);
|
||||
}
|
||||
RealD t2 = usecond();
|
||||
|
||||
/*
|
||||
* in Fortran column major notation (cuBlas order)
|
||||
*
|
||||
* Xxr = [X1(x)][..][Xn(x)]
|
||||
*
|
||||
* Yxr = [Y1(x)][..][Ym(x)]
|
||||
*
|
||||
* C_rs = X^dag Y
|
||||
*/
|
||||
deviceVector<scalar *> Xd(1);
|
||||
deviceVector<scalar *> Yd(1);
|
||||
deviceVector<scalar *> Cd(1);
|
||||
|
||||
scalar * Xh = & BLAS_X[0];
|
||||
scalar * Yh = & BLAS_Y[0];
|
||||
scalar * Ch = & BLAS_C[0];
|
||||
|
||||
acceleratorPut(Xd[0],Xh);
|
||||
acceleratorPut(Yd[0],Yh);
|
||||
acceleratorPut(Cd[0],Ch);
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
RealD t3 = usecond();
|
||||
/////////////////////////////////////////
|
||||
// C_rs = X^dag Y
|
||||
/////////////////////////////////////////
|
||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
||||
nrhs,nrhs,vw,
|
||||
ComplexD(1.0),
|
||||
Xd,
|
||||
Yd,
|
||||
ComplexD(0.0), // wipe out C
|
||||
Cd);
|
||||
BLAS.synchronise();
|
||||
RealD t4 = usecond();
|
||||
|
||||
std::vector<scalar> HOST_C(BLAS_C.size()); // nrhs . nrhs -- the coefficients
|
||||
acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
|
||||
grid->GlobalSumVector(&HOST_C[0],nrhs*nrhs);
|
||||
|
||||
RealD t5 = usecond();
|
||||
for(int rr=0;rr<nrhs;rr++){
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int off = r+nrhs*rr;
|
||||
m(r,rr)=HOST_C[off];
|
||||
}
|
||||
}
|
||||
RealD t6 = usecond();
|
||||
uint64_t M=nrhs;
|
||||
uint64_t N=nrhs;
|
||||
uint64_t K=vw;
|
||||
RealD bytes = 1.0*sizeof(ComplexD)*(M*N*2+N*K+M*K);
|
||||
RealD flops = 8.0*M*N*K;
|
||||
flops = flops/(t4-t3)/1.e3;
|
||||
bytes = bytes/(t4-t3)/1.e3;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix m,n,k "<< M<<","<<N<<","<<K<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix alloc t1 "<< t1-t0<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t2 "<< t2-t1<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix setup t3 "<< t3-t2<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas t4 "<< t4-t3<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< flops<<" GF/s"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< bytes<<" GB/s"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix gsum t5 "<< t5-t4<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t6 "<< t6-t5<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix took "<< t6-t0<<" us"<<std::endl;
|
||||
#else
|
||||
int nrhs;
|
||||
GridBase *grid;
|
||||
uint64_t vol;
|
||||
uint64_t words;
|
||||
|
||||
nrhs = X.size();
|
||||
GRID_ASSERT(X.size()==Y.size());
|
||||
conformable(X[0],Y[0]);
|
||||
|
||||
grid = X[0].Grid();
|
||||
int rd0 = grid->_rdimensions[0] * grid->_rdimensions[1];
|
||||
vol = grid->oSites()/rd0;
|
||||
words = rd0*sizeof(vector_object)/sizeof(scalar);
|
||||
int64_t vw = vol * words;
|
||||
GRID_ASSERT(vw == grid->lSites()*sizeof(scalar_object)/sizeof(scalar));
|
||||
|
||||
RealD t0 = usecond();
|
||||
BLAS_X.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_Y.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_Cred.resize(nrhs * nrhs * vol);// cost free if size doesn't change
|
||||
RealD t1 = usecond();
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// Copy in the multi-rhs sources -- layout batched BLAS ready
|
||||
/////////////////////////////////////////////
|
||||
for(int r=0;r<nrhs;r++){
|
||||
autoView(x_v,X[r],AcceleratorRead);
|
||||
autoView(y_v,Y[r],AcceleratorRead);
|
||||
scalar *from_x=(scalar *)&x_v[0];
|
||||
scalar *from_y=(scalar *)&y_v[0];
|
||||
scalar *BX = &BLAS_X[0];
|
||||
scalar *BY = &BLAS_Y[0];
|
||||
accelerator_for(ssw,vw,1,{
|
||||
uint64_t ss=ssw/words;
|
||||
uint64_t w=ssw%words;
|
||||
uint64_t offset = w+r*words+ss*nrhs*words; // [ss][rhs][words]
|
||||
BX[offset] = from_x[ssw];
|
||||
BY[offset] = from_y[ssw];
|
||||
});
|
||||
}
|
||||
RealD t2 = usecond();
|
||||
|
||||
/*
|
||||
* in Fortran column major notation (cuBlas order)
|
||||
*
|
||||
* Xxr = [X1(x)][..][Xn(x)]
|
||||
*
|
||||
* Yxr = [Y1(x)][..][Ym(x)]
|
||||
*
|
||||
* C_rs = X^dag Y
|
||||
*/
|
||||
Xdip.resize(vol);
|
||||
Ydip.resize(vol);
|
||||
Cdip.resize(vol);
|
||||
std::vector<scalar *> Xh(vol);
|
||||
std::vector<scalar *> Yh(vol);
|
||||
std::vector<scalar *> Ch(vol);
|
||||
for(uint64_t ss=0;ss<vol;ss++){
|
||||
|
||||
Xh[ss] = & BLAS_X[ss*nrhs*words];
|
||||
Yh[ss] = & BLAS_Y[ss*nrhs*words];
|
||||
Ch[ss] = & BLAS_Cred[ss*nrhs*nrhs];
|
||||
|
||||
}
|
||||
acceleratorCopyToDevice(&Xh[0],&Xdip[0],vol*sizeof(scalar *));
|
||||
acceleratorCopyToDevice(&Yh[0],&Ydip[0],vol*sizeof(scalar *));
|
||||
acceleratorCopyToDevice(&Ch[0],&Cdip[0],vol*sizeof(scalar *));
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
RealD t3 = usecond();
|
||||
/////////////////////////////////////////
|
||||
// C_rs = X^dag Y
|
||||
/////////////////////////////////////////
|
||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
||||
nrhs,nrhs,words,
|
||||
ComplexD(1.0),
|
||||
Xdip,
|
||||
Ydip,
|
||||
ComplexD(0.0), // wipe out C
|
||||
Cdip);
|
||||
BLAS.synchronise();
|
||||
RealD t4 = usecond();
|
||||
|
||||
std::vector<scalar> HOST_C(BLAS_Cred.size()); // nrhs . nrhs -- the coefficients
|
||||
acceleratorCopyFromDevice(&BLAS_Cred[0],&HOST_C[0],BLAS_Cred.size()*sizeof(scalar));
|
||||
|
||||
RealD t5 = usecond();
|
||||
m = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
for(int ss=0;ss<vol;ss++){
|
||||
Eigen::Map<Eigen::MatrixXcd> eC((std::complex<double> *)&HOST_C[ss*nrhs*nrhs],nrhs,nrhs);
|
||||
m = m + eC;
|
||||
}
|
||||
RealD t6l = usecond();
|
||||
grid->GlobalSumVector((scalar *) &m(0,0),nrhs*nrhs);
|
||||
RealD t6 = usecond();
|
||||
uint64_t M=nrhs;
|
||||
uint64_t N=nrhs;
|
||||
uint64_t K=vw;
|
||||
RealD xybytes = grid->lSites()*sizeof(scalar_object);
|
||||
RealD bytes = 1.0*sizeof(ComplexD)*(M*N*2+N*K+M*K);
|
||||
RealD flops = 8.0*M*N*K;
|
||||
flops = flops/(t4-t3)/1.e3;
|
||||
bytes = bytes/(t4-t3)/1.e3;
|
||||
xybytes = 4*xybytes/(t2-t1)/1.e3;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix m,n,k "<< M<<","<<N<<","<<K<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix alloc t1 "<< t1-t0<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t2 "<< t2-t1<<" us "<<xybytes<<" GB/s"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix setup t3 "<< t3-t2<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas t4 "<< t4-t3<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< flops<<" GF/s"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< bytes<<" GB/s"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t5 "<< t5-t4<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix lsum t6l "<< t6l-t5<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix gsum t6 "<< t6-t6l<<" us"<<std::endl;
|
||||
std::cout <<GridLogPerformance<< "InnerProductMatrix took "<< t6-t0<<" us"<<std::endl;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,513 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: MultiRHSDeflation.h
|
||||
|
||||
Copyright (C) 2023
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
/*
|
||||
MultiRHS block projection
|
||||
|
||||
Import basis -> nblock x nbasis x (block x internal)
|
||||
Import vector of fine lattice objects -> nblock x nrhs x (block x internal)
|
||||
|
||||
=> coarse_(nrhs x nbasis )^block = via batched GEMM
|
||||
|
||||
//template<class vobj,class CComplex,int nbasis,class VLattice>
|
||||
//inline void blockProject(Lattice<iVector<CComplex,nbasis > > &coarseData,
|
||||
// const VLattice &fineData,
|
||||
// const VLattice &Basis)
|
||||
*/
|
||||
|
||||
template<class Field>
|
||||
class MultiRHSBlockProject
|
||||
{
|
||||
public:
|
||||
|
||||
typedef typename Field::scalar_type scalar;
|
||||
typedef typename Field::scalar_object scalar_object;
|
||||
typedef Field Fermion;
|
||||
|
||||
int nbasis;
|
||||
GridBase *coarse_grid;
|
||||
GridBase *fine_grid;
|
||||
uint64_t block_vol;
|
||||
uint64_t fine_vol;
|
||||
uint64_t coarse_vol;
|
||||
uint64_t words;
|
||||
|
||||
// Row major layout "C" order:
|
||||
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
||||
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
||||
// BLAS_C[coarse_vol][nrhs][nbasis]
|
||||
/*
|
||||
* in Fortran column major notation (cuBlas order)
|
||||
*
|
||||
* Vxb = [v1(x)][..][vn(x)] ... x coarse vol
|
||||
*
|
||||
* Fxr = [r1(x)][..][rm(x)] ... x coarse vol
|
||||
*
|
||||
* Block project:
|
||||
* C_br = V^dag F x coarse vol
|
||||
*
|
||||
* Block promote:
|
||||
* F_xr = Vxb Cbr x coarse_vol
|
||||
*/
|
||||
deviceVector<scalar> BLAS_V; // words * block_vol * nbasis x coarse_vol
|
||||
deviceVector<scalar> BLAS_F; // nrhs x fine_vol * words -- the sources
|
||||
deviceVector<scalar> BLAS_C; // nrhs x coarse_vol * nbasis -- the coarse coeffs
|
||||
|
||||
RealD blasNorm2(deviceVector<scalar> &blas)
|
||||
{
|
||||
scalar ss(0.0);
|
||||
std::vector<scalar> tmp(blas.size());
|
||||
acceleratorCopyFromDevice(&blas[0],&tmp[0],blas.size()*sizeof(scalar));
|
||||
for(int64_t s=0;s<blas.size();s++){
|
||||
ss=ss+tmp[s]*adj(tmp[s]);
|
||||
}
|
||||
coarse_grid->GlobalSum(ss);
|
||||
return real(ss);
|
||||
}
|
||||
|
||||
MultiRHSBlockProject(){};
|
||||
~MultiRHSBlockProject(){ Deallocate(); };
|
||||
|
||||
void Deallocate(void)
|
||||
{
|
||||
nbasis=0;
|
||||
coarse_grid=nullptr;
|
||||
fine_grid=nullptr;
|
||||
fine_vol=0;
|
||||
block_vol=0;
|
||||
coarse_vol=0;
|
||||
words=0;
|
||||
BLAS_V.resize(0);
|
||||
BLAS_F.resize(0);
|
||||
BLAS_C.resize(0);
|
||||
}
|
||||
void Allocate(int _nbasis,GridBase *_fgrid,GridBase *_cgrid)
|
||||
{
|
||||
nbasis=_nbasis;
|
||||
|
||||
fine_grid=_fgrid;
|
||||
coarse_grid=_cgrid;
|
||||
|
||||
fine_vol = fine_grid->lSites();
|
||||
coarse_vol = coarse_grid->lSites();
|
||||
block_vol = fine_vol/coarse_vol;
|
||||
|
||||
words = sizeof(scalar_object)/sizeof(scalar);
|
||||
|
||||
BLAS_V.resize (fine_vol * words * nbasis );
|
||||
}
|
||||
void ImportFineGridVectors(std::vector <Field > &vecs, deviceVector<scalar> &blas)
|
||||
{
|
||||
int nvec = vecs.size();
|
||||
typedef typename Field::vector_object vobj;
|
||||
// std::cout << GridLogMessage <<" BlockProjector importing "<<nvec<< " fine grid vectors" <<std::endl;
|
||||
|
||||
GRID_ASSERT(vecs[0].Grid()==fine_grid);
|
||||
|
||||
subdivides(coarse_grid,fine_grid); // require they map
|
||||
|
||||
int _ndimension = coarse_grid->_ndimension;
|
||||
GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites());
|
||||
|
||||
Coordinate block_r (_ndimension);
|
||||
for(int d=0 ; d<_ndimension;d++){
|
||||
block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
|
||||
}
|
||||
|
||||
uint64_t sz = blas.size();
|
||||
|
||||
acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
|
||||
|
||||
Coordinate fine_rdimensions = fine_grid->_rdimensions;
|
||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||
int64_t bv= block_vol;
|
||||
for(int v=0;v<vecs.size();v++){
|
||||
|
||||
// std::cout << " BlockProjector importing vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
|
||||
autoView( fineData , vecs[v], AcceleratorRead);
|
||||
|
||||
auto blasData_p = &blas[0];
|
||||
auto fineData_p = &fineData[0];
|
||||
|
||||
int64_t osites = fine_grid->oSites();
|
||||
|
||||
// loop over fine sites
|
||||
const int Nsimd = vobj::Nsimd();
|
||||
// std::cout << "sz "<<sz<<std::endl;
|
||||
// std::cout << "prod "<<Nsimd * coarse_grid->oSites() * block_vol * nvec * words<<std::endl;
|
||||
GRID_ASSERT(sz == Nsimd * coarse_grid->oSites() * block_vol * nvec * words);
|
||||
uint64_t lwords= words; // local variable for copy in to GPU
|
||||
accelerator_for(sf,osites,Nsimd,{
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||
#else
|
||||
for(int lane=0;lane<Nsimd;lane++) {
|
||||
#endif
|
||||
// One thread per fine site
|
||||
Coordinate coor_f(_ndimension);
|
||||
Coordinate coor_b(_ndimension);
|
||||
Coordinate coor_c(_ndimension);
|
||||
|
||||
// Fine site to fine coor
|
||||
Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
|
||||
|
||||
for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
|
||||
for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
|
||||
|
||||
int sc;// coarse site
|
||||
int sb;// block site
|
||||
Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
|
||||
Lexicographic::IndexFromCoor(coor_b,sb,block_r);
|
||||
|
||||
scalar_object data = extractLane(lane,fineData[sf]);
|
||||
|
||||
// BLAS layout address calculation
|
||||
// words * block_vol * nbasis x coarse_vol
|
||||
// coarse oSite x block vole x lanes
|
||||
int64_t site = (lane*osites + sc*bv)*nvec
|
||||
+ v*bv
|
||||
+ sb;
|
||||
|
||||
// GRID_ASSERT(site*lwords<sz);
|
||||
|
||||
scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
|
||||
|
||||
*ptr = data;
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
// std::cout << " import fine Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||
// std::cout << " BlockProjector imported vector"<<v<<std::endl;
|
||||
}
|
||||
}
|
||||
void ExportFineGridVectors(std::vector <Field> &vecs, deviceVector<scalar> &blas)
|
||||
{
|
||||
typedef typename Field::vector_object vobj;
|
||||
|
||||
int nvec = vecs.size();
|
||||
|
||||
GRID_ASSERT(vecs[0].Grid()==fine_grid);
|
||||
|
||||
subdivides(coarse_grid,fine_grid); // require they map
|
||||
|
||||
int _ndimension = coarse_grid->_ndimension;
|
||||
GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites());
|
||||
|
||||
Coordinate block_r (_ndimension);
|
||||
for(int d=0 ; d<_ndimension;d++){
|
||||
block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
|
||||
}
|
||||
Coordinate fine_rdimensions = fine_grid->_rdimensions;
|
||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||
|
||||
// std::cout << " export fine Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||
|
||||
int64_t bv= block_vol;
|
||||
for(int v=0;v<vecs.size();v++){
|
||||
|
||||
autoView( fineData , vecs[v], AcceleratorWrite);
|
||||
|
||||
auto blasData_p = &blas[0];
|
||||
auto fineData_p = &fineData[0];
|
||||
|
||||
int64_t osites = fine_grid->oSites();
|
||||
uint64_t lwords = words;
|
||||
// std::cout << " Nsimd is "<<vobj::Nsimd() << std::endl;
|
||||
// std::cout << " lwords is "<<lwords << std::endl;
|
||||
// std::cout << " sizeof(scalar_object) is "<<sizeof(scalar_object) << std::endl;
|
||||
// loop over fine sites
|
||||
accelerator_for(sf,osites,vobj::Nsimd(),{
|
||||
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane=acceleratorSIMTlane(vobj::Nsimd()); // buffer lane
|
||||
#else
|
||||
for(int lane=0;lane<vobj::Nsimd();lane++) {
|
||||
#endif
|
||||
// One thread per fine site
|
||||
Coordinate coor_f(_ndimension);
|
||||
Coordinate coor_b(_ndimension);
|
||||
Coordinate coor_c(_ndimension);
|
||||
|
||||
Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
|
||||
|
||||
for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
|
||||
for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
|
||||
|
||||
int sc;
|
||||
int sb;
|
||||
Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
|
||||
Lexicographic::IndexFromCoor(coor_b,sb,block_r);
|
||||
|
||||
// BLAS layout address calculation
|
||||
// words * block_vol * nbasis x coarse_vol
|
||||
int64_t site = (lane*osites + sc*bv)*nvec
|
||||
+ v*bv
|
||||
+ sb;
|
||||
|
||||
scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
|
||||
|
||||
scalar_object data = *ptr;
|
||||
|
||||
insertLane(lane,fineData[sf],data);
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
}
|
||||
}
|
||||
template<class vobj>
|
||||
void ImportCoarseGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
|
||||
{
|
||||
int nvec = vecs.size();
|
||||
typedef typename vobj::scalar_object coarse_scalar_object;
|
||||
|
||||
// std::cout << " BlockProjector importing "<<nvec<< " coarse grid vectors" <<std::endl;
|
||||
|
||||
GRID_ASSERT(vecs[0].Grid()==coarse_grid);
|
||||
|
||||
int _ndimension = coarse_grid->_ndimension;
|
||||
|
||||
uint64_t sz = blas.size();
|
||||
|
||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||
|
||||
for(int v=0;v<vecs.size();v++){
|
||||
|
||||
// std::cout << " BlockProjector importing coarse vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
|
||||
autoView( coarseData , vecs[v], AcceleratorRead);
|
||||
|
||||
auto blasData_p = &blas[0];
|
||||
auto coarseData_p = &coarseData[0];
|
||||
|
||||
int64_t osites = coarse_grid->oSites();
|
||||
|
||||
// loop over fine sites
|
||||
const int Nsimd = vobj::Nsimd();
|
||||
uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
|
||||
GRID_ASSERT(cwords==nbasis);
|
||||
|
||||
accelerator_for(sc,osites,Nsimd,{
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||
#else
|
||||
for(int lane=0;lane<Nsimd;lane++) {
|
||||
#endif
|
||||
// C_br per site
|
||||
int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
|
||||
|
||||
coarse_scalar_object data = extractLane(lane,coarseData[sc]);
|
||||
|
||||
coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
|
||||
|
||||
*ptr = data;
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
// std::cout << " import coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||
}
|
||||
}
|
||||
template<class vobj>
|
||||
void ExportCoarseGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
|
||||
{
|
||||
int nvec = vecs.size();
|
||||
typedef typename vobj::scalar_object coarse_scalar_object;
|
||||
// std::cout << GridLogMessage<<" BlockProjector exporting "<<nvec<< " coarse grid vectors" <<std::endl;
|
||||
|
||||
GRID_ASSERT(vecs[0].Grid()==coarse_grid);
|
||||
|
||||
int _ndimension = coarse_grid->_ndimension;
|
||||
|
||||
uint64_t sz = blas.size();
|
||||
|
||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
||||
|
||||
// std::cout << " export coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
|
||||
for(int v=0;v<vecs.size();v++){
|
||||
|
||||
// std::cout << " BlockProjector exporting coarse vector"<<v<<std::endl;
|
||||
autoView( coarseData , vecs[v], AcceleratorWrite);
|
||||
|
||||
auto blasData_p = &blas[0];
|
||||
auto coarseData_p = &coarseData[0];
|
||||
|
||||
int64_t osites = coarse_grid->oSites();
|
||||
|
||||
// loop over fine sites
|
||||
const int Nsimd = vobj::Nsimd();
|
||||
uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
|
||||
GRID_ASSERT(cwords==nbasis);
|
||||
|
||||
accelerator_for(sc,osites,Nsimd,{
|
||||
// Wrap in a macro "FOR_ALL_LANES(lane,{ ... });
|
||||
#ifdef GRID_SIMT
|
||||
{
|
||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
||||
#else
|
||||
for(int lane=0;lane<Nsimd;lane++) {
|
||||
#endif
|
||||
int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
|
||||
coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
|
||||
coarse_scalar_object data = *ptr;
|
||||
insertLane(lane,coarseData[sc],data);
|
||||
#ifdef GRID_SIMT
|
||||
}
|
||||
#else
|
||||
}
|
||||
#endif
|
||||
});
|
||||
}
|
||||
}
|
||||
void ImportBasis(std::vector < Field > &vecs)
|
||||
{
|
||||
// std::cout << " BlockProjector Import basis size "<<vecs.size()<<std::endl;
|
||||
ImportFineGridVectors(vecs,BLAS_V);
|
||||
}
|
||||
|
||||
template<class cobj>
|
||||
void blockProject(std::vector<Field> &fine,std::vector< Lattice<cobj> > & coarse)
|
||||
{
|
||||
int nrhs=fine.size();
|
||||
int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
|
||||
// std::cout << "blockProject nbasis " <<nbasis<<" " << _nbasis<<std::endl;
|
||||
GRID_ASSERT(nbasis==_nbasis);
|
||||
|
||||
BLAS_F.resize (fine_vol * words * nrhs );
|
||||
BLAS_C.resize (coarse_vol * nbasis * nrhs );
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// Copy in the multi-rhs sources to same data layout
|
||||
/////////////////////////////////////////////
|
||||
// std::cout << "BlockProject import fine"<<std::endl;
|
||||
ImportFineGridVectors(fine,BLAS_F);
|
||||
|
||||
deviceVector<scalar *> Vd(coarse_vol);
|
||||
deviceVector<scalar *> Fd(coarse_vol);
|
||||
deviceVector<scalar *> Cd(coarse_vol);
|
||||
|
||||
// std::cout << "BlockProject pointers"<<std::endl;
|
||||
for(int c=0;c<coarse_vol;c++){
|
||||
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
||||
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
||||
// BLAS_C[coarse_vol][nrhs][nbasis]
|
||||
scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
|
||||
scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
|
||||
scalar * Ch = & BLAS_C[c*nrhs*nbasis];
|
||||
|
||||
acceleratorPut(Vd[c],Vh);
|
||||
acceleratorPut(Fd[c],Fh);
|
||||
acceleratorPut(Cd[c],Ch);
|
||||
}
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
// std::cout << "BlockProject BLAS"<<std::endl;
|
||||
int64_t vw = block_vol * words;
|
||||
/////////////////////////////////////////
|
||||
// C_br = V^dag R
|
||||
/////////////////////////////////////////
|
||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
||||
nbasis,nrhs,vw,
|
||||
scalar(1.0),
|
||||
Vd,
|
||||
Fd,
|
||||
scalar(0.0), // wipe out C
|
||||
Cd);
|
||||
BLAS.synchronise();
|
||||
// std::cout << "BlockProject done"<<std::endl;
|
||||
ExportCoarseGridVectors(coarse, BLAS_C);
|
||||
// std::cout << "BlockProject done"<<std::endl;
|
||||
|
||||
}
|
||||
|
||||
template<class cobj>
|
||||
void blockPromote(std::vector<Field> &fine,std::vector<Lattice<cobj> > & coarse)
|
||||
{
|
||||
int nrhs=fine.size();
|
||||
int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
|
||||
GRID_ASSERT(nbasis==_nbasis);
|
||||
|
||||
BLAS_F.resize (fine_vol * words * nrhs );
|
||||
BLAS_C.resize (coarse_vol * nbasis * nrhs );
|
||||
|
||||
ImportCoarseGridVectors(coarse, BLAS_C);
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
deviceVector<scalar *> Vd(coarse_vol);
|
||||
deviceVector<scalar *> Fd(coarse_vol);
|
||||
deviceVector<scalar *> Cd(coarse_vol);
|
||||
|
||||
for(int c=0;c<coarse_vol;c++){
|
||||
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
||||
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
||||
// BLAS_C[coarse_vol][nrhs][nbasis]
|
||||
scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
|
||||
scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
|
||||
scalar * Ch = & BLAS_C[c*nrhs*nbasis];
|
||||
acceleratorPut(Vd[c],Vh);
|
||||
acceleratorPut(Fd[c],Fh);
|
||||
acceleratorPut(Cd[c],Ch);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////
|
||||
// Block promote:
|
||||
// F_xr = Vxb Cbr (x coarse_vol)
|
||||
/////////////////////////////////////////
|
||||
|
||||
int64_t vw = block_vol * words;
|
||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
||||
vw,nrhs,nbasis,
|
||||
scalar(1.0),
|
||||
Vd,
|
||||
Cd,
|
||||
scalar(0.0), // wipe out C
|
||||
Fd);
|
||||
BLAS.synchronise();
|
||||
// std::cout << " blas call done"<<std::endl;
|
||||
|
||||
ExportFineGridVectors(fine, BLAS_F);
|
||||
// std::cout << " exported "<<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,233 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: MultiRHSDeflation.h
|
||||
|
||||
Copyright (C) 2023
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
/* Need helper object for BLAS accelerated mrhs projection
|
||||
|
||||
i) MultiRHS Deflation
|
||||
|
||||
Import Evecs -> nev x vol x internal
|
||||
Import vector of Lattice objects -> nrhs x vol x internal
|
||||
=> Cij (nrhs x Nev) via GEMM.
|
||||
=> Guess (nrhs x vol x internal) = C x evecs (via GEMM)
|
||||
Export
|
||||
|
||||
|
||||
ii) MultiRHS block projection
|
||||
|
||||
Import basis -> nblock x nbasis x (block x internal)
|
||||
Import vector of fine lattice objects -> nblock x nrhs x (block x internal)
|
||||
|
||||
=> coarse_(nrhs x nbasis )^block = via batched GEMM
|
||||
|
||||
iii) Alternate interface:
|
||||
Import higher dim Lattice object-> vol x nrhs layout
|
||||
|
||||
*/
|
||||
template<class Field>
|
||||
class MultiRHSDeflation
|
||||
{
|
||||
public:
|
||||
|
||||
typedef typename Field::scalar_type scalar;
|
||||
typedef typename Field::scalar_object scalar_object;
|
||||
|
||||
int nev;
|
||||
std::vector<RealD> eval;
|
||||
GridBase *grid;
|
||||
uint64_t vol;
|
||||
uint64_t words;
|
||||
|
||||
deviceVector<scalar> BLAS_E; // nev x vol -- the eigenbasis (up to a 1/sqrt(lambda))
|
||||
deviceVector<scalar> BLAS_R; // nrhs x vol -- the sources
|
||||
deviceVector<scalar> BLAS_G; // nrhs x vol -- the guess
|
||||
deviceVector<scalar> BLAS_C; // nrhs x nev -- the coefficients
|
||||
|
||||
MultiRHSDeflation(){};
|
||||
~MultiRHSDeflation(){ Deallocate(); };
|
||||
|
||||
void Deallocate(void)
|
||||
{
|
||||
nev=0;
|
||||
grid=nullptr;
|
||||
vol=0;
|
||||
words=0;
|
||||
BLAS_E.resize(0);
|
||||
BLAS_R.resize(0);
|
||||
BLAS_C.resize(0);
|
||||
BLAS_G.resize(0);
|
||||
}
|
||||
void Allocate(int _nev,GridBase *_grid)
|
||||
{
|
||||
nev=_nev;
|
||||
grid=_grid;
|
||||
vol = grid->lSites();
|
||||
words = sizeof(scalar_object)/sizeof(scalar);
|
||||
eval.resize(nev);
|
||||
BLAS_E.resize (vol * words * nev );
|
||||
std::cout << GridLogMessage << " Allocate for "<<nev<<" eigenvectors and volume "<<vol<<std::endl;
|
||||
}
|
||||
void ImportEigenVector(Field &evec,RealD &_eval, int ev)
|
||||
{
|
||||
// std::cout << " ev " <<ev<<" eval "<<_eval<< std::endl;
|
||||
GRID_ASSERT(ev<eval.size());
|
||||
eval[ev] = _eval;
|
||||
|
||||
int64_t offset = ev*vol*words;
|
||||
autoView(v,evec,AcceleratorRead);
|
||||
acceleratorCopyDeviceToDevice(&v[0],&BLAS_E[offset],sizeof(scalar_object)*vol);
|
||||
|
||||
}
|
||||
void ImportEigenBasis(std::vector<Field> &evec,std::vector<RealD> &_eval)
|
||||
{
|
||||
ImportEigenBasis(evec,_eval,0,evec.size());
|
||||
}
|
||||
// Could use to import a batch of eigenvectors
|
||||
void ImportEigenBasis(std::vector<Field> &evec,std::vector<RealD> &_eval, int _ev0, int _nev)
|
||||
{
|
||||
GRID_ASSERT(_ev0+_nev<=evec.size());
|
||||
|
||||
Allocate(_nev,evec[0].Grid());
|
||||
|
||||
// Imports a sub-batch of eigenvectors, _ev0, ..., _ev0+_nev-1
|
||||
for(int e=0;e<nev;e++){
|
||||
std::cout << "Importing eigenvector "<<e<<" evalue "<<_eval[_ev0+e]<<std::endl;
|
||||
ImportEigenVector(evec[_ev0+e],_eval[_ev0+e],e);
|
||||
}
|
||||
}
|
||||
void DeflateSources(std::vector<Field> &source,std::vector<Field> & guess)
|
||||
{
|
||||
int nrhs = source.size();
|
||||
GRID_ASSERT(source.size()==guess.size());
|
||||
GRID_ASSERT(grid == guess[0].Grid());
|
||||
conformable(guess[0],source[0]);
|
||||
|
||||
int64_t vw = vol * words;
|
||||
|
||||
RealD t0 = usecond();
|
||||
BLAS_R.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_G.resize(nrhs * vw); // cost free if size doesn't change
|
||||
BLAS_C.resize(nev * nrhs);// cost free if size doesn't change
|
||||
|
||||
/////////////////////////////////////////////
|
||||
// Copy in the multi-rhs sources
|
||||
/////////////////////////////////////////////
|
||||
// for(int r=0;r<nrhs;r++){
|
||||
// std::cout << " source["<<r<<"] = "<<norm2(source[r])<<std::endl;
|
||||
// }
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int64_t offset = r*vw;
|
||||
autoView(v,source[r],AcceleratorRead);
|
||||
acceleratorCopyDeviceToDevice(&v[0],&BLAS_R[offset],sizeof(scalar_object)*vol);
|
||||
}
|
||||
|
||||
/*
|
||||
* in Fortran column major notation (cuBlas order)
|
||||
*
|
||||
* Exe = [e1(x)][..][en(x)]
|
||||
*
|
||||
* Rxr = [r1(x)][..][rm(x)]
|
||||
*
|
||||
* C_er = E^dag R
|
||||
* C_er = C_er / lambda_e
|
||||
* G_xr = Exe Cer
|
||||
*/
|
||||
deviceVector<scalar *> Ed(1);
|
||||
deviceVector<scalar *> Rd(1);
|
||||
deviceVector<scalar *> Cd(1);
|
||||
deviceVector<scalar *> Gd(1);
|
||||
|
||||
scalar * Eh = & BLAS_E[0];
|
||||
scalar * Rh = & BLAS_R[0];
|
||||
scalar * Ch = & BLAS_C[0];
|
||||
scalar * Gh = & BLAS_G[0];
|
||||
|
||||
acceleratorPut(Ed[0],Eh);
|
||||
acceleratorPut(Rd[0],Rh);
|
||||
acceleratorPut(Cd[0],Ch);
|
||||
acceleratorPut(Gd[0],Gh);
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
/////////////////////////////////////////
|
||||
// C_er = E^dag R
|
||||
/////////////////////////////////////////
|
||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
||||
nev,nrhs,vw,
|
||||
scalar(1.0),
|
||||
Ed,
|
||||
Rd,
|
||||
scalar(0.0), // wipe out C
|
||||
Cd);
|
||||
BLAS.synchronise();
|
||||
|
||||
GRID_ASSERT(BLAS_C.size()==nev*nrhs);
|
||||
|
||||
std::vector<scalar> HOST_C(BLAS_C.size()); // nrhs . nev -- the coefficients
|
||||
acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
|
||||
grid->GlobalSumVector(&HOST_C[0],nev*nrhs);
|
||||
for(int e=0;e<nev;e++){
|
||||
RealD lam(1.0/eval[e]);
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int off = e+nev*r;
|
||||
HOST_C[off]=HOST_C[off] * lam;
|
||||
// std::cout << "C["<<e<<"]["<<r<<"] ="<<HOST_C[off]<< " eval[e] "<<eval[e] <<std::endl;
|
||||
}
|
||||
}
|
||||
acceleratorCopyToDevice(&HOST_C[0],&BLAS_C[0],BLAS_C.size()*sizeof(scalar));
|
||||
|
||||
|
||||
/////////////////////////////////////////
|
||||
// Guess G_xr = Exe Cer
|
||||
/////////////////////////////////////////
|
||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
||||
vw,nrhs,nev,
|
||||
scalar(1.0),
|
||||
Ed, // x . nev
|
||||
Cd, // nev . nrhs
|
||||
scalar(0.0),
|
||||
Gd);
|
||||
BLAS.synchronise();
|
||||
|
||||
///////////////////////////////////////
|
||||
// Copy out the multirhs
|
||||
///////////////////////////////////////
|
||||
for(int r=0;r<nrhs;r++){
|
||||
int64_t offset = r*vw;
|
||||
autoView(v,guess[r],AcceleratorWrite);
|
||||
acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
|
||||
}
|
||||
RealD t1 = usecond();
|
||||
std::cout << GridLogMessage << "MultiRHSDeflation for "<<nrhs<<" sources with "<<nev<<" eigenvectors took " << (t1-t0)/1e3 <<" ms"<<std::endl;
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -33,111 +33,109 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
* Script A = SolverMatrix
|
||||
* Script P = Preconditioner
|
||||
*
|
||||
* Deflation methods considered
|
||||
* -- Solve P A x = P b [ like Luscher ]
|
||||
* DEF-1 M P A x = M P b [i.e. left precon]
|
||||
* DEF-2 P^T M A x = P^T M b
|
||||
* ADEF-1 Preconditioner = M P + Q [ Q + M + M A Q]
|
||||
* ADEF-2 Preconditioner = P^T M + Q
|
||||
* BNN Preconditioner = P^T M P + Q
|
||||
* BNN2 Preconditioner = M P + P^TM +Q - M P A M
|
||||
*
|
||||
* Implement ADEF-2
|
||||
*
|
||||
* Vstart = P^Tx + Qb
|
||||
* M1 = P^TM + Q
|
||||
* M2=M3=1
|
||||
* Vout = x
|
||||
*/
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelCG : public LinearFunction<Field>
|
||||
// abstract base
|
||||
template<class Field, class CoarseField>
|
||||
class TwoLevelFlexiblePcg : public LinearFunction<Field>
|
||||
{
|
||||
public:
|
||||
int verbose;
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
const int mmax = 5;
|
||||
GridBase *grid;
|
||||
GridBase *coarsegrid;
|
||||
|
||||
// Fine operator, Smoother, CoarseSolver
|
||||
LinearOperatorBase<Field> &_FineLinop;
|
||||
LinearFunction<Field> &_Smoother;
|
||||
LinearOperatorBase<Field> *_Linop
|
||||
OperatorFunction<Field> *_Smoother,
|
||||
LinearFunction<CoarseField> *_CoarseSolver;
|
||||
|
||||
// Need somthing that knows how to get from Coarse to fine and back again
|
||||
|
||||
// more most opertor functions
|
||||
TwoLevelCG(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> &FineLinop,
|
||||
LinearFunction<Field> &Smoother,
|
||||
GridBase *fine) :
|
||||
TwoLevelFlexiblePcg(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> *Linop,
|
||||
LinearOperatorBase<Field> *SmootherLinop,
|
||||
OperatorFunction<Field> *Smoother,
|
||||
OperatorFunction<CoarseField> CoarseLinop
|
||||
) :
|
||||
Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
_FineLinop(FineLinop),
|
||||
_Smoother(Smoother)
|
||||
{
|
||||
grid = fine;
|
||||
_Linop(Linop),
|
||||
_PreconditionerLinop(PrecLinop),
|
||||
_Preconditioner(Preconditioner)
|
||||
{
|
||||
verbose=0;
|
||||
};
|
||||
|
||||
virtual void operator() (const Field &src, Field &x)
|
||||
{
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg starting single RHS"<<std::endl;
|
||||
|
||||
// The Pcg routine is common to all, but the various matrices differ from derived
|
||||
// implementation to derived implmentation
|
||||
void operator() (const Field &src, Field &psi){
|
||||
void operator() (const Field &src, Field &psi){
|
||||
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
grid = src.Grid();
|
||||
|
||||
RealD f;
|
||||
RealD rtzp,rtz,a,d,b;
|
||||
RealD rptzp;
|
||||
|
||||
RealD tn;
|
||||
RealD guess = norm2(psi);
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = ssq*Tolerance*Tolerance;
|
||||
|
||||
/////////////////////////////
|
||||
// Set up history vectors
|
||||
/////////////////////////////
|
||||
int mmax = 5;
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocating"<<std::endl;
|
||||
std::vector<Field> p(mmax,grid);
|
||||
std::vector<Field> p (mmax,grid);
|
||||
std::vector<Field> mmp(mmax,grid);
|
||||
std::vector<RealD> pAp(mmax);
|
||||
Field z(grid);
|
||||
|
||||
Field x (grid); x = psi;
|
||||
Field z (grid);
|
||||
Field tmp(grid);
|
||||
Field mp (grid);
|
||||
Field r (grid);
|
||||
Field mu (grid);
|
||||
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated"<<std::endl;
|
||||
//Initial residual computation & set up
|
||||
RealD guess = norm2(x);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg guess nrm "<<guess<<std::endl;
|
||||
RealD src_nrm = norm2(src);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg src nrm "<<src_nrm<<std::endl;
|
||||
|
||||
if ( src_nrm == 0.0 ) {
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg given trivial source norm "<<src_nrm<<std::endl;
|
||||
x=Zero();
|
||||
}
|
||||
RealD tn;
|
||||
|
||||
GridStopWatch HDCGTimer;
|
||||
HDCGTimer.Start();
|
||||
Field r (grid);
|
||||
Field mu (grid);
|
||||
|
||||
//////////////////////////
|
||||
// x0 = Vstart -- possibly modify guess
|
||||
//////////////////////////
|
||||
x=src;
|
||||
Vstart(x,src);
|
||||
|
||||
|
||||
// r0 = b -A x0
|
||||
_FineLinop.HermOp(x,mmp[0]);
|
||||
HermOp(x,mmp); // Shouldn't this be something else?
|
||||
axpy (r, -1.0,mmp[0], src); // Recomputes r=src-Ax0
|
||||
{
|
||||
double n1 = norm2(x);
|
||||
double n2 = norm2(mmp[0]);
|
||||
double n3 = norm2(r);
|
||||
std::cout<<GridLogMessage<<"x,vstart,r = "<<n1<<" "<<n2<<" "<<n3<<std::endl;
|
||||
}
|
||||
|
||||
//////////////////////////////////
|
||||
// Compute z = M1 x
|
||||
//////////////////////////////////
|
||||
PcgM1(r,z);
|
||||
M1(r,z,tmp,mp,SmootherMirs);
|
||||
rtzp =real(innerProduct(r,z));
|
||||
|
||||
|
||||
///////////////////////////////////////
|
||||
// Solve for Mss mu = P A z and set p = z-mu
|
||||
// Def2 p = 1 - Q Az = Pright z
|
||||
// Def2: p = 1 - Q Az = Pright z
|
||||
// Other algos M2 is trivial
|
||||
///////////////////////////////////////
|
||||
PcgM2(z,p[0]);
|
||||
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = ssq*Tolerance*Tolerance;
|
||||
|
||||
std::cout << GridLogMessage<<"HDCG: k=0 residual "<<rtzp<<" rsq "<<rsq<<"\n";
|
||||
|
||||
Field pp(grid);
|
||||
M2(z,p[0]);
|
||||
|
||||
for (int k=0;k<=MaxIterations;k++){
|
||||
|
||||
@@ -145,46 +143,31 @@ class TwoLevelCG : public LinearFunction<Field>
|
||||
int peri_kp = (k+1) % mmax;
|
||||
|
||||
rtz=rtzp;
|
||||
d= PcgM3(p[peri_k],mmp[peri_k]);
|
||||
d= M3(p[peri_k],mp,mmp[peri_k],tmp);
|
||||
a = rtz/d;
|
||||
|
||||
// Memorise this
|
||||
pAp[peri_k] = d;
|
||||
|
||||
|
||||
axpy(x,a,p[peri_k],x);
|
||||
RealD rn = axpy_norm(r,-a,mmp[peri_k],r);
|
||||
|
||||
// Compute z = M x
|
||||
PcgM1(r,z);
|
||||
|
||||
{
|
||||
RealD n1,n2;
|
||||
n1=norm2(r);
|
||||
n2=norm2(z);
|
||||
std::cout << GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : vector r,z "<<n1<<" "<<n2<<"\n";
|
||||
}
|
||||
M1(r,z,tmp,mp);
|
||||
|
||||
rtzp =real(innerProduct(r,z));
|
||||
std::cout << GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : inner rtzp "<<rtzp<<"\n";
|
||||
|
||||
// PcgM2(z,p[0]);
|
||||
PcgM2(z,mu); // ADEF-2 this is identity. Axpy possible to eliminate
|
||||
|
||||
p[peri_kp]=mu;
|
||||
M2(z,mu); // ADEF-2 this is identity. Axpy possible to eliminate
|
||||
|
||||
// Standard search direction p -> z + b p
|
||||
p[peri_kp]=p[peri_k];
|
||||
|
||||
// Standard search direction p -> z + b p ; b =
|
||||
b = (rtzp)/rtz;
|
||||
|
||||
int northog;
|
||||
// k=zero <=> peri_kp=1; northog = 1
|
||||
// k=1 <=> peri_kp=2; northog = 2
|
||||
// ... ... ...
|
||||
// k=mmax-2<=> peri_kp=mmax-1; northog = mmax-1
|
||||
// k=mmax-1<=> peri_kp=0; northog = 1
|
||||
|
||||
int northog;
|
||||
// northog = (peri_kp==0)?1:peri_kp; // This is the fCG(mmax) algorithm
|
||||
northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
||||
|
||||
std::cout<<GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : orthogonalising to last "<<northog<<" vectors\n";
|
||||
for(int back=0; back < northog; back++){
|
||||
int peri_back = (k-back)%mmax;
|
||||
RealD pbApk= real(innerProduct(mmp[peri_back],p[peri_kp]));
|
||||
@@ -193,324 +176,75 @@ class TwoLevelCG : public LinearFunction<Field>
|
||||
}
|
||||
|
||||
RealD rrn=sqrt(rn/ssq);
|
||||
RealD rtn=sqrt(rtz/ssq);
|
||||
RealD rtnp=sqrt(rtzp/ssq);
|
||||
|
||||
std::cout<<GridLogMessage<<"HDCG: fPcg k= "<<k<<" residual = "<<rrn<<"\n";
|
||||
std::cout<<GridLogMessage<<"TwoLevelfPcg: k= "<<k<<" residual = "<<rrn<<std::endl;
|
||||
|
||||
// Stopping condition
|
||||
if ( rn <= rsq ) {
|
||||
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: fPcg converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
||||
|
||||
_FineLinop.HermOp(x,mmp[0]);
|
||||
HermOp(x,mmp); // Shouldn't this be something else?
|
||||
axpy(tmp,-1.0,src,mmp[0]);
|
||||
|
||||
RealD mmpnorm = sqrt(norm2(mmp[0]));
|
||||
RealD xnorm = sqrt(norm2(x));
|
||||
RealD srcnorm = sqrt(norm2(src));
|
||||
RealD tmpnorm = sqrt(norm2(tmp));
|
||||
RealD true_residual = tmpnorm/srcnorm;
|
||||
std::cout<<GridLogMessage
|
||||
<<"HDCG: true residual is "<<true_residual
|
||||
<<" solution "<<xnorm
|
||||
<<" source "<<srcnorm
|
||||
<<" mmp "<<mmpnorm
|
||||
<<std::endl;
|
||||
|
||||
return;
|
||||
RealD psinorm = sqrt(norm2(x));
|
||||
RealD srcnorm = sqrt(norm2(src));
|
||||
RealD tmpnorm = sqrt(norm2(tmp));
|
||||
RealD true_residual = tmpnorm/srcnorm;
|
||||
std::cout<<GridLogMessage<<"TwoLevelfPcg: true residual is "<<true_residual<<std::endl;
|
||||
std::cout<<GridLogMessage<<"TwoLevelfPcg: target residual was"<<Tolerance<<std::endl;
|
||||
return k;
|
||||
}
|
||||
|
||||
}
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
||||
RealD xnorm = sqrt(norm2(x));
|
||||
RealD srcnorm = sqrt(norm2(src));
|
||||
std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
|
||||
// Non-convergence
|
||||
assert(0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
virtual void operator() (std::vector<Field> &src, std::vector<Field> &x)
|
||||
{
|
||||
std::cout << GridLogMessage<<"HDCG: mrhs fPcg starting"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
int nrhs = src.size();
|
||||
std::vector<RealD> f(nrhs);
|
||||
std::vector<RealD> rtzp(nrhs);
|
||||
std::vector<RealD> rtz(nrhs);
|
||||
std::vector<RealD> a(nrhs);
|
||||
std::vector<RealD> d(nrhs);
|
||||
std::vector<RealD> b(nrhs);
|
||||
std::vector<RealD> rptzp(nrhs);
|
||||
/////////////////////////////
|
||||
// Set up history vectors
|
||||
/////////////////////////////
|
||||
int mmax = 3;
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocating"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
std::vector<std::vector<Field> > p(nrhs); for(int r=0;r<nrhs;r++) p[r].resize(mmax,grid);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated p"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
std::vector<std::vector<Field> > mmp(nrhs); for(int r=0;r<nrhs;r++) mmp[r].resize(mmax,grid);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated mmp"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
std::vector<std::vector<RealD> > pAp(nrhs); for(int r=0;r<nrhs;r++) pAp[r].resize(mmax);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated pAp"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
std::vector<Field> z(nrhs,grid);
|
||||
std::vector<Field> mp (nrhs,grid);
|
||||
std::vector<Field> r (nrhs,grid);
|
||||
std::vector<Field> mu (nrhs,grid);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated z,mp,r,mu"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
|
||||
//Initial residual computation & set up
|
||||
std::vector<RealD> src_nrm(nrhs);
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
src_nrm[rhs]=norm2(src[rhs]);
|
||||
GRID_ASSERT(src_nrm[rhs]!=0.0);
|
||||
}
|
||||
std::vector<RealD> tn(nrhs);
|
||||
|
||||
GridStopWatch HDCGTimer;
|
||||
HDCGTimer.Start();
|
||||
//////////////////////////
|
||||
// x0 = Vstart -- possibly modify guess
|
||||
//////////////////////////
|
||||
Vstart(x,src);
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
// r0 = b -A x0
|
||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
||||
axpy (r[rhs], -1.0,mmp[rhs][0], src[rhs]); // Recomputes r=src-Ax0
|
||||
}
|
||||
|
||||
//////////////////////////////////
|
||||
// Compute z = M1 x
|
||||
//////////////////////////////////
|
||||
// This needs a multiRHS version for acceleration
|
||||
PcgM1(r,z);
|
||||
|
||||
std::vector<RealD> ssq(nrhs);
|
||||
std::vector<RealD> rsq(nrhs);
|
||||
std::vector<Field> pp(nrhs,grid);
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
||||
p[rhs][0]=z[rhs];
|
||||
ssq[rhs]=norm2(src[rhs]);
|
||||
rsq[rhs]= ssq[rhs]*Tolerance*Tolerance;
|
||||
std::cout << GridLogMessage<<"mrhs HDCG: "<<rhs<<" k=0 residual "<<rtzp[rhs]<<" rsq "<<rsq[rhs]<<"\n";
|
||||
}
|
||||
|
||||
std::vector<RealD> rn(nrhs);
|
||||
for (int k=0;k<=MaxIterations;k++){
|
||||
|
||||
int peri_k = k % mmax;
|
||||
int peri_kp = (k+1) % mmax;
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
rtz[rhs]=rtzp[rhs];
|
||||
d[rhs]= PcgM3(p[rhs][peri_k],mmp[rhs][peri_k]);
|
||||
a[rhs] = rtz[rhs]/d[rhs];
|
||||
|
||||
// Memorise this
|
||||
pAp[rhs][peri_k] = d[rhs];
|
||||
|
||||
axpy(x[rhs],a[rhs],p[rhs][peri_k],x[rhs]);
|
||||
rn[rhs] = axpy_norm(r[rhs],-a[rhs],mmp[rhs][peri_k],r[rhs]);
|
||||
}
|
||||
|
||||
// Compute z = M x (for *all* RHS)
|
||||
PcgM1(r,z);
|
||||
std::cout << GridLogMessage<<"HDCG::fPcg M1 complete"<<std::endl;
|
||||
grid->Barrier();
|
||||
|
||||
RealD max_rn=0.0;
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
|
||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
||||
|
||||
std::cout << GridLogMessage<<"HDCG::fPcg rhs"<<rhs<<" iteration "<<k<<" : inner rtzp "<<rtzp[rhs]<<"\n";
|
||||
|
||||
mu[rhs]=z[rhs];
|
||||
|
||||
p[rhs][peri_kp]=mu[rhs];
|
||||
|
||||
// Standard search direction p == z + b p
|
||||
b[rhs] = (rtzp[rhs])/rtz[rhs];
|
||||
|
||||
int northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
||||
std::cout<<GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : orthogonalising to last "<<northog<<" vectors\n";
|
||||
for(int back=0; back < northog; back++){
|
||||
int peri_back = (k-back)%mmax;
|
||||
RealD pbApk= real(innerProduct(mmp[rhs][peri_back],p[rhs][peri_kp]));
|
||||
RealD beta = -pbApk/pAp[rhs][peri_back];
|
||||
axpy(p[rhs][peri_kp],beta,p[rhs][peri_back],p[rhs][peri_kp]);
|
||||
}
|
||||
|
||||
RealD rrn=sqrt(rn[rhs]/ssq[rhs]);
|
||||
RealD rtn=sqrt(rtz[rhs]/ssq[rhs]);
|
||||
RealD rtnp=sqrt(rtzp[rhs]/ssq[rhs]);
|
||||
|
||||
std::cout<<GridLogMessage<<"HDCG: rhs "<<rhs<<"fPcg k= "<<k<<" residual = "<<rrn<<"\n";
|
||||
if ( rrn > max_rn ) max_rn = rrn;
|
||||
}
|
||||
|
||||
// Stopping condition based on worst case
|
||||
if ( max_rn <= Tolerance ) {
|
||||
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
||||
Field tmp(grid);
|
||||
axpy(tmp,-1.0,src[rhs],mmp[rhs][0]);
|
||||
|
||||
RealD mmpnorm = sqrt(norm2(mmp[rhs][0]));
|
||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
||||
RealD tmpnorm = sqrt(norm2(tmp));
|
||||
RealD true_residual = tmpnorm/srcnorm;
|
||||
std::cout<<GridLogMessage
|
||||
<<"HDCG: true residual ["<<rhs<<"] is "<<true_residual
|
||||
<<" solution "<<xnorm
|
||||
<<" source "<<srcnorm
|
||||
<<" mmp "<<mmpnorm
|
||||
<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
||||
std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
|
||||
virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out)
|
||||
{
|
||||
std::cout << "PcgM1 default (cheat) mrhs version"<<std::endl;
|
||||
for(int rhs=0;rhs<in.size();rhs++){
|
||||
this->PcgM1(in[rhs],out[rhs]);
|
||||
}
|
||||
}
|
||||
virtual void PcgM1(Field & in, Field & out) =0;
|
||||
virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src)
|
||||
{
|
||||
std::cout << "Vstart default (cheat) mrhs version"<<std::endl;
|
||||
for(int rhs=0;rhs<x.size();rhs++){
|
||||
this->Vstart(x[rhs],src[rhs]);
|
||||
}
|
||||
}
|
||||
virtual void Vstart(Field & x,const Field & src)=0;
|
||||
virtual void M(Field & in,Field & out,Field & tmp) {
|
||||
|
||||
virtual void PcgM2(const Field & in, Field & out) {
|
||||
out=in;
|
||||
}
|
||||
|
||||
virtual RealD PcgM3(const Field & p, Field & mmp){
|
||||
RealD dd;
|
||||
_FineLinop.HermOp(p,mmp);
|
||||
ComplexD dot = innerProduct(p,mmp);
|
||||
dd=real(dot);
|
||||
return dd;
|
||||
}
|
||||
virtual void M1(Field & in, Field & out) {// the smoother
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Only Def1 has non-trivial Vout.
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
|
||||
};
|
||||
|
||||
template<class Field, class CoarseField, class Aggregation>
|
||||
class TwoLevelADEF2 : public TwoLevelCG<Field>
|
||||
{
|
||||
public:
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
// Need something that knows how to get from Coarse to fine and back again
|
||||
// void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
|
||||
// void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
GridBase *coarsegrid;
|
||||
Aggregation &_Aggregates;
|
||||
LinearFunction<CoarseField> &_CoarseSolver;
|
||||
LinearFunction<CoarseField> &_CoarseSolverPrecise;
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// more most opertor functions
|
||||
TwoLevelADEF2(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> &FineLinop,
|
||||
LinearFunction<Field> &Smoother,
|
||||
LinearFunction<CoarseField> &CoarseSolver,
|
||||
LinearFunction<CoarseField> &CoarseSolverPrecise,
|
||||
Aggregation &Aggregates
|
||||
) :
|
||||
TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,Aggregates.FineGrid),
|
||||
_CoarseSolver(CoarseSolver),
|
||||
_CoarseSolverPrecise(CoarseSolverPrecise),
|
||||
_Aggregates(Aggregates)
|
||||
{
|
||||
coarsegrid = Aggregates.CoarseGrid;
|
||||
};
|
||||
|
||||
virtual void PcgM1(Field & in, Field & out)
|
||||
{
|
||||
GRID_TRACE("MultiGridPreconditioner ");
|
||||
// [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
|
||||
Field tmp(grid);
|
||||
Field Min(grid);
|
||||
|
||||
Field tmp(this->grid);
|
||||
Field Min(this->grid);
|
||||
CoarseField PleftProj(this->coarsegrid);
|
||||
CoarseField PleftMss_proj(this->coarsegrid);
|
||||
PcgM(in,Min); // Smoother call
|
||||
|
||||
GridStopWatch SmootherTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
SmootherTimer.Start();
|
||||
this->_Smoother(in,Min);
|
||||
SmootherTimer.Stop();
|
||||
|
||||
MatrixTimer.Start();
|
||||
this->_FineLinop.HermOp(Min,out);
|
||||
MatrixTimer.Stop();
|
||||
HermOp(Min,out);
|
||||
axpy(tmp,-1.0,out,in); // tmp = in - A Min
|
||||
|
||||
GridStopWatch ProjTimer;
|
||||
GridStopWatch CoarseTimer;
|
||||
GridStopWatch PromTimer;
|
||||
ProjTimer.Start();
|
||||
this->_Aggregates.ProjectToSubspace(PleftProj,tmp);
|
||||
ProjTimer.Stop();
|
||||
CoarseTimer.Start();
|
||||
this->_CoarseSolver(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
|
||||
CoarseTimer.Stop();
|
||||
PromTimer.Start();
|
||||
this->_Aggregates.PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
|
||||
PromTimer.Stop();
|
||||
std::cout << GridLogPerformance << "PcgM1 breakdown "<<std::endl;
|
||||
std::cout << GridLogPerformance << "\tSmoother " << SmootherTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\tProj " << ProjTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\tCoarse " << CoarseTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\tProm " << PromTimer.Elapsed() <<std::endl;
|
||||
|
||||
ProjectToSubspace(tmp,PleftProj);
|
||||
ApplyInverse(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
|
||||
PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
|
||||
axpy(out,1.0,Min,tmp); // Min+tmp
|
||||
}
|
||||
|
||||
virtual void Vstart(Field & x,const Field & src)
|
||||
{
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart "<<std::endl;
|
||||
virtual void M2(const Field & in, Field & out) {
|
||||
out=in;
|
||||
// Must override for Def2 only
|
||||
// case PcgDef2:
|
||||
// Pright(in,out);
|
||||
// break;
|
||||
}
|
||||
|
||||
virtual RealD M3(const Field & p, Field & mmp){
|
||||
double d,dd;
|
||||
HermOpAndNorm(p,mmp,d,dd);
|
||||
return dd;
|
||||
// Must override for Def1 only
|
||||
// case PcgDef1:
|
||||
// d=linop_d->Mprec(p,mmp,tmp,0,1);// Dag no
|
||||
// linop_d->Mprec(mmp,mp,tmp,1);// Dag yes
|
||||
// Pleft(mp,mmp);
|
||||
// d=real(linop_d->inner(p,mmp));
|
||||
}
|
||||
|
||||
virtual void VstartDef2(Field & xconst Field & src){
|
||||
//case PcgDef2:
|
||||
//case PcgAdef2:
|
||||
//case PcgAdef2f:
|
||||
//case PcgV11f:
|
||||
///////////////////////////////////
|
||||
// Choose x_0 such that
|
||||
// x_0 = guess + (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
|
||||
@@ -522,78 +256,142 @@ class TwoLevelADEF2 : public TwoLevelCG<Field>
|
||||
// = src_s - (A guess)_s - src_s + (A guess)_s
|
||||
// = 0
|
||||
///////////////////////////////////
|
||||
Field r(this->grid);
|
||||
Field mmp(this->grid);
|
||||
CoarseField PleftProj(this->coarsegrid);
|
||||
CoarseField PleftMss_proj(this->coarsegrid);
|
||||
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart projecting "<<std::endl;
|
||||
this->_Aggregates.ProjectToSubspace(PleftProj,src);
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart coarse solve "<<std::endl;
|
||||
this->_CoarseSolverPrecise(PleftProj,PleftMss_proj); // Ass^{-1} r_s
|
||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart promote "<<std::endl;
|
||||
this->_Aggregates.PromoteFromSubspace(PleftMss_proj,x);
|
||||
Field r(grid);
|
||||
Field mmp(grid);
|
||||
|
||||
HermOp(x,mmp);
|
||||
axpy (r, -1.0, mmp, src); // r_{-1} = src - A x
|
||||
ProjectToSubspace(r,PleftProj);
|
||||
ApplyInverseCG(PleftProj,PleftMss_proj); // Ass^{-1} r_s
|
||||
PromoteFromSubspace(PleftMss_proj,mmp);
|
||||
x=x+mmp;
|
||||
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelADEF1defl : public TwoLevelCG<Field>
|
||||
{
|
||||
public:
|
||||
const std::vector<Field> &evec;
|
||||
const std::vector<RealD> &eval;
|
||||
|
||||
TwoLevelADEF1defl(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> &FineLinop,
|
||||
LinearFunction<Field> &Smoother,
|
||||
std::vector<Field> &_evec,
|
||||
std::vector<RealD> &_eval) :
|
||||
TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,_evec[0].Grid()),
|
||||
evec(_evec),
|
||||
eval(_eval)
|
||||
{};
|
||||
|
||||
// Can just inherit existing M2
|
||||
// Can just inherit existing M3
|
||||
|
||||
// Simple vstart - do nothing
|
||||
virtual void Vstart(Field & x,const Field & src){
|
||||
x=src; // Could apply Q
|
||||
};
|
||||
|
||||
// Override PcgM1
|
||||
virtual void PcgM1(Field & in, Field & out)
|
||||
{
|
||||
GRID_TRACE("EvecPreconditioner ");
|
||||
int N=evec.size();
|
||||
Field Pin(this->grid);
|
||||
Field Qin(this->grid);
|
||||
|
||||
//MP + Q = M(1-AQ) + Q = M
|
||||
// // If we are eigenvector deflating in coarse space
|
||||
// // Q = Sum_i |phi_i> 1/lambda_i <phi_i|
|
||||
// // A Q = Sum_i |phi_i> <phi_i|
|
||||
// // M(1-AQ) = M(1-proj) + Q
|
||||
Qin.Checkerboard()=in.Checkerboard();
|
||||
Qin = Zero();
|
||||
Pin = in;
|
||||
for (int i=0;i<N;i++) {
|
||||
const Field& tmp = evec[i];
|
||||
auto ip = TensorRemove(innerProduct(tmp,in));
|
||||
axpy(Qin, ip / eval[i],tmp,Qin);
|
||||
axpy(Pin, -ip ,tmp,Pin);
|
||||
}
|
||||
|
||||
this->_Smoother(Pin,out);
|
||||
|
||||
out = out + Qin;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Only Def1 has non-trivial Vout. Override in Def1
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
virtual void Vout (Field & in, Field & out,Field & src){
|
||||
out = in;
|
||||
//case PcgDef1:
|
||||
// //Qb + PT x
|
||||
// ProjectToSubspace(src,PleftProj);
|
||||
// ApplyInverse(PleftProj,PleftMss_proj); // Ass^{-1} r_s
|
||||
// PromoteFromSubspace(PleftMss_proj,tmp);
|
||||
//
|
||||
// Pright(in,out);
|
||||
//
|
||||
// linop_d->axpy(out,tmp,out,1.0);
|
||||
// break;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Pright and Pleft are common to all implementations
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
virtual void Pright(Field & in,Field & out){
|
||||
// P_R = [ 1 0 ]
|
||||
// [ -Mss^-1 Msb 0 ]
|
||||
Field in_sbar(grid);
|
||||
|
||||
ProjectToSubspace(in,PleftProj);
|
||||
PromoteFromSubspace(PleftProj,out);
|
||||
axpy(in_sbar,-1.0,out,in); // in_sbar = in - in_s
|
||||
|
||||
HermOp(in_sbar,out);
|
||||
ProjectToSubspace(out,PleftProj); // Mssbar in_sbar (project)
|
||||
|
||||
ApplyInverse (PleftProj,PleftMss_proj); // Mss^{-1} Mssbar
|
||||
PromoteFromSubspace(PleftMss_proj,out); //
|
||||
|
||||
axpy(out,-1.0,out,in_sbar); // in_sbar - Mss^{-1} Mssbar in_sbar
|
||||
}
|
||||
virtual void Pleft (Field & in,Field & out){
|
||||
// P_L = [ 1 -Mbs Mss^-1]
|
||||
// [ 0 0 ]
|
||||
Field in_sbar(grid);
|
||||
Field tmp2(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
ProjectToSubspace(in,PleftProj);
|
||||
PromoteFromSubspace(PleftProj,out);
|
||||
axpy(in_sbar,-1.0,out,in); // in_sbar = in - in_s
|
||||
|
||||
ApplyInverse(PleftProj,PleftMss_proj); // Mss^{-1} in_s
|
||||
PromoteFromSubspace(PleftMss_proj,out);
|
||||
|
||||
HermOp(out,Mtmp);
|
||||
|
||||
ProjectToSubspace(Mtmp,PleftProj); // Msbar s Mss^{-1}
|
||||
PromoteFromSubspace(PleftProj,tmp2);
|
||||
|
||||
axpy(out,-1.0,tmp2,Mtmp);
|
||||
axpy(out,-1.0,out,in_sbar); // in_sbar - Msbars Mss^{-1} in_s
|
||||
}
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgADef2 : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp){
|
||||
|
||||
}
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp){
|
||||
|
||||
}
|
||||
virtual void M2(Field & in, Field & out){
|
||||
|
||||
}
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp){
|
||||
|
||||
}
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp){
|
||||
|
||||
}
|
||||
}
|
||||
/*
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgAD : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgDef1 : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
virtual void Vout (Field & in, Field & out,Field & src,Field & tmp);
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgDef2 : public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
}
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelFlexiblePcgV11: public TwoLevelFlexiblePcg<Field> {
|
||||
public:
|
||||
virtual void M(Field & in,Field & out,Field & tmp);
|
||||
virtual void M1(Field & in, Field & out,Field & tmp,Field & mp);
|
||||
virtual void M2(Field & in, Field & out);
|
||||
virtual RealD M3(Field & p, Field & mp,Field & mmp, Field & tmp);
|
||||
virtual void Vstart(Field & in, Field & src, Field & r, Field & mp, Field & mmp, Field & tmp);
|
||||
}
|
||||
*/
|
||||
#endif
|
||||
@@ -1,734 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/AdefGeneric.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
|
||||
/*
|
||||
* Compared to Tang-2009: P=Pleft. P^T = PRight Q=MssInv.
|
||||
* Script A = SolverMatrix
|
||||
* Script P = Preconditioner
|
||||
*
|
||||
* Implement ADEF-2
|
||||
*
|
||||
* Vstart = P^Tx + Qb
|
||||
* M1 = P^TM + Q
|
||||
* M2=M3=1
|
||||
*/
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
template<class Field>
|
||||
class TwoLevelCGmrhs
|
||||
{
|
||||
public:
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
GridBase *grid;
|
||||
|
||||
// Fine operator, Smoother, CoarseSolver
|
||||
LinearOperatorBase<Field> &_FineLinop;
|
||||
LinearFunction<Field> &_Smoother;
|
||||
MultiRHSBlockCGLinalg<Field> _BlockCGLinalg;
|
||||
|
||||
GridStopWatch ProjectTimer;
|
||||
GridStopWatch PromoteTimer;
|
||||
GridStopWatch DeflateTimer;
|
||||
GridStopWatch CoarseTimer;
|
||||
GridStopWatch FineTimer;
|
||||
GridStopWatch SmoothTimer;
|
||||
GridStopWatch InsertTimer;
|
||||
|
||||
/*
|
||||
Field rrr;
|
||||
Field sss;
|
||||
Field qqq;
|
||||
Field zzz;
|
||||
*/
|
||||
// more most opertor functions
|
||||
TwoLevelCGmrhs(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> &FineLinop,
|
||||
LinearFunction<Field> &Smoother,
|
||||
GridBase *fine) :
|
||||
Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
_FineLinop(FineLinop),
|
||||
_Smoother(Smoother)
|
||||
/*
|
||||
rrr(fine),
|
||||
sss(fine),
|
||||
qqq(fine),
|
||||
zzz(fine)
|
||||
*/
|
||||
{
|
||||
grid = fine;
|
||||
};
|
||||
|
||||
// Vector case
|
||||
virtual void operator() (std::vector<Field> &src, std::vector<Field> &x)
|
||||
{
|
||||
SolveSingleSystem(src,x);
|
||||
// SolvePrecBlockCG(src,x);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Thin QR factorisation (google it)
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//Dimensions
|
||||
// R_{ferm x Nblock} = Q_{ferm x Nblock} x C_{Nblock x Nblock} -> ferm x Nblock
|
||||
//
|
||||
// Rdag R = m_rr = Herm = L L^dag <-- Cholesky decomposition (LLT routine in Eigen)
|
||||
//
|
||||
// Q C = R => Q = R C^{-1}
|
||||
//
|
||||
// Want Ident = Q^dag Q = C^{-dag} R^dag R C^{-1} = C^{-dag} L L^dag C^{-1} = 1_{Nblock x Nblock}
|
||||
//
|
||||
// Set C = L^{dag}, and then Q^dag Q = ident
|
||||
//
|
||||
// Checks:
|
||||
// Cdag C = Rdag R ; passes.
|
||||
// QdagQ = 1 ; passes
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void ThinQRfact (Eigen::MatrixXcd &m_zz,
|
||||
Eigen::MatrixXcd &C,
|
||||
Eigen::MatrixXcd &Cinv,
|
||||
std::vector<Field> & Q,
|
||||
std::vector<Field> & MQ,
|
||||
const std::vector<Field> & Z,
|
||||
const std::vector<Field> & MZ)
|
||||
{
|
||||
RealD t0=usecond();
|
||||
_BlockCGLinalg.InnerProductMatrix(m_zz,MZ,Z);
|
||||
RealD t1=usecond();
|
||||
|
||||
m_zz = 0.5*(m_zz+m_zz.adjoint());
|
||||
|
||||
Eigen::MatrixXcd L = m_zz.llt().matrixL();
|
||||
|
||||
C = L.adjoint();
|
||||
Cinv = C.inverse();
|
||||
|
||||
RealD t3=usecond();
|
||||
_BlockCGLinalg.MulMatrix( Q,Cinv,Z);
|
||||
_BlockCGLinalg.MulMatrix(MQ,Cinv,MZ);
|
||||
RealD t4=usecond();
|
||||
std::cout << " ThinQRfact IP :"<< t1-t0<<" us"<<std::endl;
|
||||
std::cout << " ThinQRfact Eigen :"<< t3-t1<<" us"<<std::endl;
|
||||
std::cout << " ThinQRfact MulMat:"<< t4-t3<<" us"<<std::endl;
|
||||
}
|
||||
|
||||
virtual void SolvePrecBlockCG (std::vector<Field> &src, std::vector<Field> &X)
|
||||
{
|
||||
std::cout << GridLogMessage<<"HDCG: mrhs fPrecBlockcg starting"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
int nrhs = src.size();
|
||||
// std::vector<RealD> f(nrhs);
|
||||
// std::vector<RealD> rtzp(nrhs);
|
||||
// std::vector<RealD> rtz(nrhs);
|
||||
// std::vector<RealD> a(nrhs);
|
||||
// std::vector<RealD> d(nrhs);
|
||||
// std::vector<RealD> b(nrhs);
|
||||
// std::vector<RealD> rptzp(nrhs);
|
||||
|
||||
////////////////////////////////////////////
|
||||
//Initial residual computation & set up
|
||||
////////////////////////////////////////////
|
||||
std::vector<RealD> ssq(nrhs);
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
ssq[rhs]=norm2(src[rhs]); GRID_ASSERT(ssq[rhs]!=0.0);
|
||||
}
|
||||
|
||||
///////////////////////////
|
||||
// Fields -- eliminate duplicates between fPcg and block cg
|
||||
///////////////////////////
|
||||
std::vector<Field> Mtmp(nrhs,grid);
|
||||
std::vector<Field> tmp(nrhs,grid);
|
||||
std::vector<Field> Z(nrhs,grid); // Rename Z to R
|
||||
std::vector<Field> MZ(nrhs,grid); // Rename MZ to Z
|
||||
std::vector<Field> Q(nrhs,grid); //
|
||||
std::vector<Field> MQ(nrhs,grid); // Rename to P
|
||||
std::vector<Field> D(nrhs,grid);
|
||||
std::vector<Field> AD(nrhs,grid);
|
||||
|
||||
/************************************************************************
|
||||
* Preconditioned Block conjugate gradient rQ
|
||||
* Generalise Sebastien Birk Thesis, after Dubrulle 2001.
|
||||
* Introduce preconditioning following Saad Ch9
|
||||
************************************************************************
|
||||
* Dimensions:
|
||||
*
|
||||
* X,B etc... ==(Nferm x nrhs)
|
||||
* Matrix A==(Nferm x Nferm)
|
||||
*
|
||||
* Nferm = Nspin x Ncolour x Ncomplex x Nlattice_site
|
||||
* QC => Thin QR factorisation (google it)
|
||||
*
|
||||
* R = B-AX
|
||||
* Z = Mi R
|
||||
* QC = Z
|
||||
* D = Q
|
||||
* for k:
|
||||
* R = AD
|
||||
* Z = Mi R
|
||||
* M = [D^dag R]^{-1}
|
||||
* X = X + D M C
|
||||
* QS = Q - Z.M
|
||||
* D = Q + D S^dag
|
||||
* C = S C
|
||||
*/
|
||||
Eigen::MatrixXcd m_DZ = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_M = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_zz = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_rr = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
|
||||
Eigen::MatrixXcd m_C = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_Cinv = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_S = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_Sinv = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
||||
|
||||
Eigen::MatrixXcd m_tmp = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
||||
Eigen::MatrixXcd m_tmp1 = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
||||
|
||||
GridStopWatch HDCGTimer;
|
||||
|
||||
//////////////////////////
|
||||
// x0 = Vstart -- possibly modify guess
|
||||
//////////////////////////
|
||||
Vstart(X,src);
|
||||
|
||||
//////////////////////////
|
||||
// R = B-AX
|
||||
//////////////////////////
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
// r0 = b -A x0
|
||||
_FineLinop.HermOp(X[rhs],tmp[rhs]);
|
||||
axpy (Z[rhs], -1.0,tmp[rhs], src[rhs]); // Computes R=Z=src - A X0
|
||||
}
|
||||
|
||||
//////////////////////////////////
|
||||
// Compute MZ = M1 Z = M1 B - M1 A x0
|
||||
//////////////////////////////////
|
||||
PcgM1(Z,MZ);
|
||||
|
||||
//////////////////////////////////
|
||||
// QC = Z
|
||||
//////////////////////////////////
|
||||
ThinQRfact (m_zz, m_C, m_Cinv, Q, MQ, Z, MZ);
|
||||
|
||||
//////////////////////////////////
|
||||
// D=MQ
|
||||
//////////////////////////////////
|
||||
for(int b=0;b<nrhs;b++) D[b]=MQ[b]; // LLT rotation of the MZ basis of search dirs
|
||||
|
||||
std::cout << GridLogMessage<<"PrecBlockCGrQ vec computed initial residual and QR fact " <<std::endl;
|
||||
|
||||
ProjectTimer.Reset();
|
||||
PromoteTimer.Reset();
|
||||
DeflateTimer.Reset();
|
||||
CoarseTimer.Reset();
|
||||
SmoothTimer.Reset();
|
||||
FineTimer.Reset();
|
||||
InsertTimer.Reset();
|
||||
|
||||
GridStopWatch M1Timer;
|
||||
GridStopWatch M2Timer;
|
||||
GridStopWatch M3Timer;
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch InnerProdTimer;
|
||||
|
||||
HDCGTimer.Start();
|
||||
|
||||
std::vector<RealD> rn(nrhs);
|
||||
for (int k=0;k<=MaxIterations;k++){
|
||||
|
||||
////////////////////
|
||||
// Z = AD
|
||||
////////////////////
|
||||
M3Timer.Start();
|
||||
for(int b=0;b<nrhs;b++) _FineLinop.HermOp(D[b], Z[b]);
|
||||
M3Timer.Stop();
|
||||
|
||||
////////////////////
|
||||
// MZ = M1 Z <==== the Multigrid preconditioner
|
||||
////////////////////
|
||||
M1Timer.Start();
|
||||
PcgM1(Z,MZ);
|
||||
M1Timer.Stop();
|
||||
|
||||
FineTimer.Start();
|
||||
////////////////////
|
||||
// M = [D^dag Z]^{-1} = (<Ddag MZ>_M)^{-1} inner prod, generalising Saad derivation of Precon CG
|
||||
////////////////////
|
||||
InnerProdTimer.Start();
|
||||
_BlockCGLinalg.InnerProductMatrix(m_DZ,D,Z);
|
||||
InnerProdTimer.Stop();
|
||||
m_M = m_DZ.inverse();
|
||||
|
||||
///////////////////////////
|
||||
// X = X + D MC
|
||||
///////////////////////////
|
||||
m_tmp = m_M * m_C;
|
||||
LinalgTimer.Start();
|
||||
_BlockCGLinalg.MaddMatrix(X,m_tmp, D,X); // D are the search directions and X takes the updates
|
||||
LinalgTimer.Stop();
|
||||
|
||||
///////////////////////////
|
||||
// QS = Q - M Z
|
||||
// (MQ) S = MQ - M (M1Z)
|
||||
///////////////////////////
|
||||
LinalgTimer.Start();
|
||||
_BlockCGLinalg.MaddMatrix(tmp ,m_M, Z, Q,-1.0);
|
||||
_BlockCGLinalg.MaddMatrix(Mtmp,m_M,MZ,MQ,-1.0);
|
||||
ThinQRfact (m_zz, m_S, m_Sinv, Q, MQ, tmp, Mtmp);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
////////////////////////////
|
||||
// D = MQ + D S^dag
|
||||
////////////////////////////
|
||||
m_tmp = m_S.adjoint();
|
||||
LinalgTimer.Start();
|
||||
_BlockCGLinalg.MaddMatrix(D,m_tmp,D,MQ);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
////////////////////////////
|
||||
// C = S C
|
||||
////////////////////////////
|
||||
m_C = m_S*m_C;
|
||||
|
||||
////////////////////////////
|
||||
// convergence monitor
|
||||
////////////////////////////
|
||||
m_rr = m_C.adjoint() * m_C;
|
||||
|
||||
FineTimer.Stop();
|
||||
|
||||
RealD max_resid=0;
|
||||
RealD rrsum=0;
|
||||
RealD sssum=0;
|
||||
RealD rr;
|
||||
|
||||
for(int b=0;b<nrhs;b++) {
|
||||
rrsum+=real(m_rr(b,b));
|
||||
sssum+=ssq[b];
|
||||
rr = real(m_rr(b,b))/ssq[b];
|
||||
if ( rr > max_resid ) max_resid = rr;
|
||||
}
|
||||
std::cout << GridLogMessage <<
|
||||
"\t Prec BlockCGrQ Iteration "<<k<<" ave resid "<< std::sqrt(rrsum/sssum) << " max "<< std::sqrt(max_resid) <<std::endl;
|
||||
|
||||
|
||||
if ( max_resid < Tolerance*Tolerance ) {
|
||||
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Linalg "<<LinalgTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : fine H "<<M3Timer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Project "<<ProjectTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Coarse "<<CoarseTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Fine "<<FineTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Smooth "<<SmoothTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Insert "<<InsertTimer.Elapsed()<<std::endl;;
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
|
||||
_FineLinop.HermOp(X[rhs],tmp[rhs]);
|
||||
|
||||
Field mytmp(grid);
|
||||
axpy(mytmp,-1.0,src[rhs],tmp[rhs]);
|
||||
|
||||
RealD xnorm = sqrt(norm2(X[rhs]));
|
||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
||||
RealD tmpnorm = sqrt(norm2(mytmp));
|
||||
RealD true_residual = tmpnorm/srcnorm;
|
||||
std::cout<<GridLogMessage
|
||||
<<"HDCG: true residual ["<<rhs<<"] is "<<true_residual
|
||||
<<" solution "<<xnorm
|
||||
<<" source "<<srcnorm
|
||||
<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: PrecBlockCGrQ not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
|
||||
virtual void SolveSingleSystem (std::vector<Field> &src, std::vector<Field> &x)
|
||||
{
|
||||
std::cout << GridLogMessage<<"HDCG: mrhs fPcg starting"<<std::endl;
|
||||
src[0].Grid()->Barrier();
|
||||
int nrhs = src.size();
|
||||
std::vector<RealD> f(nrhs);
|
||||
std::vector<RealD> rtzp(nrhs);
|
||||
std::vector<RealD> rtz(nrhs);
|
||||
std::vector<RealD> a(nrhs);
|
||||
std::vector<RealD> d(nrhs);
|
||||
std::vector<RealD> b(nrhs);
|
||||
std::vector<RealD> rptzp(nrhs);
|
||||
/////////////////////////////
|
||||
// Set up history vectors
|
||||
/////////////////////////////
|
||||
int mmax = 3;
|
||||
|
||||
std::vector<std::vector<Field> > p(nrhs); for(int r=0;r<nrhs;r++) p[r].resize(mmax,grid);
|
||||
std::vector<std::vector<Field> > mmp(nrhs); for(int r=0;r<nrhs;r++) mmp[r].resize(mmax,grid);
|
||||
std::vector<std::vector<RealD> > pAp(nrhs); for(int r=0;r<nrhs;r++) pAp[r].resize(mmax);
|
||||
|
||||
std::vector<Field> z(nrhs,grid);
|
||||
std::vector<Field> mp (nrhs,grid);
|
||||
std::vector<Field> r (nrhs,grid);
|
||||
std::vector<Field> mu (nrhs,grid);
|
||||
|
||||
//Initial residual computation & set up
|
||||
std::vector<RealD> src_nrm(nrhs);
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
src_nrm[rhs]=norm2(src[rhs]);
|
||||
GRID_ASSERT(src_nrm[rhs]!=0.0);
|
||||
}
|
||||
std::vector<RealD> tn(nrhs);
|
||||
|
||||
GridStopWatch HDCGTimer;
|
||||
//////////////////////////
|
||||
// x0 = Vstart -- possibly modify guess
|
||||
//////////////////////////
|
||||
Vstart(x,src);
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
// r0 = b -A x0
|
||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
||||
axpy (r[rhs], -1.0,mmp[rhs][0], src[rhs]); // Recomputes r=src-Ax0
|
||||
}
|
||||
|
||||
//////////////////////////////////
|
||||
// Compute z = M1 x
|
||||
//////////////////////////////////
|
||||
// This needs a multiRHS version for acceleration
|
||||
PcgM1(r,z);
|
||||
|
||||
std::vector<RealD> ssq(nrhs);
|
||||
std::vector<RealD> rsq(nrhs);
|
||||
std::vector<Field> pp(nrhs,grid);
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
||||
p[rhs][0]=z[rhs];
|
||||
ssq[rhs]=norm2(src[rhs]);
|
||||
rsq[rhs]= ssq[rhs]*Tolerance*Tolerance;
|
||||
// std::cout << GridLogMessage<<"mrhs HDCG: "<<rhs<<" k=0 residual "<<rtzp[rhs]<<" rsq "<<rsq[rhs]<<"\n";
|
||||
}
|
||||
|
||||
ProjectTimer.Reset();
|
||||
PromoteTimer.Reset();
|
||||
DeflateTimer.Reset();
|
||||
CoarseTimer.Reset();
|
||||
SmoothTimer.Reset();
|
||||
FineTimer.Reset();
|
||||
InsertTimer.Reset();
|
||||
|
||||
GridStopWatch M1Timer;
|
||||
GridStopWatch M2Timer;
|
||||
GridStopWatch M3Timer;
|
||||
GridStopWatch LinalgTimer;
|
||||
|
||||
HDCGTimer.Start();
|
||||
|
||||
std::vector<RealD> rn(nrhs);
|
||||
for (int k=0;k<=MaxIterations;k++){
|
||||
|
||||
int peri_k = k % mmax;
|
||||
int peri_kp = (k+1) % mmax;
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
rtz[rhs]=rtzp[rhs];
|
||||
M3Timer.Start();
|
||||
d[rhs]= PcgM3(p[rhs][peri_k],mmp[rhs][peri_k]);
|
||||
M3Timer.Stop();
|
||||
a[rhs] = rtz[rhs]/d[rhs];
|
||||
|
||||
LinalgTimer.Start();
|
||||
// Memorise this
|
||||
pAp[rhs][peri_k] = d[rhs];
|
||||
|
||||
axpy(x[rhs],a[rhs],p[rhs][peri_k],x[rhs]);
|
||||
rn[rhs] = axpy_norm(r[rhs],-a[rhs],mmp[rhs][peri_k],r[rhs]);
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
|
||||
// Compute z = M x (for *all* RHS)
|
||||
M1Timer.Start();
|
||||
PcgM1(r,z);
|
||||
M1Timer.Stop();
|
||||
|
||||
RealD max_rn=0.0;
|
||||
LinalgTimer.Start();
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
|
||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
||||
|
||||
// std::cout << GridLogMessage<<"HDCG::fPcg rhs"<<rhs<<" iteration "<<k<<" : inner rtzp "<<rtzp[rhs]<<"\n";
|
||||
mu[rhs]=z[rhs];
|
||||
|
||||
p[rhs][peri_kp]=mu[rhs];
|
||||
|
||||
// Standard search direction p == z + b p
|
||||
b[rhs] = (rtzp[rhs])/rtz[rhs];
|
||||
|
||||
int northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
||||
for(int back=0; back < northog; back++){
|
||||
int peri_back = (k-back)%mmax;
|
||||
RealD pbApk= real(innerProduct(mmp[rhs][peri_back],p[rhs][peri_kp]));
|
||||
RealD beta = -pbApk/pAp[rhs][peri_back];
|
||||
axpy(p[rhs][peri_kp],beta,p[rhs][peri_back],p[rhs][peri_kp]);
|
||||
}
|
||||
|
||||
RealD rrn=sqrt(rn[rhs]/ssq[rhs]);
|
||||
RealD rtn=sqrt(rtz[rhs]/ssq[rhs]);
|
||||
RealD rtnp=sqrt(rtzp[rhs]/ssq[rhs]);
|
||||
|
||||
std::cout<<GridLogMessage<<"HDCG:fPcg rhs "<<rhs<<" k= "<<k<<" residual = "<<rrn<<"\n";
|
||||
if ( rrn > max_rn ) max_rn = rrn;
|
||||
}
|
||||
LinalgTimer.Stop();
|
||||
|
||||
// Stopping condition based on worst case
|
||||
if ( max_rn <= Tolerance ) {
|
||||
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Linalg "<<LinalgTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : fine M3 "<<M3Timer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Project "<<ProjectTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Coarse "<<CoarseTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Fine "<<FineTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Smooth "<<SmoothTimer.Elapsed()<<std::endl;;
|
||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Insert "<<InsertTimer.Elapsed()<<std::endl;;
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
||||
Field tmp(grid);
|
||||
axpy(tmp,-1.0,src[rhs],mmp[rhs][0]);
|
||||
|
||||
RealD mmpnorm = sqrt(norm2(mmp[rhs][0]));
|
||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
||||
RealD tmpnorm = sqrt(norm2(tmp));
|
||||
RealD true_residual = tmpnorm/srcnorm;
|
||||
std::cout<<GridLogMessage
|
||||
<<"HDCG: true residual ["<<rhs<<"] is "<<true_residual
|
||||
<<" solution "<<xnorm
|
||||
<<" source "<<srcnorm
|
||||
<<" mmp "<<mmpnorm
|
||||
<<std::endl;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
HDCGTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"HDCG: not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
||||
for(int rhs=0;rhs<nrhs;rhs++){
|
||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
||||
std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
|
||||
virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out) = 0;
|
||||
virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src) = 0;
|
||||
virtual void PcgM2(const Field & in, Field & out) {
|
||||
out=in;
|
||||
}
|
||||
|
||||
virtual RealD PcgM3(const Field & p, Field & mmp){
|
||||
RealD dd;
|
||||
_FineLinop.HermOp(p,mmp);
|
||||
ComplexD dot = innerProduct(p,mmp);
|
||||
dd=real(dot);
|
||||
return dd;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
template<class Field, class CoarseField>
|
||||
class TwoLevelADEF2mrhs : public TwoLevelCGmrhs<Field>
|
||||
{
|
||||
public:
|
||||
GridBase *coarsegrid;
|
||||
GridBase *coarsegridmrhs;
|
||||
LinearFunction<CoarseField> &_CoarseSolverMrhs;
|
||||
LinearFunction<CoarseField> &_CoarseSolverPreciseMrhs;
|
||||
MultiRHSBlockProject<Field> &_Projector;
|
||||
MultiRHSDeflation<CoarseField> &_Deflator;
|
||||
|
||||
|
||||
TwoLevelADEF2mrhs(RealD tol,
|
||||
Integer maxit,
|
||||
LinearOperatorBase<Field> &FineLinop,
|
||||
LinearFunction<Field> &Smoother,
|
||||
LinearFunction<CoarseField> &CoarseSolverMrhs,
|
||||
LinearFunction<CoarseField> &CoarseSolverPreciseMrhs,
|
||||
MultiRHSBlockProject<Field> &Projector,
|
||||
MultiRHSDeflation<CoarseField> &Deflator,
|
||||
GridBase *_coarsemrhsgrid) :
|
||||
TwoLevelCGmrhs<Field>(tol, maxit,FineLinop,Smoother,Projector.fine_grid),
|
||||
_CoarseSolverMrhs(CoarseSolverMrhs),
|
||||
_CoarseSolverPreciseMrhs(CoarseSolverPreciseMrhs),
|
||||
_Projector(Projector),
|
||||
_Deflator(Deflator)
|
||||
{
|
||||
coarsegrid = Projector.coarse_grid;
|
||||
coarsegridmrhs = _coarsemrhsgrid;// Thi could be in projector
|
||||
};
|
||||
|
||||
// Override Vstart
|
||||
virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src)
|
||||
{
|
||||
int nrhs=x.size();
|
||||
///////////////////////////////////
|
||||
// Choose x_0 such that
|
||||
// x_0 = guess + (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
|
||||
// = [1 - Ass_inv A] Guess + Assinv src
|
||||
// = P^T guess + Assinv src
|
||||
// = Vstart [Tang notation]
|
||||
// This gives:
|
||||
// W^T (src - A x_0) = src_s - A guess_s - r_s
|
||||
// = src_s - (A guess)_s - src_s + (A guess)_s
|
||||
// = 0
|
||||
///////////////////////////////////
|
||||
std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
|
||||
std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
|
||||
CoarseField PleftProjMrhs(this->coarsegridmrhs);
|
||||
CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
|
||||
|
||||
this->_Projector.blockProject(src,PleftProj);
|
||||
this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
|
||||
InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
|
||||
}
|
||||
|
||||
this->_CoarseSolverPreciseMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} r_s
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
|
||||
}
|
||||
this->_Projector.blockPromote(x,PleftMss_proj);
|
||||
}
|
||||
|
||||
virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out){
|
||||
|
||||
int nrhs=in.size();
|
||||
|
||||
// [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
|
||||
std::vector<Field> tmp(nrhs,this->grid);
|
||||
std::vector<Field> Min(nrhs,this->grid);
|
||||
|
||||
std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
|
||||
std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
|
||||
|
||||
CoarseField PleftProjMrhs(this->coarsegridmrhs);
|
||||
CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
|
||||
|
||||
// this->rrr=in[0];
|
||||
|
||||
#undef SMOOTHER_BLOCK_SOLVE
|
||||
#if SMOOTHER_BLOCK_SOLVE
|
||||
this->SmoothTimer.Start();
|
||||
this->_Smoother(in,Min);
|
||||
this->SmoothTimer.Stop();
|
||||
#else
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
this->SmoothTimer.Start();
|
||||
this->_Smoother(in[rhs],Min[rhs]);
|
||||
this->SmoothTimer.Stop();
|
||||
}
|
||||
#endif
|
||||
// this->sss=Min[0];
|
||||
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
|
||||
this->FineTimer.Start();
|
||||
this->_FineLinop.HermOp(Min[rhs],out[rhs]);
|
||||
axpy(tmp[rhs],-1.0,out[rhs],in[rhs]); // resid = in - A Min
|
||||
this->FineTimer.Stop();
|
||||
|
||||
}
|
||||
|
||||
this->ProjectTimer.Start();
|
||||
this->_Projector.blockProject(tmp,PleftProj);
|
||||
this->ProjectTimer.Stop();
|
||||
this->DeflateTimer.Start();
|
||||
this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
|
||||
this->DeflateTimer.Stop();
|
||||
this->InsertTimer.Start();
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
|
||||
InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
|
||||
}
|
||||
this->InsertTimer.Stop();
|
||||
|
||||
this->CoarseTimer.Start();
|
||||
this->_CoarseSolverMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} [in - A Min]_s
|
||||
this->CoarseTimer.Stop();
|
||||
|
||||
this->InsertTimer.Start();
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
|
||||
}
|
||||
this->InsertTimer.Stop();
|
||||
this->PromoteTimer.Start();
|
||||
this->_Projector.blockPromote(tmp,PleftMss_proj);// tmp= Q[in - A Min]
|
||||
this->PromoteTimer.Stop();
|
||||
this->FineTimer.Start();
|
||||
// this->qqq=tmp[0];
|
||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
||||
axpy(out[rhs],1.0,Min[rhs],tmp[rhs]); // Min+tmp
|
||||
}
|
||||
// this->zzz=out[0];
|
||||
this->FineTimer.Stop();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
@@ -77,7 +77,7 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
Linop.Op(psi, v);
|
||||
b = norm2(v);
|
||||
@@ -122,14 +122,12 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
|
||||
LinearCombTimer.Start();
|
||||
bo = beta * omega;
|
||||
{
|
||||
autoView( p_v , p, AcceleratorWrite);
|
||||
autoView( r_v , r, AcceleratorRead);
|
||||
autoView( v_v , v, AcceleratorRead);
|
||||
accelerator_for(ss, p_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(p_v[ss], beta*p_v(ss) - bo*v_v(ss) + r_v(ss));
|
||||
});
|
||||
}
|
||||
auto p_v = p.View();
|
||||
auto r_v = r.View();
|
||||
auto v_v = v.View();
|
||||
accelerator_for(ss, p_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(p_v[ss], beta*p_v(ss) - bo*v_v(ss) + r_v(ss));
|
||||
});
|
||||
LinearCombTimer.Stop();
|
||||
LinalgTimer.Stop();
|
||||
|
||||
@@ -144,20 +142,16 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
alpha = rho / Calpha.real();
|
||||
|
||||
LinearCombTimer.Start();
|
||||
{
|
||||
autoView( p_v , p, AcceleratorRead);
|
||||
autoView( r_v , r, AcceleratorRead);
|
||||
autoView( v_v , v, AcceleratorRead);
|
||||
autoView( psi_v,psi, AcceleratorRead);
|
||||
autoView( h_v , h, AcceleratorWrite);
|
||||
autoView( s_v , s, AcceleratorWrite);
|
||||
accelerator_for(ss, h_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(h_v[ss], alpha*p_v(ss) + psi_v(ss));
|
||||
});
|
||||
accelerator_for(ss, s_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(s_v[ss], -alpha*v_v(ss) + r_v(ss));
|
||||
});
|
||||
}
|
||||
auto h_v = h.View();
|
||||
auto psi_v = psi.View();
|
||||
accelerator_for(ss, h_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(h_v[ss], alpha*p_v(ss) + psi_v(ss));
|
||||
});
|
||||
|
||||
auto s_v = s.View();
|
||||
accelerator_for(ss, s_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(s_v[ss], -alpha*v_v(ss) + r_v(ss));
|
||||
});
|
||||
LinearCombTimer.Stop();
|
||||
LinalgTimer.Stop();
|
||||
|
||||
@@ -172,19 +166,13 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
omega = Comega.real() / norm2(t);
|
||||
|
||||
LinearCombTimer.Start();
|
||||
{
|
||||
autoView( psi_v,psi, AcceleratorWrite);
|
||||
autoView( r_v , r, AcceleratorWrite);
|
||||
autoView( h_v , h, AcceleratorRead);
|
||||
autoView( s_v , s, AcceleratorRead);
|
||||
autoView( t_v , t, AcceleratorRead);
|
||||
accelerator_for(ss, psi_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(psi_v[ss], h_v(ss) + omega * s_v(ss));
|
||||
coalescedWrite(r_v[ss], -omega * t_v(ss) + s_v(ss));
|
||||
});
|
||||
}
|
||||
auto t_v = t.View();
|
||||
accelerator_for(ss, psi_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(psi_v[ss], h_v(ss) + omega * s_v(ss));
|
||||
coalescedWrite(r_v[ss], -omega * t_v(ss) + s_v(ss));
|
||||
});
|
||||
LinearCombTimer.Stop();
|
||||
|
||||
|
||||
cp = norm2(r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
@@ -214,7 +202,7 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
std::cout << GridLogMessage << "\tAxpyNorm " << AxpyNormTimer.Elapsed() << std::endl;
|
||||
std::cout << GridLogMessage << "\tLinearComb " << LinearCombTimer.Elapsed() << std::endl;
|
||||
|
||||
if(ErrorOnNoConverge){ GRID_ASSERT(true_residual / Tolerance < 10000.0); }
|
||||
if(ErrorOnNoConverge){ assert(true_residual / Tolerance < 10000.0); }
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
@@ -224,7 +212,7 @@ class BiCGSTAB : public OperatorFunction<Field>
|
||||
|
||||
std::cout << GridLogMessage << "BiCGSTAB did NOT converge" << std::endl;
|
||||
|
||||
if(ErrorOnNoConverge){ GRID_ASSERT(0); }
|
||||
if(ErrorOnNoConverge){ assert(0); }
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -36,8 +36,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
template<class FieldD, class FieldF, typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionBiCGSTAB : public LinearFunction<FieldD>
|
||||
{
|
||||
public:
|
||||
using LinearFunction<FieldD>::operator();
|
||||
public:
|
||||
RealD Tolerance;
|
||||
RealD InnerTolerance; // Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||
Integer MaxInnerIterations;
|
||||
|
||||
@@ -31,58 +31,6 @@ directory
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class Field>
|
||||
void InnerProductMatrix(Eigen::MatrixXcd &m , const std::vector<Field> &X, const std::vector<Field> &Y){
|
||||
typedef typename Field::scalar_type scomplex;
|
||||
int Nblock = X.size();
|
||||
for(int b=0;b<Nblock;b++){
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
m(b,bp) = innerProduct(X[b],Y[bp]);
|
||||
}}
|
||||
}
|
||||
template<class Field>
|
||||
void MaddMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X,const std::vector<Field> &Y,RealD scale=1.0){
|
||||
// Should make this cache friendly with site outermost, parallel_for
|
||||
// Deal with case AP aliases with either Y or X
|
||||
//
|
||||
//Could pack "X" and "AP" into a Nblock x Volume dense array.
|
||||
// AP(Nrhs x vol) = Y(Nrhs x vol) + scale * m(nrhs x nrhs) * X(nrhs*vol)
|
||||
typedef typename Field::scalar_type scomplex;
|
||||
int Nblock = AP.size();
|
||||
std::vector<Field> tmp(Nblock,X[0]);
|
||||
for(int b=0;b<Nblock;b++){
|
||||
tmp[b] = Y[b];
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
tmp[b] = tmp[b] +scomplex(scale*m(bp,b))*X[bp];
|
||||
}
|
||||
}
|
||||
for(int b=0;b<Nblock;b++){
|
||||
AP[b] = tmp[b];
|
||||
}
|
||||
}
|
||||
template<class Field>
|
||||
void MulMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X){
|
||||
// Should make this cache friendly with site outermost, parallel_for
|
||||
typedef typename Field::scalar_type scomplex;
|
||||
int Nblock = AP.size();
|
||||
for(int b=0;b<Nblock;b++){
|
||||
AP[b] = Zero();
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
AP[b] += scomplex(m(bp,b))*X[bp];
|
||||
}
|
||||
}
|
||||
}
|
||||
template<class Field>
|
||||
double normv(const std::vector<Field> &P){
|
||||
int Nblock = P.size();
|
||||
double nn = 0.0;
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
nn+=norm2(P[b]);
|
||||
}
|
||||
return nn;
|
||||
}
|
||||
|
||||
|
||||
enum BlockCGtype { BlockCG, BlockCGrQ, CGmultiRHS, BlockCGVec, BlockCGrQVec };
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
@@ -98,7 +46,7 @@ class BlockConjugateGradient : public OperatorFunction<Field> {
|
||||
int Nblock;
|
||||
|
||||
BlockCGtype CGtype;
|
||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
@@ -139,19 +87,10 @@ void ThinQRfact (Eigen::MatrixXcd &m_rr,
|
||||
sliceInnerProductMatrix(m_rr,R,R,Orthog);
|
||||
|
||||
// Force manifest hermitian to avoid rounding related
|
||||
/*
|
||||
int rank=m_rr.rows();
|
||||
for(int r=0;r<rank;r++){
|
||||
for(int s=0;s<rank;s++){
|
||||
std::cout << "QR m_rr["<<r<<","<<s<<"] "<<m_rr(r,s)<<std::endl;
|
||||
}}
|
||||
*/
|
||||
m_rr = 0.5*(m_rr+m_rr.adjoint());
|
||||
|
||||
Eigen::MatrixXcd L = m_rr.llt().matrixL();
|
||||
|
||||
// ComplexD det = L.determinant();
|
||||
// std::cout << " Det m_rr "<<det<<std::endl;
|
||||
C = L.adjoint();
|
||||
Cinv = C.inverse();
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -171,20 +110,11 @@ void ThinQRfact (Eigen::MatrixXcd &m_rr,
|
||||
const std::vector<Field> & R)
|
||||
{
|
||||
InnerProductMatrix(m_rr,R,R);
|
||||
/*
|
||||
int rank=m_rr.rows();
|
||||
for(int r=0;r<rank;r++){
|
||||
for(int s=0;s<rank;s++){
|
||||
std::cout << "QRvec m_rr["<<r<<","<<s<<"] "<<m_rr(r,s)<<std::endl;
|
||||
}}
|
||||
*/
|
||||
|
||||
m_rr = 0.5*(m_rr+m_rr.adjoint());
|
||||
|
||||
Eigen::MatrixXcd L = m_rr.llt().matrixL();
|
||||
|
||||
// ComplexD det = L.determinant();
|
||||
// std::cout << " Det m_rr "<<det<<std::endl;
|
||||
|
||||
C = L.adjoint();
|
||||
Cinv = C.inverse();
|
||||
|
||||
@@ -201,7 +131,7 @@ void operator()(LinearOperatorBase<Field> &Linop, const Field &Src, Field &Psi)
|
||||
} else if (CGtype == CGmultiRHS ) {
|
||||
CGmultiRHSsolve(Linop,Src,Psi);
|
||||
} else {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
virtual void operator()(LinearOperatorBase<Field> &Linop, const std::vector<Field> &Src, std::vector<Field> &Psi)
|
||||
@@ -209,7 +139,7 @@ virtual void operator()(LinearOperatorBase<Field> &Linop, const std::vector<Fiel
|
||||
if ( CGtype == BlockCGrQVec ) {
|
||||
BlockCGrQsolveVec(Linop,Src,Psi);
|
||||
} else {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,13 +186,12 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
|
||||
sliceNorm(ssq,B,Orthog);
|
||||
RealD sssum=0;
|
||||
for(int b=0;b<Nblock;b++) sssum+=ssq[b];
|
||||
for(int b=0;b<Nblock;b++) std::cout << "src["<<b<<"]" << ssq[b] <<std::endl;
|
||||
|
||||
sliceNorm(residuals,B,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ GRID_ASSERT(std::isnan(residuals[b])==0); }
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
sliceNorm(residuals,X,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ GRID_ASSERT(std::isnan(residuals[b])==0); }
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
/************************************************************************
|
||||
* Block conjugate gradient rQ (Sebastien Birk Thesis, after Dubrulle 2001)
|
||||
@@ -292,9 +221,6 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
|
||||
Linop.HermOp(X, AD);
|
||||
tmp = B - AD;
|
||||
|
||||
sliceNorm(residuals,tmp,Orthog);
|
||||
for(int b=0;b<Nblock;b++) std::cout << "res["<<b<<"]" << residuals[b] <<std::endl;
|
||||
|
||||
ThinQRfact (m_rr, m_C, m_Cinv, Q, tmp);
|
||||
D=Q;
|
||||
|
||||
@@ -310,8 +236,6 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
RealD max_resid=0;
|
||||
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
@@ -356,7 +280,7 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
|
||||
*/
|
||||
m_rr = m_C.adjoint() * m_C;
|
||||
|
||||
max_resid=0;
|
||||
RealD max_resid=0;
|
||||
RealD rrsum=0;
|
||||
RealD rr;
|
||||
|
||||
@@ -398,11 +322,9 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
|
||||
}
|
||||
|
||||
}
|
||||
std::cout << GridLogMessage << "BlockConjugateGradient(rQ) did NOT converge" << std::endl;
|
||||
|
||||
std::cout << GridLogMessage << "BlockConjugateGradient(rQ) did NOT converge "<<k<<" / "<<MaxIterations
|
||||
<<" residual "<< std::sqrt(max_resid)<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
@@ -438,10 +360,10 @@ void CGmultiRHSsolve(LinearOperatorBase<Field> &Linop, const Field &Src, Field &
|
||||
for(int b=0;b<Nblock;b++) sssum+=ssq[b];
|
||||
|
||||
sliceNorm(residuals,Src,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ GRID_ASSERT(std::isnan(residuals[b])==0); }
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
sliceNorm(residuals,Psi,Orthog);
|
||||
for(int b=0;b<Nblock;b++){ GRID_ASSERT(std::isnan(residuals[b])==0); }
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
// Initial search dir is guess
|
||||
Linop.HermOp(Psi, AP);
|
||||
@@ -540,10 +462,47 @@ void CGmultiRHSsolve(LinearOperatorBase<Field> &Linop, const Field &Src, Field &
|
||||
}
|
||||
std::cout << GridLogMessage << "MultiRHSConjugateGradient did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
|
||||
void InnerProductMatrix(Eigen::MatrixXcd &m , const std::vector<Field> &X, const std::vector<Field> &Y){
|
||||
for(int b=0;b<Nblock;b++){
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
m(b,bp) = innerProduct(X[b],Y[bp]);
|
||||
}}
|
||||
}
|
||||
void MaddMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X,const std::vector<Field> &Y,RealD scale=1.0){
|
||||
// Should make this cache friendly with site outermost, parallel_for
|
||||
// Deal with case AP aliases with either Y or X
|
||||
std::vector<Field> tmp(Nblock,X[0]);
|
||||
for(int b=0;b<Nblock;b++){
|
||||
tmp[b] = Y[b];
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
tmp[b] = tmp[b] + scomplex(scale*m(bp,b))*X[bp];
|
||||
}
|
||||
}
|
||||
for(int b=0;b<Nblock;b++){
|
||||
AP[b] = tmp[b];
|
||||
}
|
||||
}
|
||||
void MulMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X){
|
||||
// Should make this cache friendly with site outermost, parallel_for
|
||||
for(int b=0;b<Nblock;b++){
|
||||
AP[b] = Zero();
|
||||
for(int bp=0;bp<Nblock;bp++) {
|
||||
AP[b] += scomplex(m(bp,b))*X[bp];
|
||||
}
|
||||
}
|
||||
}
|
||||
double normv(const std::vector<Field> &P){
|
||||
double nn = 0.0;
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
nn+=norm2(P[b]);
|
||||
}
|
||||
return nn;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// BlockCGrQvec implementation:
|
||||
//--------------------------
|
||||
@@ -554,7 +513,7 @@ void CGmultiRHSsolve(LinearOperatorBase<Field> &Linop, const Field &Src, Field &
|
||||
void BlockCGrQsolveVec(LinearOperatorBase<Field> &Linop, const std::vector<Field> &B, std::vector<Field> &X)
|
||||
{
|
||||
Nblock = B.size();
|
||||
GRID_ASSERT(Nblock == X.size());
|
||||
assert(Nblock == X.size());
|
||||
|
||||
std::cout<<GridLogMessage<<" Block Conjugate Gradient Vec rQ : Nblock "<<Nblock<<std::endl;
|
||||
|
||||
@@ -590,14 +549,13 @@ void BlockCGrQsolveVec(LinearOperatorBase<Field> &Linop, const std::vector<Field
|
||||
|
||||
RealD sssum=0;
|
||||
for(int b=0;b<Nblock;b++){ ssq[b] = norm2(B[b]);}
|
||||
for(int b=0;b<Nblock;b++){ std::cout << "ssq["<<b<<"] "<<ssq[b]<<std::endl;}
|
||||
for(int b=0;b<Nblock;b++) sssum+=ssq[b];
|
||||
|
||||
for(int b=0;b<Nblock;b++){ residuals[b] = norm2(B[b]);}
|
||||
for(int b=0;b<Nblock;b++){ GRID_ASSERT(std::isnan(residuals[b])==0); }
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
for(int b=0;b<Nblock;b++){ residuals[b] = norm2(X[b]);}
|
||||
for(int b=0;b<Nblock;b++){ GRID_ASSERT(std::isnan(residuals[b])==0); }
|
||||
for(int b=0;b<Nblock;b++){ assert(std::isnan(residuals[b])==0); }
|
||||
|
||||
/************************************************************************
|
||||
* Block conjugate gradient rQ (Sebastien Birk Thesis, after Dubrulle 2001)
|
||||
@@ -627,7 +585,6 @@ void BlockCGrQsolveVec(LinearOperatorBase<Field> &Linop, const std::vector<Field
|
||||
for(int b=0;b<Nblock;b++) {
|
||||
Linop.HermOp(X[b], AD[b]);
|
||||
tmp[b] = B[b] - AD[b];
|
||||
std::cout << "r0["<<b<<"] "<<norm2(tmp[b])<<std::endl;
|
||||
}
|
||||
|
||||
ThinQRfact (m_rr, m_C, m_Cinv, Q, tmp);
|
||||
@@ -731,7 +688,7 @@ void BlockCGrQsolveVec(LinearOperatorBase<Field> &Linop, const std::vector<Field
|
||||
}
|
||||
std::cout << GridLogMessage << "BlockConjugateGradient(rQ) did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when CAGMRES fails to converge,
|
||||
bool ErrorOnNoConverge; // Throw an assert when CAGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
@@ -82,7 +82,7 @@ class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
@@ -137,7 +137,7 @@ class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<
|
||||
std::cout << GridLogMessage << "CommunicationAvoidingGeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
@@ -185,7 +185,7 @@ class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<
|
||||
}
|
||||
}
|
||||
|
||||
GRID_ASSERT(0); // Never reached
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
@@ -38,14 +38,13 @@ NAMESPACE_BEGIN(Grid);
|
||||
// single input vec, single output vec.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
template <class Field>
|
||||
class ConjugateGradient : public OperatorFunction<Field> {
|
||||
public:
|
||||
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
||||
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
@@ -55,26 +54,10 @@ public:
|
||||
ConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
|
||||
: Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
ErrorOnNoConverge(err_on_no_conv)
|
||||
{};
|
||||
ErrorOnNoConverge(err_on_no_conv){};
|
||||
|
||||
virtual void LogIteration(int k,RealD a,RealD b){
|
||||
// std::cout << "ConjugageGradient::LogIteration() "<<std::endl;
|
||||
};
|
||||
virtual void LogBegin(void){
|
||||
std::cout << "ConjugageGradient::LogBegin() "<<std::endl;
|
||||
};
|
||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
||||
|
||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
||||
|
||||
this->LogBegin();
|
||||
|
||||
GRID_TRACE("ConjugateGradient");
|
||||
GridStopWatch PreambleTimer;
|
||||
GridStopWatch ConstructTimer;
|
||||
GridStopWatch NormTimer;
|
||||
GridStopWatch AssignTimer;
|
||||
PreambleTimer.Start();
|
||||
psi.Checkerboard() = src.Checkerboard();
|
||||
|
||||
conformable(psi, src);
|
||||
@@ -82,32 +65,22 @@ public:
|
||||
RealD cp, c, a, d, b, ssq, qq;
|
||||
//RealD b_pred;
|
||||
|
||||
// Was doing copies
|
||||
ConstructTimer.Start();
|
||||
Field p (src.Grid());
|
||||
Field mmp(src.Grid());
|
||||
Field r (src.Grid());
|
||||
ConstructTimer.Stop();
|
||||
Field p(src);
|
||||
Field mmp(src);
|
||||
Field r(src);
|
||||
|
||||
// Initial residual computation & set up
|
||||
NormTimer.Start();
|
||||
ssq = norm2(src);
|
||||
RealD guess = norm2(psi);
|
||||
NormTimer.Stop();
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
AssignTimer.Start();
|
||||
if ( guess == 0.0 ) {
|
||||
r = src;
|
||||
p = r;
|
||||
a = ssq;
|
||||
} else {
|
||||
Linop.HermOpAndNorm(psi, mmp, d, b);
|
||||
r = src - mmp;
|
||||
p = r;
|
||||
a = norm2(p);
|
||||
}
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
Linop.HermOpAndNorm(psi, mmp, d, b);
|
||||
|
||||
r = src - mmp;
|
||||
p = r;
|
||||
|
||||
a = norm2(p);
|
||||
cp = a;
|
||||
AssignTimer.Stop();
|
||||
ssq = norm2(src);
|
||||
|
||||
// Handle trivial case of zero src
|
||||
if (ssq == 0.){
|
||||
@@ -137,7 +110,6 @@ public:
|
||||
std::cout << GridLogIterative << std::setprecision(8)
|
||||
<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
|
||||
|
||||
PreambleTimer.Stop();
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch InnerTimer;
|
||||
GridStopWatch AxpyNormTimer;
|
||||
@@ -145,13 +117,9 @@ public:
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
|
||||
RealD usecs = -usecond();
|
||||
SolverTimer.Start();
|
||||
int k;
|
||||
for (k = 1; k <= MaxIterations; k++) {
|
||||
|
||||
GridStopWatch IterationTimer;
|
||||
IterationTimer.Start();
|
||||
c = cp;
|
||||
|
||||
MatrixTimer.Start();
|
||||
@@ -172,57 +140,43 @@ public:
|
||||
b = cp / c;
|
||||
|
||||
LinearCombTimer.Start();
|
||||
{
|
||||
autoView( psi_v , psi, AcceleratorWrite);
|
||||
autoView( p_v , p, AcceleratorWrite);
|
||||
autoView( r_v , r, AcceleratorWrite);
|
||||
accelerator_for(ss,p_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(psi_v[ss], a * p_v(ss) + psi_v(ss));
|
||||
coalescedWrite(p_v[ss] , b * p_v(ss) + r_v (ss));
|
||||
});
|
||||
}
|
||||
auto psi_v = psi.View();
|
||||
auto p_v = p.View();
|
||||
auto r_v = r.View();
|
||||
accelerator_for(ss,p_v.size(), Field::vector_object::Nsimd(),{
|
||||
coalescedWrite(psi_v[ss], a * p_v(ss) + psi_v(ss));
|
||||
coalescedWrite(p_v[ss] , b * p_v(ss) + r_v (ss));
|
||||
});
|
||||
LinearCombTimer.Stop();
|
||||
LinalgTimer.Stop();
|
||||
LogIteration(k,a,b);
|
||||
|
||||
IterationTimer.Stop();
|
||||
if ( (k % 500) == 0 ) {
|
||||
std::cout << GridLogMessage << "ConjugateGradient: Iteration " << k
|
||||
std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
|
||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << std::endl;
|
||||
} else {
|
||||
std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
|
||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << " took " << IterationTimer.Elapsed() << std::endl;
|
||||
}
|
||||
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
usecs +=usecond();
|
||||
SolverTimer.Stop();
|
||||
Linop.HermOpAndNorm(psi, mmp, d, qq);
|
||||
p = mmp - src;
|
||||
GridBase *grid = src.Grid();
|
||||
RealD DwfFlops = (1452. )*grid->gSites()*4*k
|
||||
+ (8+4+8+4+4)*12*grid->gSites()*k; // CG linear algebra
|
||||
|
||||
RealD srcnorm = std::sqrt(norm2(src));
|
||||
RealD resnorm = std::sqrt(norm2(p));
|
||||
RealD true_residual = resnorm / srcnorm;
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradient Converged on iteration " << k
|
||||
<< "\tComputed residual " << std::sqrt(cp / ssq)
|
||||
<< "\tTrue residual " << true_residual
|
||||
<< "\tTarget " << Tolerance << std::endl;
|
||||
|
||||
// std::cout << GridLogMessage << "\tPreamble " << PreambleTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver Elapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "Time breakdown "<<std::endl;
|
||||
std::cout << GridLogPerformance << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\t\tInner " << InnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\t\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\t\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogIterative << "Time breakdown "<<std::endl;
|
||||
std::cout << GridLogIterative << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogIterative << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogIterative << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogIterative << "\tInner " << InnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogIterative << "\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogIterative << "\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
||||
|
||||
std::cout << GridLogDebug << "\tMobius flop rate " << DwfFlops/ usecs<< " Gflops " <<std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) GRID_ASSERT(true_residual / Tolerance < 10000.0);
|
||||
if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
|
||||
|
||||
IterationsToComplete = k;
|
||||
TrueResidual = true_residual;
|
||||
@@ -231,143 +185,17 @@ public:
|
||||
}
|
||||
}
|
||||
// Failed. Calculate true residual before giving up
|
||||
// Linop.HermOpAndNorm(psi, mmp, d, qq);
|
||||
// p = mmp - src;
|
||||
//TrueResidual = sqrt(norm2(p)/ssq);
|
||||
// TrueResidual = 1;
|
||||
Linop.HermOpAndNorm(psi, mmp, d, qq);
|
||||
p = mmp - src;
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradient did NOT converge "<<k<<" / "<< MaxIterations
|
||||
<<" residual "<< std::sqrt(cp / ssq)<< std::endl;
|
||||
SolverTimer.Stop();
|
||||
std::cout << GridLogMessage << "\tPreamble " << PreambleTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tConstruct " << ConstructTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tNorm " << NormTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tAssign " << AssignTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "Solver breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage<< "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\t\tInner " << InnerTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\t\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogPerformance << "\t\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
||||
TrueResidual = sqrt(norm2(p)/ssq);
|
||||
|
||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
||||
std::cout << GridLogMessage << "ConjugateGradient did NOT converge "<<k<<" / "<< MaxIterations<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template <class Field>
|
||||
class ConjugateGradientPolynomial : public ConjugateGradient<Field> {
|
||||
public:
|
||||
// Optionally record the CG polynomial
|
||||
std::vector<double> ak;
|
||||
std::vector<double> bk;
|
||||
std::vector<double> poly_p;
|
||||
std::vector<double> poly_r;
|
||||
std::vector<double> poly_Ap;
|
||||
std::vector<double> polynomial;
|
||||
|
||||
public:
|
||||
ConjugateGradientPolynomial(RealD tol, Integer maxit, bool err_on_no_conv = true)
|
||||
: ConjugateGradient<Field>(tol,maxit,err_on_no_conv)
|
||||
{ };
|
||||
void PolyHermOp(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
|
||||
{
|
||||
Field tmp(src.Grid());
|
||||
Field AtoN(src.Grid());
|
||||
AtoN = src;
|
||||
psi=AtoN*polynomial[0];
|
||||
for(int n=1;n<polynomial.size();n++){
|
||||
tmp = AtoN;
|
||||
Linop.HermOp(tmp,AtoN);
|
||||
psi = psi + polynomial[n]*AtoN;
|
||||
}
|
||||
}
|
||||
void CGsequenceHermOp(LinearOperatorBase<Field> &Linop, const Field &src, Field &x)
|
||||
{
|
||||
Field Ap(src.Grid());
|
||||
Field r(src.Grid());
|
||||
Field p(src.Grid());
|
||||
p=src;
|
||||
r=src;
|
||||
x=Zero();
|
||||
x.Checkerboard()=src.Checkerboard();
|
||||
for(int k=0;k<ak.size();k++){
|
||||
x = x + ak[k]*p;
|
||||
Linop.HermOp(p,Ap);
|
||||
r = r - ak[k] * Ap;
|
||||
p = r + bk[k] * p;
|
||||
}
|
||||
}
|
||||
void Solve(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
|
||||
{
|
||||
psi=Zero();
|
||||
this->operator ()(Linop,src,psi);
|
||||
}
|
||||
virtual void LogBegin(void)
|
||||
{
|
||||
std::cout << "ConjugageGradientPolynomial::LogBegin() "<<std::endl;
|
||||
ak.resize(0);
|
||||
bk.resize(0);
|
||||
polynomial.resize(0);
|
||||
poly_Ap.resize(0);
|
||||
poly_Ap.resize(0);
|
||||
poly_p.resize(1);
|
||||
poly_r.resize(1);
|
||||
poly_p[0]=1.0;
|
||||
poly_r[0]=1.0;
|
||||
};
|
||||
virtual void LogIteration(int k,RealD a,RealD b)
|
||||
{
|
||||
// With zero guess,
|
||||
// p = r = src
|
||||
//
|
||||
// iterate:
|
||||
// x = x + a p
|
||||
// r = r - a A p
|
||||
// p = r + b p
|
||||
//
|
||||
// [0]
|
||||
// r = x
|
||||
// p = x
|
||||
// Ap=0
|
||||
//
|
||||
// [1]
|
||||
// Ap = A x + 0 ==> shift poly P right by 1 and add 0.
|
||||
// x = x + a p ==> add polynomials term by term
|
||||
// r = r - a A p ==> add polynomials term by term
|
||||
// p = r + b p ==> add polynomials term by term
|
||||
//
|
||||
std::cout << "ConjugageGradientPolynomial::LogIteration() "<<k<<std::endl;
|
||||
ak.push_back(a);
|
||||
bk.push_back(b);
|
||||
// Ap= right_shift(p)
|
||||
poly_Ap.resize(k+1);
|
||||
poly_Ap[0]=0.0;
|
||||
for(int i=0;i<k;i++){
|
||||
poly_Ap[i+1]=poly_p[i];
|
||||
}
|
||||
|
||||
// x = x + a p
|
||||
polynomial.resize(k);
|
||||
polynomial[k-1]=0.0;
|
||||
for(int i=0;i<k;i++){
|
||||
polynomial[i] = polynomial[i] + a * poly_p[i];
|
||||
}
|
||||
|
||||
// r = r - a Ap
|
||||
// p = r + b p
|
||||
poly_r.resize(k+1);
|
||||
poly_p.resize(k+1);
|
||||
poly_r[k] = poly_p[k] = 0.0;
|
||||
for(int i=0;i<k+1;i++){
|
||||
poly_r[i] = poly_r[i] - a * poly_Ap[i];
|
||||
poly_p[i] = poly_r[i] + b * poly_p[i];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -35,8 +35,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionConjugateGradient : public LinearFunction<FieldD> {
|
||||
public:
|
||||
using LinearFunction<FieldD>::operator();
|
||||
public:
|
||||
RealD Tolerance;
|
||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||
Integer MaxInnerIterations;
|
||||
@@ -49,7 +48,6 @@ NAMESPACE_BEGIN(Grid);
|
||||
Integer TotalInnerIterations; //Number of inner CG iterations
|
||||
Integer TotalOuterIterations; //Number of restarts
|
||||
Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
|
||||
RealD TrueResidual;
|
||||
|
||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
||||
LinearFunction<FieldF> *guesser;
|
||||
@@ -69,7 +67,6 @@ NAMESPACE_BEGIN(Grid);
|
||||
}
|
||||
|
||||
void operator() (const FieldD &src_d_in, FieldD &sol_d){
|
||||
std::cout << GridLogMessage << "MixedPrecisionConjugateGradient: Starting mixed precision CG with outer tolerance " << Tolerance << " and inner tolerance " << InnerTolerance << std::endl;
|
||||
TotalInnerIterations = 0;
|
||||
|
||||
GridStopWatch TotalTimer;
|
||||
@@ -99,7 +96,6 @@ NAMESPACE_BEGIN(Grid);
|
||||
FieldF sol_f(SinglePrecGrid);
|
||||
sol_f.Checkerboard() = cb;
|
||||
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting initial inner CG with tolerance " << inner_tol << std::endl;
|
||||
ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
||||
CG_f.ErrorOnNoConverge = false;
|
||||
|
||||
@@ -108,25 +104,22 @@ NAMESPACE_BEGIN(Grid);
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
||||
|
||||
precisionChangeWorkspace pc_wk_sp_to_dp(DoublePrecGrid, SinglePrecGrid);
|
||||
precisionChangeWorkspace pc_wk_dp_to_sp(SinglePrecGrid, DoublePrecGrid);
|
||||
|
||||
|
||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
|
||||
//Compute double precision rsd and also new RHS vector.
|
||||
Linop_d.HermOp(sol_d, tmp_d);
|
||||
RealD norm = axpy_norm(src_d, -1., tmp_d, src_d_in); //src_d is residual vector
|
||||
std::cout<<GridLogMessage<<" rsd norm "<<norm<<std::endl;
|
||||
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " <<outer_iter<<" residual "<< norm<< " target "<< stop<<std::endl;
|
||||
|
||||
if(norm < OuterLoopNormMult * stop){
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration converged on iteration " <<outer_iter <<std::endl;
|
||||
break;
|
||||
}
|
||||
while(norm * inner_tol * inner_tol < stop*1.01) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
||||
while(norm * inner_tol * inner_tol < stop) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(src_f, src_d, pc_wk_dp_to_sp);
|
||||
precisionChange(src_f, src_d);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
sol_f = Zero();
|
||||
@@ -136,7 +129,6 @@ NAMESPACE_BEGIN(Grid);
|
||||
(*guesser)(src_f, sol_f);
|
||||
|
||||
//Inner CG
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " << outer_iter << " starting inner CG with tolerance " << inner_tol << std::endl;
|
||||
CG_f.Tolerance = inner_tol;
|
||||
InnerCGtimer.Start();
|
||||
CG_f(Linop_f, src_f, sol_f);
|
||||
@@ -145,7 +137,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//Convert sol back to double and add to double prec solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(tmp_d, sol_f, pc_wk_sp_to_dp);
|
||||
precisionChange(tmp_d, sol_f);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
axpy(sol_d, 1.0, tmp_d, sol_d);
|
||||
@@ -157,7 +149,6 @@ NAMESPACE_BEGIN(Grid);
|
||||
ConjugateGradient<FieldD> CG_d(Tolerance, MaxInnerIterations);
|
||||
CG_d(Linop_d, src_d_in, sol_d);
|
||||
TotalFinalStepIterations = CG_d.IterationsToComplete;
|
||||
TrueResidual = CG_d.TrueResidual;
|
||||
|
||||
TotalTimer.Stop();
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Inner CG iterations " << TotalInnerIterations << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations << std::endl;
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMixedPrecBatched.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Raoul Hodgson <raoul.hodgson@ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_MIXED_PREC_BATCHED_H
|
||||
#define GRID_CONJUGATE_GRADIENT_MIXED_PREC_BATCHED_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//Mixed precision restarted defect correction CG
|
||||
template<class FieldD,class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class MixedPrecisionConjugateGradientBatched : public LinearFunction<FieldD> {
|
||||
public:
|
||||
using LinearFunction<FieldD>::operator();
|
||||
RealD Tolerance;
|
||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
||||
Integer MaxInnerIterations;
|
||||
Integer MaxOuterIterations;
|
||||
Integer MaxPatchupIterations;
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
||||
LinearOperatorBase<FieldF> &Linop_f;
|
||||
LinearOperatorBase<FieldD> &Linop_d;
|
||||
|
||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
||||
LinearFunction<FieldF> *guesser;
|
||||
bool updateResidual;
|
||||
|
||||
MixedPrecisionConjugateGradientBatched(RealD tol,
|
||||
Integer maxinnerit,
|
||||
Integer maxouterit,
|
||||
Integer maxpatchit,
|
||||
GridBase* _sp_grid,
|
||||
LinearOperatorBase<FieldF> &_Linop_f,
|
||||
LinearOperatorBase<FieldD> &_Linop_d,
|
||||
bool _updateResidual=true) :
|
||||
Linop_f(_Linop_f), Linop_d(_Linop_d),
|
||||
Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), MaxPatchupIterations(maxpatchit), SinglePrecGrid(_sp_grid),
|
||||
OuterLoopNormMult(100.), guesser(NULL), updateResidual(_updateResidual) { };
|
||||
|
||||
void useGuesser(LinearFunction<FieldF> &g){
|
||||
guesser = &g;
|
||||
}
|
||||
|
||||
void operator() (const FieldD &src_d_in, FieldD &sol_d){
|
||||
std::vector<FieldD> srcs_d_in{src_d_in};
|
||||
std::vector<FieldD> sols_d{sol_d};
|
||||
|
||||
(*this)(srcs_d_in,sols_d);
|
||||
|
||||
sol_d = sols_d[0];
|
||||
}
|
||||
|
||||
void operator() (const std::vector<FieldD> &src_d_in, std::vector<FieldD> &sol_d){
|
||||
GRID_ASSERT(src_d_in.size() == sol_d.size());
|
||||
int NBatch = src_d_in.size();
|
||||
|
||||
std::cout << GridLogMessage << "NBatch = " << NBatch << std::endl;
|
||||
|
||||
Integer TotalOuterIterations = 0; //Number of restarts
|
||||
std::vector<Integer> TotalInnerIterations(NBatch,0); //Number of inner CG iterations
|
||||
std::vector<Integer> TotalFinalStepIterations(NBatch,0); //Number of CG iterations in final patch-up step
|
||||
|
||||
GridStopWatch TotalTimer;
|
||||
TotalTimer.Start();
|
||||
|
||||
GridStopWatch InnerCGtimer;
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
int cb = src_d_in[0].Checkerboard();
|
||||
|
||||
std::vector<RealD> src_norm;
|
||||
std::vector<RealD> norm;
|
||||
std::vector<RealD> stop;
|
||||
|
||||
GridBase* DoublePrecGrid = src_d_in[0].Grid();
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
tmp_d.Checkerboard() = cb;
|
||||
|
||||
FieldD tmp2_d(DoublePrecGrid);
|
||||
tmp2_d.Checkerboard() = cb;
|
||||
|
||||
std::vector<FieldD> src_d;
|
||||
std::vector<FieldF> src_f;
|
||||
std::vector<FieldF> sol_f;
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
sol_d[i].Checkerboard() = cb;
|
||||
|
||||
src_norm.push_back(norm2(src_d_in[i]));
|
||||
norm.push_back(0.);
|
||||
stop.push_back(src_norm[i] * Tolerance*Tolerance);
|
||||
|
||||
src_d.push_back(src_d_in[i]); //source for next inner iteration, computed from residual during operation
|
||||
|
||||
src_f.push_back(SinglePrecGrid);
|
||||
src_f[i].Checkerboard() = cb;
|
||||
|
||||
sol_f.push_back(SinglePrecGrid);
|
||||
sol_f[i].Checkerboard() = cb;
|
||||
}
|
||||
|
||||
RealD inner_tol = InnerTolerance;
|
||||
|
||||
ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
||||
CG_f.ErrorOnNoConverge = false;
|
||||
|
||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
||||
|
||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout << GridLogMessage << "Outer iteration " << outer_iter << std::endl;
|
||||
|
||||
bool allConverged = true;
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
//Compute double precision rsd and also new RHS vector.
|
||||
Linop_d.HermOp(sol_d[i], tmp_d);
|
||||
norm[i] = axpy_norm(src_d[i], -1., tmp_d, src_d_in[i]); //src_d is residual vector
|
||||
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer iteration " << outer_iter <<" solve " << i << " residual "<< norm[i] << " target "<< stop[i] <<std::endl;
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(src_f[i], src_d[i]);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
sol_f[i] = Zero();
|
||||
|
||||
if(norm[i] > OuterLoopNormMult * stop[i]) {
|
||||
allConverged = false;
|
||||
}
|
||||
}
|
||||
if (allConverged) break;
|
||||
|
||||
if (updateResidual) {
|
||||
RealD normMax = *std::max_element(std::begin(norm), std::end(norm));
|
||||
RealD stopMax = *std::max_element(std::begin(stop), std::end(stop));
|
||||
while( normMax * inner_tol * inner_tol < stopMax) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
||||
CG_f.Tolerance = inner_tol;
|
||||
}
|
||||
|
||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
||||
if(guesser != NULL) {
|
||||
(*guesser)(src_f, sol_f);
|
||||
}
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
//Inner CG
|
||||
InnerCGtimer.Start();
|
||||
CG_f(Linop_f, src_f[i], sol_f[i]);
|
||||
InnerCGtimer.Stop();
|
||||
TotalInnerIterations[i] += CG_f.IterationsToComplete;
|
||||
|
||||
//Convert sol back to double and add to double prec solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(tmp_d, sol_f[i]);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
axpy(sol_d[i], 1.0, tmp_d, sol_d[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//Final trial CG
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Starting final patch-up double-precision solve"<<std::endl;
|
||||
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
ConjugateGradient<FieldD> CG_d(Tolerance, MaxPatchupIterations);
|
||||
CG_d(Linop_d, src_d_in[i], sol_d[i]);
|
||||
TotalFinalStepIterations[i] += CG_d.IterationsToComplete;
|
||||
}
|
||||
|
||||
TotalTimer.Stop();
|
||||
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
for (int i=0; i<NBatch; i++) {
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: solve " << i << " Inner CG iterations " << TotalInnerIterations[i] << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations[i] << std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -44,7 +44,7 @@ public:
|
||||
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
// RealD Tolerance;
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
||||
@@ -52,7 +52,7 @@ public:
|
||||
MultiShiftFunction shifts;
|
||||
std::vector<RealD> TrueResidualShift;
|
||||
|
||||
ConjugateGradientMultiShift(Integer maxit, const MultiShiftFunction &_shifts) :
|
||||
ConjugateGradientMultiShift(Integer maxit,MultiShiftFunction &_shifts) :
|
||||
MaxIterations(maxit),
|
||||
shifts(_shifts)
|
||||
{
|
||||
@@ -84,7 +84,6 @@ public:
|
||||
|
||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector<Field> &psi)
|
||||
{
|
||||
GRID_TRACE("ConjugateGradientMultiShift");
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
|
||||
@@ -98,15 +97,15 @@ public:
|
||||
std::vector<RealD> alpha(nshift,1.0);
|
||||
std::vector<Field> ps(nshift,grid);// Search directions
|
||||
|
||||
GRID_ASSERT(psi.size()==nshift);
|
||||
GRID_ASSERT(mass.size()==nshift);
|
||||
GRID_ASSERT(mresidual.size()==nshift);
|
||||
assert(psi.size()==nshift);
|
||||
assert(mass.size()==nshift);
|
||||
assert(mresidual.size()==nshift);
|
||||
|
||||
// remove dynamic sized arrays on stack; 2d is a pain with vector
|
||||
std::vector<RealD> bs(nshift);
|
||||
std::vector<RealD> rsq(nshift);
|
||||
std::vector<std::array<RealD,2> > z(nshift);
|
||||
std::vector<int> converged(nshift);
|
||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
||||
RealD bs[nshift];
|
||||
RealD rsq[nshift];
|
||||
RealD z[nshift][2];
|
||||
int converged[nshift];
|
||||
|
||||
const int primary =0;
|
||||
|
||||
@@ -122,7 +121,7 @@ public:
|
||||
|
||||
// Check lightest mass
|
||||
for(int s=0;s<nshift;s++){
|
||||
GRID_ASSERT( mass[s]>= mass[primary] );
|
||||
assert( mass[s]>= mass[primary] );
|
||||
converged[s]=0;
|
||||
}
|
||||
|
||||
@@ -144,7 +143,7 @@ public:
|
||||
for(int s=0;s<nshift;s++){
|
||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShift: shift "<<s
|
||||
<<" target resid^2 "<<rsq[s]<<std::endl;
|
||||
<<" target resid "<<rsq[s]<<std::endl;
|
||||
ps[s] = src;
|
||||
}
|
||||
// r and p for primary
|
||||
@@ -183,9 +182,6 @@ public:
|
||||
for(int s=0;s<nshift;s++) {
|
||||
axpby(psi[s],0.,-bs[s]*alpha[s],src,src);
|
||||
}
|
||||
|
||||
std::cout << GridLogIterative << "ConjugateGradientMultiShift: initial rn (|src|^2) =" << rn << " qq (|MdagM src|^2) =" << qq << " d ( dot(src, [MdagM + m_0]src) ) =" << d << " c=" << c << std::endl;
|
||||
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
@@ -325,8 +321,8 @@ public:
|
||||
|
||||
std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMarix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
@@ -338,7 +334,7 @@ public:
|
||||
}
|
||||
// ugly hack
|
||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
||||
// GRID_ASSERT(0);
|
||||
// assert(0);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
@@ -1,373 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christopher Kelly <ckelly@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//CK 2020: A variant of the multi-shift conjugate gradient with the matrix multiplication in single precision.
|
||||
//The residual is stored in single precision, but the search directions and solution are stored in double precision.
|
||||
//Every update_freq iterations the residual is corrected in double precision.
|
||||
//For safety the a final regular CG is applied to clean up if necessary
|
||||
|
||||
//PB Pure single, then double fixup
|
||||
|
||||
template<class FieldD, class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class ConjugateGradientMultiShiftMixedPrecCleanup : public OperatorMultiFunction<FieldD>,
|
||||
public OperatorFunction<FieldD>
|
||||
{
|
||||
public:
|
||||
|
||||
using OperatorFunction<FieldD>::operator();
|
||||
|
||||
RealD Tolerance;
|
||||
Integer MaxIterationsMshift;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
||||
int verbose;
|
||||
MultiShiftFunction shifts;
|
||||
std::vector<RealD> TrueResidualShift;
|
||||
|
||||
int ReliableUpdateFreq; //number of iterations between reliable updates
|
||||
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
LinearOperatorBase<FieldF> &Linop_f; //single precision
|
||||
|
||||
ConjugateGradientMultiShiftMixedPrecCleanup(Integer maxit, const MultiShiftFunction &_shifts,
|
||||
GridBase* _SinglePrecGrid, LinearOperatorBase<FieldF> &_Linop_f,
|
||||
int _ReliableUpdateFreq) :
|
||||
MaxIterationsMshift(maxit), shifts(_shifts), SinglePrecGrid(_SinglePrecGrid), Linop_f(_Linop_f), ReliableUpdateFreq(_ReliableUpdateFreq),
|
||||
MaxIterations(20000)
|
||||
{
|
||||
verbose=1;
|
||||
IterationsToCompleteShift.resize(_shifts.order);
|
||||
TrueResidualShift.resize(_shifts.order);
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, FieldD &psi)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
int nshift = shifts.order;
|
||||
std::vector<FieldD> results(nshift,grid);
|
||||
(*this)(Linop,src,results,psi);
|
||||
}
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, std::vector<FieldD> &results, FieldD &psi)
|
||||
{
|
||||
int nshift = shifts.order;
|
||||
|
||||
(*this)(Linop,src,results);
|
||||
|
||||
psi = shifts.norm*src;
|
||||
for(int i=0;i<nshift;i++){
|
||||
psi = psi + shifts.residues[i]*results[i];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop_d, const FieldD &src_d, std::vector<FieldD> &psi_d)
|
||||
{
|
||||
GRID_TRACE("ConjugateGradientMultiShiftMixedPrecCleanup");
|
||||
GridBase *DoublePrecGrid = src_d.Grid();
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Convenience references to the info stored in "MultiShiftFunction"
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
int nshift = shifts.order;
|
||||
|
||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
||||
std::vector<RealD> alpha(nshift,1.0);
|
||||
|
||||
//Double precision search directions
|
||||
FieldD p_d(DoublePrecGrid);
|
||||
std::vector<FieldF> ps_f (nshift, SinglePrecGrid);// Search directions (single precision)
|
||||
std::vector<FieldF> psi_f(nshift, SinglePrecGrid);// solutions (single precision)
|
||||
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
FieldD r_d(DoublePrecGrid);
|
||||
FieldF r_f(SinglePrecGrid);
|
||||
FieldD mmp_d(DoublePrecGrid);
|
||||
|
||||
GRID_ASSERT(psi_d.size()==nshift);
|
||||
GRID_ASSERT(mass.size()==nshift);
|
||||
GRID_ASSERT(mresidual.size()==nshift);
|
||||
|
||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
||||
std::vector<RealD> bs(nshift);
|
||||
std::vector<RealD> rsq(nshift);
|
||||
std::vector<RealD> rsqf(nshift);
|
||||
std::vector<std::array<RealD,2> > z(nshift);
|
||||
std::vector<int> converged(nshift);
|
||||
|
||||
const int primary =0;
|
||||
|
||||
//Primary shift fields CG iteration
|
||||
RealD a,b,c,d;
|
||||
RealD cp,bp,qq; //prev
|
||||
|
||||
// Matrix mult fields
|
||||
FieldF p_f(SinglePrecGrid);
|
||||
FieldF mmp_f(SinglePrecGrid);
|
||||
|
||||
// Check lightest mass
|
||||
for(int s=0;s<nshift;s++){
|
||||
GRID_ASSERT( mass[s]>= mass[primary] );
|
||||
converged[s]=0;
|
||||
}
|
||||
|
||||
// Wire guess to zero
|
||||
// Residuals "r" are src
|
||||
// First search direction "p" is also src
|
||||
cp = norm2(src_d);
|
||||
|
||||
// Handle trivial case of zero src.
|
||||
if( cp == 0. ){
|
||||
for(int s=0;s<nshift;s++){
|
||||
psi_d[s] = Zero();
|
||||
psi_f[s] = Zero();
|
||||
IterationsToCompleteShift[s] = 1;
|
||||
TrueResidualShift[s] = 0.;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
||||
rsqf[s] =rsq[s];
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: shift "<< s <<" target resid "<<rsq[s]<<std::endl;
|
||||
// ps_d[s] = src_d;
|
||||
precisionChange(ps_f[s],src_d);
|
||||
}
|
||||
// r and p for primary
|
||||
p_d = src_d; //primary copy --- make this a reference to ps_d to save axpys
|
||||
r_d = p_d;
|
||||
|
||||
//MdagM+m[0]
|
||||
precisionChange(p_f,p_d);
|
||||
Linop_f.HermOpAndNorm(p_f,mmp_f,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
precisionChange(tmp_d,mmp_f);
|
||||
Linop_d.HermOpAndNorm(p_d,mmp_d,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
tmp_d = tmp_d - mmp_d;
|
||||
std::cout << " Testing operators match "<<norm2(mmp_d)<<" f "<<norm2(mmp_f)<<" diff "<< norm2(tmp_d)<<std::endl;
|
||||
// GRID_ASSERT(norm2(tmp_d)< 1.0e-4);
|
||||
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
b = -cp /d;
|
||||
|
||||
// Set up the various shift variables
|
||||
int iz=0;
|
||||
z[0][1-iz] = 1.0;
|
||||
z[0][iz] = 1.0;
|
||||
bs[0] = b;
|
||||
for(int s=1;s<nshift;s++){
|
||||
z[s][1-iz] = 1.0;
|
||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
||||
bs[s] = b*z[s][iz];
|
||||
}
|
||||
|
||||
// r += b[0] A.p[0]
|
||||
// c= norm(r)
|
||||
c=axpy_norm(r_d,b,mmp_d,r_d);
|
||||
|
||||
for(int s=0;s<nshift;s++) {
|
||||
axpby(psi_d[s],0.,-bs[s]*alpha[s],src_d,src_d);
|
||||
precisionChange(psi_f[s],psi_d[s]);
|
||||
}
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch AXPYTimer, ShiftTimer, QRTimer, MatrixTimer, SolverTimer, PrecChangeTimer, CleanupTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
|
||||
// Iteration loop
|
||||
int k;
|
||||
|
||||
for (k=1;k<=MaxIterationsMshift;k++){
|
||||
|
||||
a = c /cp;
|
||||
AXPYTimer.Start();
|
||||
axpy(p_d,a,p_d,r_d);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(r_f, r_d);
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
if ( ! converged[s] ) {
|
||||
if (s==0){
|
||||
axpy(ps_f[s],a,ps_f[s],r_f);
|
||||
} else{
|
||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
||||
axpby(ps_f[s],z[s][iz],as,r_f,ps_f[s]);
|
||||
}
|
||||
}
|
||||
}
|
||||
AXPYTimer.Stop();
|
||||
|
||||
cp=c;
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(p_f, p_d); //get back single prec search direction for linop
|
||||
PrecChangeTimer.Stop();
|
||||
MatrixTimer.Start();
|
||||
Linop_f.HermOp(p_f,mmp_f);
|
||||
MatrixTimer.Stop();
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp_d, mmp_f); // From Float to Double
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
d=real(innerProduct(p_d,mmp_d));
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
bp=b;
|
||||
b=-cp/d;
|
||||
|
||||
// Toggle the recurrence history
|
||||
bs[0] = b;
|
||||
iz = 1-iz;
|
||||
ShiftTimer.Start();
|
||||
for(int s=1;s<nshift;s++){
|
||||
if((!converged[s])){
|
||||
RealD z0 = z[s][1-iz];
|
||||
RealD z1 = z[s][iz];
|
||||
z[s][iz] = z0*z1*bp
|
||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
||||
}
|
||||
}
|
||||
ShiftTimer.Stop();
|
||||
|
||||
//Update single precision solutions
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
int ss = s;
|
||||
if( (!converged[s]) ) {
|
||||
axpy(psi_f[ss],-bs[s]*alpha[s],ps_f[s],psi_f[ss]);
|
||||
}
|
||||
}
|
||||
c = axpy_norm(r_d,b,mmp_d,r_d);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
// Convergence checks
|
||||
int all_converged = 1;
|
||||
for(int s=0;s<nshift;s++){
|
||||
|
||||
if ( (!converged[s]) ){
|
||||
IterationsToCompleteShift[s] = k;
|
||||
|
||||
RealD css = c * z[s][iz]* z[s][iz];
|
||||
|
||||
if(css<rsqf[s]){
|
||||
if ( ! converged[s] )
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
||||
converged[s]=1;
|
||||
} else {
|
||||
all_converged=0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if ( all_converged || k == MaxIterationsMshift-1){
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
precisionChange(psi_d[s],psi_f[s]);
|
||||
}
|
||||
|
||||
|
||||
if ( all_converged ){
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: All shifts have converged iteration "<<k<<std::endl;
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: Checking solutions"<<std::endl;
|
||||
} else {
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: Not all shifts have converged iteration "<<k<<std::endl;
|
||||
}
|
||||
|
||||
// Check answers
|
||||
for(int s=0; s < nshift; s++) {
|
||||
Linop_d.HermOpAndNorm(psi_d[s],mmp_d,d,qq);
|
||||
axpy(tmp_d,mass[s],psi_d[s],mmp_d);
|
||||
axpy(r_d,-alpha[s],src_d,tmp_d);
|
||||
RealD rn = norm2(r_d);
|
||||
RealD cn = norm2(src_d);
|
||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: shift["<<s<<"] true residual "<< TrueResidualShift[s] << " target " << mresidual[s] << std::endl;
|
||||
|
||||
//If we have not reached the desired tolerance, do a (mixed precision) CG cleanup
|
||||
if(rn >= rsq[s]){
|
||||
CleanupTimer.Start();
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: performing cleanup step for shift " << s << std::endl;
|
||||
|
||||
//Setup linear operators for final cleanup
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldD> Linop_shift_d(Linop_d, mass[s]);
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldF> Linop_shift_f(Linop_f, mass[s]);
|
||||
|
||||
MixedPrecisionConjugateGradient<FieldD,FieldF> cg(mresidual[s], MaxIterations, MaxIterations, SinglePrecGrid, Linop_shift_f, Linop_shift_d);
|
||||
cg(src_d, psi_d[s]);
|
||||
|
||||
TrueResidualShift[s] = cg.TrueResidual;
|
||||
CleanupTimer.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientMultiShiftMixedPrecCleanup: Time Breakdown for body"<<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tPrecision Change " << PrecChangeTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tFinal Cleanup " << CleanupTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver+Cleanup " << SolverTimer.Elapsed() + CleanupTimer.Elapsed() << std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,416 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christopher Kelly <ckelly@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_CONJUGATE_GRADIENT_MULTI_SHIFT_MIXEDPREC_H
|
||||
#define GRID_CONJUGATE_GRADIENT_MULTI_SHIFT_MIXEDPREC_H
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
//CK 2020: A variant of the multi-shift conjugate gradient with the matrix multiplication in single precision.
|
||||
//The residual is stored in single precision, but the search directions and solution are stored in double precision.
|
||||
//Every update_freq iterations the residual is corrected in double precision.
|
||||
|
||||
//For safety the a final regular CG is applied to clean up if necessary
|
||||
|
||||
//Linop to add shift to input linop, used in cleanup CG
|
||||
namespace ConjugateGradientMultiShiftMixedPrecSupport{
|
||||
template<typename Field>
|
||||
class ShiftedLinop: public LinearOperatorBase<Field>{
|
||||
public:
|
||||
LinearOperatorBase<Field> &linop_base;
|
||||
RealD shift;
|
||||
|
||||
ShiftedLinop(LinearOperatorBase<Field> &_linop_base, RealD _shift): linop_base(_linop_base), shift(_shift){}
|
||||
|
||||
void OpDiag (const Field &in, Field &out){ GRID_ASSERT(0); }
|
||||
void OpDir (const Field &in, Field &out,int dir,int disp){ GRID_ASSERT(0); }
|
||||
void OpDirAll (const Field &in, std::vector<Field> &out){ GRID_ASSERT(0); }
|
||||
|
||||
void Op (const Field &in, Field &out){ GRID_ASSERT(0); }
|
||||
void AdjOp (const Field &in, Field &out){ GRID_ASSERT(0); }
|
||||
|
||||
void HermOp(const Field &in, Field &out){
|
||||
linop_base.HermOp(in, out);
|
||||
axpy(out, shift, in, out);
|
||||
}
|
||||
|
||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
||||
HermOp(in,out);
|
||||
ComplexD dot = innerProduct(in,out);
|
||||
n1=real(dot);
|
||||
n2=norm2(out);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
template<class FieldD, class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class ConjugateGradientMultiShiftMixedPrec : public OperatorMultiFunction<FieldD>,
|
||||
public OperatorFunction<FieldD>
|
||||
{
|
||||
public:
|
||||
|
||||
using OperatorFunction<FieldD>::operator();
|
||||
|
||||
RealD Tolerance;
|
||||
Integer MaxIterationsMshift;
|
||||
Integer MaxIterations;
|
||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
||||
int verbose;
|
||||
MultiShiftFunction shifts;
|
||||
std::vector<RealD> TrueResidualShift;
|
||||
|
||||
int ReliableUpdateFreq; //number of iterations between reliable updates
|
||||
|
||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
||||
LinearOperatorBase<FieldF> &Linop_f; //single precision
|
||||
|
||||
ConjugateGradientMultiShiftMixedPrec(Integer maxit, const MultiShiftFunction &_shifts,
|
||||
GridBase* _SinglePrecGrid, LinearOperatorBase<FieldF> &_Linop_f,
|
||||
int _ReliableUpdateFreq) :
|
||||
MaxIterationsMshift(maxit), shifts(_shifts), SinglePrecGrid(_SinglePrecGrid), Linop_f(_Linop_f), ReliableUpdateFreq(_ReliableUpdateFreq),
|
||||
MaxIterations(20000)
|
||||
{
|
||||
verbose=1;
|
||||
IterationsToCompleteShift.resize(_shifts.order);
|
||||
TrueResidualShift.resize(_shifts.order);
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, FieldD &psi)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
int nshift = shifts.order;
|
||||
std::vector<FieldD> results(nshift,grid);
|
||||
(*this)(Linop,src,results,psi);
|
||||
}
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, std::vector<FieldD> &results, FieldD &psi)
|
||||
{
|
||||
int nshift = shifts.order;
|
||||
|
||||
(*this)(Linop,src,results);
|
||||
|
||||
psi = shifts.norm*src;
|
||||
for(int i=0;i<nshift;i++){
|
||||
psi = psi + shifts.residues[i]*results[i];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
void operator() (LinearOperatorBase<FieldD> &Linop_d, const FieldD &src_d, std::vector<FieldD> &psi_d)
|
||||
{
|
||||
GRID_TRACE("ConjugateGradientMultiShiftMixedPrec");
|
||||
GridBase *DoublePrecGrid = src_d.Grid();
|
||||
|
||||
precisionChangeWorkspace pc_wk_s_to_d(DoublePrecGrid,SinglePrecGrid);
|
||||
precisionChangeWorkspace pc_wk_d_to_s(SinglePrecGrid,DoublePrecGrid);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Convenience references to the info stored in "MultiShiftFunction"
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
int nshift = shifts.order;
|
||||
|
||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
||||
std::vector<RealD> alpha(nshift,1.0);
|
||||
|
||||
//Double precision search directions
|
||||
FieldD p_d(DoublePrecGrid);
|
||||
std::vector<FieldD> ps_d(nshift, DoublePrecGrid);// Search directions (double precision)
|
||||
|
||||
FieldD tmp_d(DoublePrecGrid);
|
||||
FieldD r_d(DoublePrecGrid);
|
||||
FieldD mmp_d(DoublePrecGrid);
|
||||
|
||||
GRID_ASSERT(psi_d.size()==nshift);
|
||||
GRID_ASSERT(mass.size()==nshift);
|
||||
GRID_ASSERT(mresidual.size()==nshift);
|
||||
|
||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
||||
std::vector<RealD> bs(nshift);
|
||||
std::vector<RealD> rsq(nshift);
|
||||
std::vector<RealD> rsqf(nshift);
|
||||
std::vector<std::array<RealD,2> > z(nshift);
|
||||
std::vector<int> converged(nshift);
|
||||
|
||||
const int primary =0;
|
||||
|
||||
//Primary shift fields CG iteration
|
||||
RealD a,b,c,d;
|
||||
RealD cp,bp,qq; //prev
|
||||
|
||||
// Matrix mult fields
|
||||
FieldF p_f(SinglePrecGrid);
|
||||
FieldF mmp_f(SinglePrecGrid);
|
||||
|
||||
// Check lightest mass
|
||||
for(int s=0;s<nshift;s++){
|
||||
GRID_ASSERT( mass[s]>= mass[primary] );
|
||||
converged[s]=0;
|
||||
}
|
||||
|
||||
// Wire guess to zero
|
||||
// Residuals "r" are src
|
||||
// First search direction "p" is also src
|
||||
cp = norm2(src_d);
|
||||
|
||||
// Handle trivial case of zero src.
|
||||
if( cp == 0. ){
|
||||
for(int s=0;s<nshift;s++){
|
||||
psi_d[s] = Zero();
|
||||
IterationsToCompleteShift[s] = 1;
|
||||
TrueResidualShift[s] = 0.;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
||||
rsqf[s] =rsq[s];
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: shift "<< s <<" target resid "<<rsq[s]<<std::endl;
|
||||
ps_d[s] = src_d;
|
||||
}
|
||||
// r and p for primary
|
||||
p_d = src_d; //primary copy --- make this a reference to ps_d to save axpys
|
||||
r_d = p_d;
|
||||
|
||||
//MdagM+m[0]
|
||||
precisionChange(p_f, p_d, pc_wk_d_to_s);
|
||||
|
||||
Linop_f.HermOpAndNorm(p_f,mmp_f,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
precisionChange(tmp_d, mmp_f, pc_wk_s_to_d);
|
||||
Linop_d.HermOpAndNorm(p_d,mmp_d,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
||||
tmp_d = tmp_d - mmp_d;
|
||||
std::cout << " Testing operators match "<<norm2(mmp_d)<<" f "<<norm2(mmp_f)<<" diff "<< norm2(tmp_d)<<std::endl;
|
||||
GRID_ASSERT(norm2(tmp_d)< 1.0);
|
||||
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
b = -cp /d;
|
||||
|
||||
// Set up the various shift variables
|
||||
int iz=0;
|
||||
z[0][1-iz] = 1.0;
|
||||
z[0][iz] = 1.0;
|
||||
bs[0] = b;
|
||||
for(int s=1;s<nshift;s++){
|
||||
z[s][1-iz] = 1.0;
|
||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
||||
bs[s] = b*z[s][iz];
|
||||
}
|
||||
|
||||
// r += b[0] A.p[0]
|
||||
// c= norm(r)
|
||||
c=axpy_norm(r_d,b,mmp_d,r_d);
|
||||
|
||||
for(int s=0;s<nshift;s++) {
|
||||
axpby(psi_d[s],0.,-bs[s]*alpha[s],src_d,src_d);
|
||||
}
|
||||
|
||||
///////////////////////////////////////
|
||||
// Timers
|
||||
///////////////////////////////////////
|
||||
GridStopWatch AXPYTimer, ShiftTimer, QRTimer, MatrixTimer, SolverTimer, PrecChangeTimer, CleanupTimer;
|
||||
|
||||
SolverTimer.Start();
|
||||
|
||||
// Iteration loop
|
||||
int k;
|
||||
|
||||
for (k=1;k<=MaxIterationsMshift;k++){
|
||||
|
||||
a = c /cp;
|
||||
AXPYTimer.Start();
|
||||
axpy(p_d,a,p_d,r_d);
|
||||
|
||||
for(int s=0;s<nshift;s++){
|
||||
if ( ! converged[s] ) {
|
||||
if (s==0){
|
||||
axpy(ps_d[s],a,ps_d[s],r_d);
|
||||
} else{
|
||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
||||
axpby(ps_d[s],z[s][iz],as,r_d,ps_d[s]);
|
||||
}
|
||||
}
|
||||
}
|
||||
AXPYTimer.Stop();
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(p_f, p_d, pc_wk_d_to_s); //get back single prec search direction for linop
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
cp=c;
|
||||
MatrixTimer.Start();
|
||||
Linop_f.HermOp(p_f,mmp_f);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp_d, mmp_f, pc_wk_s_to_d); // From Float to Double
|
||||
PrecChangeTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
d=real(innerProduct(p_d,mmp_d));
|
||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
||||
AXPYTimer.Stop();
|
||||
RealD rn = norm2(p_d);
|
||||
d += rn*mass[0];
|
||||
|
||||
bp=b;
|
||||
b=-cp/d;
|
||||
|
||||
// Toggle the recurrence history
|
||||
bs[0] = b;
|
||||
iz = 1-iz;
|
||||
ShiftTimer.Start();
|
||||
for(int s=1;s<nshift;s++){
|
||||
if((!converged[s])){
|
||||
RealD z0 = z[s][1-iz];
|
||||
RealD z1 = z[s][iz];
|
||||
z[s][iz] = z0*z1*bp
|
||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
||||
}
|
||||
}
|
||||
ShiftTimer.Stop();
|
||||
|
||||
//Update double precision solutions
|
||||
AXPYTimer.Start();
|
||||
for(int s=0;s<nshift;s++){
|
||||
int ss = s;
|
||||
if( (!converged[s]) ) {
|
||||
axpy(psi_d[ss],-bs[s]*alpha[s],ps_d[s],psi_d[ss]);
|
||||
}
|
||||
}
|
||||
|
||||
//Perform reliable update if necessary; otherwise update residual from single-prec mmp
|
||||
c = axpy_norm(r_d,b,mmp_d,r_d);
|
||||
|
||||
AXPYTimer.Stop();
|
||||
|
||||
if(k % ReliableUpdateFreq == 0){
|
||||
RealD c_old = c;
|
||||
//Replace r with true residual
|
||||
MatrixTimer.Start();
|
||||
Linop_d.HermOp(psi_d[0],mmp_d);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
AXPYTimer.Start();
|
||||
axpy(mmp_d,mass[0],psi_d[0],mmp_d);
|
||||
|
||||
c = axpy_norm(r_d, -1.0, mmp_d, src_d);
|
||||
AXPYTimer.Stop();
|
||||
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec k="<<k<< ", replaced |r|^2 = "<<c_old <<" with |r|^2 = "<<c<<std::endl;
|
||||
}
|
||||
|
||||
// Convergence checks
|
||||
int all_converged = 1;
|
||||
for(int s=0;s<nshift;s++){
|
||||
|
||||
if ( (!converged[s]) ){
|
||||
IterationsToCompleteShift[s] = k;
|
||||
|
||||
RealD css = c * z[s][iz]* z[s][iz];
|
||||
|
||||
if(css<rsqf[s]){
|
||||
if ( ! converged[s] )
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
||||
converged[s]=1;
|
||||
} else {
|
||||
all_converged=0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if ( all_converged || k == MaxIterationsMshift-1){
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
if ( all_converged ){
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: All shifts have converged iteration "<<k<<std::endl;
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: Checking solutions"<<std::endl;
|
||||
} else {
|
||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: Not all shifts have converged iteration "<<k<<std::endl;
|
||||
}
|
||||
|
||||
// Check answers
|
||||
for(int s=0; s < nshift; s++) {
|
||||
Linop_d.HermOpAndNorm(psi_d[s],mmp_d,d,qq);
|
||||
axpy(tmp_d,mass[s],psi_d[s],mmp_d);
|
||||
axpy(r_d,-alpha[s],src_d,tmp_d);
|
||||
RealD rn = norm2(r_d);
|
||||
RealD cn = norm2(src_d);
|
||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: shift["<<s<<"] true residual "<< TrueResidualShift[s] << " target " << mresidual[s] << std::endl;
|
||||
|
||||
//If we have not reached the desired tolerance, do a (mixed precision) CG cleanup
|
||||
if(rn >= rsq[s]){
|
||||
CleanupTimer.Start();
|
||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: performing cleanup step for shift " << s << std::endl;
|
||||
|
||||
//Setup linear operators for final cleanup
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldD> Linop_shift_d(Linop_d, mass[s]);
|
||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldF> Linop_shift_f(Linop_f, mass[s]);
|
||||
|
||||
MixedPrecisionConjugateGradient<FieldD,FieldF> cg(mresidual[s], MaxIterations, MaxIterations, SinglePrecGrid, Linop_shift_f, Linop_shift_d);
|
||||
cg(src_d, psi_d[s]);
|
||||
|
||||
TrueResidualShift[s] = cg.TrueResidual;
|
||||
CleanupTimer.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientMultiShiftMixedPrec: Time Breakdown for body"<<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\t\tPrecision Change " << PrecChangeTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tFinal Cleanup " << CleanupTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tSolver+Cleanup " << SolverTimer.Elapsed() + CleanupTimer.Elapsed() << std::endl;
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -35,7 +35,7 @@ template<class FieldD,class FieldF,
|
||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
||||
class ConjugateGradientReliableUpdate : public LinearFunction<FieldD> {
|
||||
public:
|
||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
@@ -48,7 +48,7 @@ public:
|
||||
LinearOperatorBase<FieldF> &Linop_f;
|
||||
LinearOperatorBase<FieldD> &Linop_d;
|
||||
GridBase* SinglePrecGrid;
|
||||
RealD Delta; //reliable update parameter. A reliable update is performed when the residual drops by a factor of Delta relative to its value at the last update
|
||||
RealD Delta; //reliable update parameter
|
||||
|
||||
//Optional ability to switch to a different linear operator once the tolerance reaches a certain point. Useful for single/half -> single/single
|
||||
LinearOperatorBase<FieldF> *Linop_fallback;
|
||||
@@ -65,9 +65,7 @@ public:
|
||||
ErrorOnNoConverge(err_on_no_conv),
|
||||
DoFinalCleanup(true),
|
||||
Linop_fallback(NULL)
|
||||
{
|
||||
GRID_ASSERT(Delta > 0. && Delta < 1. && "Expect 0 < Delta < 1");
|
||||
};
|
||||
{};
|
||||
|
||||
void setFallbackLinop(LinearOperatorBase<FieldF> &_Linop_fallback, const RealD _fallback_transition_tol){
|
||||
Linop_fallback = &_Linop_fallback;
|
||||
@@ -75,7 +73,6 @@ public:
|
||||
}
|
||||
|
||||
void operator()(const FieldD &src, FieldD &psi) {
|
||||
GRID_TRACE("ConjugateGradientReliableUpdate");
|
||||
LinearOperatorBase<FieldF> *Linop_f_use = &Linop_f;
|
||||
bool using_fallback = false;
|
||||
|
||||
@@ -90,7 +87,7 @@ public:
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
Linop_d.HermOpAndNorm(psi, mmp, d, b);
|
||||
|
||||
@@ -118,12 +115,9 @@ public:
|
||||
}
|
||||
|
||||
//Single prec initialization
|
||||
precisionChangeWorkspace pc_wk_sp_to_dp(src.Grid(), SinglePrecGrid);
|
||||
precisionChangeWorkspace pc_wk_dp_to_sp(SinglePrecGrid, src.Grid());
|
||||
|
||||
FieldF r_f(SinglePrecGrid);
|
||||
r_f.Checkerboard() = r.Checkerboard();
|
||||
precisionChange(r_f, r, pc_wk_dp_to_sp);
|
||||
precisionChange(r_f, r);
|
||||
|
||||
FieldF psi_f(r_f);
|
||||
psi_f = Zero();
|
||||
@@ -139,8 +133,7 @@ public:
|
||||
GridStopWatch LinalgTimer;
|
||||
GridStopWatch MatrixTimer;
|
||||
GridStopWatch SolverTimer;
|
||||
GridStopWatch PrecChangeTimer;
|
||||
|
||||
|
||||
SolverTimer.Start();
|
||||
int k = 0;
|
||||
int l = 0;
|
||||
@@ -179,9 +172,7 @@ public:
|
||||
// Stopping condition
|
||||
if (cp <= rsq) {
|
||||
//Although not written in the paper, I assume that I have to add on the final solution
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp, psi_f, pc_wk_sp_to_dp);
|
||||
PrecChangeTimer.Stop();
|
||||
precisionChange(mmp, psi_f);
|
||||
psi = psi + mmp;
|
||||
|
||||
|
||||
@@ -202,10 +193,7 @@ public:
|
||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tPrecChange " << PrecChangeTimer.Elapsed() <<std::endl;
|
||||
std::cout << GridLogMessage << "\tPrecChange avg time " << PrecChangeTimer.Elapsed()/(2*l+1) <<std::endl;
|
||||
|
||||
|
||||
IterationsToComplete = k;
|
||||
ReliableUpdatesPerformed = l;
|
||||
|
||||
@@ -217,7 +205,7 @@ public:
|
||||
CG(Linop_d,src,psi);
|
||||
IterationsToCleanup = CG.IterationsToComplete;
|
||||
}
|
||||
else if (ErrorOnNoConverge) GRID_ASSERT(true_residual / Tolerance < 10000.0);
|
||||
else if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
|
||||
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate complete.\n";
|
||||
return;
|
||||
@@ -225,21 +213,14 @@ public:
|
||||
else if(cp < Delta * MaxResidSinceLastRelUp) { //reliable update
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate "
|
||||
<< cp << "(residual) < " << Delta << "(Delta) * " << MaxResidSinceLastRelUp << "(MaxResidSinceLastRelUp) on iteration " << k << " : performing reliable update\n";
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(mmp, psi_f, pc_wk_sp_to_dp);
|
||||
PrecChangeTimer.Stop();
|
||||
precisionChange(mmp, psi_f);
|
||||
psi = psi + mmp;
|
||||
|
||||
MatrixTimer.Start();
|
||||
Linop_d.HermOpAndNorm(psi, mmp, d, qq);
|
||||
MatrixTimer.Stop();
|
||||
|
||||
r = src - mmp;
|
||||
|
||||
psi_f = Zero();
|
||||
PrecChangeTimer.Start();
|
||||
precisionChange(r_f, r, pc_wk_dp_to_sp);
|
||||
PrecChangeTimer.Stop();
|
||||
precisionChange(r_f, r);
|
||||
cp = norm2(r);
|
||||
MaxResidSinceLastRelUp = cp;
|
||||
|
||||
@@ -263,7 +244,7 @@ public:
|
||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate did NOT converge"
|
||||
<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
||||
if (ErrorOnNoConverge) assert(0);
|
||||
IterationsToComplete = k;
|
||||
ReliableUpdatesPerformed = l;
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ public:
|
||||
}
|
||||
|
||||
std::cout<<GridLogMessage<<"ConjugateResidual did NOT converge"<<std::endl;
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -33,19 +33,16 @@ namespace Grid {
|
||||
template<class Field>
|
||||
class ZeroGuesser: public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field &guess) { guess = Zero(); };
|
||||
};
|
||||
template<class Field>
|
||||
class DoNothingGuesser: public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field &guess) { };
|
||||
};
|
||||
template<class Field>
|
||||
class SourceGuesser: public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
virtual void operator()(const Field &src, Field &guess) { guess = src; };
|
||||
};
|
||||
|
||||
@@ -57,24 +54,15 @@ class DeflatedGuesser: public LinearFunction<Field> {
|
||||
private:
|
||||
const std::vector<Field> &evec;
|
||||
const std::vector<RealD> &eval;
|
||||
const unsigned int N;
|
||||
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
|
||||
DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval)
|
||||
: DeflatedGuesser(_evec, _eval, _evec.size())
|
||||
{}
|
||||
|
||||
DeflatedGuesser(const std::vector<Field> & _evec, const std::vector<RealD> & _eval, const unsigned int _N)
|
||||
: evec(_evec), eval(_eval), N(_N)
|
||||
{
|
||||
GRID_ASSERT(evec.size()==eval.size());
|
||||
GRID_ASSERT(N <= evec.size());
|
||||
}
|
||||
DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval) : evec(_evec), eval(_eval) {};
|
||||
|
||||
virtual void operator()(const Field &src,Field &guess) {
|
||||
guess = Zero();
|
||||
assert(evec.size()==eval.size());
|
||||
auto N = evec.size();
|
||||
for (int i=0;i<N;i++) {
|
||||
const Field& tmp = evec[i];
|
||||
axpy(guess,TensorRemove(innerProduct(tmp,src)) / eval[i],tmp,guess);
|
||||
@@ -91,7 +79,6 @@ private:
|
||||
const std::vector<RealD> &eval_coarse;
|
||||
public:
|
||||
|
||||
using LinearFunction<FineField>::operator();
|
||||
LocalCoherenceDeflatedGuesser(const std::vector<FineField> &_subspace,
|
||||
const std::vector<CoarseField> &_evec_coarse,
|
||||
const std::vector<RealD> &_eval_coarse)
|
||||
@@ -113,42 +100,7 @@ public:
|
||||
blockPromote(guess_coarse,guess,subspace);
|
||||
guess.Checkerboard() = src.Checkerboard();
|
||||
};
|
||||
|
||||
void operator()(const std::vector<FineField> &src,std::vector<FineField> &guess) {
|
||||
int Nevec = (int)evec_coarse.size();
|
||||
int Nsrc = (int)src.size();
|
||||
// make temp variables
|
||||
std::vector<CoarseField> src_coarse(Nsrc,evec_coarse[0].Grid());
|
||||
std::vector<CoarseField> guess_coarse(Nsrc,evec_coarse[0].Grid());
|
||||
//Preporcessing
|
||||
std::cout << GridLogMessage << "Start BlockProject for loop" << std::endl;
|
||||
for (int j=0;j<Nsrc;j++)
|
||||
{
|
||||
guess_coarse[j] = Zero();
|
||||
std::cout << GridLogMessage << "BlockProject iter: " << j << std::endl;
|
||||
blockProject(src_coarse[j],src[j],subspace);
|
||||
}
|
||||
//deflation set up for eigen vector batchsize 1 and source batch size equal number of sources
|
||||
std::cout << GridLogMessage << "Start ProjectAccum for loop" << std::endl;
|
||||
for (int i=0;i<Nevec;i++)
|
||||
{
|
||||
std::cout << GridLogMessage << "ProjectAccum Nvec: " << i << std::endl;
|
||||
const CoarseField & tmp = evec_coarse[i];
|
||||
for (int j=0;j<Nsrc;j++)
|
||||
{
|
||||
axpy(guess_coarse[j],TensorRemove(innerProduct(tmp,src_coarse[j])) / eval_coarse[i],tmp,guess_coarse[j]);
|
||||
}
|
||||
}
|
||||
//postprocessing
|
||||
std::cout << GridLogMessage << "Start BlockPromote for loop" << std::endl;
|
||||
for (int j=0;j<Nsrc;j++) {
|
||||
std::cout << GridLogMessage << "BlockProject iter: " << j << std::endl;
|
||||
blockPromote(guess_coarse[j],guess[j],subspace);
|
||||
guess[j].Checkerboard() = src[j].Checkerboard();
|
||||
}
|
||||
};
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorF
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when FCAGMRES fails to converge,
|
||||
bool ErrorOnNoConverge; // Throw an assert when FCAGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
@@ -87,7 +87,7 @@ class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorF
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
@@ -144,7 +144,7 @@ class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorF
|
||||
std::cout << GridLogMessage << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
@@ -191,7 +191,7 @@ class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorF
|
||||
}
|
||||
}
|
||||
|
||||
GRID_ASSERT(0); // Never reached
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ class FlexibleGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when FGMRES fails to converge,
|
||||
bool ErrorOnNoConverge; // Throw an assert when FGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
@@ -85,7 +85,7 @@ class FlexibleGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
@@ -142,7 +142,7 @@ class FlexibleGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
std::cout << GridLogMessage << "FlexibleGeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
@@ -189,7 +189,7 @@ class FlexibleGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
}
|
||||
}
|
||||
|
||||
GRID_ASSERT(0); // Never reached
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ class GeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when GMRES fails to converge,
|
||||
bool ErrorOnNoConverge; // Throw an assert when GMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
@@ -80,7 +80,7 @@ class GeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
@@ -135,7 +135,7 @@ class GeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
std::cout << GridLogMessage << "GeneralisedMinimalResidual did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
||||
@@ -181,7 +181,7 @@ class GeneralisedMinimalResidual : public OperatorFunction<Field> {
|
||||
}
|
||||
}
|
||||
|
||||
GRID_ASSERT(0); // Never reached
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -79,16 +79,14 @@ template<class Field> class ImplicitlyRestartedLanczosHermOpTester : public Imp
|
||||
RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
|
||||
|
||||
std::cout.precision(13);
|
||||
|
||||
int conv=0;
|
||||
if( (vv<eresid*eresid) ) conv = 1;
|
||||
|
||||
std::cout<<GridLogIRL << "[" << std::setw(3)<<j<<"] "
|
||||
<<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
|
||||
<<" target " << eresid*eresid << " conv " <<conv
|
||||
<<std::endl;
|
||||
|
||||
int conv=0;
|
||||
if( (vv<eresid*eresid) ) conv = 1;
|
||||
|
||||
return conv;
|
||||
}
|
||||
};
|
||||
@@ -211,7 +209,7 @@ until convergence
|
||||
void calc(std::vector<RealD>& eval, std::vector<Field>& evec, const Field& src, int& Nconv, bool reverse=false)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
GRID_ASSERT(grid == evec[0].Grid());
|
||||
assert(grid == evec[0].Grid());
|
||||
|
||||
// GridLogIRL.TimingMode(1);
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
@@ -231,7 +229,7 @@ until convergence
|
||||
}
|
||||
std::cout << GridLogIRL <<"**************************************************************************"<< std::endl;
|
||||
|
||||
GRID_ASSERT(Nm <= evec.size() && Nm <= eval.size());
|
||||
assert(Nm <= evec.size() && Nm <= eval.size());
|
||||
|
||||
// quickly get an idea of the largest eigenvalue to more properly normalize the residuum
|
||||
RealD evalMaxApprox = 0.0;
|
||||
@@ -245,10 +243,9 @@ until convergence
|
||||
_HermOp(src_n,tmp);
|
||||
// std::cout << GridLogMessage<< tmp<<std::endl; exit(0);
|
||||
// std::cout << GridLogIRL << " _HermOp " << norm2(tmp) << std::endl;
|
||||
// RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
|
||||
RealD vnum = real(innerProduct(tmp,tmp)); // HermOp^2.
|
||||
RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
|
||||
RealD vden = norm2(src_n);
|
||||
RealD na = std::sqrt(vnum/vden);
|
||||
RealD na = vnum/vden;
|
||||
if (fabs(evalMaxApprox/na - 1.0) < 0.0001)
|
||||
i=_MAX_ITER_IRL_MEVAPP_;
|
||||
evalMaxApprox = na;
|
||||
@@ -256,7 +253,6 @@ until convergence
|
||||
src_n = tmp;
|
||||
}
|
||||
}
|
||||
std::cout << GridLogIRL << " Final evalMaxApprox " << evalMaxApprox << std::endl;
|
||||
|
||||
std::vector<RealD> lme(Nm);
|
||||
std::vector<RealD> lme2(Nm);
|
||||
@@ -337,7 +333,7 @@ until convergence
|
||||
}
|
||||
std::cout<<GridLogIRL <<"QR decomposed "<<std::endl;
|
||||
|
||||
GRID_ASSERT(k2<Nm); GRID_ASSERT(k2<Nm); GRID_ASSERT(k1>0);
|
||||
assert(k2<Nm); assert(k2<Nm); assert(k1>0);
|
||||
|
||||
basisRotate(evec,Qt,k1-1,k2+1,0,Nm,Nm); /// big constraint on the basis
|
||||
std::cout<<GridLogIRL <<"basisRotated by Qt *"<<k1-1<<","<<k2+1<<")"<<std::endl;
|
||||
@@ -423,15 +419,14 @@ until convergence
|
||||
}
|
||||
}
|
||||
|
||||
if ( Nconv < Nstop ) {
|
||||
if ( Nconv < Nstop )
|
||||
std::cout << GridLogIRL << "Nconv ("<<Nconv<<") < Nstop ("<<Nstop<<")"<<std::endl;
|
||||
std::cout << GridLogIRL << "returning Nstop vectors, the last "<< Nstop-Nconv << "of which might meet convergence criterion only approximately" <<std::endl;
|
||||
}
|
||||
|
||||
eval=eval2;
|
||||
|
||||
//Keep only converged
|
||||
eval.resize(Nstop);// was Nconv
|
||||
evec.resize(Nstop,grid);// was Nconv
|
||||
eval.resize(Nconv);// Nstop?
|
||||
evec.resize(Nconv,grid);// Nstop?
|
||||
basisSortInPlace(evec,eval,reverse);
|
||||
|
||||
}
|
||||
@@ -461,15 +456,15 @@ until convergence
|
||||
std::vector<Field>& evec,
|
||||
Field& w,int Nm,int k)
|
||||
{
|
||||
std::cout<<GridLogDebug << "Lanczos step " <<k<<std::endl;
|
||||
std::cout<<GridLogIRL << "Lanczos step " <<k<<std::endl;
|
||||
const RealD tiny = 1.0e-20;
|
||||
GRID_ASSERT( k< Nm );
|
||||
assert( k< Nm );
|
||||
|
||||
GridStopWatch gsw_op,gsw_o;
|
||||
|
||||
Field& evec_k = evec[k];
|
||||
|
||||
_PolyOp(evec_k,w); std::cout<<GridLogDebug << "PolyOp" <<std::endl;
|
||||
_PolyOp(evec_k,w); std::cout<<GridLogIRL << "PolyOp" <<std::endl;
|
||||
|
||||
if(k>0) w -= lme[k-1] * evec[k-1];
|
||||
|
||||
@@ -484,18 +479,18 @@ until convergence
|
||||
lme[k] = beta;
|
||||
|
||||
if ( (k>0) && ( (k % orth_period) == 0 )) {
|
||||
std::cout<<GridLogDebug << "Orthogonalising " <<k<<std::endl;
|
||||
std::cout<<GridLogIRL << "Orthogonalising " <<k<<std::endl;
|
||||
orthogonalize(w,evec,k); // orthonormalise
|
||||
std::cout<<GridLogDebug << "Orthogonalised " <<k<<std::endl;
|
||||
std::cout<<GridLogIRL << "Orthogonalised " <<k<<std::endl;
|
||||
}
|
||||
|
||||
if(k < Nm-1) evec[k+1] = w;
|
||||
|
||||
std::cout<<GridLogIRL << "Lanczos step alpha[" << k << "] = " << zalph << " beta[" << k << "] = "<<beta<<std::endl;
|
||||
std::cout<<GridLogIRL << "alpha[" << k << "] = " << zalph << " beta[" << k << "] = "<<beta<<std::endl;
|
||||
if ( beta < tiny )
|
||||
std::cout<<GridLogIRL << " beta is tiny "<<beta<<std::endl;
|
||||
|
||||
std::cout<<GridLogDebug << "Lanczos step complete " <<k<<std::endl;
|
||||
std::cout<<GridLogIRL << "Lanczos step complete " <<k<<std::endl;
|
||||
}
|
||||
|
||||
void diagonalize_Eigen(std::vector<RealD>& lmd, std::vector<RealD>& lme,
|
||||
@@ -597,7 +592,7 @@ until convergence
|
||||
} else if ( diagonalisation == IRLdiagonaliseWithEigen ) {
|
||||
diagonalize_Eigen(lmd,lme,Nk,Nm,Qt,grid);
|
||||
} else {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -687,7 +682,7 @@ void diagonalize_lapack(std::vector<RealD>& lmd,
|
||||
}
|
||||
}
|
||||
#else
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,6 @@ public:
|
||||
int, MinRes); // Must restart
|
||||
};
|
||||
|
||||
//This class is the input parameter class for some testing programs
|
||||
struct LocalCoherenceLanczosParams : Serializable {
|
||||
public:
|
||||
GRID_SERIALIZABLE_CLASS_MEMBERS(LocalCoherenceLanczosParams,
|
||||
@@ -68,7 +67,6 @@ public:
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class ProjectedHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
|
||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||
@@ -80,7 +78,7 @@ public:
|
||||
ProjectedHermOp(LinearOperatorBase<FineField>& linop, std::vector<FineField> & _subspace) :
|
||||
_Linop(linop), subspace(_subspace)
|
||||
{
|
||||
GRID_ASSERT(subspace.size() >0);
|
||||
assert(subspace.size() >0);
|
||||
};
|
||||
|
||||
void operator()(const CoarseField& in, CoarseField& out) {
|
||||
@@ -99,7 +97,6 @@ public:
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class ProjectedFunctionHermOp : public LinearFunction<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
using LinearFunction<Lattice<iVector<CComplex,nbasis > > >::operator();
|
||||
typedef iVector<CComplex,nbasis > CoarseSiteVector;
|
||||
typedef Lattice<CoarseSiteVector> CoarseField;
|
||||
typedef Lattice<CComplex> CoarseScalar; // used for inner products on fine field
|
||||
@@ -146,24 +143,16 @@ public:
|
||||
LinearOperatorBase<FineField> &_Linop;
|
||||
RealD _coarse_relax_tol;
|
||||
std::vector<FineField> &_subspace;
|
||||
|
||||
int _largestEvalIdxForReport; //The convergence of the LCL is based on the evals of the coarse grid operator, not those of the underlying fine grid operator
|
||||
//As a result we do not know what the eval range of the fine operator is until the very end, making tuning the Cheby bounds very difficult
|
||||
//To work around this issue, every restart we separately reconstruct the fine operator eval for the lowest and highest evec and print these
|
||||
//out alongside the evals of the coarse operator. To do so we need to know the index of the largest eval (i.e. Nstop-1)
|
||||
//NOTE: If largestEvalIdxForReport=-1 (default) then this is not performed
|
||||
|
||||
ImplicitlyRestartedLanczosSmoothedTester(LinearFunction<CoarseField> &Poly,
|
||||
OperatorFunction<FineField> &smoother,
|
||||
LinearOperatorBase<FineField> &Linop,
|
||||
std::vector<FineField> &subspace,
|
||||
RealD coarse_relax_tol=5.0e3,
|
||||
int largestEvalIdxForReport=-1)
|
||||
RealD coarse_relax_tol=5.0e3)
|
||||
: _smoother(smoother), _Linop(Linop), _Poly(Poly), _subspace(subspace),
|
||||
_coarse_relax_tol(coarse_relax_tol), _largestEvalIdxForReport(largestEvalIdxForReport)
|
||||
_coarse_relax_tol(coarse_relax_tol)
|
||||
{ };
|
||||
|
||||
//evalMaxApprox: approximation of largest eval of the fine Chebyshev operator (suitably wrapped by block projection)
|
||||
int TestConvergence(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
|
||||
{
|
||||
CoarseField v(B);
|
||||
@@ -186,26 +175,12 @@ public:
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
|
||||
<<std::endl;
|
||||
|
||||
if(_largestEvalIdxForReport != -1 && (j==0 || j==_largestEvalIdxForReport)){
|
||||
std::cout<<GridLogIRL << "Estimating true eval of fine grid operator for eval idx " << j << std::endl;
|
||||
RealD tmp_eval;
|
||||
ReconstructEval(j,eresid,B,tmp_eval,1.0); //don't use evalMaxApprox of coarse operator! (cf below)
|
||||
}
|
||||
|
||||
int conv=0;
|
||||
if( (vv<eresid*eresid) ) conv = 1;
|
||||
return conv;
|
||||
}
|
||||
|
||||
//This function is called at the end of the coarse grid Lanczos. It promotes the coarse eigenvector 'B' to the fine grid,
|
||||
//applies a smoother to the result then computes the computes the *fine grid* eigenvalue (output as 'eval').
|
||||
|
||||
//evalMaxApprox should be the approximation of the largest eval of the fine Hermop. However when this function is called by IRL it actually passes the largest eval of the *Chebyshev* operator (as this is the max approx used for the TestConvergence above)
|
||||
//As the largest eval of the Chebyshev is typically several orders of magnitude larger this makes the convergence test pass even when it should not.
|
||||
//We therefore ignore evalMaxApprox here and use a value of 1.0 (note this value is already used by TestCoarse)
|
||||
int ReconstructEval(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
|
||||
int ReconstructEval(int j,RealD eresid,CoarseField &B, RealD &eval,RealD evalMaxApprox)
|
||||
{
|
||||
evalMaxApprox = 1.0; //cf above
|
||||
GridBase *FineGrid = _subspace[0].Grid();
|
||||
int checkerboard = _subspace[0].Checkerboard();
|
||||
FineField fB(FineGrid);fB.Checkerboard() =checkerboard;
|
||||
@@ -224,13 +199,13 @@ public:
|
||||
eval = vnum/vden;
|
||||
fv -= eval*fB;
|
||||
RealD vv = norm2(fv) / ::pow(evalMaxApprox,2.0);
|
||||
if ( j > nbasis ) eresid = eresid*_coarse_relax_tol;
|
||||
|
||||
|
||||
std::cout.precision(13);
|
||||
std::cout<<GridLogIRL << "[" << std::setw(3)<<j<<"] "
|
||||
<<"eval = "<<std::setw(25)<< eval << " (" << eval_poly << ")"
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv << " target " << eresid*eresid
|
||||
<<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25) << vv
|
||||
<<std::endl;
|
||||
if ( j > nbasis ) eresid = eresid*_coarse_relax_tol;
|
||||
if( (vv<eresid*eresid) ) return 1;
|
||||
return 0;
|
||||
}
|
||||
@@ -308,10 +283,6 @@ public:
|
||||
evals_coarse.resize(0);
|
||||
};
|
||||
|
||||
//The block inner product is the inner product on the fine grid locally summed over the blocks
|
||||
//to give a Lattice<Scalar> on the coarse grid. This function orthnormalizes the fine-grid subspace
|
||||
//vectors under the block inner product. This step must be performed after computing the fine grid
|
||||
//eigenvectors and before computing the coarse grid eigenvectors.
|
||||
void Orthogonalise(void ) {
|
||||
CoarseScalar InnerProd(_CoarseGrid);
|
||||
std::cout << GridLogMessage <<" Gramm-Schmidt pass 1"<<std::endl;
|
||||
@@ -346,21 +317,19 @@ public:
|
||||
|
||||
void testFine(RealD resid)
|
||||
{
|
||||
GRID_ASSERT(evals_fine.size() == nbasis);
|
||||
GRID_ASSERT(subspace.size() == nbasis);
|
||||
assert(evals_fine.size() == nbasis);
|
||||
assert(subspace.size() == nbasis);
|
||||
PlainHermOp<FineField> Op(_FineOp);
|
||||
ImplicitlyRestartedLanczosHermOpTester<FineField> SimpleTester(Op);
|
||||
for(int k=0;k<nbasis;k++){
|
||||
GRID_ASSERT(SimpleTester.ReconstructEval(k,resid,subspace[k],evals_fine[k],1.0)==1);
|
||||
assert(SimpleTester.ReconstructEval(k,resid,subspace[k],evals_fine[k],1.0)==1);
|
||||
}
|
||||
}
|
||||
|
||||
//While this method serves to check the coarse eigenvectors, it also recomputes the eigenvalues from the smoothed reconstructed eigenvectors
|
||||
//hence the smoother can be tuned after running the coarse Lanczos by using a different smoother here
|
||||
void testCoarse(RealD resid,ChebyParams cheby_smooth,RealD relax)
|
||||
{
|
||||
GRID_ASSERT(evals_fine.size() == nbasis);
|
||||
GRID_ASSERT(subspace.size() == nbasis);
|
||||
assert(evals_fine.size() == nbasis);
|
||||
assert(subspace.size() == nbasis);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -380,7 +349,7 @@ public:
|
||||
void calcFine(ChebyParams cheby_parms,int Nstop,int Nk,int Nm,RealD resid,
|
||||
RealD MaxIt, RealD betastp, int MinRes)
|
||||
{
|
||||
GRID_ASSERT(nbasis<=Nm);
|
||||
assert(nbasis<=Nm);
|
||||
Chebyshev<FineField> Cheby(cheby_parms);
|
||||
FunctionHermOp<FineField> ChebyOp(Cheby,_FineOp);
|
||||
PlainHermOp<FineField> Op(_FineOp);
|
||||
@@ -400,54 +369,40 @@ public:
|
||||
IRL.calc(evals_fine,subspace,src,Nconv,false);
|
||||
|
||||
// Shrink down to number saved
|
||||
GRID_ASSERT(Nstop>=nbasis);
|
||||
GRID_ASSERT(Nconv>=nbasis);
|
||||
assert(Nstop>=nbasis);
|
||||
assert(Nconv>=nbasis);
|
||||
evals_fine.resize(nbasis);
|
||||
subspace.resize(nbasis,_FineGrid);
|
||||
}
|
||||
|
||||
|
||||
//cheby_op: Parameters of the fine grid Chebyshev polynomial used for the Lanczos acceleration
|
||||
//cheby_smooth: Parameters of a separate Chebyshev polynomial used after the Lanczos has completed to smooth out high frequency noise in the reconstructed fine grid eigenvectors prior to computing the eigenvalue
|
||||
//relax: Reconstructed eigenvectors (post smoothing) are naturally not as precise as true eigenvectors. This factor acts as a multiplier on the stopping condition when determining whether the results satisfy the user provided stopping condition
|
||||
void calcCoarse(ChebyParams cheby_op,ChebyParams cheby_smooth,RealD relax,
|
||||
int Nstop, int Nk, int Nm,RealD resid,
|
||||
RealD MaxIt, RealD betastp, int MinRes)
|
||||
{
|
||||
Chebyshev<FineField> Cheby(cheby_op); //Chebyshev of fine operator on fine grid
|
||||
ProjectedHermOp<Fobj,CComplex,nbasis> Op(_FineOp,subspace); //Fine operator on coarse grid with intermediate fine grid conversion
|
||||
ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (Cheby,_FineOp,subspace); //Chebyshev of fine operator on coarse grid with intermediate fine grid conversion
|
||||
Chebyshev<FineField> Cheby(cheby_op);
|
||||
ProjectedHermOp<Fobj,CComplex,nbasis> Op(_FineOp,subspace);
|
||||
ProjectedFunctionHermOp<Fobj,CComplex,nbasis> ChebyOp (Cheby,_FineOp,subspace);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// create a smoother and see if we can get a cheap convergence test and smooth inside the IRL
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
Chebyshev<FineField> ChebySmooth(cheby_smooth); //lower order Chebyshev of fine operator on fine grid used to smooth regenerated eigenvectors
|
||||
ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,subspace,relax,Nstop-1);
|
||||
Chebyshev<FineField> ChebySmooth(cheby_smooth);
|
||||
ImplicitlyRestartedLanczosSmoothedTester<Fobj,CComplex,nbasis> ChebySmoothTester(ChebyOp,ChebySmooth,_FineOp,subspace,relax);
|
||||
|
||||
evals_coarse.resize(Nm);
|
||||
evec_coarse.resize(Nm,_CoarseGrid);
|
||||
|
||||
CoarseField src(_CoarseGrid); src=1.0;
|
||||
|
||||
//Note the "tester" here is also responsible for generating the fine grid eigenvalues which are output into the "evals_coarse" array
|
||||
ImplicitlyRestartedLanczos<CoarseField> IRL(ChebyOp,ChebyOp,ChebySmoothTester,Nstop,Nk,Nm,resid,MaxIt,betastp,MinRes);
|
||||
int Nconv=0;
|
||||
IRL.calc(evals_coarse,evec_coarse,src,Nconv,false);
|
||||
GRID_ASSERT(Nconv>=Nstop);
|
||||
assert(Nconv>=Nstop);
|
||||
evals_coarse.resize(Nstop);
|
||||
evec_coarse.resize (Nstop,_CoarseGrid);
|
||||
for (int i=0;i<Nstop;i++){
|
||||
std::cout << i << " Coarse eval = " << evals_coarse[i] << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
//Get the fine eigenvector 'i' by reconstruction
|
||||
void getFineEvecEval(FineField &evec, RealD &eval, const int i) const{
|
||||
blockPromote(evec_coarse[i],evec,subspace);
|
||||
eval = evals_coarse[i];
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -35,7 +35,7 @@ template<class Field> class MinimalResidual : public OperatorFunction<Field> {
|
||||
public:
|
||||
using OperatorFunction<Field>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the MR fails to converge.
|
||||
bool ErrorOnNoConverge; // throw an assert when the MR fails to converge.
|
||||
// Defaults true.
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
@@ -59,7 +59,7 @@ template<class Field> class MinimalResidual : public OperatorFunction<Field> {
|
||||
|
||||
// Initial residual computation & set up
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD ssq = norm2(src);
|
||||
RealD rsq = Tolerance * Tolerance * ssq;
|
||||
@@ -136,7 +136,7 @@ template<class Field> class MinimalResidual : public OperatorFunction<Field> {
|
||||
std::cout << GridLogMessage << "MR Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(true_residual / Tolerance < 10000.0);
|
||||
assert(true_residual / Tolerance < 10000.0);
|
||||
|
||||
IterationsToComplete = k;
|
||||
|
||||
@@ -148,7 +148,7 @@ template<class Field> class MinimalResidual : public OperatorFunction<Field> {
|
||||
<< std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
|
||||
IterationsToComplete = k;
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction
|
||||
|
||||
using OperatorFunction<FieldD>::operator();
|
||||
|
||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when MPFGMRES fails to converge,
|
||||
bool ErrorOnNoConverge; // Throw an assert when MPFGMRES fails to converge,
|
||||
// defaults to true
|
||||
|
||||
RealD Tolerance;
|
||||
@@ -91,7 +91,7 @@ class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction
|
||||
conformable(psi, src);
|
||||
|
||||
RealD guess = norm2(psi);
|
||||
GRID_ASSERT(std::isnan(guess) == 0);
|
||||
assert(std::isnan(guess) == 0);
|
||||
|
||||
RealD cp;
|
||||
RealD ssq = norm2(src);
|
||||
@@ -150,7 +150,7 @@ class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction
|
||||
std::cout << GridLogMessage << "MPFGMRES did NOT converge" << std::endl;
|
||||
|
||||
if (ErrorOnNoConverge)
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
RealD outerLoopBody(LinearOperatorBase<FieldD> &LinOp, const FieldD &src, FieldD &psi, RealD rsq) {
|
||||
@@ -197,7 +197,7 @@ class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction
|
||||
}
|
||||
}
|
||||
|
||||
GRID_ASSERT(0); // Never reached
|
||||
assert(0); // Never reached
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Take a matrix and form an NE solver calling a Herm solver
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class NormalEquations : public LinearFunction<Field>{
|
||||
template<class Field> class NormalEquations {
|
||||
private:
|
||||
SparseMatrixBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
@@ -60,33 +60,7 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field> class NormalResidual : public LinearFunction<Field>{
|
||||
private:
|
||||
SparseMatrixBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
LinearFunction<Field> & _Guess;
|
||||
public:
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations trick
|
||||
/////////////////////////////////////////////////////
|
||||
NormalResidual(SparseMatrixBase<Field> &Matrix, OperatorFunction<Field> &HermitianSolver,
|
||||
LinearFunction<Field> &Guess)
|
||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
||||
|
||||
void operator() (const Field &in, Field &out){
|
||||
|
||||
Field res(in.Grid());
|
||||
Field tmp(in.Grid());
|
||||
|
||||
MMdagLinearOperator<SparseMatrixBase<Field>,Field> MMdagOp(_Matrix);
|
||||
_Guess(in,res);
|
||||
_HermitianSolver(MMdagOp,in,res); // M Mdag res = in ;
|
||||
_Matrix.Mdag(res,out); // out = Mdag res
|
||||
}
|
||||
};
|
||||
|
||||
template<class Field> class HPDSolver : public LinearFunction<Field> {
|
||||
template<class Field> class HPDSolver {
|
||||
private:
|
||||
LinearOperatorBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
@@ -104,13 +78,13 @@ public:
|
||||
void operator() (const Field &in, Field &out){
|
||||
|
||||
_Guess(in,out);
|
||||
_HermitianSolver(_Matrix,in,out); //M out = in
|
||||
_HermitianSolver(_Matrix,in,out); // Mdag M out = Mdag in
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template<class Field> class MdagMSolver : public LinearFunction<Field> {
|
||||
template<class Field> class MdagMSolver {
|
||||
private:
|
||||
SparseMatrixBase<Field> & _Matrix;
|
||||
OperatorFunction<Field> & _HermitianSolver;
|
||||
|
||||
@@ -20,7 +20,7 @@ template<class Field> class PowerMethod
|
||||
RealD evalMaxApprox = 0.0;
|
||||
auto src_n = src;
|
||||
auto tmp = src;
|
||||
const int _MAX_ITER_EST_ = 200;
|
||||
const int _MAX_ITER_EST_ = 50;
|
||||
|
||||
for (int i=0;i<_MAX_ITER_EST_;i++) {
|
||||
|
||||
@@ -29,18 +29,17 @@ template<class Field> class PowerMethod
|
||||
RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
|
||||
RealD vden = norm2(src_n);
|
||||
RealD na = vnum/vden;
|
||||
|
||||
std::cout << GridLogMessage << "PowerMethod: Current approximation of largest eigenvalue " << na << std::endl;
|
||||
|
||||
// if ( (fabs(evalMaxApprox/na - 1.0) < 0.0001) || (i==_MAX_ITER_EST_-1) ) {
|
||||
// evalMaxApprox = na;
|
||||
// return evalMaxApprox;
|
||||
// }
|
||||
if ( (fabs(evalMaxApprox/na - 1.0) < 0.001) || (i==_MAX_ITER_EST_-1) ) {
|
||||
evalMaxApprox = na;
|
||||
std::cout << GridLogMessage << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
|
||||
return evalMaxApprox;
|
||||
}
|
||||
evalMaxApprox = na;
|
||||
src_n = tmp;
|
||||
}
|
||||
std::cout << GridLogMessage << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
|
||||
return evalMaxApprox;
|
||||
assert(0);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
#pragma once
|
||||
namespace Grid {
|
||||
|
||||
class Band
|
||||
{
|
||||
RealD lo, hi;
|
||||
public:
|
||||
Band(RealD _lo,RealD _hi)
|
||||
{
|
||||
lo=_lo;
|
||||
hi=_hi;
|
||||
}
|
||||
RealD operator() (RealD x){
|
||||
if ( x>lo && x<hi ){
|
||||
return 1.0;
|
||||
} else {
|
||||
return 0.0;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class PowerSpectrum
|
||||
{
|
||||
public:
|
||||
|
||||
template<typename T> static RealD normalise(T& v)
|
||||
{
|
||||
RealD nn = norm2(v);
|
||||
nn = sqrt(nn);
|
||||
v = v * (1.0/nn);
|
||||
return nn;
|
||||
}
|
||||
|
||||
std::vector<RealD> ranges;
|
||||
std::vector<int> order;
|
||||
|
||||
PowerSpectrum( std::vector<RealD> &bins, std::vector<int> &_order ) : ranges(bins), order(_order) { };
|
||||
|
||||
template<class Field>
|
||||
RealD operator()(LinearOperatorBase<Field> &HermOp, const Field &src)
|
||||
{
|
||||
GridBase *grid = src.Grid();
|
||||
int N=ranges.size();
|
||||
RealD hi = ranges[N-1];
|
||||
|
||||
RealD lo_band = 0.0;
|
||||
RealD hi_band;
|
||||
RealD nn=norm2(src);
|
||||
RealD ss=0.0;
|
||||
|
||||
Field tmp = src;
|
||||
|
||||
for(int b=0;b<N;b++){
|
||||
hi_band = ranges[b];
|
||||
Band Notch(lo_band,hi_band);
|
||||
|
||||
Chebyshev<Field> polynomial;
|
||||
polynomial.Init(0.0,hi,order[b],Notch);
|
||||
polynomial.JacksonSmooth();
|
||||
|
||||
polynomial(HermOp,src,tmp) ;
|
||||
|
||||
RealD p=norm2(tmp);
|
||||
ss=ss+p;
|
||||
std::cout << GridLogMessage << " PowerSpectrum Band["<<lo_band<<","<<hi_band<<"] power "<<norm2(tmp)/nn<<std::endl;
|
||||
|
||||
lo_band=hi_band;
|
||||
}
|
||||
std::cout << GridLogMessage << " PowerSpectrum total power "<<ss/nn<<std::endl;
|
||||
std::cout << GridLogMessage << " PowerSpectrum total power (unnormalised) "<<nn<<std::endl;
|
||||
|
||||
return 0;
|
||||
};
|
||||
};
|
||||
|
||||
}
|
||||
@@ -112,7 +112,7 @@ public:
|
||||
}
|
||||
|
||||
std::cout<<GridLogMessage<<"PrecConjugateResidual did NOT converge"<<std::endl;
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -43,7 +43,7 @@ NAMESPACE_BEGIN(Grid);
|
||||
template<class Field>
|
||||
class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
int verbose;
|
||||
@@ -118,7 +118,7 @@ public:
|
||||
|
||||
}
|
||||
GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
|
||||
// GRID_ASSERT(0);
|
||||
// assert(0);
|
||||
}
|
||||
|
||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
||||
@@ -221,7 +221,7 @@ public:
|
||||
int northog = ((kp)>(mmax-1))?(mmax-1):(kp); // if more than mmax done, we orthog all mmax history.
|
||||
for(int back=0;back<northog;back++){
|
||||
|
||||
int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
|
||||
int peri_back=(k-back)%mmax; assert((k-back)>=0);
|
||||
|
||||
b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
|
||||
p[peri_kp]=p[peri_kp]+b*p[peri_back];
|
||||
@@ -231,7 +231,7 @@ public:
|
||||
qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
GRID_ASSERT(0); // never reached
|
||||
assert(0); // never reached
|
||||
return cp;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,242 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/iterative/PrecGeneralisedConjugateResidual.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_PREC_GCR_NON_HERM_H
|
||||
#define GRID_PREC_GCR_NON_HERM_H
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//VPGCR Abe and Zhang, 2005.
|
||||
//INTERNATIONAL JOURNAL OF NUMERICAL ANALYSIS AND MODELING
|
||||
//Computing and Information Volume 2, Number 2, Pages 147-161
|
||||
//NB. Likely not original reference since they are focussing on a preconditioner variant.
|
||||
// but VPGCR was nicely written up in their paper
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define GCRLogLevel std::cout << GridLogMessage <<std::string(level,'\t')<< " Level "<<level<<" "
|
||||
|
||||
template<class Field>
|
||||
class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
|
||||
public:
|
||||
using LinearFunction<Field>::operator();
|
||||
RealD Tolerance;
|
||||
Integer MaxIterations;
|
||||
int verbose;
|
||||
int mmax;
|
||||
int nstep;
|
||||
int steps;
|
||||
int level;
|
||||
GridStopWatch PrecTimer;
|
||||
GridStopWatch MatTimer;
|
||||
GridStopWatch LinalgTimer;
|
||||
|
||||
LinearFunction<Field> &Preconditioner;
|
||||
LinearOperatorBase<Field> &Linop;
|
||||
|
||||
void Level(int lv) { level=lv; };
|
||||
|
||||
PrecGeneralisedConjugateResidualNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) :
|
||||
Tolerance(tol),
|
||||
MaxIterations(maxit),
|
||||
Linop(_Linop),
|
||||
Preconditioner(Prec),
|
||||
mmax(_mmax),
|
||||
nstep(_nstep)
|
||||
{
|
||||
level=1;
|
||||
verbose=1;
|
||||
};
|
||||
|
||||
void operator() (const Field &src, Field &psi){
|
||||
|
||||
// psi=Zero();
|
||||
RealD cp, ssq,rsq;
|
||||
ssq=norm2(src);
|
||||
rsq=Tolerance*Tolerance*ssq;
|
||||
|
||||
Field r(src.Grid());
|
||||
|
||||
PrecTimer.Reset();
|
||||
MatTimer.Reset();
|
||||
LinalgTimer.Reset();
|
||||
|
||||
GridStopWatch SolverTimer;
|
||||
SolverTimer.Start();
|
||||
|
||||
steps=0;
|
||||
for(int k=0;k<MaxIterations;k++){
|
||||
|
||||
cp=GCRnStep(src,psi,rsq);
|
||||
|
||||
GCRLogLevel <<"PGCR("<<mmax<<","<<nstep<<") "<< steps <<" steps cp = "<<cp<<" target "<<rsq <<std::endl;
|
||||
|
||||
if(cp<rsq) {
|
||||
|
||||
SolverTimer.Stop();
|
||||
|
||||
Linop.Op(psi,r);
|
||||
axpy(r,-1.0,src,r);
|
||||
RealD tr = norm2(r);
|
||||
GCRLogLevel<<"PGCR: Converged on iteration " <<steps
|
||||
<< " computed residual "<<sqrt(cp/ssq)
|
||||
<< " true residual " <<sqrt(tr/ssq)
|
||||
<< " target " <<Tolerance <<std::endl;
|
||||
|
||||
GCRLogLevel<<"PGCR Time elapsed: Total "<< SolverTimer.Elapsed() <<std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
|
||||
// GRID_ASSERT(0);
|
||||
}
|
||||
|
||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
||||
|
||||
RealD cp;
|
||||
ComplexD a, b;
|
||||
// ComplexD zAz;
|
||||
RealD zAAz;
|
||||
ComplexD rq;
|
||||
|
||||
GridBase *grid = src.Grid();
|
||||
|
||||
Field r(grid);
|
||||
Field z(grid);
|
||||
Field tmp(grid);
|
||||
Field ttmp(grid);
|
||||
Field Az(grid);
|
||||
|
||||
////////////////////////////////
|
||||
// history for flexible orthog
|
||||
////////////////////////////////
|
||||
std::vector<Field> q(mmax,grid);
|
||||
std::vector<Field> p(mmax,grid);
|
||||
std::vector<RealD> qq(mmax);
|
||||
|
||||
GCRLogLevel<< "PGCR nStep("<<nstep<<")"<<std::endl;
|
||||
|
||||
//////////////////////////////////
|
||||
// initial guess x0 is taken as nonzero.
|
||||
// r0=src-A x0 = src
|
||||
//////////////////////////////////
|
||||
MatTimer.Start();
|
||||
Linop.Op(psi,Az);
|
||||
// zAz = innerProduct(Az,psi);
|
||||
zAAz= norm2(Az);
|
||||
MatTimer.Stop();
|
||||
|
||||
|
||||
LinalgTimer.Start();
|
||||
r=src-Az;
|
||||
LinalgTimer.Stop();
|
||||
GCRLogLevel<< "PGCR true residual r = src - A psi "<<norm2(r) <<std::endl;
|
||||
|
||||
/////////////////////
|
||||
// p = Prec(r)
|
||||
/////////////////////
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(r,z);
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatTimer.Start();
|
||||
Linop.Op(z,Az);
|
||||
MatTimer.Stop();
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
// zAz = innerProduct(Az,psi);
|
||||
zAAz= norm2(Az);
|
||||
|
||||
//p[0],q[0],qq[0]
|
||||
p[0]= z;
|
||||
q[0]= Az;
|
||||
qq[0]= zAAz;
|
||||
|
||||
cp =norm2(r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
for(int k=0;k<nstep;k++){
|
||||
|
||||
steps++;
|
||||
|
||||
int kp = k+1;
|
||||
int peri_k = k %mmax;
|
||||
int peri_kp= kp%mmax;
|
||||
|
||||
LinalgTimer.Start();
|
||||
rq= innerProduct(q[peri_k],r); // what if rAr not real?
|
||||
a = rq/qq[peri_k];
|
||||
|
||||
axpy(psi,a,p[peri_k],psi);
|
||||
|
||||
cp = axpy_norm(r,-a,q[peri_k],r);
|
||||
LinalgTimer.Stop();
|
||||
|
||||
GCRLogLevel<< "PGCR step["<<steps<<"] resid " << cp << " target " <<rsq<<std::endl;
|
||||
|
||||
if((k==nstep-1)||(cp<rsq)){
|
||||
return cp;
|
||||
}
|
||||
|
||||
|
||||
PrecTimer.Start();
|
||||
Preconditioner(r,z);// solve Az = r
|
||||
PrecTimer.Stop();
|
||||
|
||||
MatTimer.Start();
|
||||
Linop.Op(z,Az);
|
||||
MatTimer.Stop();
|
||||
// zAz = innerProduct(Az,psi);
|
||||
zAAz= norm2(Az);
|
||||
|
||||
LinalgTimer.Start();
|
||||
|
||||
q[peri_kp]=Az;
|
||||
p[peri_kp]=z;
|
||||
|
||||
int northog = ((kp)>(mmax-1))?(mmax-1):(kp); // if more than mmax done, we orthog all mmax history.
|
||||
for(int back=0;back<northog;back++){
|
||||
|
||||
int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
|
||||
|
||||
b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
|
||||
p[peri_kp]=p[peri_kp]+b*p[peri_back];
|
||||
q[peri_kp]=q[peri_kp]+b*q[peri_back];
|
||||
|
||||
}
|
||||
qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
|
||||
LinalgTimer.Stop();
|
||||
}
|
||||
GRID_ASSERT(0); // never reached
|
||||
return cp;
|
||||
}
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -79,7 +79,7 @@ class QuasiMinimalResidual : public OperatorFunction<Field> {
|
||||
|
||||
LinOp.Op(x,r); r = b - r;
|
||||
|
||||
GRID_ASSERT(normb> 0.0);
|
||||
assert(normb> 0.0);
|
||||
|
||||
resid = norm2(r)/normb;
|
||||
if (resid <= Tolerance) {
|
||||
@@ -105,8 +105,8 @@ class QuasiMinimalResidual : public OperatorFunction<Field> {
|
||||
for (int i = 1; i <= MaxIterations; i++) {
|
||||
|
||||
// Breakdown tests
|
||||
GRID_ASSERT( rho != 0.0);
|
||||
GRID_ASSERT( xi != 0.0);
|
||||
assert( rho != 0.0);
|
||||
assert( xi != 0.0);
|
||||
|
||||
v = (1. / rho) * v_tld;
|
||||
y = (1. / rho) * y;
|
||||
@@ -134,10 +134,10 @@ class QuasiMinimalResidual : public OperatorFunction<Field> {
|
||||
ep=Zep.real();
|
||||
std::cout << "Zep "<<Zep <<std::endl;
|
||||
// Complex Audit
|
||||
GRID_ASSERT(abs(ep)>0);
|
||||
assert(abs(ep)>0);
|
||||
|
||||
beta = ep / delta;
|
||||
GRID_ASSERT(abs(beta)>0);
|
||||
assert(abs(beta)>0);
|
||||
|
||||
v_tld = p_tld - beta * v;
|
||||
y = v_tld;
|
||||
@@ -158,7 +158,7 @@ class QuasiMinimalResidual : public OperatorFunction<Field> {
|
||||
std::cout << "theta "<<theta<<std::endl;
|
||||
std::cout << "gamma "<<gamma<<std::endl;
|
||||
|
||||
GRID_ASSERT(abs(gamma)> 0.0);
|
||||
assert(abs(gamma)> 0.0);
|
||||
|
||||
eta = -eta * rho_1 * gamma* gamma / (beta * gamma_1 * gamma_1);
|
||||
|
||||
@@ -178,7 +178,7 @@ class QuasiMinimalResidual : public OperatorFunction<Field> {
|
||||
}
|
||||
std::cout << "Iteration "<<i<<" resid " << resid<<std::endl;
|
||||
}
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
return; // no convergence
|
||||
}
|
||||
#else
|
||||
|
||||
@@ -132,31 +132,6 @@ namespace Grid {
|
||||
(*this)(_Matrix,in,out,guess);
|
||||
}
|
||||
|
||||
void RedBlackSource(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &src_o)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
Field tmp(grid);
|
||||
int nblock = in.size();
|
||||
for(int b=0;b<nblock;b++){
|
||||
RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||
}
|
||||
}
|
||||
// James can write his own deflated guesser
|
||||
// with optimised code for the inner products
|
||||
// RedBlackSolveSplitGrid();
|
||||
// RedBlackSolve(_Matrix,src_o,sol_o);
|
||||
|
||||
void RedBlackSolution(Matrix &_Matrix, const std::vector<Field> &in, const std::vector<Field> &sol_o, std::vector<Field> &out)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
Field tmp(grid);
|
||||
int nblock = in.size();
|
||||
for(int b=0;b<nblock;b++) {
|
||||
pickCheckerboard(Even,tmp,in[b]);
|
||||
RedBlackSolution(_Matrix,sol_o[b],tmp,out[b]);
|
||||
}
|
||||
}
|
||||
|
||||
template<class Guesser>
|
||||
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out,Guesser &guess)
|
||||
{
|
||||
@@ -175,29 +150,24 @@ namespace Grid {
|
||||
////////////////////////////////////////////////
|
||||
// Prepare RedBlack source
|
||||
////////////////////////////////////////////////
|
||||
RedBlackSource(_Matrix,in,src_o);
|
||||
// for(int b=0;b<nblock;b++){
|
||||
// RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||
// }
|
||||
|
||||
for(int b=0;b<nblock;b++){
|
||||
RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
||||
}
|
||||
////////////////////////////////////////////////
|
||||
// Make the guesses
|
||||
////////////////////////////////////////////////
|
||||
if ( subGuess ) guess_save.resize(nblock,grid);
|
||||
|
||||
|
||||
if(useSolnAsInitGuess) {
|
||||
for(int b=0;b<nblock;b++){
|
||||
for(int b=0;b<nblock;b++){
|
||||
if(useSolnAsInitGuess) {
|
||||
pickCheckerboard(Odd, sol_o[b], out[b]);
|
||||
} else {
|
||||
guess(src_o[b],sol_o[b]);
|
||||
}
|
||||
} else {
|
||||
guess(src_o, sol_o);
|
||||
}
|
||||
|
||||
if ( subGuess ) {
|
||||
for(int b=0;b<nblock;b++){
|
||||
guess_save[b] = sol_o[b];
|
||||
}
|
||||
if ( subGuess ) {
|
||||
guess_save[b] = sol_o[b];
|
||||
}
|
||||
}
|
||||
//////////////////////////////////////////////////////////////
|
||||
// Call the block solver
|
||||
@@ -327,9 +297,9 @@ namespace Grid {
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
||||
_Matrix.MooeeInv(src_e,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); assert( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; assert( tmp.Checkerboard() ==Odd);
|
||||
|
||||
_Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source from dumb choice of matrix norm.
|
||||
}
|
||||
@@ -347,17 +317,17 @@ namespace Grid {
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
src_e = src_e-tmp; GRID_ASSERT( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(src_e,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
_Matrix.Meooe(sol_o,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
src_e = src_e-tmp; assert( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(src_e,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); GRID_ASSERT( sol_o.Checkerboard() ==Odd );
|
||||
setCheckerboard(sol,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); assert( sol_o.Checkerboard() ==Odd );
|
||||
}
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
{
|
||||
SchurStaggeredOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); GRID_ASSERT(sol_o.Checkerboard()==Odd);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); assert(sol_o.Checkerboard()==Odd);
|
||||
};
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
||||
{
|
||||
@@ -396,13 +366,13 @@ namespace Grid {
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
||||
_Matrix.MooeeInv(src_e,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); assert( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; assert( tmp.Checkerboard() ==Odd);
|
||||
|
||||
// get the right MpcDag
|
||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
_HermOpEO.MpcDag(tmp,src_o); GRID_ASSERT(src_o.Checkerboard() ==Odd);
|
||||
_HermOpEO.MpcDag(tmp,src_o); assert(src_o.Checkerboard() ==Odd);
|
||||
|
||||
}
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
||||
@@ -416,17 +386,17 @@ namespace Grid {
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
src_e_i = src_e-tmp; GRID_ASSERT( src_e_i.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(src_e_i,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
_Matrix.Meooe(sol_o,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
src_e_i = src_e-tmp; assert( src_e_i.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(src_e_i,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); GRID_ASSERT( sol_o.Checkerboard() ==Odd );
|
||||
setCheckerboard(sol,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); assert( sol_o.Checkerboard() ==Odd );
|
||||
}
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
{
|
||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); GRID_ASSERT(sol_o.Checkerboard()==Odd);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); assert(sol_o.Checkerboard()==Odd);
|
||||
};
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
||||
{
|
||||
@@ -461,9 +431,9 @@ namespace Grid {
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
||||
_Matrix.Meooe (tmp, Mtmp); GRID_ASSERT( Mtmp.Checkerboard() == Odd );
|
||||
src_o -= Mtmp; GRID_ASSERT( src_o.Checkerboard() == Odd );
|
||||
_Matrix.MooeeInv(src_e, tmp); assert( tmp.Checkerboard() == Even );
|
||||
_Matrix.Meooe (tmp, Mtmp); assert( Mtmp.Checkerboard() == Odd );
|
||||
src_o -= Mtmp; assert( src_o.Checkerboard() == Odd );
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix& _Matrix, const Field& sol_o, const Field& src_e, Field& sol)
|
||||
@@ -478,18 +448,18 @@ namespace Grid {
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
||||
src_e_i = src_e - tmp; GRID_ASSERT( src_e_i.Checkerboard() == Even );
|
||||
_Matrix.MooeeInv(src_e_i, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
||||
_Matrix.Meooe(sol_o, tmp); assert( tmp.Checkerboard() == Even );
|
||||
src_e_i = src_e - tmp; assert( src_e_i.Checkerboard() == Even );
|
||||
_Matrix.MooeeInv(src_e_i, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
|
||||
setCheckerboard(sol, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
||||
setCheckerboard(sol, sol_o); GRID_ASSERT( sol_o.Checkerboard() == Odd );
|
||||
setCheckerboard(sol, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
setCheckerboard(sol, sol_o); assert( sol_o.Checkerboard() == Odd );
|
||||
}
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const Field& src_o, Field& sol_o)
|
||||
{
|
||||
NonHermitianSchurDiagMooeeOperator<Matrix,Field> _OpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o); GRID_ASSERT(sol_o.Checkerboard() == Odd);
|
||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o); assert(sol_o.Checkerboard() == Odd);
|
||||
}
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const std::vector<Field>& src_o, std::vector<Field>& sol_o)
|
||||
@@ -499,87 +469,6 @@ namespace Grid {
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Site diagonal is identity, left preconditioned by Mee^inv
|
||||
// ( 1 - Mee^inv Meo Moo^inv Moe ) phi = Mee_inv ( Mee - Meo Moo^inv Moe Mee^inv ) phi = Mee_inv eta
|
||||
//
|
||||
// Solve:
|
||||
// ( 1 - Mee^inv Meo Moo^inv Moe )^dag ( 1 - Mee^inv Meo Moo^inv Moe ) phi = ( 1 - Mee^inv Meo Moo^inv Moe )^dag Mee_inv eta
|
||||
//
|
||||
// Old notation e<->o
|
||||
//
|
||||
// Left precon by Moo^-1
|
||||
// b) (Doo^{dag} M_oo^-dag) (Moo^-1 Doo) psi_o = [ (D_oo)^dag M_oo^-dag ] Moo^-1 L^{-1} eta_o
|
||||
// eta_o' = (D_oo)^dag M_oo^-dag Moo^-1 (eta_o - Moe Mee^{-1} eta_e)
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template<class Field> class SchurRedBlackDiagOneSolve : public SchurRedBlackBase<Field> {
|
||||
public:
|
||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// Wrap the usual normal equations Schur trick
|
||||
/////////////////////////////////////////////////////
|
||||
SchurRedBlackDiagOneSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
||||
const bool _solnAsInitGuess = false)
|
||||
: SchurRedBlackBase<Field>(HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
|
||||
|
||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
SchurDiagOneOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
|
||||
Field tmp(grid);
|
||||
Field Mtmp(grid);
|
||||
|
||||
pickCheckerboard(Even,src_e,src);
|
||||
pickCheckerboard(Odd ,src_o,src);
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mpcdag *MooeeInv * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
||||
Mtmp=src_o-Mtmp;
|
||||
_Matrix.MooeeInv(Mtmp,tmp); GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
||||
|
||||
// get the right MpcDag
|
||||
_HermOpEO.MpcDag(tmp,src_o); GRID_ASSERT(src_o.Checkerboard() ==Odd);
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
||||
{
|
||||
GridBase *grid = _Matrix.RedBlackGrid();
|
||||
GridBase *fgrid= _Matrix.Grid();
|
||||
|
||||
Field tmp(grid);
|
||||
Field sol_e(grid);
|
||||
|
||||
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
tmp = src_e-tmp; GRID_ASSERT( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(tmp,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o); GRID_ASSERT( sol_o.Checkerboard() ==Odd );
|
||||
};
|
||||
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
{
|
||||
SchurDiagOneOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
||||
};
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
||||
{
|
||||
SchurDiagOneOperator<Matrix,Field> _HermOpEO(_Matrix);
|
||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Site diagonal is identity, right preconditioned by Mee^inv
|
||||
// ( 1 - Meo Moo^inv Moe Mee^inv ) phi =( 1 - Meo Moo^inv Moe Mee^inv ) Mee psi = = eta = eta
|
||||
@@ -612,12 +501,12 @@ namespace Grid {
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
||||
_Matrix.MooeeInv(src_e,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
_Matrix.Meooe (tmp,Mtmp); assert( Mtmp.Checkerboard() ==Odd);
|
||||
tmp=src_o-Mtmp; assert( tmp.Checkerboard() ==Odd);
|
||||
|
||||
// get the right MpcDag
|
||||
_HermOpEO.MpcDag(tmp,src_o); GRID_ASSERT(src_o.Checkerboard() ==Odd);
|
||||
_HermOpEO.MpcDag(tmp,src_o); assert(src_o.Checkerboard() ==Odd);
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
||||
@@ -638,12 +527,12 @@ namespace Grid {
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o_i,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
||||
tmp = src_e-tmp; GRID_ASSERT( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(tmp,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
_Matrix.Meooe(sol_o_i,tmp); assert( tmp.Checkerboard() ==Even);
|
||||
tmp = src_e-tmp; assert( src_e.Checkerboard() ==Even);
|
||||
_Matrix.MooeeInv(tmp,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
|
||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o_i); GRID_ASSERT( sol_o_i.Checkerboard() ==Odd );
|
||||
setCheckerboard(sol,sol_e); assert( sol_e.Checkerboard() ==Even);
|
||||
setCheckerboard(sol,sol_o_i); assert( sol_o_i.Checkerboard() ==Odd );
|
||||
};
|
||||
|
||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
||||
@@ -684,9 +573,9 @@ namespace Grid {
|
||||
/////////////////////////////////////////////////////
|
||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
||||
/////////////////////////////////////////////////////
|
||||
_Matrix.MooeeInv(src_e, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
||||
_Matrix.Meooe (tmp, Mtmp); GRID_ASSERT( Mtmp.Checkerboard() == Odd );
|
||||
src_o -= Mtmp; GRID_ASSERT( src_o.Checkerboard() == Odd );
|
||||
_Matrix.MooeeInv(src_e, tmp); assert( tmp.Checkerboard() == Even );
|
||||
_Matrix.Meooe (tmp, Mtmp); assert( Mtmp.Checkerboard() == Odd );
|
||||
src_o -= Mtmp; assert( src_o.Checkerboard() == Odd );
|
||||
}
|
||||
|
||||
virtual void RedBlackSolution(Matrix& _Matrix, const Field& sol_o, const Field& src_e, Field& sol)
|
||||
@@ -707,12 +596,12 @@ namespace Grid {
|
||||
///////////////////////////////////////////////////
|
||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
||||
///////////////////////////////////////////////////
|
||||
_Matrix.Meooe(sol_o_i, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
||||
tmp = src_e - tmp; GRID_ASSERT( src_e.Checkerboard() == Even );
|
||||
_Matrix.MooeeInv(tmp, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
||||
_Matrix.Meooe(sol_o_i, tmp); assert( tmp.Checkerboard() == Even );
|
||||
tmp = src_e - tmp; assert( src_e.Checkerboard() == Even );
|
||||
_Matrix.MooeeInv(tmp, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
|
||||
setCheckerboard(sol, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
||||
setCheckerboard(sol, sol_o_i); GRID_ASSERT( sol_o_i.Checkerboard() == Odd );
|
||||
setCheckerboard(sol, sol_e); assert( sol_e.Checkerboard() == Even );
|
||||
setCheckerboard(sol, sol_o_i); assert( sol_o_i.Checkerboard() == Odd );
|
||||
};
|
||||
|
||||
virtual void RedBlackSolve(Matrix& _Matrix, const Field& src_o, Field& sol_o)
|
||||
|
||||
@@ -1,611 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/Aggregates.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
inline RealD AggregatePowerLaw(RealD x)
|
||||
{
|
||||
// return std::pow(x,-4);
|
||||
// return std::pow(x,-3);
|
||||
return std::pow(x,-5);
|
||||
}
|
||||
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class Aggregation {
|
||||
public:
|
||||
constexpr int Nbasis(void) { return nbasis; };
|
||||
|
||||
typedef iVector<CComplex,nbasis > siteVector;
|
||||
typedef Lattice<siteVector> CoarseVector;
|
||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
||||
|
||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj > FineField;
|
||||
|
||||
GridBase *CoarseGrid;
|
||||
GridBase *FineGrid;
|
||||
std::vector<Lattice<Fobj> > subspace;
|
||||
int checkerboard;
|
||||
int Checkerboard(void){return checkerboard;}
|
||||
Aggregation(GridBase *_CoarseGrid,GridBase *_FineGrid,int _checkerboard) :
|
||||
CoarseGrid(_CoarseGrid),
|
||||
FineGrid(_FineGrid),
|
||||
subspace(nbasis,_FineGrid),
|
||||
checkerboard(_checkerboard)
|
||||
{
|
||||
};
|
||||
|
||||
|
||||
void Orthogonalise(void){
|
||||
CoarseScalar InnerProd(CoarseGrid);
|
||||
// std::cout << GridLogMessage <<" Block Gramm-Schmidt pass 1"<<std::endl;
|
||||
blockOrthogonalise(InnerProd,subspace);
|
||||
}
|
||||
void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
|
||||
blockProject(CoarseVec,FineVec,subspace);
|
||||
}
|
||||
void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
|
||||
FineVec.Checkerboard() = subspace[0].Checkerboard();
|
||||
blockPromote(CoarseVec,FineVec,subspace);
|
||||
}
|
||||
|
||||
virtual void CreateSubspaceRandom(GridParallelRNG &RNG) {
|
||||
int nn=nbasis;
|
||||
RealD scale;
|
||||
FineField noise(FineGrid);
|
||||
for(int b=0;b<nn;b++){
|
||||
subspace[b] = Zero();
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
subspace[b] = noise;
|
||||
}
|
||||
}
|
||||
virtual void CreateSubspace(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,int nn=nbasis)
|
||||
{
|
||||
|
||||
RealD scale;
|
||||
|
||||
ConjugateGradient<FineField> CG(1.0e-4,2000,false);
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
|
||||
for(int b=0;b<nn;b++){
|
||||
|
||||
subspace[b] = Zero();
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise ["<<b<<"] <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
for(int i=0;i<4;i++){
|
||||
|
||||
CG(hermop,noise,subspace[b]);
|
||||
|
||||
noise = subspace[b];
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
}
|
||||
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "filtered["<<b<<"] <f|MdagM|f> "<<norm2(Mn)<<std::endl;
|
||||
subspace[b] = noise;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
virtual void CreateSubspaceGCR(GridParallelRNG &RNG,LinearOperatorBase<FineField> &DiracOp,int nn=nbasis)
|
||||
{
|
||||
RealD scale;
|
||||
|
||||
TrivialPrecon<FineField> simple_fine;
|
||||
// PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,10,DiracOp,simple_fine,30,30);
|
||||
// PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,10,DiracOp,simple_fine,12,12);
|
||||
// PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,30,DiracOp,simple_fine,12,12);
|
||||
PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,30,DiracOp,simple_fine,10,10);
|
||||
FineField noise(FineGrid);
|
||||
FineField src(FineGrid);
|
||||
FineField guess(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
|
||||
for(int b=0;b<nn;b++){
|
||||
|
||||
subspace[b] = Zero();
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
DiracOp.Op(noise,Mn); std::cout<<GridLogMessage << "noise ["<<b<<"] <n|Op|n> "<<innerProduct(noise,Mn)<<std::endl;
|
||||
|
||||
for(int i=0;i<3;i++){
|
||||
// void operator() (const Field &src, Field &psi){
|
||||
#if 1
|
||||
if (i==0)std::cout << GridLogMessage << " inverting on noise "<<std::endl;
|
||||
src = noise;
|
||||
guess=Zero();
|
||||
GCR(src,guess);
|
||||
subspace[b] = guess;
|
||||
#else
|
||||
if (i==0)std::cout << GridLogMessage << " inverting on zero "<<std::endl;
|
||||
src=Zero();
|
||||
guess = noise;
|
||||
GCR(src,guess);
|
||||
subspace[b] = guess;
|
||||
#endif
|
||||
noise = subspace[b];
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
}
|
||||
|
||||
DiracOp.Op(noise,Mn); std::cout<<GridLogMessage << "filtered["<<b<<"] <f|Op|f> "<<innerProduct(noise,Mn)<<" <f|OpDagOp|f>"<<norm2(Mn)<<std::endl;
|
||||
subspace[b] = noise;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// World of possibilities here. But have tried quite a lot of experiments (250+ jobs run on Summit)
|
||||
// and this is the best I found
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
double lo,
|
||||
int orderfilter,
|
||||
int ordermin,
|
||||
int orderstep,
|
||||
double filterlo
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
std::cout << GridLogMessage<<" Chebyshev subspace pass-1 : ord "<<orderfilter<<" ["<<lo<<","<<hi<<"]"<<std::endl;
|
||||
std::cout << GridLogMessage<<" Chebyshev subspace pass-2 : nbasis"<<nn<<" min "
|
||||
<<ordermin<<" step "<<orderstep
|
||||
<<" lo"<<filterlo<<std::endl;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
int b =0;
|
||||
{
|
||||
ComplexD ip;
|
||||
// Filter
|
||||
Chebyshev<FineField> Cheb(lo,hi,orderfilter);
|
||||
Cheb(hermop,noise,Mn);
|
||||
// normalise
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
|
||||
hermop.Op(Mn,tmp);
|
||||
ip= innerProduct(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|Op|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
||||
|
||||
hermop.AdjOp(Mn,tmp);
|
||||
ip = innerProduct(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|AdjOp|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
||||
b++;
|
||||
}
|
||||
|
||||
// Generate a full sequence of Chebyshevs
|
||||
{
|
||||
lo=filterlo;
|
||||
noise=Mn;
|
||||
|
||||
FineField T0(FineGrid); T0 = noise;
|
||||
FineField T1(FineGrid);
|
||||
FineField T2(FineGrid);
|
||||
FineField y(FineGrid);
|
||||
|
||||
FineField *Tnm = &T0;
|
||||
FineField *Tn = &T1;
|
||||
FineField *Tnp = &T2;
|
||||
|
||||
// Tn=T1 = (xscale M + mscale)in
|
||||
RealD xscale = 2.0/(hi-lo);
|
||||
RealD mscale = -(hi+lo)/(hi-lo);
|
||||
hermop.HermOp(T0,y);
|
||||
T1=y*xscale+noise*mscale;
|
||||
|
||||
for(int n=2;n<=ordermin+orderstep*(nn-2);n++){
|
||||
|
||||
hermop.HermOp(*Tn,y);
|
||||
|
||||
autoView( y_v , y, AcceleratorWrite);
|
||||
autoView( Tn_v , (*Tn), AcceleratorWrite);
|
||||
autoView( Tnp_v , (*Tnp), AcceleratorWrite);
|
||||
autoView( Tnm_v , (*Tnm), AcceleratorWrite);
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
accelerator_for(ss, FineGrid->oSites(), Nsimd, {
|
||||
coalescedWrite(y_v[ss],xscale*y_v(ss)+mscale*Tn_v(ss));
|
||||
coalescedWrite(Tnp_v[ss],2.0*y_v(ss)-Tnm_v(ss));
|
||||
});
|
||||
|
||||
// Possible more fine grained control is needed than a linear sweep,
|
||||
// but huge productivity gain if this is simple algorithm and not a tunable
|
||||
int m =1;
|
||||
if ( n>=ordermin ) m=n-ordermin;
|
||||
if ( (m%orderstep)==0 ) {
|
||||
Mn=*Tnp;
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
|
||||
|
||||
ComplexD ip;
|
||||
|
||||
hermop.Op(Mn,tmp);
|
||||
ip= innerProduct(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|Op|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
||||
|
||||
hermop.AdjOp(Mn,tmp);
|
||||
ip = innerProduct(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|AdjOp|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
||||
|
||||
b++;
|
||||
}
|
||||
|
||||
// Cycle pointers to avoid copies
|
||||
FineField *swizzle = Tnm;
|
||||
Tnm =Tn;
|
||||
Tn =Tnp;
|
||||
Tnp =swizzle;
|
||||
|
||||
}
|
||||
}
|
||||
GRID_ASSERT(b==nn);
|
||||
}
|
||||
|
||||
|
||||
virtual void CreateSubspacePolyCheby(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
double lo1,
|
||||
int orderfilter,
|
||||
double lo2,
|
||||
int orderstep)
|
||||
{
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
std::cout << GridLogMessage<<" CreateSubspacePolyCheby "<<std::endl;
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn);
|
||||
std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
int b =0;
|
||||
{
|
||||
// Filter
|
||||
std::cout << GridLogMessage << "Cheby "<<lo1<<","<<hi<<" "<<orderstep<<std::endl;
|
||||
Chebyshev<FineField> Cheb(lo1,hi,orderfilter);
|
||||
Cheb(hermop,noise,Mn);
|
||||
// normalise
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|n> "<<norm2(Mn)<<std::endl;
|
||||
}
|
||||
|
||||
// Generate a full sequence of Chebyshevs
|
||||
for(int n=1;n<nn;n++){
|
||||
std::cout << GridLogMessage << "Cheby "<<lo2<<","<<hi<<" "<<orderstep<<std::endl;
|
||||
Chebyshev<FineField> Cheb(lo2,hi,orderstep);
|
||||
Cheb(hermop,subspace[n-1],Mn);
|
||||
|
||||
for(int m=0;m<n;m++){
|
||||
ComplexD c = innerProduct(subspace[m],Mn);
|
||||
Mn = Mn - c*subspace[m];
|
||||
}
|
||||
|
||||
// normalise
|
||||
scale = std::pow(norm2(Mn),-0.5);
|
||||
Mn=Mn*scale;
|
||||
|
||||
subspace[n]=Mn;
|
||||
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<n<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
std::cout<<GridLogMessage << "filt ["<<n<<"] <n|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
double lo,
|
||||
int orderfilter
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : ord "<<orderfilter<<" ["<<lo<<","<<hi<<"]"<<std::endl;
|
||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : nbasis "<<nn<<std::endl;
|
||||
|
||||
|
||||
for(int b =0;b<nbasis;b++)
|
||||
{
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn);
|
||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
// Filter
|
||||
Chebyshev<FineField> Cheb(lo,hi,orderfilter);
|
||||
Cheb(hermop,noise,Mn);
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
|
||||
// Refine
|
||||
Chebyshev<FineField> PowerLaw(lo,hi,1000,AggregatePowerLaw);
|
||||
noise = Mn;
|
||||
PowerLaw(hermop,noise,Mn);
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
|
||||
// normalise
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
virtual void CreateSubspaceChebyshevPowerLaw(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
int nn,
|
||||
double hi,
|
||||
int orderfilter
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : ord "<<orderfilter<<" [0,"<<hi<<"]"<<std::endl;
|
||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : nbasis "<<nn<<std::endl;
|
||||
|
||||
for(int b =0;b<nbasis;b++)
|
||||
{
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn);
|
||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
// Filter
|
||||
Chebyshev<FineField> Cheb(0.0,hi,orderfilter,AggregatePowerLaw);
|
||||
Cheb(hermop,noise,Mn);
|
||||
// normalise
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
virtual void CreateSubspaceChebyshevNew(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
double hi
|
||||
) {
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
for(int b =0;b<nbasis;b++)
|
||||
{
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn);
|
||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
// Filter
|
||||
//#opt2(x) = acheb(x,3,90,300)* acheb(x,1,90,50) * acheb(x,0.5,90,200) * acheb(x,0.05,90,400) * acheb(x,0.01,90,1500)
|
||||
/*266
|
||||
Chebyshev<FineField> Cheb1(3.0,hi,300);
|
||||
Chebyshev<FineField> Cheb2(1.0,hi,50);
|
||||
Chebyshev<FineField> Cheb3(0.5,hi,300);
|
||||
Chebyshev<FineField> Cheb4(0.05,hi,500);
|
||||
Chebyshev<FineField> Cheb5(0.01,hi,2000);
|
||||
*/
|
||||
/* 242 */
|
||||
/*
|
||||
Chebyshev<FineField> Cheb3(0.1,hi,300);
|
||||
Chebyshev<FineField> Cheb2(0.02,hi,1000);
|
||||
Chebyshev<FineField> Cheb1(0.003,hi,2000);
|
||||
8?
|
||||
*/
|
||||
/* How many??
|
||||
*/
|
||||
Chebyshev<FineField> Cheb2(0.001,hi,2500); // 169 iters on HDCG after refine
|
||||
Chebyshev<FineField> Cheb1(0.02,hi,600);
|
||||
|
||||
// Chebyshev<FineField> Cheb2(0.001,hi,1500);
|
||||
// Chebyshev<FineField> Cheb1(0.02,hi,600);
|
||||
Cheb1(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
||||
hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb1 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
Cheb2(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
||||
hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb2 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
// Cheb3(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
||||
// hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb3 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
// Cheb4(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
||||
// hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb4 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
// Cheb5(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
||||
// hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb5 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
subspace[b] = noise;
|
||||
hermop.Op(subspace[b],tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<< " norm " << norm2(noise)<<std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
virtual void CreateSubspaceMultishift(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
||||
double Lo,double tol,int maxit)
|
||||
{
|
||||
|
||||
RealD scale;
|
||||
|
||||
FineField noise(FineGrid);
|
||||
FineField Mn(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
|
||||
// New normalised noise
|
||||
std::cout << GridLogMessage<<" Multishift subspace : Lo "<<Lo<<std::endl;
|
||||
|
||||
// Filter
|
||||
// [ 1/6(x+Lo) - 1/2(x+2Lo) + 1/2(x+3Lo) -1/6(x+4Lo) = Lo^3 /[ (x+1Lo)(x+2Lo)(x+3Lo)(x+4Lo) ]
|
||||
//
|
||||
// 1/(x+Lo) - 1/(x+2 Lo)
|
||||
double epsilon = Lo/3;
|
||||
std::vector<RealD> alpha({1.0/6.0,-1.0/2.0,1.0/2.0,-1.0/6.0});
|
||||
std::vector<RealD> shifts({Lo,Lo+epsilon,Lo+2*epsilon,Lo+3*epsilon});
|
||||
std::vector<RealD> tols({tol,tol,tol,tol});
|
||||
std::cout << "sizes "<<alpha.size()<<" "<<shifts.size()<<" "<<tols.size()<<std::endl;
|
||||
|
||||
MultiShiftFunction msf(4,0.0,95.0);
|
||||
std::cout << "msf constructed "<<std::endl;
|
||||
msf.poles=shifts;
|
||||
msf.residues=alpha;
|
||||
msf.tolerances=tols;
|
||||
msf.norm=0.0;
|
||||
msf.order=alpha.size();
|
||||
ConjugateGradientMultiShift<FineField> MSCG(maxit,msf);
|
||||
|
||||
for(int b =0;b<nbasis;b++)
|
||||
{
|
||||
gaussian(RNG,noise);
|
||||
scale = std::pow(norm2(noise),-0.5);
|
||||
noise=noise*scale;
|
||||
|
||||
// Initial matrix element
|
||||
hermop.Op(noise,Mn);
|
||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
||||
|
||||
MSCG(hermop,noise,Mn);
|
||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
||||
subspace[b] = Mn;
|
||||
hermop.Op(Mn,tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
virtual void RefineSubspace(LinearOperatorBase<FineField> &hermop,
|
||||
double Lo,double tol,int maxit)
|
||||
{
|
||||
FineField tmp(FineGrid);
|
||||
for(int b =0;b<nbasis;b++)
|
||||
{
|
||||
ConjugateGradient<FineField> CGsloppy(tol,maxit,false);
|
||||
ShiftedHermOpLinearOperator<FineField> ShiftedFineHermOp(hermop,Lo);
|
||||
tmp=Zero();
|
||||
CGsloppy(hermop,subspace[b],tmp);
|
||||
RealD scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
||||
subspace[b]=tmp;
|
||||
hermop.Op(subspace[b],tmp);
|
||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
}
|
||||
}
|
||||
virtual void RefineSubspaceHDCG(LinearOperatorBase<FineField> &hermop,
|
||||
TwoLevelADEF2mrhs<FineField,CoarseVector> & theHDCG,
|
||||
int nrhs)
|
||||
{
|
||||
std::vector<FineField> src_mrhs(nrhs,FineGrid);
|
||||
std::vector<FineField> res_mrhs(nrhs,FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
for(int b =0;b<nbasis;b+=nrhs)
|
||||
{
|
||||
tmp = subspace[b];
|
||||
RealD scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
||||
subspace[b] =tmp;
|
||||
hermop.Op(subspace[b],tmp);
|
||||
std::cout<<GridLogMessage << "before filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
|
||||
for(int r=0;r<MIN(nbasis-b,nrhs);r++){
|
||||
src_mrhs[r] = subspace[b+r];
|
||||
}
|
||||
for(int r=0;r<nrhs;r++){
|
||||
res_mrhs[r] = Zero();
|
||||
}
|
||||
theHDCG(src_mrhs,res_mrhs);
|
||||
|
||||
for(int r=0;r<MIN(nbasis-b,nrhs);r++){
|
||||
tmp = res_mrhs[r];
|
||||
RealD scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
||||
subspace[b+r]=tmp;
|
||||
}
|
||||
hermop.Op(subspace[b],tmp);
|
||||
std::cout<<GridLogMessage << "after filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
};
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,837 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/CoarsenedMatrix.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#ifndef GRID_ALGORITHM_COARSENED_MATRIX_H
|
||||
#define GRID_ALGORITHM_COARSENED_MATRIX_H
|
||||
|
||||
#include <Grid/qcd/QCD.h> // needed for Dagger(Yes|No), Inverse(Yes|No)
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
template<class vobj,class CComplex>
|
||||
inline void blockMaskedInnerProduct(Lattice<CComplex> &CoarseInner,
|
||||
const Lattice<decltype(innerProduct(vobj(),vobj()))> &FineMask,
|
||||
const Lattice<vobj> &fineX,
|
||||
const Lattice<vobj> &fineY)
|
||||
{
|
||||
typedef decltype(innerProduct(vobj(),vobj())) dotp;
|
||||
|
||||
GridBase *coarse(CoarseInner.Grid());
|
||||
GridBase *fine (fineX.Grid());
|
||||
|
||||
Lattice<dotp> fine_inner(fine); fine_inner.Checkerboard() = fineX.Checkerboard();
|
||||
Lattice<dotp> fine_inner_msk(fine);
|
||||
|
||||
// Multiply could be fused with innerProduct
|
||||
// Single block sum kernel could do both masks.
|
||||
fine_inner = localInnerProduct(fineX,fineY);
|
||||
mult(fine_inner_msk, fine_inner,FineMask);
|
||||
blockSum(CoarseInner,fine_inner_msk);
|
||||
}
|
||||
|
||||
// Fine Object == (per site) type of fine field
|
||||
// nbasis == number of deflation vectors
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class CoarsenedMatrix : public CheckerBoardedSparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
|
||||
typedef iVector<CComplex,nbasis > siteVector;
|
||||
typedef Lattice<CComplex > CoarseComplexField;
|
||||
typedef Lattice<siteVector> CoarseVector;
|
||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj > FineField;
|
||||
typedef CoarseVector FermionField;
|
||||
|
||||
// enrich interface, use default implementation as in FermionOperator ///////
|
||||
void Dminus(CoarseVector const& in, CoarseVector& out) { out = in; }
|
||||
void DminusDag(CoarseVector const& in, CoarseVector& out) { out = in; }
|
||||
void ImportPhysicalFermionSource(CoarseVector const& input, CoarseVector& imported) { imported = input; }
|
||||
void ImportUnphysicalFermion(CoarseVector const& input, CoarseVector& imported) { imported = input; }
|
||||
void ExportPhysicalFermionSolution(CoarseVector const& solution, CoarseVector& exported) { exported = solution; };
|
||||
void ExportPhysicalFermionSource(CoarseVector const& solution, CoarseVector& exported) { exported = solution; };
|
||||
|
||||
////////////////////
|
||||
// Data members
|
||||
////////////////////
|
||||
Geometry geom;
|
||||
GridBase * _grid;
|
||||
GridBase* _cbgrid;
|
||||
int hermitian;
|
||||
|
||||
CartesianStencil<siteVector,siteVector,DefaultImplParams> Stencil;
|
||||
CartesianStencil<siteVector,siteVector,DefaultImplParams> StencilEven;
|
||||
CartesianStencil<siteVector,siteVector,DefaultImplParams> StencilOdd;
|
||||
|
||||
std::vector<CoarseMatrix> A;
|
||||
std::vector<CoarseMatrix> Aeven;
|
||||
std::vector<CoarseMatrix> Aodd;
|
||||
|
||||
CoarseMatrix AselfInv;
|
||||
CoarseMatrix AselfInvEven;
|
||||
CoarseMatrix AselfInvOdd;
|
||||
|
||||
deviceVector<RealD> dag_factor;
|
||||
|
||||
///////////////////////
|
||||
// Interface
|
||||
///////////////////////
|
||||
GridBase * Grid(void) { return _grid; }; // this is all the linalg routines need to know
|
||||
GridBase * RedBlackGrid() { return _cbgrid; };
|
||||
|
||||
int ConstEE() { return 0; }
|
||||
|
||||
void M (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
conformable(_grid,in.Grid());
|
||||
conformable(in.Grid(),out.Grid());
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
|
||||
SimpleCompressor<siteVector> compressor;
|
||||
|
||||
Stencil.HaloExchange(in,compressor);
|
||||
autoView( in_v , in, AcceleratorRead);
|
||||
autoView( out_v , out, AcceleratorWrite);
|
||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
||||
int npoint = geom.npoint;
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
|
||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++) {
|
||||
hAcceleratorViewContainer[p] = A[p].View(AcceleratorRead);
|
||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
||||
}
|
||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
int osites=Grid()->oSites();
|
||||
|
||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
for(int point=0;point<npoint;point++){
|
||||
|
||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(Stencil_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
|
||||
for(int p=0;p<geom.npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
||||
};
|
||||
|
||||
void Mdag (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
if(hermitian) {
|
||||
// corresponds to Petrov-Galerkin coarsening
|
||||
return M(in,out);
|
||||
} else {
|
||||
// corresponds to Galerkin coarsening
|
||||
return MdagNonHermitian(in, out);
|
||||
}
|
||||
};
|
||||
|
||||
void MdagNonHermitian(const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
conformable(_grid,in.Grid());
|
||||
conformable(in.Grid(),out.Grid());
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
|
||||
SimpleCompressor<siteVector> compressor;
|
||||
|
||||
Stencil.HaloExchange(in,compressor);
|
||||
autoView( in_v , in, AcceleratorRead);
|
||||
autoView( out_v , out, AcceleratorWrite);
|
||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
||||
int npoint = geom.npoint;
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
|
||||
|
||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++) {
|
||||
hAcceleratorViewContainer[p] = A[p].View(AcceleratorRead);
|
||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
||||
}
|
||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
int osites=Grid()->oSites();
|
||||
|
||||
deviceVector<int> points(geom.npoint);
|
||||
for(int p=0; p<geom.npoint; p++) {
|
||||
acceleratorPut(points[p],geom.points_dagger[p]);
|
||||
}
|
||||
auto points_p = &points[0];
|
||||
|
||||
RealD* dag_factor_p = &dag_factor[0];
|
||||
|
||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
for(int p=0;p<npoint;p++){
|
||||
int point = points_p[p];
|
||||
|
||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(Stencil_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + dag_factor_p[b*nbasis+bb]*coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
|
||||
for(int p=0;p<geom.npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
||||
}
|
||||
|
||||
void MdirComms(const CoarseVector &in)
|
||||
{
|
||||
SimpleCompressor<siteVector> compressor;
|
||||
Stencil.HaloExchange(in,compressor);
|
||||
}
|
||||
void MdirCalc(const CoarseVector &in, CoarseVector &out, int point)
|
||||
{
|
||||
conformable(_grid,in.Grid());
|
||||
conformable(_grid,out.Grid());
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
|
||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++) {
|
||||
hAcceleratorViewContainer[p] = A[p].View(AcceleratorRead);
|
||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
||||
}
|
||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||
|
||||
autoView( out_v , out, AcceleratorWrite);
|
||||
autoView( in_v , in, AcceleratorRead);
|
||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(Stencil_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
for(int p=0;p<geom.npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
||||
}
|
||||
void MdirAll(const CoarseVector &in,std::vector<CoarseVector> &out)
|
||||
{
|
||||
this->MdirComms(in);
|
||||
int ndir=geom.npoint-1;
|
||||
if ((out.size()!=ndir)&&(out.size()!=ndir+1)) {
|
||||
std::cout <<"MdirAll out size "<< out.size()<<std::endl;
|
||||
std::cout <<"MdirAll ndir "<< ndir<<std::endl;
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
for(int p=0;p<ndir;p++){
|
||||
MdirCalc(in,out[p],p);
|
||||
}
|
||||
};
|
||||
void Mdir(const CoarseVector &in, CoarseVector &out, int dir, int disp){
|
||||
|
||||
this->MdirComms(in);
|
||||
|
||||
MdirCalc(in,out,geom.point(dir,disp));
|
||||
};
|
||||
|
||||
void Mdiag(const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
int point=geom.npoint-1;
|
||||
MdirCalc(in, out, point); // No comms
|
||||
};
|
||||
|
||||
void Mooee(const CoarseVector &in, CoarseVector &out) {
|
||||
MooeeInternal(in, out, DaggerNo, InverseNo);
|
||||
}
|
||||
|
||||
void MooeeInv(const CoarseVector &in, CoarseVector &out) {
|
||||
MooeeInternal(in, out, DaggerNo, InverseYes);
|
||||
}
|
||||
|
||||
void MooeeDag(const CoarseVector &in, CoarseVector &out) {
|
||||
MooeeInternal(in, out, DaggerYes, InverseNo);
|
||||
}
|
||||
|
||||
void MooeeInvDag(const CoarseVector &in, CoarseVector &out) {
|
||||
MooeeInternal(in, out, DaggerYes, InverseYes);
|
||||
}
|
||||
|
||||
void Meooe(const CoarseVector &in, CoarseVector &out) {
|
||||
if(in.Checkerboard() == Odd) {
|
||||
DhopEO(in, out, DaggerNo);
|
||||
} else {
|
||||
DhopOE(in, out, DaggerNo);
|
||||
}
|
||||
}
|
||||
|
||||
void MeooeDag(const CoarseVector &in, CoarseVector &out) {
|
||||
if(in.Checkerboard() == Odd) {
|
||||
DhopEO(in, out, DaggerYes);
|
||||
} else {
|
||||
DhopOE(in, out, DaggerYes);
|
||||
}
|
||||
}
|
||||
|
||||
void Dhop(const CoarseVector &in, CoarseVector &out, int dag) {
|
||||
conformable(in.Grid(), _grid); // verifies full grid
|
||||
conformable(in.Grid(), out.Grid());
|
||||
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
|
||||
DhopInternal(Stencil, A, in, out, dag);
|
||||
}
|
||||
|
||||
void DhopOE(const CoarseVector &in, CoarseVector &out, int dag) {
|
||||
conformable(in.Grid(), _cbgrid); // verifies half grid
|
||||
conformable(in.Grid(), out.Grid()); // drops the cb check
|
||||
|
||||
GRID_ASSERT(in.Checkerboard() == Even);
|
||||
out.Checkerboard() = Odd;
|
||||
|
||||
DhopInternal(StencilEven, Aodd, in, out, dag);
|
||||
}
|
||||
|
||||
void DhopEO(const CoarseVector &in, CoarseVector &out, int dag) {
|
||||
conformable(in.Grid(), _cbgrid); // verifies half grid
|
||||
conformable(in.Grid(), out.Grid()); // drops the cb check
|
||||
|
||||
GRID_ASSERT(in.Checkerboard() == Odd);
|
||||
out.Checkerboard() = Even;
|
||||
|
||||
DhopInternal(StencilOdd, Aeven, in, out, dag);
|
||||
}
|
||||
|
||||
void MooeeInternal(const CoarseVector &in, CoarseVector &out, int dag, int inv) {
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
GRID_ASSERT(in.Checkerboard() == Odd || in.Checkerboard() == Even);
|
||||
|
||||
CoarseMatrix *Aself = nullptr;
|
||||
if(in.Grid()->_isCheckerBoarded) {
|
||||
if(in.Checkerboard() == Odd) {
|
||||
Aself = (inv) ? &AselfInvOdd : &Aodd[geom.npoint-1];
|
||||
DselfInternal(StencilOdd, *Aself, in, out, dag);
|
||||
} else {
|
||||
Aself = (inv) ? &AselfInvEven : &Aeven[geom.npoint-1];
|
||||
DselfInternal(StencilEven, *Aself, in, out, dag);
|
||||
}
|
||||
} else {
|
||||
Aself = (inv) ? &AselfInv : &A[geom.npoint-1];
|
||||
DselfInternal(Stencil, *Aself, in, out, dag);
|
||||
}
|
||||
GRID_ASSERT(Aself != nullptr);
|
||||
}
|
||||
|
||||
void DselfInternal(CartesianStencil<siteVector,siteVector,DefaultImplParams> &st, CoarseMatrix &a,
|
||||
const CoarseVector &in, CoarseVector &out, int dag) {
|
||||
int point = geom.npoint-1;
|
||||
autoView( out_v, out, AcceleratorWrite);
|
||||
autoView( in_v, in, AcceleratorRead);
|
||||
autoView( st_v, st, AcceleratorRead);
|
||||
autoView( a_v, a, AcceleratorRead);
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
RealD* dag_factor_p = &dag_factor[0];
|
||||
|
||||
if(dag) {
|
||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
SE=st_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + dag_factor_p[b*nbasis+bb]*coalescedRead(a_v[ss](b,bb))*nbr(bb);
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
} else {
|
||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
SE=st_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(a_v[ss](b,bb))*nbr(bb);
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void DhopInternal(CartesianStencil<siteVector,siteVector,DefaultImplParams> &st, std::vector<CoarseMatrix> &a,
|
||||
const CoarseVector &in, CoarseVector &out, int dag) {
|
||||
SimpleCompressor<siteVector> compressor;
|
||||
|
||||
st.HaloExchange(in,compressor);
|
||||
autoView( in_v, in, AcceleratorRead);
|
||||
autoView( out_v, out, AcceleratorWrite);
|
||||
autoView( st_v , st, AcceleratorRead);
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
|
||||
// determine in what order we need the points
|
||||
int npoint = geom.npoint-1;
|
||||
deviceVector<int> points(npoint);
|
||||
for(int p=0; p<npoint; p++) {
|
||||
int val = (dag && !hermitian) ? geom.points_dagger[p] : p;
|
||||
acceleratorPut(points[p], val);
|
||||
}
|
||||
auto points_p = &points[0];
|
||||
|
||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++) {
|
||||
hAcceleratorViewContainer[p] = a[p].View(AcceleratorRead);
|
||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
||||
}
|
||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
|
||||
RealD* dag_factor_p = &dag_factor[0];
|
||||
|
||||
if(dag) {
|
||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
for(int p=0;p<npoint;p++){
|
||||
int point = points_p[p];
|
||||
SE=st_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + dag_factor_p[b*nbasis+bb]*coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
} else {
|
||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
||||
int ss = sss/nbasis;
|
||||
int b = sss%nbasis;
|
||||
calcComplex res = Zero();
|
||||
calcVector nbr;
|
||||
int ptype;
|
||||
StencilEntry *SE;
|
||||
|
||||
for(int p=0;p<npoint;p++){
|
||||
int point = points_p[p];
|
||||
SE=st_v.GetEntry(ptype,point,ss);
|
||||
|
||||
if(SE->_is_local) {
|
||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
||||
} else {
|
||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
||||
}
|
||||
acceleratorSynchronise();
|
||||
|
||||
for(int bb=0;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
||||
}
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
}
|
||||
|
||||
for(int p=0;p<npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
||||
}
|
||||
|
||||
CoarsenedMatrix(GridCartesian &CoarseGrid, int hermitian_=0) :
|
||||
_grid(&CoarseGrid),
|
||||
_cbgrid(new GridRedBlackCartesian(&CoarseGrid)),
|
||||
geom(CoarseGrid._ndimension),
|
||||
hermitian(hermitian_),
|
||||
Stencil(&CoarseGrid,geom.npoint,Even,geom.directions,geom.displacements),
|
||||
StencilEven(_cbgrid,geom.npoint,Even,geom.directions,geom.displacements),
|
||||
StencilOdd(_cbgrid,geom.npoint,Odd,geom.directions,geom.displacements),
|
||||
A(geom.npoint,&CoarseGrid),
|
||||
Aeven(geom.npoint,_cbgrid),
|
||||
Aodd(geom.npoint,_cbgrid),
|
||||
AselfInv(&CoarseGrid),
|
||||
AselfInvEven(_cbgrid),
|
||||
AselfInvOdd(_cbgrid),
|
||||
dag_factor(nbasis*nbasis)
|
||||
{
|
||||
fillFactor();
|
||||
};
|
||||
|
||||
CoarsenedMatrix(GridCartesian &CoarseGrid, GridRedBlackCartesian &CoarseRBGrid, int hermitian_=0) :
|
||||
|
||||
_grid(&CoarseGrid),
|
||||
_cbgrid(&CoarseRBGrid),
|
||||
geom(CoarseGrid._ndimension),
|
||||
hermitian(hermitian_),
|
||||
Stencil(&CoarseGrid,geom.npoint,Even,geom.directions,geom.displacements),
|
||||
StencilEven(&CoarseRBGrid,geom.npoint,Even,geom.directions,geom.displacements),
|
||||
StencilOdd(&CoarseRBGrid,geom.npoint,Odd,geom.directions,geom.displacements),
|
||||
A(geom.npoint,&CoarseGrid),
|
||||
Aeven(geom.npoint,&CoarseRBGrid),
|
||||
Aodd(geom.npoint,&CoarseRBGrid),
|
||||
AselfInv(&CoarseGrid),
|
||||
AselfInvEven(&CoarseRBGrid),
|
||||
AselfInvOdd(&CoarseRBGrid),
|
||||
dag_factor(nbasis*nbasis)
|
||||
{
|
||||
fillFactor();
|
||||
};
|
||||
|
||||
void fillFactor() {
|
||||
Eigen::MatrixXd dag_factor_eigen = Eigen::MatrixXd::Ones(nbasis, nbasis);
|
||||
if(!hermitian) {
|
||||
const int nb = nbasis/2;
|
||||
dag_factor_eigen.block(0,nb,nb,nb) *= -1.0;
|
||||
dag_factor_eigen.block(nb,0,nb,nb) *= -1.0;
|
||||
}
|
||||
|
||||
// GPU readable prefactor
|
||||
std::vector<RealD> h_dag_factor(nbasis*nbasis);
|
||||
thread_for(i, nbasis*nbasis, {
|
||||
int j = i/nbasis;
|
||||
int k = i%nbasis;
|
||||
h_dag_factor[i] = dag_factor_eigen(j, k);
|
||||
});
|
||||
acceleratorCopyToDevice(&h_dag_factor[0],&dag_factor[0],dag_factor.size()*sizeof(RealD));
|
||||
}
|
||||
|
||||
void CoarsenOperator(GridBase *FineGrid,LinearOperatorBase<Lattice<Fobj> > &linop,
|
||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
||||
{
|
||||
typedef Lattice<typename Fobj::tensor_reduced> FineComplexField;
|
||||
typedef typename Fobj::scalar_type scalar_type;
|
||||
|
||||
std::cout << GridLogMessage<< "CoarsenMatrix "<< std::endl;
|
||||
|
||||
FineComplexField one(FineGrid); one=scalar_type(1.0,0.0);
|
||||
FineComplexField zero(FineGrid); zero=scalar_type(0.0,0.0);
|
||||
|
||||
std::vector<FineComplexField> masks(geom.npoint,FineGrid);
|
||||
FineComplexField imask(FineGrid); // contributions from within this block
|
||||
FineComplexField omask(FineGrid); // contributions from outwith this block
|
||||
|
||||
FineComplexField evenmask(FineGrid);
|
||||
FineComplexField oddmask(FineGrid);
|
||||
|
||||
FineField phi(FineGrid);
|
||||
FineField tmp(FineGrid);
|
||||
FineField zz(FineGrid); zz=Zero();
|
||||
FineField Mphi(FineGrid);
|
||||
FineField Mphie(FineGrid);
|
||||
FineField Mphio(FineGrid);
|
||||
std::vector<FineField> Mphi_p(geom.npoint,FineGrid);
|
||||
|
||||
Lattice<iScalar<vInteger> > coor (FineGrid);
|
||||
Lattice<iScalar<vInteger> > bcoor(FineGrid);
|
||||
Lattice<iScalar<vInteger> > bcb (FineGrid); bcb = Zero();
|
||||
|
||||
CoarseVector iProj(Grid());
|
||||
CoarseVector oProj(Grid());
|
||||
CoarseVector SelfProj(Grid());
|
||||
CoarseComplexField iZProj(Grid());
|
||||
CoarseComplexField oZProj(Grid());
|
||||
|
||||
CoarseScalar InnerProd(Grid());
|
||||
|
||||
std::cout << GridLogMessage<< "CoarsenMatrix Orthog "<< std::endl;
|
||||
// Orthogonalise the subblocks over the basis
|
||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
||||
|
||||
// Compute the matrix elements of linop between this orthonormal
|
||||
// set of vectors.
|
||||
std::cout << GridLogMessage<< "CoarsenMatrix masks "<< std::endl;
|
||||
int self_stencil=-1;
|
||||
for(int p=0;p<geom.npoint;p++)
|
||||
{
|
||||
int dir = geom.directions[p];
|
||||
int disp = geom.displacements[p];
|
||||
A[p]=Zero();
|
||||
if( geom.displacements[p]==0){
|
||||
self_stencil=p;
|
||||
}
|
||||
|
||||
Integer block=(FineGrid->_rdimensions[dir])/(Grid()->_rdimensions[dir]);
|
||||
|
||||
LatticeCoordinate(coor,dir);
|
||||
|
||||
///////////////////////////////////////////////////////
|
||||
// Work out even and odd block checkerboarding for fast diagonal term
|
||||
///////////////////////////////////////////////////////
|
||||
if ( disp==1 ) {
|
||||
bcb = bcb + div(coor,block);
|
||||
}
|
||||
|
||||
if ( disp==0 ) {
|
||||
masks[p]= Zero();
|
||||
} else if ( disp==1 ) {
|
||||
masks[p] = where(mod(coor,block)==(block-1),one,zero);
|
||||
} else if ( disp==-1 ) {
|
||||
masks[p] = where(mod(coor,block)==(Integer)0,one,zero);
|
||||
}
|
||||
}
|
||||
evenmask = where(mod(bcb,2)==(Integer)0,one,zero);
|
||||
oddmask = one-evenmask;
|
||||
|
||||
GRID_ASSERT(self_stencil!=-1);
|
||||
|
||||
for(int i=0;i<nbasis;i++){
|
||||
|
||||
phi=Subspace.subspace[i];
|
||||
|
||||
std::cout << GridLogMessage<< "CoarsenMatrix vector "<<i << std::endl;
|
||||
linop.OpDirAll(phi,Mphi_p);
|
||||
linop.OpDiag (phi,Mphi_p[geom.npoint-1]);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
|
||||
Mphi = Mphi_p[p];
|
||||
|
||||
int dir = geom.directions[p];
|
||||
int disp = geom.displacements[p];
|
||||
|
||||
if ( (disp==-1) || (!hermitian ) ) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// Pick out contributions coming from this cell and neighbour cell
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
omask = masks[p];
|
||||
imask = one-omask;
|
||||
|
||||
for(int j=0;j<nbasis;j++){
|
||||
|
||||
blockMaskedInnerProduct(oZProj,omask,Subspace.subspace[j],Mphi);
|
||||
|
||||
autoView( iZProj_v , iZProj, AcceleratorRead) ;
|
||||
autoView( oZProj_v , oZProj, AcceleratorRead) ;
|
||||
autoView( A_p , A[p], AcceleratorWrite);
|
||||
autoView( A_self , A[self_stencil], AcceleratorWrite);
|
||||
|
||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_p[ss](j,i),oZProj_v(ss)); });
|
||||
if ( hermitian && (disp==-1) ) {
|
||||
for(int pp=0;pp<geom.npoint;pp++){// Find the opposite link and set <j|A|i> = <i|A|j>*
|
||||
int dirp = geom.directions[pp];
|
||||
int dispp = geom.displacements[pp];
|
||||
if ( (dirp==dir) && (dispp==1) ){
|
||||
auto sft = conjugate(Cshift(oZProj,dir,1));
|
||||
autoView( sft_v , sft , AcceleratorWrite);
|
||||
autoView( A_pp , A[pp], AcceleratorWrite);
|
||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_pp[ss](i,j),sft_v(ss)); });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////
|
||||
// Faster alternate self coupling.. use hermiticity to save 2x
|
||||
///////////////////////////////////////////
|
||||
{
|
||||
mult(tmp,phi,evenmask); linop.Op(tmp,Mphie);
|
||||
mult(tmp,phi,oddmask ); linop.Op(tmp,Mphio);
|
||||
|
||||
{
|
||||
autoView( tmp_ , tmp, AcceleratorWrite);
|
||||
autoView( evenmask_ , evenmask, AcceleratorRead);
|
||||
autoView( oddmask_ , oddmask, AcceleratorRead);
|
||||
autoView( Mphie_ , Mphie, AcceleratorRead);
|
||||
autoView( Mphio_ , Mphio, AcceleratorRead);
|
||||
accelerator_for(ss, FineGrid->oSites(), Fobj::Nsimd(),{
|
||||
coalescedWrite(tmp_[ss],evenmask_(ss)*Mphie_(ss) + oddmask_(ss)*Mphio_(ss));
|
||||
});
|
||||
}
|
||||
|
||||
blockProject(SelfProj,tmp,Subspace.subspace);
|
||||
|
||||
autoView( SelfProj_ , SelfProj, AcceleratorRead);
|
||||
autoView( A_self , A[self_stencil], AcceleratorWrite);
|
||||
|
||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{
|
||||
for(int j=0;j<nbasis;j++){
|
||||
coalescedWrite(A_self[ss](j,i), SelfProj_(ss)(j));
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
if(hermitian) {
|
||||
std::cout << GridLogMessage << " ForceHermitian, new code "<<std::endl;
|
||||
}
|
||||
|
||||
InvertSelfStencilLink(); std::cout << GridLogMessage << "Coarse self link inverted" << std::endl;
|
||||
FillHalfCbs(); std::cout << GridLogMessage << "Coarse half checkerboards filled" << std::endl;
|
||||
}
|
||||
|
||||
void InvertSelfStencilLink() {
|
||||
std::cout << GridLogDebug << "CoarsenedMatrix::InvertSelfStencilLink" << std::endl;
|
||||
int localVolume = Grid()->lSites();
|
||||
|
||||
typedef typename Cobj::scalar_object scalar_object;
|
||||
|
||||
autoView(Aself_v, A[geom.npoint-1], CpuRead);
|
||||
autoView(AselfInv_v, AselfInv, CpuWrite);
|
||||
thread_for(site, localVolume, { // NOTE: Not able to bring this to GPU because of Eigen + peek/poke
|
||||
Eigen::MatrixXcd selfLinkEigen = Eigen::MatrixXcd::Zero(nbasis, nbasis);
|
||||
Eigen::MatrixXcd selfLinkInvEigen = Eigen::MatrixXcd::Zero(nbasis, nbasis);
|
||||
|
||||
scalar_object selfLink = Zero();
|
||||
scalar_object selfLinkInv = Zero();
|
||||
|
||||
Coordinate lcoor;
|
||||
|
||||
Grid()->LocalIndexToLocalCoor(site, lcoor);
|
||||
peekLocalSite(selfLink, Aself_v, lcoor);
|
||||
|
||||
for (int i = 0; i < nbasis; ++i)
|
||||
for (int j = 0; j < nbasis; ++j)
|
||||
selfLinkEigen(i, j) = static_cast<ComplexD>(TensorRemove(selfLink(i, j)));
|
||||
|
||||
selfLinkInvEigen = selfLinkEigen.inverse();
|
||||
|
||||
for(int i = 0; i < nbasis; ++i)
|
||||
for(int j = 0; j < nbasis; ++j)
|
||||
selfLinkInv(i, j) = selfLinkInvEigen(i, j);
|
||||
|
||||
pokeLocalSite(selfLinkInv, AselfInv_v, lcoor);
|
||||
});
|
||||
}
|
||||
|
||||
void FillHalfCbs() {
|
||||
std::cout << GridLogDebug << "CoarsenedMatrix::FillHalfCbs" << std::endl;
|
||||
for(int p = 0; p < geom.npoint; ++p) {
|
||||
pickCheckerboard(Even, Aeven[p], A[p]);
|
||||
pickCheckerboard(Odd, Aodd[p], A[p]);
|
||||
}
|
||||
pickCheckerboard(Even, AselfInvEven, AselfInv);
|
||||
pickCheckerboard(Odd, AselfInvOdd, AselfInv);
|
||||
}
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,629 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/GeneralCoarsenedMatrix.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
#include <Grid/qcd/QCD.h> // needed for Dagger(Yes|No), Inverse(Yes|No)
|
||||
|
||||
#include <Grid/lattice/PaddedCell.h>
|
||||
#include <Grid/stencil/GeneralLocalStencil.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// Fine Object == (per site) type of fine field
|
||||
// nbasis == number of deflation vectors
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class GeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
|
||||
typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
|
||||
typedef iVector<CComplex,nbasis > siteVector;
|
||||
typedef iMatrix<CComplex,nbasis > siteMatrix;
|
||||
typedef Lattice<iScalar<CComplex> > CoarseComplexField;
|
||||
typedef Lattice<siteVector> CoarseVector;
|
||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
||||
typedef iVector<CComplex,nbasis > Cvec;
|
||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj > FineField;
|
||||
typedef Lattice<CComplex > FineComplexField;
|
||||
typedef CoarseVector Field;
|
||||
////////////////////
|
||||
// Data members
|
||||
////////////////////
|
||||
int hermitian;
|
||||
GridBase * _FineGrid;
|
||||
GridCartesian * _CoarseGrid;
|
||||
NonLocalStencilGeometry &geom;
|
||||
PaddedCell Cell;
|
||||
GeneralLocalStencil Stencil;
|
||||
|
||||
std::vector<CoarseMatrix> _A;
|
||||
std::vector<CoarseMatrix> _Adag;
|
||||
std::vector<CoarseVector> MultTemporaries;
|
||||
|
||||
///////////////////////
|
||||
// Interface
|
||||
///////////////////////
|
||||
GridBase * Grid(void) { return _CoarseGrid; }; // this is all the linalg routines need to know
|
||||
GridBase * FineGrid(void) { return _FineGrid; }; // this is all the linalg routines need to know
|
||||
GridCartesian * CoarseGrid(void) { return _CoarseGrid; }; // this is all the linalg routines need to know
|
||||
|
||||
/* void ShiftMatrix(RealD shift)
|
||||
{
|
||||
int Nd=_FineGrid->Nd();
|
||||
Coordinate zero_shift(Nd,0);
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
if ( zero_shift==geom.shifts[p] ) {
|
||||
_A[p] = _A[p]+shift;
|
||||
// _Adag[p] = _Adag[p]+shift;
|
||||
}
|
||||
}
|
||||
}
|
||||
void ProjectNearestNeighbour(RealD shift, GeneralCoarseOp &CopyMe)
|
||||
{
|
||||
int nfound=0;
|
||||
std::cout << GridLogMessage <<"GeneralCoarsenedMatrix::ProjectNearestNeighbour "<< CopyMe._A[0].Grid()<<std::endl;
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
for(int pp=0;pp<CopyMe.geom.npoint;pp++){
|
||||
// Search for the same relative shift
|
||||
// Avoids brutal handling of Grid pointers
|
||||
if ( CopyMe.geom.shifts[pp]==geom.shifts[p] ) {
|
||||
_A[p] = CopyMe.Cell.Extract(CopyMe._A[pp]);
|
||||
// _Adag[p] = CopyMe.Cell.Extract(CopyMe._Adag[pp]);
|
||||
nfound++;
|
||||
}
|
||||
}
|
||||
}
|
||||
GRID_ASSERT(nfound==geom.npoint);
|
||||
ExchangeCoarseLinks();
|
||||
}
|
||||
*/
|
||||
|
||||
GeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridBase *FineGrid, GridCartesian * CoarseGrid)
|
||||
: geom(_geom),
|
||||
_FineGrid(FineGrid),
|
||||
_CoarseGrid(CoarseGrid),
|
||||
hermitian(1),
|
||||
Cell(_geom.Depth(),_CoarseGrid),
|
||||
Stencil(Cell.grids.back(),geom.shifts)
|
||||
{
|
||||
{
|
||||
int npoint = _geom.npoint;
|
||||
}
|
||||
_A.resize(geom.npoint,CoarseGrid);
|
||||
// _Adag.resize(geom.npoint,CoarseGrid);
|
||||
}
|
||||
void M (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
Mult(_A,in,out);
|
||||
}
|
||||
void Mdag (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
GRID_ASSERT(hermitian);
|
||||
Mult(_A,in,out);
|
||||
// if ( hermitian ) M(in,out);
|
||||
// else Mult(_Adag,in,out);
|
||||
}
|
||||
void Mult (std::vector<CoarseMatrix> &A,const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
RealD tviews=0; RealD ttot=0; RealD tmult=0; RealD texch=0; RealD text=0; RealD ttemps=0; RealD tcopy=0;
|
||||
RealD tmult2=0;
|
||||
|
||||
ttot=-usecond();
|
||||
conformable(CoarseGrid(),in.Grid());
|
||||
conformable(in.Grid(),out.Grid());
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
CoarseVector tin=in;
|
||||
|
||||
texch-=usecond();
|
||||
CoarseVector pin = Cell.ExchangePeriodic(tin);
|
||||
texch+=usecond();
|
||||
|
||||
CoarseVector pout(pin.Grid());
|
||||
|
||||
int npoint = geom.npoint;
|
||||
typedef LatticeView<Cobj> Aview;
|
||||
typedef LatticeView<Cvec> Vview;
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
|
||||
int64_t osites=pin.Grid()->oSites();
|
||||
|
||||
RealD flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
|
||||
RealD bytes = 1.0*osites*sizeof(siteMatrix)*npoint
|
||||
+ 2.0*osites*sizeof(siteVector)*npoint;
|
||||
|
||||
{
|
||||
tviews-=usecond();
|
||||
autoView( in_v , pin, AcceleratorRead);
|
||||
autoView( out_v , pout, AcceleratorWriteDiscard);
|
||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
||||
tviews+=usecond();
|
||||
|
||||
// Static and prereserve to keep UVM region live and not resized across multiple calls
|
||||
ttemps-=usecond();
|
||||
MultTemporaries.resize(npoint,pin.Grid());
|
||||
ttemps+=usecond();
|
||||
std::vector<Aview> AcceleratorViewContainer_h;
|
||||
std::vector<Vview> AcceleratorVecViewContainer_h;
|
||||
|
||||
tviews-=usecond();
|
||||
for(int p=0;p<npoint;p++) {
|
||||
AcceleratorViewContainer_h.push_back( A[p].View(AcceleratorRead));
|
||||
AcceleratorVecViewContainer_h.push_back(MultTemporaries[p].View(AcceleratorWrite));
|
||||
}
|
||||
tviews+=usecond();
|
||||
|
||||
static deviceVector<Aview> AcceleratorViewContainer; AcceleratorViewContainer.resize(npoint);
|
||||
static deviceVector<Vview> AcceleratorVecViewContainer; AcceleratorVecViewContainer.resize(npoint);
|
||||
|
||||
auto Aview_p = &AcceleratorViewContainer[0];
|
||||
auto Vview_p = &AcceleratorVecViewContainer[0];
|
||||
tcopy-=usecond();
|
||||
acceleratorCopyToDevice(&AcceleratorViewContainer_h[0],&AcceleratorViewContainer[0],npoint *sizeof(Aview));
|
||||
acceleratorCopyToDevice(&AcceleratorVecViewContainer_h[0],&AcceleratorVecViewContainer[0],npoint *sizeof(Vview));
|
||||
tcopy+=usecond();
|
||||
|
||||
tmult-=usecond();
|
||||
accelerator_for(spb, osites*nbasis*npoint, Nsimd, {
|
||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
||||
int32_t ss = spb/(nbasis*npoint);
|
||||
int32_t bp = spb%(nbasis*npoint);
|
||||
int32_t point= bp/nbasis;
|
||||
int32_t b = bp%nbasis;
|
||||
auto SE = Stencil_v.GetEntry(point,ss);
|
||||
auto nbr = coalescedReadGeneralPermute(in_v[SE->_offset],SE->_permute,Nd);
|
||||
auto res = coalescedRead(Aview_p[point][ss](0,b))*nbr(0);
|
||||
for(int bb=1;bb<nbasis;bb++) {
|
||||
res = res + coalescedRead(Aview_p[point][ss](bb,b))*nbr(bb);
|
||||
}
|
||||
coalescedWrite(Vview_p[point][ss](b),res);
|
||||
});
|
||||
tmult2-=usecond();
|
||||
accelerator_for(sb, osites*nbasis, Nsimd, {
|
||||
int ss = sb/nbasis;
|
||||
int b = sb%nbasis;
|
||||
auto res = coalescedRead(Vview_p[0][ss](b));
|
||||
for(int point=1;point<npoint;point++){
|
||||
res = res + coalescedRead(Vview_p[point][ss](b));
|
||||
}
|
||||
coalescedWrite(out_v[ss](b),res);
|
||||
});
|
||||
tmult2+=usecond();
|
||||
tmult+=usecond();
|
||||
for(int p=0;p<npoint;p++) {
|
||||
AcceleratorViewContainer_h[p].ViewClose();
|
||||
AcceleratorVecViewContainer_h[p].ViewClose();
|
||||
}
|
||||
}
|
||||
|
||||
text-=usecond();
|
||||
out = Cell.Extract(pout);
|
||||
text+=usecond();
|
||||
ttot+=usecond();
|
||||
|
||||
std::cout << GridLogPerformance<<"Coarse 1rhs Mult Aviews "<<tviews<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Mult exch "<<texch<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Mult mult "<<tmult<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<" of which mult2 "<<tmult2<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Mult ext "<<text<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Mult temps "<<ttemps<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Mult copy "<<tcopy<<" us"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Mult tot "<<ttot<<" us"<<std::endl;
|
||||
// std::cout << GridLogPerformance<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Kernel flops "<< flops<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Kernel flop/s "<< flops/tmult<<" mflop/s"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse Kernel bytes/s "<< bytes/tmult<<" MB/s"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse overall flops/s "<< flops/ttot<<" mflop/s"<<std::endl;
|
||||
std::cout << GridLogPerformance<<"Coarse total bytes "<< bytes/1e6<<" MB"<<std::endl;
|
||||
|
||||
};
|
||||
|
||||
void PopulateAdag(void)
|
||||
{
|
||||
for(int64_t bidx=0;bidx<CoarseGrid()->gSites() ;bidx++){
|
||||
Coordinate bcoor;
|
||||
CoarseGrid()->GlobalIndexToGlobalCoor(bidx,bcoor);
|
||||
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
Coordinate scoor = bcoor;
|
||||
for(int mu=0;mu<bcoor.size();mu++){
|
||||
int L = CoarseGrid()->GlobalDimensions()[mu];
|
||||
scoor[mu] = (bcoor[mu] - geom.shifts[p][mu] + L) % L; // Modulo arithmetic
|
||||
}
|
||||
// Flip to poke/peekLocalSite and not too bad
|
||||
auto link = peekSite(_A[p],scoor);
|
||||
int pp = geom.Reverse(p);
|
||||
pokeSite(adj(link),_Adag[pp],bcoor);
|
||||
}
|
||||
}
|
||||
}
|
||||
/////////////////////////////////////////////////////////////
|
||||
//
|
||||
// A) Only reduced flops option is to use a padded cell of depth 4
|
||||
// and apply MpcDagMpc in the padded cell.
|
||||
//
|
||||
// Makes for ONE application of MpcDagMpc per vector instead of 30 or 80.
|
||||
// With the effective cell size around (B+8)^4 perhaps 12^4/4^4 ratio
|
||||
// Cost is 81x more, same as stencil size.
|
||||
//
|
||||
// But: can eliminate comms and do as local dirichlet.
|
||||
//
|
||||
// Local exchange gauge field once.
|
||||
// Apply to all vectors, local only computation.
|
||||
// Must exchange ghost subcells in reverse process of PaddedCell to take inner products
|
||||
//
|
||||
// B) Can reduce cost: pad by 1, apply Deo (4^4+6^4+8^4+8^4 )/ (4x 4^4)
|
||||
// pad by 2, apply Doe
|
||||
// pad by 3, apply Deo
|
||||
// then break out 8x directions; cost is ~10x MpcDagMpc per vector
|
||||
//
|
||||
// => almost factor of 10 in setup cost, excluding data rearrangement
|
||||
//
|
||||
// Intermediates -- ignore the corner terms, leave approximate and force Hermitian
|
||||
// Intermediates -- pad by 2 and apply 1+8+24 = 33 times.
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// BFM HDCG style approach: Solve a system of equations to get Aij
|
||||
//////////////////////////////////////////////////////////
|
||||
/*
|
||||
* Here, k,l index which possible shift within the 3^Nd "ball" connected by MdagM.
|
||||
*
|
||||
* conj(phases[block]) proj[k][ block*Nvec+j ] = \sum_ball e^{i q_k . delta} < phi_{block,j} | MdagM | phi_{(block+delta),i} >
|
||||
* = \sum_ball e^{iqk.delta} A_ji
|
||||
*
|
||||
* Must invert matrix M_k,l = e^[i q_k . delta_l]
|
||||
*
|
||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
||||
*/
|
||||
#if 0
|
||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
||||
{
|
||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrix "<< std::endl;
|
||||
GridBase *grid = FineGrid();
|
||||
|
||||
RealD tproj=0.0;
|
||||
RealD teigen=0.0;
|
||||
RealD tmat=0.0;
|
||||
RealD tphase=0.0;
|
||||
RealD tinv=0.0;
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Orthogonalise the subblocks over the basis
|
||||
/////////////////////////////////////////////////////////////
|
||||
CoarseScalar InnerProd(CoarseGrid());
|
||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
||||
|
||||
const int npoint = geom.npoint;
|
||||
|
||||
Coordinate clatt = CoarseGrid()->GlobalDimensions();
|
||||
int Nd = CoarseGrid()->Nd();
|
||||
|
||||
/*
|
||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
||||
* Matrix index i is mapped to this shift via
|
||||
* geom.shifts[i]
|
||||
*
|
||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
||||
* = M_{kl} A_ji^{b.b+l}
|
||||
*
|
||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
||||
*
|
||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
||||
*
|
||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
||||
*/
|
||||
teigen-=usecond();
|
||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
ComplexD ci(0.0,1.0);
|
||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
||||
|
||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
||||
ComplexD phase(0.0,0.0);
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
phase=phase+TwoPiL*geom.shifts[k][mu]*geom.shifts[l][mu];
|
||||
}
|
||||
phase=exp(phase*ci);
|
||||
Mkl(k,l) = phase;
|
||||
}
|
||||
}
|
||||
invMkl = Mkl.inverse();
|
||||
teigen+=usecond();
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// Now compute the matrix elements of linop between the orthonormal
|
||||
// set of vectors.
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
FineField phaV(grid); // Phased block basis vector
|
||||
FineField MphaV(grid);// Matrix applied
|
||||
CoarseVector coarseInner(CoarseGrid());
|
||||
|
||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid());
|
||||
std::vector<CoarseVector> FT(npoint,CoarseGrid());
|
||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
||||
/////////////////////////////////////////////////////
|
||||
// Stick a phase on every block
|
||||
/////////////////////////////////////////////////////
|
||||
tphase-=usecond();
|
||||
CoarseComplexField coor(CoarseGrid());
|
||||
CoarseComplexField pha(CoarseGrid()); pha=Zero();
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
LatticeCoordinate(coor,mu);
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
pha = pha + (TwoPiL * geom.shifts[p][mu]) * coor;
|
||||
}
|
||||
pha =exp(pha*ci);
|
||||
phaV=Zero();
|
||||
blockZAXPY(phaV,pha,Subspace.subspace[i],phaV);
|
||||
tphase+=usecond();
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Multiple phased subspace vector by matrix and project to subspace
|
||||
// Remove local bulk phase to leave relative phases
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
tmat-=usecond();
|
||||
linop.Op(phaV,MphaV);
|
||||
tmat+=usecond();
|
||||
|
||||
tproj-=usecond();
|
||||
blockProject(coarseInner,MphaV,Subspace.subspace);
|
||||
coarseInner = conjugate(pha) * coarseInner;
|
||||
|
||||
ComputeProj[p] = coarseInner;
|
||||
tproj+=usecond();
|
||||
|
||||
}
|
||||
|
||||
tinv-=usecond();
|
||||
for(int k=0;k<npoint;k++){
|
||||
FT[k] = Zero();
|
||||
for(int l=0;l<npoint;l++){
|
||||
FT[k]= FT[k]+ invMkl(l,k)*ComputeProj[l];
|
||||
}
|
||||
|
||||
int osites=CoarseGrid()->oSites();
|
||||
autoView( A_v , _A[k], AcceleratorWrite);
|
||||
autoView( FT_v , FT[k], AcceleratorRead);
|
||||
accelerator_for(sss, osites, 1, {
|
||||
for(int j=0;j<nbasis;j++){
|
||||
A_v[sss](i,j) = FT_v[sss](j);
|
||||
}
|
||||
});
|
||||
}
|
||||
tinv+=usecond();
|
||||
}
|
||||
|
||||
// Only needed if nonhermitian
|
||||
if ( ! hermitian ) {
|
||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
||||
// PopulateAdag();
|
||||
}
|
||||
|
||||
// Need to write something to populate Adag from A
|
||||
ExchangeCoarseLinks();
|
||||
std::cout << GridLogMessage<<"CoarsenOperator eigen "<<teigen<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator phase "<<tphase<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator mat "<<tmat <<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator proj "<<tproj<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator inv "<<tinv<<" us"<<std::endl;
|
||||
}
|
||||
#else
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Galerkin projection of matrix
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
||||
{
|
||||
CoarsenOperator(linop,Subspace,Subspace);
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Petrov - Galerkin projection of matrix
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
||||
Aggregation<Fobj,CComplex,nbasis> & U,
|
||||
Aggregation<Fobj,CComplex,nbasis> & V)
|
||||
{
|
||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrix "<< std::endl;
|
||||
GridBase *grid = FineGrid();
|
||||
|
||||
RealD tproj=0.0;
|
||||
RealD teigen=0.0;
|
||||
RealD tmat=0.0;
|
||||
RealD tphase=0.0;
|
||||
RealD tphaseBZ=0.0;
|
||||
RealD tinv=0.0;
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Orthogonalise the subblocks over the basis
|
||||
/////////////////////////////////////////////////////////////
|
||||
CoarseScalar InnerProd(CoarseGrid());
|
||||
blockOrthogonalise(InnerProd,V.subspace);
|
||||
blockOrthogonalise(InnerProd,U.subspace);
|
||||
|
||||
const int npoint = geom.npoint;
|
||||
|
||||
Coordinate clatt = CoarseGrid()->GlobalDimensions();
|
||||
int Nd = CoarseGrid()->Nd();
|
||||
|
||||
/*
|
||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
||||
* Matrix index i is mapped to this shift via
|
||||
* geom.shifts[i]
|
||||
*
|
||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
||||
* = M_{kl} A_ji^{b.b+l}
|
||||
*
|
||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
||||
*
|
||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
||||
*
|
||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
||||
*/
|
||||
teigen-=usecond();
|
||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
ComplexD ci(0.0,1.0);
|
||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
||||
|
||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
||||
ComplexD phase(0.0,0.0);
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
phase=phase+TwoPiL*geom.shifts[k][mu]*geom.shifts[l][mu];
|
||||
}
|
||||
phase=exp(phase*ci);
|
||||
Mkl(k,l) = phase;
|
||||
}
|
||||
}
|
||||
invMkl = Mkl.inverse();
|
||||
teigen+=usecond();
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// Now compute the matrix elements of linop between the orthonormal
|
||||
// set of vectors.
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
FineField phaV(grid); // Phased block basis vector
|
||||
FineField MphaV(grid);// Matrix applied
|
||||
std::vector<FineComplexField> phaF(npoint,grid);
|
||||
std::vector<CoarseComplexField> pha(npoint,CoarseGrid());
|
||||
|
||||
CoarseVector coarseInner(CoarseGrid());
|
||||
|
||||
typedef typename CComplex::scalar_type SComplex;
|
||||
FineComplexField one(grid); one=SComplex(1.0);
|
||||
FineComplexField zz(grid); zz = Zero();
|
||||
tphase=-usecond();
|
||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
||||
/////////////////////////////////////////////////////
|
||||
// Stick a phase on every block
|
||||
/////////////////////////////////////////////////////
|
||||
CoarseComplexField coor(CoarseGrid());
|
||||
pha[p]=Zero();
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
LatticeCoordinate(coor,mu);
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
pha[p] = pha[p] + (TwoPiL * geom.shifts[p][mu]) * coor;
|
||||
}
|
||||
pha[p] =exp(pha[p]*ci);
|
||||
|
||||
blockZAXPY(phaF[p],pha[p],one,zz);
|
||||
|
||||
}
|
||||
tphase+=usecond();
|
||||
|
||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid());
|
||||
std::vector<CoarseVector> FT(npoint,CoarseGrid());
|
||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
||||
tphaseBZ-=usecond();
|
||||
phaV = phaF[p]*V.subspace[i];
|
||||
tphaseBZ+=usecond();
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Multiple phased subspace vector by matrix and project to subspace
|
||||
// Remove local bulk phase to leave relative phases
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
tmat-=usecond();
|
||||
linop.Op(phaV,MphaV);
|
||||
tmat+=usecond();
|
||||
// std::cout << i << " " <<p << " MphaV "<<norm2(MphaV)<<" "<<norm2(phaV)<<std::endl;
|
||||
|
||||
tproj-=usecond();
|
||||
blockProject(coarseInner,MphaV,U.subspace);
|
||||
coarseInner = conjugate(pha[p]) * coarseInner;
|
||||
|
||||
ComputeProj[p] = coarseInner;
|
||||
tproj+=usecond();
|
||||
// std::cout << i << " " <<p << " ComputeProj "<<norm2(ComputeProj[p])<<std::endl;
|
||||
|
||||
}
|
||||
|
||||
tinv-=usecond();
|
||||
for(int k=0;k<npoint;k++){
|
||||
FT[k] = Zero();
|
||||
for(int l=0;l<npoint;l++){
|
||||
FT[k]= FT[k]+ invMkl(l,k)*ComputeProj[l];
|
||||
}
|
||||
|
||||
int osites=CoarseGrid()->oSites();
|
||||
autoView( A_v , _A[k], AcceleratorWrite);
|
||||
autoView( FT_v , FT[k], AcceleratorRead);
|
||||
accelerator_for(sss, osites, 1, {
|
||||
for(int j=0;j<nbasis;j++){
|
||||
A_v[sss](i,j) = FT_v[sss](j);
|
||||
}
|
||||
});
|
||||
}
|
||||
tinv+=usecond();
|
||||
}
|
||||
|
||||
// Only needed if nonhermitian
|
||||
if ( ! hermitian ) {
|
||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
||||
// PopulateAdag();
|
||||
}
|
||||
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
std::cout << " _A["<<p<<"] "<<norm2(_A[p])<<std::endl;
|
||||
}
|
||||
|
||||
// Need to write something to populate Adag from A
|
||||
ExchangeCoarseLinks();
|
||||
std::cout << GridLogMessage<<"CoarsenOperator eigen "<<teigen<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator phase "<<tphase<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator mat "<<tmat <<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator proj "<<tproj<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator inv "<<tinv<<" us"<<std::endl;
|
||||
}
|
||||
#endif
|
||||
void ExchangeCoarseLinks(void){
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
_A[p] = Cell.ExchangePeriodic(_A[p]);
|
||||
// _Adag[p]= Cell.ExchangePeriodic(_Adag[p]);
|
||||
}
|
||||
}
|
||||
virtual void Mdiag (const Field &in, Field &out){ GRID_ASSERT(0);};
|
||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);};
|
||||
virtual void MdirAll (const Field &in, std::vector<Field> &out){assert(0);};
|
||||
};
|
||||
|
||||
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,729 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/GeneralCoarsenedMatrixMultiRHS.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
// Fine Object == (per site) type of fine field
|
||||
// nbasis == number of deflation vectors
|
||||
template<class Fobj,class CComplex,int nbasis>
|
||||
class MultiGeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
||||
public:
|
||||
typedef typename CComplex::scalar_object SComplex;
|
||||
typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
|
||||
typedef MultiGeneralCoarsenedMatrix<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
|
||||
|
||||
typedef iVector<CComplex,nbasis > siteVector;
|
||||
typedef iMatrix<CComplex,nbasis > siteMatrix;
|
||||
typedef iVector<SComplex,nbasis > calcVector;
|
||||
typedef iMatrix<SComplex,nbasis > calcMatrix;
|
||||
typedef Lattice<iScalar<CComplex> > CoarseComplexField;
|
||||
typedef Lattice<siteVector> CoarseVector;
|
||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
||||
typedef iVector<CComplex,nbasis > Cvec;
|
||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
||||
typedef Lattice<Fobj > FineField;
|
||||
typedef Lattice<CComplex > FineComplexField;
|
||||
typedef CoarseVector Field;
|
||||
|
||||
////////////////////
|
||||
// Data members
|
||||
////////////////////
|
||||
GridCartesian * _CoarseGridMulti;
|
||||
NonLocalStencilGeometry geom;
|
||||
NonLocalStencilGeometry geom_srhs;
|
||||
PaddedCell Cell;
|
||||
GeneralLocalStencil Stencil;
|
||||
|
||||
deviceVector<calcVector> BLAS_B;
|
||||
deviceVector<calcVector> BLAS_C;
|
||||
std::vector<deviceVector<calcMatrix> > BLAS_A;
|
||||
|
||||
std::vector<deviceVector<ComplexD *> > BLAS_AP;
|
||||
std::vector<deviceVector<ComplexD *> > BLAS_BP;
|
||||
deviceVector<ComplexD *> BLAS_CP;
|
||||
|
||||
///////////////////////
|
||||
// Interface
|
||||
///////////////////////
|
||||
GridBase * Grid(void) { return _CoarseGridMulti; }; // this is all the linalg routines need to know
|
||||
GridCartesian * CoarseGrid(void) { return _CoarseGridMulti; }; // this is all the linalg routines need to know
|
||||
|
||||
// Can be used to do I/O on the operator matrices externally
|
||||
void SetMatrix (int p,CoarseMatrix & A)
|
||||
{
|
||||
GRID_ASSERT(A.size()==geom_srhs.npoint);
|
||||
GridtoBLAS(A[p],BLAS_A[p]);
|
||||
}
|
||||
void GetMatrix (int p,CoarseMatrix & A)
|
||||
{
|
||||
GRID_ASSERT(A.size()==geom_srhs.npoint);
|
||||
BLAStoGrid(A[p],BLAS_A[p]);
|
||||
}
|
||||
void CopyMatrix (GeneralCoarseOp &_Op)
|
||||
{
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
auto Aup = _Op.Cell.Extract(_Op._A[p]);
|
||||
//Unpadded
|
||||
GridtoBLAS(Aup,BLAS_A[p]);
|
||||
}
|
||||
}
|
||||
/*
|
||||
void CheckMatrix (GeneralCoarseOp &_Op)
|
||||
{
|
||||
std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
//Unpadded
|
||||
auto Aup = _Op.Cell.Extract(_Op._A[p]);
|
||||
auto Ack = Aup;
|
||||
BLAStoGrid(Ack,BLAS_A[p]);
|
||||
std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
|
||||
std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
|
||||
}
|
||||
std::cout <<"************* "<<std::endl;
|
||||
}
|
||||
*/
|
||||
|
||||
MultiGeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridCartesian *CoarseGridMulti) :
|
||||
_CoarseGridMulti(CoarseGridMulti),
|
||||
geom_srhs(_geom),
|
||||
geom(_CoarseGridMulti,_geom.hops,_geom.skip+1),
|
||||
Cell(geom.Depth(),_CoarseGridMulti),
|
||||
Stencil(Cell.grids.back(),geom.shifts) // padded cell stencil
|
||||
{
|
||||
int32_t padded_sites = Cell.grids.back()->lSites();
|
||||
int32_t unpadded_sites = CoarseGridMulti->lSites();
|
||||
|
||||
int32_t nrhs = CoarseGridMulti->FullDimensions()[0]; // # RHS
|
||||
int32_t orhs = nrhs/CComplex::Nsimd();
|
||||
|
||||
padded_sites = padded_sites/nrhs;
|
||||
unpadded_sites = unpadded_sites/nrhs;
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Device data vector storage
|
||||
/////////////////////////////////////////////////
|
||||
BLAS_A.resize(geom.npoint);
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
BLAS_A[p].resize (unpadded_sites); // no ghost zone, npoint elements
|
||||
}
|
||||
|
||||
BLAS_B.resize(nrhs *padded_sites); // includes ghost zone
|
||||
BLAS_C.resize(nrhs *unpadded_sites); // no ghost zone
|
||||
BLAS_AP.resize(geom.npoint);
|
||||
BLAS_BP.resize(geom.npoint);
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
BLAS_AP[p].resize(unpadded_sites);
|
||||
BLAS_BP[p].resize(unpadded_sites);
|
||||
}
|
||||
BLAS_CP.resize(unpadded_sites);
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Pointers to data
|
||||
/////////////////////////////////////////////////
|
||||
|
||||
// Site identity mapping for A
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
for(int ss=0;ss<unpadded_sites;ss++){
|
||||
ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
|
||||
acceleratorPut(BLAS_AP[p][ss],ptr);
|
||||
}
|
||||
}
|
||||
// Site identity mapping for C
|
||||
for(int ss=0;ss<unpadded_sites;ss++){
|
||||
ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
|
||||
acceleratorPut(BLAS_CP[ss],ptr);
|
||||
}
|
||||
|
||||
// Neighbour table is more complicated
|
||||
int32_t j=0; // Interior point counter (unpadded)
|
||||
for(int32_t s=0;s<padded_sites;s++){ // 4 volume, padded
|
||||
int ghost_zone=0;
|
||||
for(int32_t point = 0 ; point < geom.npoint; point++){
|
||||
int i=s*orhs*geom.npoint+point;
|
||||
if( Stencil._entries[i]._wrap ) { // stencil is indexed by the oSite of the CoarseGridMulti, hence orhs factor
|
||||
ghost_zone=1; // If general stencil wrapped in any direction, wrap=1
|
||||
}
|
||||
}
|
||||
|
||||
if( ghost_zone==0) {
|
||||
for(int32_t point = 0 ; point < geom.npoint; point++){
|
||||
int i=s*orhs*geom.npoint+point;
|
||||
int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite
|
||||
GRID_ASSERT(nbr<BLAS_B.size());
|
||||
ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
|
||||
acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
|
||||
}
|
||||
j++;
|
||||
}
|
||||
}
|
||||
GRID_ASSERT(j==unpadded_sites);
|
||||
}
|
||||
template<class vobj> void GridtoBLAS(const Lattice<vobj> &from,deviceVector<typename vobj::scalar_object> &to)
|
||||
{
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
|
||||
GridBase *Fg = from.Grid();
|
||||
GRID_ASSERT(!Fg->_isCheckerBoarded);
|
||||
int nd = Fg->_ndimension;
|
||||
|
||||
to.resize(Fg->lSites());
|
||||
|
||||
Coordinate LocalLatt = Fg->LocalDimensions();
|
||||
size_t nsite = 1;
|
||||
for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// do the index calc on the GPU
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Coordinate f_ostride = Fg->_ostride;
|
||||
Coordinate f_istride = Fg->_istride;
|
||||
Coordinate f_rdimensions = Fg->_rdimensions;
|
||||
|
||||
autoView(from_v,from,AcceleratorRead);
|
||||
auto to_v = &to[0];
|
||||
|
||||
const int words=sizeof(vobj)/sizeof(vector_type);
|
||||
accelerator_for(idx,nsite,1,{
|
||||
|
||||
Coordinate from_coor, base;
|
||||
Lexicographic::CoorFromIndex(base,idx,LocalLatt);
|
||||
for(int i=0;i<nd;i++){
|
||||
from_coor[i] = base[i];
|
||||
}
|
||||
int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*(from_coor[d]%f_rdimensions[d]);
|
||||
int from_lane = 0; for(int d=0;d<nd;d++) from_lane+=f_istride[d]*(from_coor[d]/f_rdimensions[d]);
|
||||
|
||||
const vector_type* from = (const vector_type *)&from_v[from_oidx];
|
||||
scalar_type* to = (scalar_type *)&to_v[idx];
|
||||
|
||||
scalar_type stmp;
|
||||
for(int w=0;w<words;w++){
|
||||
stmp = getlane(from[w], from_lane);
|
||||
to[w] = stmp;
|
||||
}
|
||||
});
|
||||
}
|
||||
template<class vobj> void BLAStoGrid(Lattice<vobj> &grid,deviceVector<typename vobj::scalar_object> &in)
|
||||
{
|
||||
typedef typename vobj::scalar_object sobj;
|
||||
typedef typename vobj::scalar_type scalar_type;
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
|
||||
GridBase *Tg = grid.Grid();
|
||||
GRID_ASSERT(!Tg->_isCheckerBoarded);
|
||||
int nd = Tg->_ndimension;
|
||||
|
||||
GRID_ASSERT(in.size()==Tg->lSites());
|
||||
|
||||
Coordinate LocalLatt = Tg->LocalDimensions();
|
||||
size_t nsite = 1;
|
||||
for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// do the index calc on the GPU
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Coordinate t_ostride = Tg->_ostride;
|
||||
Coordinate t_istride = Tg->_istride;
|
||||
Coordinate t_rdimensions = Tg->_rdimensions;
|
||||
|
||||
autoView(to_v,grid,AcceleratorWrite);
|
||||
auto from_v = &in[0];
|
||||
|
||||
const int words=sizeof(vobj)/sizeof(vector_type);
|
||||
accelerator_for(idx,nsite,1,{
|
||||
|
||||
Coordinate to_coor, base;
|
||||
Lexicographic::CoorFromIndex(base,idx,LocalLatt);
|
||||
for(int i=0;i<nd;i++){
|
||||
to_coor[i] = base[i];
|
||||
}
|
||||
int to_oidx = 0; for(int d=0;d<nd;d++) to_oidx+=t_ostride[d]*(to_coor[d]%t_rdimensions[d]);
|
||||
int to_lane = 0; for(int d=0;d<nd;d++) to_lane+=t_istride[d]*(to_coor[d]/t_rdimensions[d]);
|
||||
|
||||
vector_type* to = (vector_type *)&to_v[to_oidx];
|
||||
scalar_type* from = (scalar_type *)&from_v[idx];
|
||||
|
||||
scalar_type stmp;
|
||||
for(int w=0;w<words;w++){
|
||||
stmp=from[w];
|
||||
putlane(to[w], stmp, to_lane);
|
||||
}
|
||||
});
|
||||
}
|
||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
||||
Aggregation<Fobj,CComplex,nbasis> & Subspace,
|
||||
GridBase *CoarseGrid)
|
||||
{
|
||||
#if 0
|
||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
|
||||
|
||||
GridBase *grid = Subspace.FineGrid;
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Orthogonalise the subblocks over the basis
|
||||
/////////////////////////////////////////////////////////////
|
||||
CoarseScalar InnerProd(CoarseGrid);
|
||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
||||
|
||||
const int npoint = geom_srhs.npoint;
|
||||
|
||||
Coordinate clatt = CoarseGrid->GlobalDimensions();
|
||||
int Nd = CoarseGrid->Nd();
|
||||
/*
|
||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
||||
* Matrix index i is mapped to this shift via
|
||||
* geom.shifts[i]
|
||||
*
|
||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
||||
* = M_{kl} A_ji^{b.b+l}
|
||||
*
|
||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
||||
*
|
||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
||||
*
|
||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
||||
*/
|
||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
ComplexD ci(0.0,1.0);
|
||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
||||
|
||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
||||
ComplexD phase(0.0,0.0);
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
|
||||
}
|
||||
phase=exp(phase*ci);
|
||||
Mkl(k,l) = phase;
|
||||
}
|
||||
}
|
||||
invMkl = Mkl.inverse();
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// Now compute the matrix elements of linop between the orthonormal
|
||||
// set of vectors.
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
FineField phaV(grid); // Phased block basis vector
|
||||
FineField MphaV(grid);// Matrix applied
|
||||
std::vector<FineComplexField> phaF(npoint,grid);
|
||||
std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
|
||||
|
||||
CoarseVector coarseInner(CoarseGrid);
|
||||
|
||||
typedef typename CComplex::scalar_type SComplex;
|
||||
FineComplexField one(grid); one=SComplex(1.0);
|
||||
FineComplexField zz(grid); zz = Zero();
|
||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
||||
/////////////////////////////////////////////////////
|
||||
// Stick a phase on every block
|
||||
/////////////////////////////////////////////////////
|
||||
CoarseComplexField coor(CoarseGrid);
|
||||
pha[p]=Zero();
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
LatticeCoordinate(coor,mu);
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
|
||||
}
|
||||
pha[p] =exp(pha[p]*ci);
|
||||
|
||||
blockZAXPY(phaF[p],pha[p],one,zz);
|
||||
}
|
||||
|
||||
// Could save on temporary storage here
|
||||
std::vector<CoarseMatrix> _A;
|
||||
_A.resize(geom_srhs.npoint,CoarseGrid);
|
||||
|
||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
|
||||
CoarseVector FT(CoarseGrid);
|
||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
||||
|
||||
phaV = phaF[p]*Subspace.subspace[i];
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Multiple phased subspace vector by matrix and project to subspace
|
||||
// Remove local bulk phase to leave relative phases
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
linop.Op(phaV,MphaV);
|
||||
|
||||
// Fixme, could use batched block projector here
|
||||
blockProject(coarseInner,MphaV,Subspace.subspace);
|
||||
|
||||
coarseInner = conjugate(pha[p]) * coarseInner;
|
||||
|
||||
ComputeProj[p] = coarseInner;
|
||||
}
|
||||
|
||||
// Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
|
||||
for(int k=0;k<npoint;k++){
|
||||
|
||||
FT = Zero();
|
||||
for(int l=0;l<npoint;l++){
|
||||
FT= FT+ invMkl(l,k)*ComputeProj[l];
|
||||
}
|
||||
|
||||
int osites=CoarseGrid->oSites();
|
||||
autoView( A_v , _A[k], AcceleratorWrite);
|
||||
autoView( FT_v , FT, AcceleratorRead);
|
||||
accelerator_for(sss, osites, 1, {
|
||||
for(int j=0;j<nbasis;j++){
|
||||
A_v[sss](i,j) = FT_v[sss](j);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Only needed if nonhermitian
|
||||
// if ( ! hermitian ) {
|
||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
||||
// PopulateAdag();
|
||||
// }
|
||||
// Need to write something to populate Adag from A
|
||||
|
||||
for(int p=0;p<geom_srhs.npoint;p++){
|
||||
GridtoBLAS(_A[p],BLAS_A[p]);
|
||||
}
|
||||
/*
|
||||
Grid : Message : 11698.730546 s : CoarsenOperator eigen 1334 us
|
||||
Grid : Message : 11698.730563 s : CoarsenOperator phase 34729 us
|
||||
Grid : Message : 11698.730565 s : CoarsenOperator phaseBZ 2423814 us
|
||||
Grid : Message : 11698.730566 s : CoarsenOperator mat 127890998 us
|
||||
Grid : Message : 11698.730567 s : CoarsenOperator proj 515840840 us
|
||||
Grid : Message : 11698.730568 s : CoarsenOperator inv 103948313 us
|
||||
Takes 600s to compute matrix elements, DOMINATED by the block project.
|
||||
Easy to speed up with the batched block project.
|
||||
Store npoint vectors, get npoint x Nbasis block projection, and 81 fold faster.
|
||||
|
||||
// Block project below taks to 240s
|
||||
Grid : Message : 328.193418 s : CoarsenOperator phase 38338 us
|
||||
Grid : Message : 328.193434 s : CoarsenOperator phaseBZ 1711226 us
|
||||
Grid : Message : 328.193436 s : CoarsenOperator mat 122213270 us
|
||||
//Grid : Message : 328.193438 s : CoarsenOperator proj 1181154 us <-- this is mistimed
|
||||
//Grid : Message : 11698.730568 s : CoarsenOperator inv 103948313 us <-- Cut this ~10x if lucky by loop fusion
|
||||
*/
|
||||
#else
|
||||
RealD tproj=0.0;
|
||||
RealD tmat=0.0;
|
||||
RealD tphase=0.0;
|
||||
RealD tphaseBZ=0.0;
|
||||
RealD tinv=0.0;
|
||||
|
||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
|
||||
|
||||
GridBase *grid = Subspace.FineGrid;
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// Orthogonalise the subblocks over the basis
|
||||
/////////////////////////////////////////////////////////////
|
||||
CoarseScalar InnerProd(CoarseGrid);
|
||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
||||
|
||||
|
||||
MultiRHSBlockProject<Lattice<Fobj> > Projector;
|
||||
Projector.Allocate(nbasis,grid,CoarseGrid);
|
||||
Projector.ImportBasis(Subspace.subspace);
|
||||
|
||||
const int npoint = geom_srhs.npoint;
|
||||
|
||||
Coordinate clatt = CoarseGrid->GlobalDimensions();
|
||||
int Nd = CoarseGrid->Nd();
|
||||
/*
|
||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
||||
* Matrix index i is mapped to this shift via
|
||||
* geom.shifts[i]
|
||||
*
|
||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
||||
* = M_{kl} A_ji^{b.b+l}
|
||||
*
|
||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
||||
*
|
||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
||||
*
|
||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
||||
*/
|
||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
||||
ComplexD ci(0.0,1.0);
|
||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
||||
|
||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
||||
ComplexD phase(0.0,0.0);
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
|
||||
}
|
||||
phase=exp(phase*ci);
|
||||
Mkl(k,l) = phase;
|
||||
}
|
||||
}
|
||||
invMkl = Mkl.inverse();
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// Now compute the matrix elements of linop between the orthonormal
|
||||
// set of vectors.
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
FineField phaV(grid); // Phased block basis vector
|
||||
FineField MphaV(grid);// Matrix applied
|
||||
std::vector<FineComplexField> phaF(npoint,grid);
|
||||
std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
|
||||
|
||||
CoarseVector coarseInner(CoarseGrid);
|
||||
|
||||
tphase=-usecond();
|
||||
typedef typename CComplex::scalar_type SComplex;
|
||||
FineComplexField one(grid); one=SComplex(1.0);
|
||||
FineComplexField zz(grid); zz = Zero();
|
||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
||||
/////////////////////////////////////////////////////
|
||||
// Stick a phase on every block
|
||||
/////////////////////////////////////////////////////
|
||||
CoarseComplexField coor(CoarseGrid);
|
||||
pha[p]=Zero();
|
||||
for(int mu=0;mu<Nd;mu++){
|
||||
LatticeCoordinate(coor,mu);
|
||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
||||
pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
|
||||
}
|
||||
pha[p] =exp(pha[p]*ci);
|
||||
|
||||
blockZAXPY(phaF[p],pha[p],one,zz);
|
||||
}
|
||||
tphase+=usecond();
|
||||
|
||||
// Could save on temporary storage here
|
||||
std::vector<CoarseMatrix> _A;
|
||||
_A.resize(geom_srhs.npoint,CoarseGrid);
|
||||
|
||||
// Count use small chunks than npoint == 81 and save memory
|
||||
int batch = 9;
|
||||
std::vector<FineField> _MphaV(batch,grid);
|
||||
std::vector<CoarseVector> TmpProj(batch,CoarseGrid);
|
||||
|
||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
|
||||
CoarseVector FT(CoarseGrid);
|
||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
||||
|
||||
// std::cout << GridLogMessage << " phasing the fine vector "<<std::endl;
|
||||
// Fixme : do this in batches
|
||||
for(int p=0;p<npoint;p+=batch){ // Loop over momenta in npoint
|
||||
|
||||
for(int b=0;b<MIN(batch,npoint-p);b++){
|
||||
tphaseBZ-=usecond();
|
||||
phaV = phaF[p+b]*Subspace.subspace[i];
|
||||
tphaseBZ+=usecond();
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Multiple phased subspace vector by matrix and project to subspace
|
||||
// Remove local bulk phase to leave relative phases
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Memory footprint was an issue
|
||||
tmat-=usecond();
|
||||
linop.Op(phaV,MphaV);
|
||||
_MphaV[b] = MphaV;
|
||||
tmat+=usecond();
|
||||
}
|
||||
|
||||
// std::cout << GridLogMessage << " Calling block project "<<std::endl;
|
||||
tproj-=usecond();
|
||||
Projector.blockProject(_MphaV,TmpProj);
|
||||
tproj+=usecond();
|
||||
|
||||
// std::cout << GridLogMessage << " conj phasing the coarse vectors "<<std::endl;
|
||||
for(int b=0;b<MIN(batch,npoint-p);b++){
|
||||
ComputeProj[p+b] = conjugate(pha[p+b])*TmpProj[b];
|
||||
}
|
||||
}
|
||||
|
||||
// Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
|
||||
|
||||
// std::cout << GridLogMessage << " Starting FT inv "<<std::endl;
|
||||
tinv-=usecond();
|
||||
for(int k=0;k<npoint;k++){
|
||||
FT = Zero();
|
||||
// 81 kernel calls as many ComputeProj vectors
|
||||
// Could fuse with a vector of views, but ugly
|
||||
// Could unroll the expression and run fewer kernels -- much more attractive
|
||||
// Could also do non blocking.
|
||||
#if 0
|
||||
for(int l=0;l<npoint;l++){
|
||||
FT= FT+ invMkl(l,k)*ComputeProj[l];
|
||||
}
|
||||
#else
|
||||
const int radix = 9;
|
||||
int ll;
|
||||
for(ll=0;ll+radix-1<npoint;ll+=radix){
|
||||
// When ll = npoint-radix, ll+radix-1 = npoint-1, and we do it all.
|
||||
FT = FT
|
||||
+ invMkl(ll+0,k)*ComputeProj[ll+0]
|
||||
+ invMkl(ll+1,k)*ComputeProj[ll+1]
|
||||
+ invMkl(ll+2,k)*ComputeProj[ll+2]
|
||||
+ invMkl(ll+3,k)*ComputeProj[ll+3]
|
||||
+ invMkl(ll+4,k)*ComputeProj[ll+4]
|
||||
+ invMkl(ll+5,k)*ComputeProj[ll+5]
|
||||
+ invMkl(ll+6,k)*ComputeProj[ll+6]
|
||||
+ invMkl(ll+7,k)*ComputeProj[ll+7]
|
||||
+ invMkl(ll+8,k)*ComputeProj[ll+8];
|
||||
}
|
||||
for(int l=ll;l<npoint;l++){
|
||||
FT= FT+ invMkl(l,k)*ComputeProj[l];
|
||||
}
|
||||
#endif
|
||||
|
||||
// 1 kernel call -- must be cheaper
|
||||
int osites=CoarseGrid->oSites();
|
||||
autoView( A_v , _A[k], AcceleratorWrite);
|
||||
autoView( FT_v , FT, AcceleratorRead);
|
||||
accelerator_for(sss, osites, 1, {
|
||||
for(int j=0;j<nbasis;j++){
|
||||
A_v[sss](i,j) = FT_v[sss](j);
|
||||
}
|
||||
});
|
||||
}
|
||||
tinv+=usecond();
|
||||
}
|
||||
|
||||
// Only needed if nonhermitian
|
||||
// if ( ! hermitian ) {
|
||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
||||
// PopulateAdag();
|
||||
// }
|
||||
// Need to write something to populate Adag from A
|
||||
// std::cout << GridLogMessage << " Calling GridtoBLAS "<<std::endl;
|
||||
for(int p=0;p<geom_srhs.npoint;p++){
|
||||
GridtoBLAS(_A[p],BLAS_A[p]);
|
||||
}
|
||||
std::cout << GridLogMessage<<"CoarsenOperator phase "<<tphase<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator mat "<<tmat <<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator proj "<<tproj<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"CoarsenOperator inv "<<tinv<<" us"<<std::endl;
|
||||
#endif
|
||||
}
|
||||
void Mdag(const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
this->M(in,out);
|
||||
}
|
||||
void M (const CoarseVector &in, CoarseVector &out)
|
||||
{
|
||||
// std::cout << GridLogMessage << "New Mrhs coarse"<<std::endl;
|
||||
conformable(CoarseGrid(),in.Grid());
|
||||
conformable(in.Grid(),out.Grid());
|
||||
out.Checkerboard() = in.Checkerboard();
|
||||
|
||||
RealD t_tot;
|
||||
RealD t_exch;
|
||||
RealD t_GtoB;
|
||||
RealD t_BtoG;
|
||||
RealD t_mult;
|
||||
|
||||
t_tot=-usecond();
|
||||
CoarseVector tin=in;
|
||||
t_exch=-usecond();
|
||||
CoarseVector pin = Cell.ExchangePeriodic(tin); //padded input
|
||||
t_exch+=usecond();
|
||||
|
||||
CoarseVector pout(pin.Grid());
|
||||
|
||||
int npoint = geom.npoint;
|
||||
typedef calcMatrix* Aview;
|
||||
typedef LatticeView<Cvec> Vview;
|
||||
|
||||
const int Nsimd = CComplex::Nsimd();
|
||||
|
||||
int64_t nrhs =pin.Grid()->GlobalDimensions()[0];
|
||||
GRID_ASSERT(nrhs>=1);
|
||||
|
||||
RealD flops,bytes;
|
||||
int64_t osites=in.Grid()->oSites(); // unpadded
|
||||
int64_t unpadded_vol = CoarseGrid()->lSites()/nrhs;
|
||||
|
||||
flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
|
||||
bytes = 1.0*osites*sizeof(siteMatrix)*npoint/pin.Grid()->GlobalDimensions()[0]
|
||||
+ 2.0*osites*sizeof(siteVector)*npoint;
|
||||
|
||||
|
||||
t_GtoB=-usecond();
|
||||
GridtoBLAS(pin,BLAS_B);
|
||||
t_GtoB+=usecond();
|
||||
|
||||
GridBLAS BLAS;
|
||||
|
||||
t_mult=-usecond();
|
||||
for(int p=0;p<geom.npoint;p++){
|
||||
RealD c = 1.0;
|
||||
if (p==0) c = 0.0;
|
||||
ComplexD beta(c);
|
||||
|
||||
BLAS.gemmBatched(nbasis,nrhs,nbasis,
|
||||
ComplexD(1.0),
|
||||
BLAS_AP[p],
|
||||
BLAS_BP[p],
|
||||
ComplexD(c),
|
||||
BLAS_CP);
|
||||
}
|
||||
BLAS.synchronise();
|
||||
t_mult+=usecond();
|
||||
|
||||
t_BtoG=-usecond();
|
||||
BLAStoGrid(out,BLAS_C);
|
||||
t_BtoG+=usecond();
|
||||
t_tot+=usecond();
|
||||
/*
|
||||
std::cout << GridLogMessage << "New Mrhs coarse DONE "<<std::endl;
|
||||
std::cout << GridLogMessage<<"Coarse Mult exch "<<t_exch<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"Coarse Mult mult "<<t_mult<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"Coarse Mult GtoB "<<t_GtoB<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"Coarse Mult BtoG "<<t_BtoG<<" us"<<std::endl;
|
||||
std::cout << GridLogMessage<<"Coarse Mult tot "<<t_tot<<" us"<<std::endl;
|
||||
*/
|
||||
// std::cout << GridLogMessage<<std::endl;
|
||||
// std::cout << GridLogMessage<<"Coarse Kernel flops "<< flops<<std::endl;
|
||||
// std::cout << GridLogMessage<<"Coarse Kernel flop/s "<< flops/t_mult<<" mflop/s"<<std::endl;
|
||||
// std::cout << GridLogMessage<<"Coarse Kernel bytes/s "<< bytes/t_mult/1000<<" GB/s"<<std::endl;
|
||||
// std::cout << GridLogMessage<<"Coarse overall flops/s "<< flops/t_tot<<" mflop/s"<<std::endl;
|
||||
// std::cout << GridLogMessage<<"Coarse total bytes "<< bytes/1e6<<" MB"<<std::endl;
|
||||
};
|
||||
virtual void Mdiag (const Field &in, Field &out){ GRID_ASSERT(0);};
|
||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);};
|
||||
virtual void MdirAll (const Field &in, std::vector<Field> &out){assert(0);};
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,238 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/algorithms/GeneralCoarsenedMatrix.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Geometry class in cartesian case
|
||||
/////////////////////////////////////////////////////////////////
|
||||
|
||||
class Geometry {
|
||||
public:
|
||||
int npoint;
|
||||
int base;
|
||||
std::vector<int> directions ;
|
||||
std::vector<int> displacements;
|
||||
std::vector<int> points_dagger;
|
||||
|
||||
Geometry(int _d) {
|
||||
|
||||
base = (_d==5) ? 1:0;
|
||||
|
||||
// make coarse grid stencil for 4d , not 5d
|
||||
if ( _d==5 ) _d=4;
|
||||
|
||||
npoint = 2*_d+1;
|
||||
directions.resize(npoint);
|
||||
displacements.resize(npoint);
|
||||
points_dagger.resize(npoint);
|
||||
for(int d=0;d<_d;d++){
|
||||
directions[d ] = d+base;
|
||||
directions[d+_d] = d+base;
|
||||
displacements[d ] = +1;
|
||||
displacements[d+_d]= -1;
|
||||
points_dagger[d ] = d+_d;
|
||||
points_dagger[d+_d] = d;
|
||||
}
|
||||
directions [2*_d]=0;
|
||||
displacements[2*_d]=0;
|
||||
points_dagger[2*_d]=2*_d;
|
||||
}
|
||||
|
||||
int point(int dir, int disp) {
|
||||
GRID_ASSERT(disp == -1 || disp == 0 || disp == 1);
|
||||
GRID_ASSERT(base+0 <= dir && dir < base+4);
|
||||
|
||||
// directions faster index = new indexing
|
||||
// 4d (base = 0):
|
||||
// point 0 1 2 3 4 5 6 7 8
|
||||
// dir 0 1 2 3 0 1 2 3 0
|
||||
// disp +1 +1 +1 +1 -1 -1 -1 -1 0
|
||||
// 5d (base = 1):
|
||||
// point 0 1 2 3 4 5 6 7 8
|
||||
// dir 1 2 3 4 1 2 3 4 0
|
||||
// disp +1 +1 +1 +1 -1 -1 -1 -1 0
|
||||
|
||||
// displacements faster index = old indexing
|
||||
// 4d (base = 0):
|
||||
// point 0 1 2 3 4 5 6 7 8
|
||||
// dir 0 0 1 1 2 2 3 3 0
|
||||
// disp +1 -1 +1 -1 +1 -1 +1 -1 0
|
||||
// 5d (base = 1):
|
||||
// point 0 1 2 3 4 5 6 7 8
|
||||
// dir 1 1 2 2 3 3 4 4 0
|
||||
// disp +1 -1 +1 -1 +1 -1 +1 -1 0
|
||||
|
||||
if(dir == 0 and disp == 0)
|
||||
return 8;
|
||||
else // New indexing
|
||||
return (1 - disp) / 2 * 4 + dir - base;
|
||||
// else // Old indexing
|
||||
// return (4 * (dir - base) + 1 - disp) / 2;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// Less local equivalent of Geometry class in cartesian case
|
||||
/////////////////////////////////////////////////////////////////
|
||||
class NonLocalStencilGeometry {
|
||||
public:
|
||||
// int depth;
|
||||
int skip;
|
||||
int hops;
|
||||
int npoint;
|
||||
std::vector<Coordinate> shifts;
|
||||
Coordinate stencil_size;
|
||||
Coordinate stencil_lo;
|
||||
Coordinate stencil_hi;
|
||||
GridCartesian *grid;
|
||||
GridCartesian *Grid() {return grid;};
|
||||
int Depth(void){return 1;}; // Ghost zone depth
|
||||
int Hops(void){return hops;}; // # of hops=> level of corner fill in in stencil
|
||||
int DimSkip(void){return skip;};
|
||||
|
||||
virtual ~NonLocalStencilGeometry() {};
|
||||
|
||||
int Reverse(int point)
|
||||
{
|
||||
int Nd = Grid()->Nd();
|
||||
Coordinate shft = shifts[point];
|
||||
Coordinate rev(Nd);
|
||||
for(int mu=0;mu<Nd;mu++) rev[mu]= -shft[mu];
|
||||
for(int p=0;p<npoint;p++){
|
||||
if(rev==shifts[p]){
|
||||
return p;
|
||||
}
|
||||
}
|
||||
GRID_ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
void BuildShifts(void)
|
||||
{
|
||||
this->shifts.resize(0);
|
||||
int Nd = this->grid->Nd();
|
||||
|
||||
int dd = this->DimSkip();
|
||||
for(int s0=this->stencil_lo[dd+0];s0<=this->stencil_hi[dd+0];s0++){
|
||||
for(int s1=this->stencil_lo[dd+1];s1<=this->stencil_hi[dd+1];s1++){
|
||||
for(int s2=this->stencil_lo[dd+2];s2<=this->stencil_hi[dd+2];s2++){
|
||||
for(int s3=this->stencil_lo[dd+3];s3<=this->stencil_hi[dd+3];s3++){
|
||||
Coordinate sft(Nd,0);
|
||||
sft[dd+0] = s0;
|
||||
sft[dd+1] = s1;
|
||||
sft[dd+2] = s2;
|
||||
sft[dd+3] = s3;
|
||||
int nhops = abs(s0)+abs(s1)+abs(s2)+abs(s3);
|
||||
if(nhops<=this->hops) this->shifts.push_back(sft);
|
||||
}}}}
|
||||
this->npoint = this->shifts.size();
|
||||
std::cout << GridLogMessage << "NonLocalStencilGeometry has "<< this->npoint << " terms in stencil "<<std::endl;
|
||||
}
|
||||
|
||||
NonLocalStencilGeometry(GridCartesian *_coarse_grid,int _hops,int _skip) : grid(_coarse_grid), hops(_hops), skip(_skip)
|
||||
{
|
||||
Coordinate latt = grid->GlobalDimensions();
|
||||
stencil_size.resize(grid->Nd());
|
||||
stencil_lo.resize(grid->Nd());
|
||||
stencil_hi.resize(grid->Nd());
|
||||
for(int d=0;d<grid->Nd();d++){
|
||||
if ( latt[d] == 1 ) {
|
||||
stencil_lo[d] = 0;
|
||||
stencil_hi[d] = 0;
|
||||
stencil_size[d]= 1;
|
||||
} else if ( latt[d] == 2 ) {
|
||||
stencil_lo[d] = -1;
|
||||
stencil_hi[d] = 0;
|
||||
stencil_size[d]= 2;
|
||||
} else if ( latt[d] > 2 ) {
|
||||
stencil_lo[d] = -1;
|
||||
stencil_hi[d] = 1;
|
||||
stencil_size[d]= 3;
|
||||
}
|
||||
}
|
||||
this->BuildShifts();
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
// Need to worry about red-black now
|
||||
class NonLocalStencilGeometry4D : public NonLocalStencilGeometry {
|
||||
public:
|
||||
virtual int DerivedDimSkip(void) { return 0;};
|
||||
NonLocalStencilGeometry4D(GridCartesian *Coarse,int _hops) : NonLocalStencilGeometry(Coarse,_hops,0) { };
|
||||
virtual ~NonLocalStencilGeometry4D() {};
|
||||
};
|
||||
class NonLocalStencilGeometry5D : public NonLocalStencilGeometry {
|
||||
public:
|
||||
virtual int DerivedDimSkip(void) { return 1; };
|
||||
NonLocalStencilGeometry5D(GridCartesian *Coarse,int _hops) : NonLocalStencilGeometry(Coarse,_hops,1) { };
|
||||
virtual ~NonLocalStencilGeometry5D() {};
|
||||
};
|
||||
/*
|
||||
* Bunch of different options classes
|
||||
*/
|
||||
class NextToNextToNextToNearestStencilGeometry4D : public NonLocalStencilGeometry4D {
|
||||
public:
|
||||
NextToNextToNextToNearestStencilGeometry4D(GridCartesian *Coarse) : NonLocalStencilGeometry4D(Coarse,4)
|
||||
{
|
||||
};
|
||||
};
|
||||
class NextToNextToNextToNearestStencilGeometry5D : public NonLocalStencilGeometry5D {
|
||||
public:
|
||||
NextToNextToNextToNearestStencilGeometry5D(GridCartesian *Coarse) : NonLocalStencilGeometry5D(Coarse,4)
|
||||
{
|
||||
};
|
||||
};
|
||||
class NextToNearestStencilGeometry4D : public NonLocalStencilGeometry4D {
|
||||
public:
|
||||
NextToNearestStencilGeometry4D(GridCartesian *Coarse) : NonLocalStencilGeometry4D(Coarse,2)
|
||||
{
|
||||
};
|
||||
};
|
||||
class NextToNearestStencilGeometry5D : public NonLocalStencilGeometry5D {
|
||||
public:
|
||||
NextToNearestStencilGeometry5D(GridCartesian *Coarse) : NonLocalStencilGeometry5D(Coarse,2)
|
||||
{
|
||||
};
|
||||
};
|
||||
class NearestStencilGeometry4D : public NonLocalStencilGeometry4D {
|
||||
public:
|
||||
NearestStencilGeometry4D(GridCartesian *Coarse) : NonLocalStencilGeometry4D(Coarse,1)
|
||||
{
|
||||
};
|
||||
};
|
||||
class NearestStencilGeometry5D : public NonLocalStencilGeometry5D {
|
||||
public:
|
||||
NearestStencilGeometry5D(GridCartesian *Coarse) : NonLocalStencilGeometry5D(Coarse,1)
|
||||
{
|
||||
};
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
@@ -1,34 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: Grid/algorithms/multigrid/MultiGrid.h
|
||||
|
||||
Copyright (C) 2023
|
||||
|
||||
Author: Peter Boyle <pboyle@bnl.gov>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
#include <Grid/algorithms/multigrid/Aggregates.h>
|
||||
#include <Grid/algorithms/multigrid/Geometry.h>
|
||||
#include <Grid/algorithms/multigrid/CoarsenedMatrix.h>
|
||||
#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrix.h>
|
||||
#include <Grid/algorithms/multigrid/GeneralCoarsenedMatrixMultiRHS.h>
|
||||
@@ -0,0 +1,154 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
MemoryStats *MemoryProfiler::stats = nullptr;
|
||||
bool MemoryProfiler::debug = false;
|
||||
|
||||
int PointerCache::NcacheSmall = PointerCache::NcacheSmallMax;
|
||||
#ifdef GRID_CUDA
|
||||
int PointerCache::Ncache = 32;
|
||||
#else
|
||||
int PointerCache::Ncache = 8;
|
||||
#endif
|
||||
int PointerCache::Victim;
|
||||
int PointerCache::VictimSmall;
|
||||
PointerCache::PointerCacheEntry PointerCache::Entries[PointerCache::NcacheMax];
|
||||
PointerCache::PointerCacheEntry PointerCache::EntriesSmall[PointerCache::NcacheSmallMax];
|
||||
|
||||
void PointerCache::Init(void)
|
||||
{
|
||||
char * str;
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_LARGE");
|
||||
if ( str ) Ncache = atoi(str);
|
||||
if ( (Ncache<0) || (Ncache > NcacheMax)) Ncache = NcacheMax;
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_SMALL");
|
||||
if ( str ) NcacheSmall = atoi(str);
|
||||
if ( (NcacheSmall<0) || (NcacheSmall > NcacheSmallMax)) NcacheSmall = NcacheSmallMax;
|
||||
|
||||
// printf("Aligned alloocator cache: large %d/%d small %d/%d\n",Ncache,NcacheMax,NcacheSmall,NcacheSmallMax);
|
||||
}
|
||||
void *PointerCache::Insert(void *ptr,size_t bytes)
|
||||
{
|
||||
if (bytes < GRID_ALLOC_SMALL_LIMIT )
|
||||
return Insert(ptr,bytes,EntriesSmall,NcacheSmall,VictimSmall);
|
||||
return Insert(ptr,bytes,Entries,Ncache,Victim);
|
||||
}
|
||||
void *PointerCache::Insert(void *ptr,size_t bytes,PointerCacheEntry *entries,int ncache,int &victim)
|
||||
{
|
||||
#ifdef GRID_OMP
|
||||
assert(omp_in_parallel()==0);
|
||||
#endif
|
||||
|
||||
void * ret = NULL;
|
||||
int v = -1;
|
||||
|
||||
for(int e=0;e<ncache;e++) {
|
||||
if ( entries[e].valid==0 ) {
|
||||
v=e;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ( v==-1 ) {
|
||||
v=victim;
|
||||
victim = (victim+1)%ncache;
|
||||
}
|
||||
|
||||
if ( entries[v].valid ) {
|
||||
ret = entries[v].address;
|
||||
entries[v].valid = 0;
|
||||
entries[v].address = NULL;
|
||||
entries[v].bytes = 0;
|
||||
}
|
||||
|
||||
entries[v].address=ptr;
|
||||
entries[v].bytes =bytes;
|
||||
entries[v].valid =1;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *PointerCache::Lookup(size_t bytes)
|
||||
{
|
||||
if (bytes < GRID_ALLOC_SMALL_LIMIT )
|
||||
return Lookup(bytes,EntriesSmall,NcacheSmall);
|
||||
return Lookup(bytes,Entries,Ncache);
|
||||
}
|
||||
void *PointerCache::Lookup(size_t bytes,PointerCacheEntry *entries,int ncache)
|
||||
{
|
||||
#ifdef GRID_OMP
|
||||
assert(omp_in_parallel()==0);
|
||||
#endif
|
||||
for(int e=0;e<ncache;e++){
|
||||
if ( entries[e].valid && ( entries[e].bytes == bytes ) ) {
|
||||
entries[e].valid = 0;
|
||||
return entries[e].address;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
void check_huge_pages(void *Buf,uint64_t BYTES)
|
||||
{
|
||||
#ifdef __linux__
|
||||
int fd = open("/proc/self/pagemap", O_RDONLY);
|
||||
assert(fd >= 0);
|
||||
const int page_size = 4096;
|
||||
uint64_t virt_pfn = (uint64_t)Buf / page_size;
|
||||
off_t offset = sizeof(uint64_t) * virt_pfn;
|
||||
uint64_t npages = (BYTES + page_size-1) / page_size;
|
||||
uint64_t pagedata[npages];
|
||||
uint64_t ret = lseek(fd, offset, SEEK_SET);
|
||||
assert(ret == offset);
|
||||
ret = ::read(fd, pagedata, sizeof(uint64_t)*npages);
|
||||
assert(ret == sizeof(uint64_t) * npages);
|
||||
int nhugepages = npages / 512;
|
||||
int n4ktotal, nnothuge;
|
||||
n4ktotal = 0;
|
||||
nnothuge = 0;
|
||||
for (int i = 0; i < nhugepages; ++i) {
|
||||
uint64_t baseaddr = (pagedata[i*512] & 0x7fffffffffffffULL) * page_size;
|
||||
for (int j = 0; j < 512; ++j) {
|
||||
uint64_t pageaddr = (pagedata[i*512+j] & 0x7fffffffffffffULL) * page_size;
|
||||
++n4ktotal;
|
||||
if (pageaddr != baseaddr + j * page_size)
|
||||
++nnothuge;
|
||||
}
|
||||
}
|
||||
int rank = CartesianCommunicator::RankWorld();
|
||||
printf("rank %d Allocated %d 4k pages, %d not in huge pages\n", rank, n4ktotal, nnothuge);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string sizeString(const size_t bytes)
|
||||
{
|
||||
constexpr unsigned int bufSize = 256;
|
||||
const char *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
|
||||
char buf[256];
|
||||
size_t s = 0;
|
||||
double count = bytes;
|
||||
|
||||
while (count >= 1024 && s < 7)
|
||||
{
|
||||
s++;
|
||||
count /= 1024;
|
||||
}
|
||||
if (count - floor(count) == 0.0)
|
||||
{
|
||||
snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
|
||||
}
|
||||
else
|
||||
{
|
||||
snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
|
||||
}
|
||||
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
+187
-149
@@ -26,10 +26,129 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
#ifndef GRID_ALIGNED_ALLOCATOR_H
|
||||
#define GRID_ALIGNED_ALLOCATOR_H
|
||||
|
||||
#ifdef HAVE_MALLOC_MALLOC_H
|
||||
#include <malloc/malloc.h>
|
||||
#endif
|
||||
#ifdef HAVE_MALLOC_H
|
||||
#include <malloc.h>
|
||||
#endif
|
||||
|
||||
#ifdef HAVE_MM_MALLOC_H
|
||||
#include <mm_malloc.h>
|
||||
#endif
|
||||
|
||||
#define POINTER_CACHE
|
||||
#define GRID_ALLOC_ALIGN (2*1024*1024)
|
||||
#define GRID_ALLOC_SMALL_LIMIT (4096)
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// Move control to configure.ac and Config.h?
|
||||
|
||||
class PointerCache {
|
||||
private:
|
||||
/*Pinning pages is costly*/
|
||||
/*Could maintain separate large and small allocation caches*/
|
||||
/* Could make these configurable, perhaps up to a max size*/
|
||||
static const int NcacheSmallMax=128;
|
||||
static const int NcacheMax=16;
|
||||
static int NcacheSmall;
|
||||
static int Ncache;
|
||||
|
||||
typedef struct {
|
||||
void *address;
|
||||
size_t bytes;
|
||||
int valid;
|
||||
} PointerCacheEntry;
|
||||
|
||||
static PointerCacheEntry Entries[NcacheMax];
|
||||
static int Victim;
|
||||
static PointerCacheEntry EntriesSmall[NcacheSmallMax];
|
||||
static int VictimSmall;
|
||||
|
||||
public:
|
||||
static void Init(void);
|
||||
static void *Insert(void *ptr,size_t bytes) ;
|
||||
static void *Insert(void *ptr,size_t bytes,PointerCacheEntry *entries,int ncache,int &victim) ;
|
||||
static void *Lookup(size_t bytes) ;
|
||||
static void *Lookup(size_t bytes,PointerCacheEntry *entries,int ncache) ;
|
||||
};
|
||||
|
||||
std::string sizeString(size_t bytes);
|
||||
|
||||
struct MemoryStats
|
||||
{
|
||||
size_t totalAllocated{0}, maxAllocated{0},
|
||||
currentlyAllocated{0}, totalFreed{0};
|
||||
};
|
||||
|
||||
class MemoryProfiler
|
||||
{
|
||||
public:
|
||||
static MemoryStats *stats;
|
||||
static bool debug;
|
||||
};
|
||||
|
||||
#ifdef GRID_NVCC
|
||||
#define profilerCudaMeminfo \
|
||||
{ size_t f, t ; cudaMemGetInfo ( &f,&t); std::cout << GridLogDebug << "[Memory debug] Cuda free "<<f<<"/"<<t << std::endl;}
|
||||
#else
|
||||
#define profilerCudaMeminfo
|
||||
#endif
|
||||
|
||||
#define memString(bytes) std::to_string(bytes) + " (" + sizeString(bytes) + ")"
|
||||
#define profilerDebugPrint \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
std::cout << GridLogDebug << "[Memory debug] Stats " << MemoryProfiler::stats << std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] total : " << memString(s->totalAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] max : " << memString(s->maxAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] current: " << memString(s->currentlyAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] freed : " << memString(s->totalFreed) \
|
||||
<< std::endl; \
|
||||
} \
|
||||
profilerCudaMeminfo;
|
||||
|
||||
#define profilerAllocate(bytes) \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
s->totalAllocated += (bytes); \
|
||||
s->currentlyAllocated += (bytes); \
|
||||
s->maxAllocated = std::max(s->maxAllocated, s->currentlyAllocated); \
|
||||
} \
|
||||
if (MemoryProfiler::debug) \
|
||||
{ \
|
||||
std::cout << GridLogDebug << "[Memory debug] allocating " << memString(bytes) << std::endl; \
|
||||
profilerDebugPrint; \
|
||||
}
|
||||
|
||||
#define profilerFree(bytes) \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
s->totalFreed += (bytes); \
|
||||
s->currentlyAllocated -= (bytes); \
|
||||
} \
|
||||
if (MemoryProfiler::debug) \
|
||||
{ \
|
||||
std::cout << GridLogDebug << "[Memory debug] freeing " << memString(bytes) << std::endl; \
|
||||
profilerDebugPrint; \
|
||||
}
|
||||
|
||||
void check_huge_pages(void *Buf,uint64_t BYTES);
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// A lattice of something, but assume the something is SIMDized.
|
||||
////////////////////////////////////////////////////////////////////
|
||||
|
||||
template<typename _Tp>
|
||||
class alignedAllocator {
|
||||
public:
|
||||
@@ -53,170 +172,89 @@ public:
|
||||
{
|
||||
size_type bytes = __n*sizeof(_Tp);
|
||||
profilerAllocate(bytes);
|
||||
_Tp *ptr = (_Tp*) MemoryManager::CpuAllocate(bytes);
|
||||
if ( (_Tp*)ptr == (_Tp *) NULL ) {
|
||||
printf("Grid CPU Allocator got NULL for %lu bytes\n",(unsigned long) bytes );
|
||||
}
|
||||
GRID_ASSERT( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
||||
|
||||
|
||||
#ifdef POINTER_CACHE
|
||||
_Tp *ptr = (_Tp *) PointerCache::Lookup(bytes);
|
||||
#else
|
||||
pointer ptr = nullptr;
|
||||
#endif
|
||||
|
||||
#ifdef GRID_NVCC
|
||||
////////////////////////////////////
|
||||
// Unified (managed) memory
|
||||
////////////////////////////////////
|
||||
if ( ptr == (_Tp *) NULL ) {
|
||||
// printf(" alignedAllocater cache miss %ld bytes ",bytes); BACKTRACEFP(stdout);
|
||||
auto err = cudaMallocManaged((void **)&ptr,bytes);
|
||||
if( err != cudaSuccess ) {
|
||||
ptr = (_Tp *) NULL;
|
||||
std::cerr << " cudaMallocManaged failed for " << bytes<<" bytes " <<cudaGetErrorString(err)<< std::endl;
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
assert( ptr != (_Tp *)NULL);
|
||||
#else
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// 2MB align; could make option probably doesn't need configurability
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef HAVE_MM_MALLOC_H
|
||||
if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) _mm_malloc(bytes,GRID_ALLOC_ALIGN);
|
||||
#else
|
||||
if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) memalign(GRID_ALLOC_ALIGN,bytes);
|
||||
#endif
|
||||
assert( ptr != (_Tp *)NULL);
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// First touch optimise in threaded loop
|
||||
//////////////////////////////////////////////////
|
||||
uint64_t *cp = (uint64_t *)ptr;
|
||||
thread_for(n,bytes/sizeof(uint64_t), { // need only one touch per page
|
||||
cp[n]=0;
|
||||
});
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deallocate(pointer __p, size_type __n)
|
||||
{
|
||||
void deallocate(pointer __p, size_type __n) {
|
||||
size_type bytes = __n * sizeof(_Tp);
|
||||
|
||||
profilerFree(bytes);
|
||||
MemoryManager::CpuFree((void *)__p,bytes);
|
||||
|
||||
#ifdef POINTER_CACHE
|
||||
pointer __freeme = (pointer)PointerCache::Insert((void *)__p,bytes);
|
||||
#else
|
||||
pointer __freeme = __p;
|
||||
#endif
|
||||
|
||||
#ifdef GRID_NVCC
|
||||
if ( __freeme ) cudaFree((void *)__freeme);
|
||||
#else
|
||||
#ifdef HAVE_MM_MALLOC_H
|
||||
if ( __freeme ) _mm_free((void *)__freeme);
|
||||
#else
|
||||
if ( __freeme ) free((void *)__freeme);
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
// FIXME: hack for the copy constructor: it must be avoided to avoid single thread loop
|
||||
void construct(pointer __p, const _Tp& __val) { };
|
||||
// FIXME: hack for the copy constructor, eventually it must be avoided
|
||||
void construct(pointer __p, const _Tp& __val) { new((void *)__p) _Tp(__val); };
|
||||
//void construct(pointer __p, const _Tp& __val) { };
|
||||
void construct(pointer __p) { };
|
||||
void destroy(pointer __p) { };
|
||||
};
|
||||
template<typename _Tp> inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
|
||||
template<typename _Tp> inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// Unified virtual memory
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
template<typename _Tp>
|
||||
class uvmAllocator {
|
||||
public:
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef _Tp* pointer;
|
||||
typedef const _Tp* const_pointer;
|
||||
typedef _Tp& reference;
|
||||
typedef const _Tp& const_reference;
|
||||
typedef _Tp value_type;
|
||||
|
||||
template<typename _Tp1> struct rebind { typedef uvmAllocator<_Tp1> other; };
|
||||
uvmAllocator() throw() { }
|
||||
uvmAllocator(const uvmAllocator&) throw() { }
|
||||
template<typename _Tp1> uvmAllocator(const uvmAllocator<_Tp1>&) throw() { }
|
||||
~uvmAllocator() throw() { }
|
||||
pointer address(reference __x) const { return &__x; }
|
||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
||||
|
||||
pointer allocate(size_type __n, const void* _p= 0)
|
||||
{
|
||||
size_type bytes = __n*sizeof(_Tp);
|
||||
profilerAllocate(bytes);
|
||||
_Tp *ptr = (_Tp*) MemoryManager::SharedAllocate(bytes);
|
||||
if ( (_Tp*)ptr == (_Tp *) NULL ) {
|
||||
printf("Grid Shared Allocator got NULL for %lu bytes\n",(unsigned long) bytes );
|
||||
}
|
||||
GRID_ASSERT( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deallocate(pointer __p, size_type __n)
|
||||
{
|
||||
size_type bytes = __n * sizeof(_Tp);
|
||||
profilerFree(bytes);
|
||||
MemoryManager::SharedFree((void *)__p,bytes);
|
||||
}
|
||||
|
||||
void construct(pointer __p, const _Tp& __val) { new((void *)__p) _Tp(__val); };
|
||||
void construct(pointer __p) { };
|
||||
void destroy(pointer __p) { };
|
||||
};
|
||||
template<typename _Tp> inline bool operator==(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return true; }
|
||||
template<typename _Tp> inline bool operator!=(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return false; }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Device memory
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
template<typename _Tp>
|
||||
class devAllocator {
|
||||
public:
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef _Tp* pointer;
|
||||
typedef const _Tp* const_pointer;
|
||||
typedef _Tp& reference;
|
||||
typedef const _Tp& const_reference;
|
||||
typedef _Tp value_type;
|
||||
|
||||
template<typename _Tp1> struct rebind { typedef devAllocator<_Tp1> other; };
|
||||
devAllocator() throw() { }
|
||||
devAllocator(const devAllocator&) throw() { }
|
||||
template<typename _Tp1> devAllocator(const devAllocator<_Tp1>&) throw() { }
|
||||
~devAllocator() throw() { }
|
||||
pointer address(reference __x) const { return &__x; }
|
||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
||||
|
||||
pointer allocate(size_type __n, const void* _p= 0)
|
||||
{
|
||||
size_type bytes = __n*sizeof(_Tp);
|
||||
profilerAllocate(bytes);
|
||||
_Tp *ptr = (_Tp*) MemoryManager::AcceleratorAllocate(bytes);
|
||||
if ( (_Tp*)ptr == (_Tp *) NULL ) {
|
||||
printf("Grid Device Allocator got NULL for %lu bytes\n",(unsigned long) bytes );
|
||||
}
|
||||
GRID_ASSERT( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deallocate(pointer __p, size_type __n)
|
||||
{
|
||||
size_type bytes = __n * sizeof(_Tp);
|
||||
profilerFree(bytes);
|
||||
MemoryManager::AcceleratorFree((void *)__p,bytes);
|
||||
}
|
||||
void construct(pointer __p, const _Tp& __val) { };
|
||||
void construct(pointer __p) { };
|
||||
void destroy(pointer __p) { };
|
||||
};
|
||||
template<typename _Tp> inline bool operator==(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return true; }
|
||||
template<typename _Tp> inline bool operator!=(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return false; }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Template typedefs
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
template<class T> using hostVector = std::vector<T,alignedAllocator<T> >; // Needs autoview
|
||||
template<class T> using Vector = std::vector<T,uvmAllocator<T> >; // Really want to deprecate
|
||||
template<class T> using uvmVector = std::vector<T,uvmAllocator<T> >; // auto migrating page
|
||||
template<class T> using deviceVector = std::vector<T,devAllocator<T> >; // device vector
|
||||
|
||||
/*
|
||||
template<class T> class vecView
|
||||
{
|
||||
protected:
|
||||
T * data;
|
||||
uint64_t size;
|
||||
ViewMode mode;
|
||||
void * cpu_ptr;
|
||||
public:
|
||||
// Rvalue accessor
|
||||
accelerator_inline T & operator[](size_t i) const { return this->data[i]; };
|
||||
vecView(Vector<T> &refer_to_me,ViewMode _mode)
|
||||
{
|
||||
cpu_ptr = &refer_to_me[0];
|
||||
size = refer_to_me.size();
|
||||
mode = _mode;
|
||||
data =(T *) MemoryManager::ViewOpen(cpu_ptr,
|
||||
size*sizeof(T),
|
||||
mode,
|
||||
AdviseDefault);
|
||||
}
|
||||
void ViewClose(void)
|
||||
{ // Inform the manager
|
||||
MemoryManager::ViewClose(this->cpu_ptr,this->mode);
|
||||
}
|
||||
};
|
||||
|
||||
template<class T> vecView<T> VectorView(Vector<T> &vec,ViewMode _mode)
|
||||
{
|
||||
vecView<T> ret(vec,_mode); // does the open
|
||||
return ret; // must be closed
|
||||
}
|
||||
|
||||
#define autoVecView(v_v,v,mode) \
|
||||
auto v_v = VectorView(v,mode); \
|
||||
ViewCloser<decltype(v_v)> _autoView##v_v(v_v);
|
||||
*/
|
||||
template<class T> using commAllocator = alignedAllocator<T>;
|
||||
template<class T> using Vector = std::vector<T,alignedAllocator<T> >;
|
||||
template<class T> using commVector = std::vector<T,alignedAllocator<T> >;
|
||||
template<class T> using Matrix = std::vector<std::vector<T,alignedAllocator<T> > >;
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
#endif
|
||||
@@ -1,4 +0,0 @@
|
||||
#pragma once
|
||||
#include <Grid/allocator/MemoryStats.h>
|
||||
#include <Grid/allocator/MemoryManager.h>
|
||||
#include <Grid/allocator/AlignedAllocator.h>
|
||||
@@ -1,362 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
/*Allocation types, saying which pointer cache should be used*/
|
||||
#define Cpu (0)
|
||||
#define CpuHuge (1)
|
||||
#define CpuSmall (2)
|
||||
#define Acc (3)
|
||||
#define AccHuge (4)
|
||||
#define AccSmall (5)
|
||||
#define Shared (6)
|
||||
#define SharedHuge (7)
|
||||
#define SharedSmall (8)
|
||||
#undef GRID_MM_VERBOSE
|
||||
uint64_t total_shared;
|
||||
uint64_t total_device;
|
||||
uint64_t total_host;;
|
||||
|
||||
#if defined(__has_feature)
|
||||
#if __has_feature(leak_sanitizer)
|
||||
#define ASAN_LEAK_CHECK
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef ASAN_LEAK_CHECK
|
||||
#include <sanitizer/asan_interface.h>
|
||||
#include <sanitizer/common_interface_defs.h>
|
||||
#include <sanitizer/lsan_interface.h>
|
||||
#define LEAK_CHECK(A) { __lsan_do_recoverable_leak_check(); }
|
||||
#else
|
||||
#define LEAK_CHECK(A) { }
|
||||
#endif
|
||||
|
||||
void MemoryManager::DisplayMallinfo(void)
|
||||
{
|
||||
#ifdef __linux__
|
||||
struct mallinfo mi; // really want mallinfo2, but glibc version isn't uniform
|
||||
|
||||
mi = mallinfo();
|
||||
|
||||
std::cout << "MemoryManager: Total non-mmapped bytes (arena): "<< (size_t)mi.arena<<std::endl;
|
||||
std::cout << "MemoryManager: # of free chunks (ordblks): "<< (size_t)mi.ordblks<<std::endl;
|
||||
std::cout << "MemoryManager: # of free fastbin blocks (smblks): "<< (size_t)mi.smblks<<std::endl;
|
||||
std::cout << "MemoryManager: # of mapped regions (hblks): "<< (size_t)mi.hblks<<std::endl;
|
||||
std::cout << "MemoryManager: Bytes in mapped regions (hblkhd): "<< (size_t)mi.hblkhd<<std::endl;
|
||||
std::cout << "MemoryManager: Max. total allocated space (usmblks): "<< (size_t)mi.usmblks<<std::endl;
|
||||
std::cout << "MemoryManager: Free bytes held in fastbins (fsmblks): "<< (size_t)mi.fsmblks<<std::endl;
|
||||
std::cout << "MemoryManager: Total allocated space (uordblks): "<< (size_t)mi.uordblks<<std::endl;
|
||||
std::cout << "MemoryManager: Total free space (fordblks): "<< (size_t)mi.fordblks<<std::endl;
|
||||
std::cout << "MemoryManager: Topmost releasable block (keepcost): "<< (size_t)mi.keepcost<<std::endl;
|
||||
#endif
|
||||
LEAK_CHECK();
|
||||
|
||||
}
|
||||
|
||||
void MemoryManager::PrintBytes(void)
|
||||
{
|
||||
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
||||
std::cout << " MemoryManager : PrintBytes "<<std::endl;
|
||||
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
||||
std::cout << " MemoryManager : "<<(total_shared>>20)<<" shared Mbytes "<<std::endl;
|
||||
std::cout << " MemoryManager : "<<(total_device>>20)<<" accelerator Mbytes "<<std::endl;
|
||||
std::cout << " MemoryManager : "<<(total_host>>20) <<" cpu Mbytes "<<std::endl;
|
||||
uint64_t cacheBytes;
|
||||
cacheBytes = CacheBytes[Cpu];
|
||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" cpu cache Mbytes "<<std::endl;
|
||||
cacheBytes = CacheBytes[Acc];
|
||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" acc cache Mbytes "<<std::endl;
|
||||
cacheBytes = CacheBytes[Shared];
|
||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" shared cache Mbytes "<<std::endl;
|
||||
|
||||
#ifdef GRID_CUDA
|
||||
cuda_mem();
|
||||
#endif
|
||||
DisplayMallinfo();
|
||||
}
|
||||
|
||||
uint64_t MemoryManager::DeviceCacheBytes() { return CacheBytes[Acc] + CacheBytes[AccHuge] + CacheBytes[AccSmall]; }
|
||||
uint64_t MemoryManager::HostCacheBytes() { return CacheBytes[Cpu] + CacheBytes[CpuHuge] + CacheBytes[CpuSmall]; }
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Data tables for recently freed pooiniter caches
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
MemoryManager::AllocationCacheEntry MemoryManager::Entries[MemoryManager::NallocType][MemoryManager::NallocCacheMax];
|
||||
int MemoryManager::Victim[MemoryManager::NallocType];
|
||||
int MemoryManager::Ncache[MemoryManager::NallocType] = { 2, 0, 8, 8, 0, 16, 8, 0, 16 };
|
||||
uint64_t MemoryManager::CacheBytes[MemoryManager::NallocType];
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Actual allocation and deallocation utils
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
void *MemoryManager::AcceleratorAllocate(size_t bytes)
|
||||
{
|
||||
total_device+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Acc);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocDevice(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"AcceleratorAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::AcceleratorFree (void *ptr,size_t bytes)
|
||||
{
|
||||
total_device-=bytes;
|
||||
void *__freeme = Insert(ptr,bytes,Acc);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeDevice(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"AcceleratorFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
void *MemoryManager::SharedAllocate(size_t bytes)
|
||||
{
|
||||
total_shared+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Shared);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocShared(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"SharedAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::SharedFree (void *ptr,size_t bytes)
|
||||
{
|
||||
total_shared-=bytes;
|
||||
void *__freeme = Insert(ptr,bytes,Shared);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeShared(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"SharedFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
#ifdef GRID_UVM
|
||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
||||
{
|
||||
total_host+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocShared(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
||||
{
|
||||
total_host-=bytes;
|
||||
NotifyDeletion(_ptr);
|
||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeShared(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
#else
|
||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
||||
{
|
||||
total_host+=bytes;
|
||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
||||
if ( ptr == (void *) NULL ) {
|
||||
ptr = (void *) acceleratorAllocCpu(bytes);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuAllocate "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
return ptr;
|
||||
}
|
||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
||||
{
|
||||
total_host-=bytes;
|
||||
NotifyDeletion(_ptr);
|
||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
||||
if ( __freeme ) {
|
||||
acceleratorFreeCpu(__freeme);
|
||||
}
|
||||
#ifdef GRID_MM_VERBOSE
|
||||
std::cout <<"CpuFree "<<std::endl;
|
||||
PrintBytes();
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
//////////////////////////////////////////
|
||||
// call only once
|
||||
//////////////////////////////////////////
|
||||
void MemoryManager::Init(void)
|
||||
{
|
||||
|
||||
char * str;
|
||||
int Nc;
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_LARGE");
|
||||
if ( str ) {
|
||||
Nc = atoi(str);
|
||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
||||
Ncache[Cpu]=Nc;
|
||||
Ncache[Acc]=Nc;
|
||||
Ncache[Shared]=Nc;
|
||||
}
|
||||
}
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_HUGE");
|
||||
if ( str ) {
|
||||
Nc = atoi(str);
|
||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
||||
Ncache[CpuHuge]=Nc;
|
||||
Ncache[AccHuge]=Nc;
|
||||
Ncache[SharedHuge]=Nc;
|
||||
}
|
||||
}
|
||||
|
||||
str= getenv("GRID_ALLOC_NCACHE_SMALL");
|
||||
if ( str ) {
|
||||
Nc = atoi(str);
|
||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
||||
Ncache[CpuSmall]=Nc;
|
||||
Ncache[AccSmall]=Nc;
|
||||
Ncache[SharedSmall]=Nc;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void MemoryManager::InitMessage(void) {
|
||||
|
||||
#ifndef GRID_UVM
|
||||
std::cout << GridLogMessage << "MemoryManager Cache "<< MemoryManager::DeviceMaxBytes <<" bytes "<<std::endl;
|
||||
#endif
|
||||
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() setting up"<<std::endl;
|
||||
#ifdef ALLOCATION_CACHE
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent host allocations: SMALL "<<Ncache[CpuSmall]<<" LARGE "<<Ncache[Cpu]<<" HUGE "<<Ncache[CpuHuge]<<std::endl;
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent device allocations: SMALL "<<Ncache[AccSmall]<<" LARGE "<<Ncache[Acc]<<" Huge "<<Ncache[AccHuge]<<std::endl;
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent shared allocations: SMALL "<<Ncache[SharedSmall]<<" LARGE "<<Ncache[Shared]<<" Huge "<<Ncache[SharedHuge]<<std::endl;
|
||||
#endif
|
||||
|
||||
#ifdef GRID_UVM
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Unified memory space"<<std::endl;
|
||||
#ifdef GRID_CUDA
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMallocManaged"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMallocManaged"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_SYCL
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_shared"<<std::endl;
|
||||
#endif
|
||||
#else
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory"<<std::endl;
|
||||
#ifdef GRID_CUDA
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMalloc"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMalloc"<<std::endl;
|
||||
#endif
|
||||
#ifdef GRID_SYCL
|
||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
void *MemoryManager::Insert(void *ptr,size_t bytes,int type)
|
||||
{
|
||||
#ifdef ALLOCATION_CACHE
|
||||
int cache;
|
||||
if (bytes < GRID_ALLOC_SMALL_LIMIT) cache = type + 2;
|
||||
else if (bytes >= GRID_ALLOC_HUGE_LIMIT) cache = type + 1;
|
||||
else cache = type;
|
||||
|
||||
return Insert(ptr,bytes,Entries[cache],Ncache[cache],Victim[cache],CacheBytes[cache]);
|
||||
#else
|
||||
return ptr;
|
||||
#endif
|
||||
}
|
||||
|
||||
void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim, uint64_t &cacheBytes)
|
||||
{
|
||||
#ifdef GRID_OMP
|
||||
GRID_ASSERT(omp_in_parallel()==0);
|
||||
#endif
|
||||
|
||||
if (ncache == 0) return ptr;
|
||||
|
||||
void * ret = NULL;
|
||||
int v = -1;
|
||||
|
||||
for(int e=0;e<ncache;e++) {
|
||||
if ( entries[e].valid==0 ) {
|
||||
v=e;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ( v==-1 ) {
|
||||
v=victim;
|
||||
victim = (victim+1)%ncache;
|
||||
}
|
||||
|
||||
if ( entries[v].valid ) {
|
||||
ret = entries[v].address;
|
||||
cacheBytes -= entries[v].bytes;
|
||||
entries[v].valid = 0;
|
||||
entries[v].address = NULL;
|
||||
entries[v].bytes = 0;
|
||||
}
|
||||
|
||||
entries[v].address=ptr;
|
||||
entries[v].bytes =bytes;
|
||||
entries[v].valid =1;
|
||||
cacheBytes += bytes;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *MemoryManager::Lookup(size_t bytes,int type)
|
||||
{
|
||||
#ifdef ALLOCATION_CACHE
|
||||
int cache;
|
||||
if (bytes < GRID_ALLOC_SMALL_LIMIT) cache = type + 2;
|
||||
else if (bytes >= GRID_ALLOC_HUGE_LIMIT) cache = type + 1;
|
||||
else cache = type;
|
||||
|
||||
return Lookup(bytes,Entries[cache],Ncache[cache],CacheBytes[cache]);
|
||||
#else
|
||||
return NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
void *MemoryManager::Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t & cacheBytes)
|
||||
{
|
||||
#ifdef GRID_OMP
|
||||
GRID_ASSERT(omp_in_parallel()==0);
|
||||
#endif
|
||||
for(int e=0;e<ncache;e++){
|
||||
if ( entries[e].valid && ( entries[e].bytes == bytes ) ) {
|
||||
entries[e].valid = 0;
|
||||
cacheBytes -= entries[e].bytes;
|
||||
return entries[e].address;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/MemoryManager.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
#include <list>
|
||||
#include <unordered_map>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
// Move control to configure.ac and Config.h?
|
||||
|
||||
#define GRID_ALLOC_SMALL_LIMIT (4096)
|
||||
#define GRID_ALLOC_HUGE_LIMIT (2147483648)
|
||||
|
||||
#define STRINGIFY(x) #x
|
||||
#define TOSTRING(x) STRINGIFY(x)
|
||||
#define FILE_LINE __FILE__ ":" TOSTRING(__LINE__)
|
||||
#define AUDIT(a) MemoryManager::Audit(FILE_LINE)
|
||||
|
||||
/*Pinning pages is costly*/
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Advise the LatticeAccelerator class
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
enum ViewAdvise {
|
||||
AdviseDefault = 0x0, // Regular data
|
||||
AdviseInfrequentUse = 0x1 // Advise that the data is used infrequently. This can
|
||||
// significantly influence performance of bulk storage.
|
||||
|
||||
// AdviseTransient = 0x2, // Data will mostly be read. On some architectures
|
||||
// enables read-only copies of memory to be kept on
|
||||
// host and device.
|
||||
|
||||
// AdviseAcceleratorWriteDiscard = 0x4 // Field will be written in entirety on device
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// View Access Mode
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
enum ViewMode {
|
||||
AcceleratorRead = 0x01,
|
||||
AcceleratorWrite = 0x02,
|
||||
AcceleratorWriteDiscard = 0x04,
|
||||
CpuRead = 0x08,
|
||||
CpuWrite = 0x10,
|
||||
CpuWriteDiscard = 0x10 // same for now
|
||||
};
|
||||
|
||||
struct MemoryStatus {
|
||||
uint64_t DeviceBytes;
|
||||
uint64_t DeviceLRUBytes;
|
||||
uint64_t DeviceMaxBytes;
|
||||
uint64_t HostToDeviceBytes;
|
||||
uint64_t DeviceToHostBytes;
|
||||
uint64_t HostToDeviceXfer;
|
||||
uint64_t DeviceToHostXfer;
|
||||
uint64_t DeviceEvictions;
|
||||
uint64_t DeviceDestroy;
|
||||
uint64_t DeviceAllocCacheBytes;
|
||||
uint64_t HostAllocCacheBytes;
|
||||
};
|
||||
|
||||
|
||||
class MemoryManager {
|
||||
private:
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// For caching recently freed allocations
|
||||
////////////////////////////////////////////////////////////
|
||||
typedef struct {
|
||||
void *address;
|
||||
size_t bytes;
|
||||
int valid;
|
||||
} AllocationCacheEntry;
|
||||
|
||||
static const int NallocCacheMax=128;
|
||||
static const int NallocType=9;
|
||||
static AllocationCacheEntry Entries[NallocType][NallocCacheMax];
|
||||
static int Victim[NallocType];
|
||||
static int Ncache[NallocType];
|
||||
static uint64_t CacheBytes[NallocType];
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Free pool
|
||||
/////////////////////////////////////////////////
|
||||
static void *Insert(void *ptr,size_t bytes,int type) ;
|
||||
static void *Lookup(size_t bytes,int type) ;
|
||||
static void *Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim,uint64_t &cbytes) ;
|
||||
static void *Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t &cbytes) ;
|
||||
|
||||
public:
|
||||
static void PrintBytes(void);
|
||||
static void Audit(std::string s);
|
||||
static void Init(void);
|
||||
static void InitMessage(void);
|
||||
static void *AcceleratorAllocate(size_t bytes);
|
||||
static void AcceleratorFree (void *ptr,size_t bytes);
|
||||
static void *SharedAllocate(size_t bytes);
|
||||
static void SharedFree (void *ptr,size_t bytes);
|
||||
static void *CpuAllocate(size_t bytes);
|
||||
static void CpuFree (void *ptr,size_t bytes);
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// Footprint tracking
|
||||
////////////////////////////////////////////////////////
|
||||
static uint64_t DeviceBytes;
|
||||
static uint64_t DeviceLRUBytes;
|
||||
static uint64_t DeviceMaxBytes;
|
||||
static uint64_t HostToDeviceBytes;
|
||||
static uint64_t DeviceToHostBytes;
|
||||
static uint64_t HostToDeviceXfer;
|
||||
static uint64_t DeviceToHostXfer;
|
||||
static uint64_t DeviceEvictions;
|
||||
static uint64_t DeviceDestroy;
|
||||
|
||||
static uint64_t DeviceCacheBytes();
|
||||
static uint64_t HostCacheBytes();
|
||||
|
||||
static MemoryStatus GetFootprint(void) {
|
||||
MemoryStatus stat;
|
||||
stat.DeviceBytes = DeviceBytes;
|
||||
stat.DeviceLRUBytes = DeviceLRUBytes;
|
||||
stat.DeviceMaxBytes = DeviceMaxBytes;
|
||||
stat.HostToDeviceBytes = HostToDeviceBytes;
|
||||
stat.DeviceToHostBytes = DeviceToHostBytes;
|
||||
stat.HostToDeviceXfer = HostToDeviceXfer;
|
||||
stat.DeviceToHostXfer = DeviceToHostXfer;
|
||||
stat.DeviceEvictions = DeviceEvictions;
|
||||
stat.DeviceDestroy = DeviceDestroy;
|
||||
stat.DeviceAllocCacheBytes = DeviceCacheBytes();
|
||||
stat.HostAllocCacheBytes = HostCacheBytes();
|
||||
return stat;
|
||||
};
|
||||
|
||||
private:
|
||||
#ifndef GRID_UVM
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Data tables for ViewCache
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
typedef std::list<uint64_t> LRU_t;
|
||||
typedef typename LRU_t::iterator LRUiterator;
|
||||
typedef struct {
|
||||
int LRU_valid;
|
||||
LRUiterator LRU_entry;
|
||||
uint64_t CpuPtr;
|
||||
uint64_t AccPtr;
|
||||
size_t bytes;
|
||||
uint32_t transient;
|
||||
uint32_t state;
|
||||
uint32_t accLock;
|
||||
uint32_t cpuLock;
|
||||
} AcceleratorViewEntry;
|
||||
|
||||
typedef std::unordered_map<uint64_t,AcceleratorViewEntry> AccViewTable_t;
|
||||
typedef typename AccViewTable_t::iterator AccViewTableIterator ;
|
||||
|
||||
static AccViewTable_t AccViewTable;
|
||||
static LRU_t LRU;
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Device motion
|
||||
/////////////////////////////////////////////////
|
||||
static void Create(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
static void EvictVictims(uint64_t bytes); // Frees up <bytes>
|
||||
static void Evict(AcceleratorViewEntry &AccCache);
|
||||
static void Flush(AcceleratorViewEntry &AccCache);
|
||||
static void Clone(AcceleratorViewEntry &AccCache);
|
||||
static void AccDiscard(AcceleratorViewEntry &AccCache);
|
||||
static void CpuDiscard(AcceleratorViewEntry &AccCache);
|
||||
|
||||
// static void LRUupdate(AcceleratorViewEntry &AccCache);
|
||||
static void LRUinsert(AcceleratorViewEntry &AccCache);
|
||||
static void LRUremove(AcceleratorViewEntry &AccCache);
|
||||
|
||||
// manage entries in the table
|
||||
static int EntryPresent(uint64_t CpuPtr);
|
||||
static void EntryCreate(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
static void EntryErase (uint64_t CpuPtr);
|
||||
static AccViewTableIterator EntryLookup(uint64_t CpuPtr);
|
||||
static void EntrySet (uint64_t CpuPtr,AcceleratorViewEntry &entry);
|
||||
|
||||
static void AcceleratorViewClose(uint64_t AccPtr);
|
||||
static uint64_t AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
static void CpuViewClose(uint64_t Ptr);
|
||||
static uint64_t CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
#endif
|
||||
|
||||
public:
|
||||
static void DisplayMallinfo(void);
|
||||
static void NotifyDeletion(void * CpuPtr);
|
||||
static void Print(void);
|
||||
static void PrintAll(void);
|
||||
static void PrintState( void* CpuPtr);
|
||||
static int isOpen (void* CpuPtr);
|
||||
static void ViewClose(void* CpuPtr,ViewMode mode);
|
||||
static void *ViewOpen (void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
||||
|
||||
};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -1,606 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#ifndef GRID_UVM
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#define MAXLINE 512
|
||||
static char print_buffer [ MAXLINE ];
|
||||
|
||||
#define mprintf(...) snprintf (print_buffer,MAXLINE, __VA_ARGS__ ); std::cout << GridLogMemory << print_buffer << std::endl;
|
||||
#define dprintf(...) snprintf (print_buffer,MAXLINE, __VA_ARGS__ ); std::cout << GridLogDebug << print_buffer << std::endl;
|
||||
//#define dprintf(...)
|
||||
//#define mprintf(...)
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// For caching copies of data on device
|
||||
////////////////////////////////////////////////////////////
|
||||
MemoryManager::AccViewTable_t MemoryManager::AccViewTable;
|
||||
MemoryManager::LRU_t MemoryManager::LRU;
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// Footprint tracking
|
||||
////////////////////////////////////////////////////////
|
||||
uint64_t MemoryManager::DeviceBytes;
|
||||
uint64_t MemoryManager::DeviceLRUBytes;
|
||||
uint64_t MemoryManager::DeviceMaxBytes = 1024*1024*128;
|
||||
uint64_t MemoryManager::HostToDeviceBytes;
|
||||
uint64_t MemoryManager::DeviceToHostBytes;
|
||||
uint64_t MemoryManager::HostToDeviceXfer;
|
||||
uint64_t MemoryManager::DeviceToHostXfer;
|
||||
uint64_t MemoryManager::DeviceEvictions;
|
||||
uint64_t MemoryManager::DeviceDestroy;
|
||||
|
||||
////////////////////////////////////
|
||||
// Priority ordering for unlocked entries
|
||||
// Empty
|
||||
// CpuDirty
|
||||
// Consistent
|
||||
// AccDirty
|
||||
////////////////////////////////////
|
||||
#define Empty (0x0) /*Entry unoccupied */
|
||||
#define CpuDirty (0x1) /*CPU copy is golden, Acc buffer MAY not be allocated*/
|
||||
#define Consistent (0x2) /*ACC copy AND CPU copy are valid */
|
||||
#define AccDirty (0x4) /*ACC copy is golden */
|
||||
#define EvictNext (0x8) /*Priority for eviction*/
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// Mechanics of data table maintenance
|
||||
/////////////////////////////////////////////////
|
||||
int MemoryManager::EntryPresent(uint64_t CpuPtr)
|
||||
{
|
||||
if(AccViewTable.empty()) return 0;
|
||||
|
||||
auto count = AccViewTable.count(CpuPtr); GRID_ASSERT((count==0)||(count==1));
|
||||
return count;
|
||||
}
|
||||
void MemoryManager::EntryCreate(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
||||
{
|
||||
GRID_ASSERT(!EntryPresent(CpuPtr));
|
||||
AcceleratorViewEntry AccCache;
|
||||
AccCache.CpuPtr = CpuPtr;
|
||||
AccCache.AccPtr = (uint64_t)NULL;
|
||||
AccCache.bytes = bytes;
|
||||
AccCache.state = CpuDirty;
|
||||
AccCache.LRU_valid=0;
|
||||
AccCache.transient=0;
|
||||
AccCache.accLock=0;
|
||||
AccCache.cpuLock=0;
|
||||
AccViewTable[CpuPtr] = AccCache;
|
||||
}
|
||||
MemoryManager::AccViewTableIterator MemoryManager::EntryLookup(uint64_t CpuPtr)
|
||||
{
|
||||
GRID_ASSERT(EntryPresent(CpuPtr));
|
||||
auto AccCacheIterator = AccViewTable.find(CpuPtr);
|
||||
GRID_ASSERT(AccCacheIterator!=AccViewTable.end());
|
||||
return AccCacheIterator;
|
||||
}
|
||||
void MemoryManager::EntryErase(uint64_t CpuPtr)
|
||||
{
|
||||
auto AccCache = EntryLookup(CpuPtr);
|
||||
AccViewTable.erase(CpuPtr);
|
||||
}
|
||||
void MemoryManager::LRUinsert(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
GRID_ASSERT(AccCache.LRU_valid==0);
|
||||
if (AccCache.transient) {
|
||||
LRU.push_back(AccCache.CpuPtr);
|
||||
AccCache.LRU_entry = --LRU.end();
|
||||
} else {
|
||||
LRU.push_front(AccCache.CpuPtr);
|
||||
AccCache.LRU_entry = LRU.begin();
|
||||
}
|
||||
AccCache.LRU_valid = 1;
|
||||
DeviceLRUBytes+=AccCache.bytes;
|
||||
}
|
||||
void MemoryManager::LRUremove(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
GRID_ASSERT(AccCache.LRU_valid==1);
|
||||
LRU.erase(AccCache.LRU_entry);
|
||||
AccCache.LRU_valid = 0;
|
||||
DeviceLRUBytes-=AccCache.bytes;
|
||||
}
|
||||
/////////////////////////////////////////////////
|
||||
// Accelerator cache motion & consistency logic
|
||||
/////////////////////////////////////////////////
|
||||
void MemoryManager::AccDiscard(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
///////////////////////////////////////////////////////////
|
||||
// Remove from Accelerator, remove entry, without flush
|
||||
// Cannot be locked. If allocated Must be in LRU pool.
|
||||
///////////////////////////////////////////////////////////
|
||||
GRID_ASSERT(AccCache.state!=Empty);
|
||||
|
||||
dprintf("MemoryManager: Discard(%lx) %lx",(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr);
|
||||
GRID_ASSERT(AccCache.accLock==0);
|
||||
GRID_ASSERT(AccCache.cpuLock==0);
|
||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
if(AccCache.AccPtr) {
|
||||
AcceleratorFree((void *)AccCache.AccPtr,AccCache.bytes);
|
||||
DeviceDestroy++;
|
||||
DeviceBytes -=AccCache.bytes;
|
||||
LRUremove(AccCache);
|
||||
AccCache.AccPtr=(uint64_t) NULL;
|
||||
dprintf("MemoryManager: Free(%lx) LRU %ld Total %ld",(uint64_t)AccCache.AccPtr,DeviceLRUBytes,DeviceBytes);
|
||||
}
|
||||
uint64_t CpuPtr = AccCache.CpuPtr;
|
||||
EntryErase(CpuPtr);
|
||||
}
|
||||
|
||||
void MemoryManager::Evict(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// Make CPU consistent, remove from Accelerator, remove from LRU, LEAVE CPU only entry
|
||||
// Cannot be acclocked. If allocated must be in LRU pool.
|
||||
//
|
||||
// Nov 2022... Felix issue: Allocating two CpuPtrs, can have an entry in LRU-q with CPUlock.
|
||||
// and require to evict the AccPtr copy. Eviction was a mistake in CpuViewOpen
|
||||
// but there is a weakness where CpuLock entries are attempted for erase
|
||||
// Take these OUT LRU queue when CPU locked?
|
||||
// Cannot take out the table as cpuLock data is important.
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
GRID_ASSERT(AccCache.state!=Empty);
|
||||
|
||||
mprintf("MemoryManager: Evict CpuPtr %lx AccPtr %lx cpuLock %ld accLock %ld",
|
||||
(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr,
|
||||
(uint64_t)AccCache.cpuLock,(uint64_t)AccCache.accLock);
|
||||
if (AccCache.accLock!=0) return;
|
||||
if (AccCache.cpuLock!=0) return;
|
||||
if(AccCache.state==AccDirty) {
|
||||
Flush(AccCache);
|
||||
}
|
||||
if(AccCache.AccPtr) {
|
||||
AcceleratorFree((void *)AccCache.AccPtr,AccCache.bytes);
|
||||
LRUremove(AccCache);
|
||||
AccCache.AccPtr=(uint64_t)NULL;
|
||||
AccCache.state=CpuDirty; // CPU primary now
|
||||
DeviceBytes -=AccCache.bytes;
|
||||
dprintf("MemoryManager: Free(AccPtr %lx) footprint now %ld ",(uint64_t)AccCache.AccPtr,DeviceBytes);
|
||||
}
|
||||
// uint64_t CpuPtr = AccCache.CpuPtr;
|
||||
DeviceEvictions++;
|
||||
// EntryErase(CpuPtr);
|
||||
}
|
||||
void MemoryManager::Flush(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
GRID_ASSERT(AccCache.state==AccDirty);
|
||||
GRID_ASSERT(AccCache.cpuLock==0);
|
||||
GRID_ASSERT(AccCache.accLock==0);
|
||||
GRID_ASSERT(AccCache.AccPtr!=(uint64_t)NULL);
|
||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
acceleratorCopyFromDevice((void *)AccCache.AccPtr,(void *)AccCache.CpuPtr,AccCache.bytes);
|
||||
mprintf("MemoryManager: acceleratorCopyFromDevice Flush size %ld AccPtr %lx -> CpuPtr %lx",(uint64_t)AccCache.bytes,(uint64_t)AccCache.AccPtr,(uint64_t)AccCache.CpuPtr); fflush(stdout);
|
||||
DeviceToHostBytes+=AccCache.bytes;
|
||||
DeviceToHostXfer++;
|
||||
AccCache.state=Consistent;
|
||||
}
|
||||
void MemoryManager::Clone(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
GRID_ASSERT(AccCache.state==CpuDirty);
|
||||
GRID_ASSERT(AccCache.cpuLock==0);
|
||||
GRID_ASSERT(AccCache.accLock==0);
|
||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
if(AccCache.AccPtr==(uint64_t)NULL){
|
||||
AccCache.AccPtr=(uint64_t)AcceleratorAllocate(AccCache.bytes);
|
||||
DeviceBytes+=AccCache.bytes;
|
||||
}
|
||||
mprintf("MemoryManager: acceleratorCopyToDevice Clone size %ld AccPtr %lx <- CpuPtr %lx",
|
||||
(uint64_t)AccCache.bytes,
|
||||
(uint64_t)AccCache.AccPtr,(uint64_t)AccCache.CpuPtr); fflush(stdout);
|
||||
acceleratorCopyToDevice((void *)AccCache.CpuPtr,(void *)AccCache.AccPtr,AccCache.bytes);
|
||||
HostToDeviceBytes+=AccCache.bytes;
|
||||
HostToDeviceXfer++;
|
||||
AccCache.state=Consistent;
|
||||
}
|
||||
|
||||
void MemoryManager::CpuDiscard(AcceleratorViewEntry &AccCache)
|
||||
{
|
||||
GRID_ASSERT(AccCache.state!=Empty);
|
||||
GRID_ASSERT(AccCache.cpuLock==0);
|
||||
GRID_ASSERT(AccCache.accLock==0);
|
||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
||||
if(AccCache.AccPtr==(uint64_t)NULL){
|
||||
AccCache.AccPtr=(uint64_t)AcceleratorAllocate(AccCache.bytes);
|
||||
DeviceBytes+=AccCache.bytes;
|
||||
}
|
||||
AccCache.state=AccDirty;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// View management
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
void MemoryManager::ViewClose(void* Ptr,ViewMode mode)
|
||||
{
|
||||
if( (mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard) ){
|
||||
dprintf("AcceleratorViewClose %lx",(uint64_t)Ptr);
|
||||
AcceleratorViewClose((uint64_t)Ptr);
|
||||
} else if( (mode==CpuRead)||(mode==CpuWrite)){
|
||||
CpuViewClose((uint64_t)Ptr);
|
||||
} else {
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
}
|
||||
void *MemoryManager::ViewOpen(void* _CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
||||
{
|
||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||
if( (mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard) ){
|
||||
dprintf("AcceleratorViewOpen %lx",(uint64_t)CpuPtr);
|
||||
return (void *) AcceleratorViewOpen(CpuPtr,bytes,mode,hint);
|
||||
} else if( (mode==CpuRead)||(mode==CpuWrite)){
|
||||
return (void *)CpuViewOpen(CpuPtr,bytes,mode,hint);
|
||||
} else {
|
||||
GRID_ASSERT(0);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
void MemoryManager::EvictVictims(uint64_t bytes)
|
||||
{
|
||||
if(bytes>=DeviceMaxBytes) {
|
||||
printf("EvictVictims bytes %ld DeviceMaxBytes %ld\n",bytes,DeviceMaxBytes);
|
||||
}
|
||||
GRID_ASSERT(bytes<DeviceMaxBytes);
|
||||
while(bytes+DeviceLRUBytes > DeviceMaxBytes){
|
||||
if ( DeviceLRUBytes > 0){
|
||||
GRID_ASSERT(LRU.size()>0);
|
||||
uint64_t victim = LRU.back(); // From the LRU
|
||||
auto AccCacheIterator = EntryLookup(victim);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
Evict(AccCache);
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
uint64_t MemoryManager::AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Find if present, otherwise get or force an empty
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
if ( EntryPresent(CpuPtr)==0 ){
|
||||
EntryCreate(CpuPtr,bytes,mode,hint);
|
||||
}
|
||||
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
if (!AccCache.AccPtr) {
|
||||
EvictVictims(bytes);
|
||||
}
|
||||
GRID_ASSERT((mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard));
|
||||
|
||||
GRID_ASSERT(AccCache.cpuLock==0); // Programming error
|
||||
|
||||
if(AccCache.state!=Empty) {
|
||||
dprintf("ViewOpen found entry %lx %lx : sizes %ld %ld accLock %ld",
|
||||
(uint64_t)AccCache.CpuPtr,
|
||||
(uint64_t)CpuPtr,
|
||||
(uint64_t)AccCache.bytes,
|
||||
(uint64_t)bytes,
|
||||
(uint64_t)AccCache.accLock);
|
||||
GRID_ASSERT(AccCache.CpuPtr == CpuPtr);
|
||||
GRID_ASSERT(AccCache.bytes ==bytes);
|
||||
}
|
||||
/*
|
||||
* State transitions and actions
|
||||
*
|
||||
* Action State StateNext Flush Clone
|
||||
*
|
||||
* AccRead Empty Consistent - Y
|
||||
* AccWrite Empty AccDirty - Y
|
||||
* AccRead CpuDirty Consistent - Y
|
||||
* AccWrite CpuDirty AccDirty - Y
|
||||
* AccRead Consistent Consistent - -
|
||||
* AccWrite Consistent AccDirty - -
|
||||
* AccRead AccDirty AccDirty - -
|
||||
* AccWrite AccDirty AccDirty - -
|
||||
*/
|
||||
if(AccCache.state==Empty) {
|
||||
GRID_ASSERT(AccCache.LRU_valid==0);
|
||||
AccCache.CpuPtr = CpuPtr;
|
||||
AccCache.AccPtr = (uint64_t)NULL;
|
||||
AccCache.bytes = bytes;
|
||||
AccCache.state = CpuDirty; // Cpu starts primary
|
||||
if(mode==AcceleratorWriteDiscard){
|
||||
CpuDiscard(AccCache);
|
||||
AccCache.state = AccDirty; // Empty + AcceleratorWrite=> AccDirty
|
||||
} else if(mode==AcceleratorWrite){
|
||||
Clone(AccCache);
|
||||
AccCache.state = AccDirty; // Empty + AcceleratorWrite=> AccDirty
|
||||
} else {
|
||||
Clone(AccCache);
|
||||
AccCache.state = Consistent; // Empty + AccRead => Consistent
|
||||
}
|
||||
AccCache.accLock= 1;
|
||||
dprintf("Copied Empty entry into device accLock= %d",AccCache.accLock);
|
||||
} else if(AccCache.state==CpuDirty ){
|
||||
if(mode==AcceleratorWriteDiscard) {
|
||||
CpuDiscard(AccCache);
|
||||
AccCache.state = AccDirty; // CpuDirty + AcceleratorWrite=> AccDirty
|
||||
} else if(mode==AcceleratorWrite) {
|
||||
Clone(AccCache);
|
||||
AccCache.state = AccDirty; // CpuDirty + AcceleratorWrite=> AccDirty
|
||||
} else {
|
||||
Clone(AccCache);
|
||||
AccCache.state = Consistent; // CpuDirty + AccRead => Consistent
|
||||
}
|
||||
AccCache.accLock++;
|
||||
dprintf("CpuDirty entry into device ++accLock= %d",AccCache.accLock);
|
||||
} else if(AccCache.state==Consistent) {
|
||||
if((mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard))
|
||||
AccCache.state = AccDirty; // Consistent + AcceleratorWrite=> AccDirty
|
||||
else
|
||||
AccCache.state = Consistent; // Consistent + AccRead => Consistent
|
||||
AccCache.accLock++;
|
||||
dprintf("Consistent entry into device ++accLock= %d",AccCache.accLock);
|
||||
} else if(AccCache.state==AccDirty) {
|
||||
if((mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard))
|
||||
AccCache.state = AccDirty; // AccDirty + AcceleratorWrite=> AccDirty
|
||||
else
|
||||
AccCache.state = AccDirty; // AccDirty + AccRead => AccDirty
|
||||
AccCache.accLock++;
|
||||
dprintf("AccDirty entry ++accLock= %d",AccCache.accLock);
|
||||
} else {
|
||||
GRID_ASSERT(0);
|
||||
}
|
||||
|
||||
GRID_ASSERT(AccCache.accLock>0);
|
||||
// If view is opened on device must remove from LRU
|
||||
if(AccCache.LRU_valid==1){
|
||||
// must possibly remove from LRU as now locked on GPU
|
||||
dprintf("AccCache entry removed from LRU ");
|
||||
LRUremove(AccCache);
|
||||
}
|
||||
|
||||
int transient =hint;
|
||||
AccCache.transient= transient? EvictNext : 0;
|
||||
|
||||
return AccCache.AccPtr;
|
||||
}
|
||||
////////////////////////////////////
|
||||
// look up & decrement lock count
|
||||
////////////////////////////////////
|
||||
void MemoryManager::AcceleratorViewClose(uint64_t CpuPtr)
|
||||
{
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
|
||||
GRID_ASSERT(AccCache.cpuLock==0);
|
||||
GRID_ASSERT(AccCache.accLock>0);
|
||||
|
||||
AccCache.accLock--;
|
||||
// Move to LRU queue if not locked and close on device
|
||||
if(AccCache.accLock==0) {
|
||||
dprintf("AccleratorViewClose %lx AccLock decremented to %ld move to LRU queue",(uint64_t)CpuPtr,(uint64_t)AccCache.accLock);
|
||||
LRUinsert(AccCache);
|
||||
} else {
|
||||
dprintf("AccleratorViewClose %lx AccLock decremented to %ld",(uint64_t)CpuPtr,(uint64_t)AccCache.accLock);
|
||||
}
|
||||
}
|
||||
void MemoryManager::CpuViewClose(uint64_t CpuPtr)
|
||||
{
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
|
||||
GRID_ASSERT(AccCache.cpuLock>0);
|
||||
GRID_ASSERT(AccCache.accLock==0);
|
||||
|
||||
AccCache.cpuLock--;
|
||||
}
|
||||
/*
|
||||
* Action State StateNext Flush Clone
|
||||
*
|
||||
* CpuRead Empty CpuDirty - -
|
||||
* CpuWrite Empty CpuDirty - -
|
||||
* CpuRead CpuDirty CpuDirty - -
|
||||
* CpuWrite CpuDirty CpuDirty - -
|
||||
* CpuRead Consistent Consistent - -
|
||||
* CpuWrite Consistent CpuDirty - -
|
||||
* CpuRead AccDirty Consistent Y -
|
||||
* CpuWrite AccDirty CpuDirty Y -
|
||||
*/
|
||||
uint64_t MemoryManager::CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise transient)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// Find if present, otherwise get or force an empty
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
if ( EntryPresent(CpuPtr)==0 ){
|
||||
EntryCreate(CpuPtr,bytes,mode,transient);
|
||||
}
|
||||
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
|
||||
// CPU doesn't need to free space
|
||||
// if (!AccCache.AccPtr) {
|
||||
// EvictVictims(bytes);
|
||||
// }
|
||||
|
||||
GRID_ASSERT((mode==CpuRead)||(mode==CpuWrite));
|
||||
GRID_ASSERT(AccCache.accLock==0); // Programming error
|
||||
|
||||
if(AccCache.state!=Empty) {
|
||||
GRID_ASSERT(AccCache.CpuPtr == CpuPtr);
|
||||
GRID_ASSERT(AccCache.bytes==bytes);
|
||||
}
|
||||
|
||||
if(AccCache.state==Empty) {
|
||||
AccCache.CpuPtr = CpuPtr;
|
||||
AccCache.AccPtr = (uint64_t)NULL;
|
||||
AccCache.bytes = bytes;
|
||||
AccCache.state = CpuDirty; // Empty + CpuRead/CpuWrite => CpuDirty
|
||||
AccCache.accLock= 0;
|
||||
AccCache.cpuLock= 1;
|
||||
} else if(AccCache.state==CpuDirty ){
|
||||
// AccPtr dont care, deferred allocate
|
||||
AccCache.state = CpuDirty; // CpuDirty +CpuRead/CpuWrite => CpuDirty
|
||||
AccCache.cpuLock++;
|
||||
} else if(AccCache.state==Consistent) {
|
||||
GRID_ASSERT(AccCache.AccPtr != (uint64_t)NULL);
|
||||
if(mode==CpuWrite)
|
||||
AccCache.state = CpuDirty; // Consistent +CpuWrite => CpuDirty
|
||||
else
|
||||
AccCache.state = Consistent; // Consistent +CpuRead => Consistent
|
||||
AccCache.cpuLock++;
|
||||
} else if(AccCache.state==AccDirty) {
|
||||
GRID_ASSERT(AccCache.AccPtr != (uint64_t)NULL);
|
||||
Flush(AccCache);
|
||||
if(mode==CpuWrite) AccCache.state = CpuDirty; // AccDirty +CpuWrite => CpuDirty, Flush
|
||||
else AccCache.state = Consistent; // AccDirty +CpuRead => Consistent, Flush
|
||||
AccCache.cpuLock++;
|
||||
} else {
|
||||
GRID_ASSERT(0); // should be unreachable
|
||||
}
|
||||
|
||||
AccCache.transient= transient? EvictNext : 0;
|
||||
|
||||
return AccCache.CpuPtr;
|
||||
}
|
||||
void MemoryManager::NotifyDeletion(void *_ptr)
|
||||
{
|
||||
// Look up in ViewCache
|
||||
uint64_t ptr = (uint64_t)_ptr;
|
||||
if(EntryPresent(ptr)) {
|
||||
auto e = EntryLookup(ptr);
|
||||
AccDiscard(e->second);
|
||||
}
|
||||
}
|
||||
void MemoryManager::Print(void)
|
||||
{
|
||||
PrintBytes();
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
std::cout << GridLogMessage << "Memory Manager " << std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
std::cout << GridLogMessage << DeviceBytes << " bytes allocated on device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceLRUBytes<< " bytes evictable on device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceMaxBytes<< " bytes max on device " << std::endl;
|
||||
std::cout << GridLogMessage << HostToDeviceXfer << " transfers to device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceToHostXfer << " transfers from device " << std::endl;
|
||||
std::cout << GridLogMessage << HostToDeviceBytes<< " bytes transfered to device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceToHostBytes<< " bytes transfered from device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceEvictions << " Evictions from device " << std::endl;
|
||||
std::cout << GridLogMessage << DeviceDestroy << " Destroyed vectors on device " << std::endl;
|
||||
std::cout << GridLogMessage << AccViewTable.size()<< " vectors " << LRU.size()<<" evictable"<< std::endl;
|
||||
acceleratorMem();
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
}
|
||||
void MemoryManager::PrintAll(void)
|
||||
{
|
||||
Print();
|
||||
std::cout << GridLogMessage << std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
for(auto it=AccViewTable.begin();it!=AccViewTable.end();it++){
|
||||
auto &AccCache = it->second;
|
||||
|
||||
std::string str;
|
||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||
|
||||
std::cout << GridLogMessage << "0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||
<< "\t" << AccCache.cpuLock
|
||||
<< "\t" << AccCache.accLock
|
||||
<< "\t" << AccCache.LRU_valid<<std::endl;
|
||||
}
|
||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
||||
|
||||
};
|
||||
int MemoryManager::isOpen (void* _CpuPtr)
|
||||
{
|
||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||
if ( EntryPresent(CpuPtr) ){
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
return AccCache.cpuLock+AccCache.accLock;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
void MemoryManager::Audit(std::string s)
|
||||
{
|
||||
uint64_t CpuBytes=0;
|
||||
uint64_t AccBytes=0;
|
||||
uint64_t LruBytes1=0;
|
||||
uint64_t LruBytes2=0;
|
||||
uint64_t LruCnt=0;
|
||||
|
||||
std::cout << " Memory Manager::Audit() from "<<s<<std::endl;
|
||||
for(auto it=LRU.begin();it!=LRU.end();it++){
|
||||
uint64_t cpuPtr = *it;
|
||||
GRID_ASSERT(EntryPresent(cpuPtr));
|
||||
auto AccCacheIterator = EntryLookup(cpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
LruBytes2+=AccCache.bytes;
|
||||
GRID_ASSERT(AccCache.LRU_valid==1);
|
||||
GRID_ASSERT(AccCache.LRU_entry==it);
|
||||
}
|
||||
std::cout << " Memory Manager::Audit() LRU queue matches table entries "<<std::endl;
|
||||
|
||||
for(auto it=AccViewTable.begin();it!=AccViewTable.end();it++){
|
||||
auto &AccCache = it->second;
|
||||
|
||||
std::string str;
|
||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||
|
||||
CpuBytes+=AccCache.bytes;
|
||||
if( AccCache.AccPtr ) AccBytes+=AccCache.bytes;
|
||||
if( AccCache.LRU_valid ) LruBytes1+=AccCache.bytes;
|
||||
if( AccCache.LRU_valid ) LruCnt++;
|
||||
|
||||
if ( AccCache.cpuLock || AccCache.accLock ) {
|
||||
GRID_ASSERT(AccCache.LRU_valid==0);
|
||||
|
||||
std::cout << GridLogError << s<< "\n\t 0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||
<< "\t cpuLock " << AccCache.cpuLock
|
||||
<< "\t accLock " << AccCache.accLock
|
||||
<< "\t LRUvalid " << AccCache.LRU_valid<<std::endl;
|
||||
}
|
||||
|
||||
GRID_ASSERT( AccCache.cpuLock== 0 ) ;
|
||||
GRID_ASSERT( AccCache.accLock== 0 ) ;
|
||||
}
|
||||
std::cout << " Memory Manager::Audit() no locked table entries "<<std::endl;
|
||||
GRID_ASSERT(LruBytes1==LruBytes2);
|
||||
GRID_ASSERT(LruBytes1==DeviceLRUBytes);
|
||||
std::cout << " Memory Manager::Audit() evictable bytes matches sum over table "<<std::endl;
|
||||
GRID_ASSERT(AccBytes==DeviceBytes);
|
||||
std::cout << " Memory Manager::Audit() device bytes matches sum over table "<<std::endl;
|
||||
GRID_ASSERT(LruCnt == LRU.size());
|
||||
std::cout << " Memory Manager::Audit() LRU entry count matches "<<std::endl;
|
||||
|
||||
}
|
||||
|
||||
void MemoryManager::PrintState(void* _CpuPtr)
|
||||
{
|
||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
||||
|
||||
if ( EntryPresent(CpuPtr) ){
|
||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
||||
auto & AccCache = AccCacheIterator->second;
|
||||
std::string str;
|
||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
||||
if ( AccCache.state==EvictNext) str = std::string("EvictNext");
|
||||
|
||||
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
||||
std::cout << GridLogMessage << "\tx"<<std::hex<<AccCache.CpuPtr<<std::dec
|
||||
<< "\tx"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
||||
<< "\t" << AccCache.cpuLock
|
||||
<< "\t" << AccCache.accLock
|
||||
<< "\t" << AccCache.LRU_valid<<std::endl;
|
||||
|
||||
} else {
|
||||
std::cout << GridLogMessage << "No Entry in AccCache table." << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
#endif
|
||||
@@ -1,31 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#ifdef GRID_UVM
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// View management is 1:1 address space mapping
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
uint64_t MemoryManager::DeviceBytes;
|
||||
uint64_t MemoryManager::DeviceLRUBytes;
|
||||
uint64_t MemoryManager::DeviceMaxBytes = 1024*1024*128;
|
||||
uint64_t MemoryManager::HostToDeviceBytes;
|
||||
uint64_t MemoryManager::DeviceToHostBytes;
|
||||
uint64_t MemoryManager::HostToDeviceXfer;
|
||||
uint64_t MemoryManager::DeviceToHostXfer;
|
||||
uint64_t MemoryManager::DeviceEvictions;
|
||||
uint64_t MemoryManager::DeviceDestroy;
|
||||
|
||||
void MemoryManager::Audit(std::string s){};
|
||||
void MemoryManager::ViewClose(void* AccPtr,ViewMode mode){};
|
||||
void *MemoryManager::ViewOpen(void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint){ return CpuPtr; };
|
||||
int MemoryManager::isOpen (void* CpuPtr) { return 0;}
|
||||
void MemoryManager::PrintState(void* CpuPtr)
|
||||
{
|
||||
std::cout << GridLogMessage << "Host<->Device memory movement not currently managed by Grid." << std::endl;
|
||||
};
|
||||
void MemoryManager::Print(void){};
|
||||
void MemoryManager::PrintAll(void){};
|
||||
void MemoryManager::NotifyDeletion(void *ptr){};
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
#endif
|
||||
@@ -1,67 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
MemoryStats *MemoryProfiler::stats = nullptr;
|
||||
bool MemoryProfiler::debug = false;
|
||||
|
||||
void check_huge_pages(void *Buf,uint64_t BYTES)
|
||||
{
|
||||
#ifdef __linux__
|
||||
int fd = open("/proc/self/pagemap", O_RDONLY);
|
||||
GRID_ASSERT(fd >= 0);
|
||||
const int page_size = 4096;
|
||||
uint64_t virt_pfn = (uint64_t)Buf / page_size;
|
||||
off_t offset = sizeof(uint64_t) * virt_pfn;
|
||||
uint64_t npages = (BYTES + page_size-1) / page_size;
|
||||
std::vector<uint64_t> pagedata(npages);
|
||||
uint64_t ret = lseek(fd, offset, SEEK_SET);
|
||||
GRID_ASSERT(ret == offset);
|
||||
ret = ::read(fd, &pagedata[0], sizeof(uint64_t)*npages);
|
||||
GRID_ASSERT(ret == sizeof(uint64_t) * npages);
|
||||
int nhugepages = npages / 512;
|
||||
int n4ktotal, nnothuge;
|
||||
n4ktotal = 0;
|
||||
nnothuge = 0;
|
||||
for (int i = 0; i < nhugepages; ++i) {
|
||||
uint64_t baseaddr = (pagedata[i*512] & 0x7fffffffffffffULL) * page_size;
|
||||
for (int j = 0; j < 512; ++j) {
|
||||
uint64_t pageaddr = (pagedata[i*512+j] & 0x7fffffffffffffULL) * page_size;
|
||||
++n4ktotal;
|
||||
if (pageaddr != baseaddr + j * page_size)
|
||||
++nnothuge;
|
||||
}
|
||||
}
|
||||
int rank = CartesianCommunicator::RankWorld();
|
||||
printf("rank %d Allocated %d 4k pages, %d not in huge pages\n", rank, n4ktotal, nnothuge);
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string sizeString(const size_t bytes)
|
||||
{
|
||||
constexpr unsigned int bufSize = 256;
|
||||
const char *suffixes[7] = {"", "K", "M", "G", "T", "P", "E"};
|
||||
char buf[256];
|
||||
size_t s = 0;
|
||||
double count = bytes;
|
||||
|
||||
while (count >= 1024 && s < 7)
|
||||
{
|
||||
s++;
|
||||
count /= 1024;
|
||||
}
|
||||
if (count - floor(count) == 0.0)
|
||||
{
|
||||
snprintf(buf, bufSize, "%d %sB", (int)count, suffixes[s]);
|
||||
}
|
||||
else
|
||||
{
|
||||
snprintf(buf, bufSize, "%.1f %sB", count, suffixes[s]);
|
||||
}
|
||||
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/MemoryStats.h
|
||||
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
See the full license in the file "LICENSE" in the top level distribution directory
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
#pragma once
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
std::string sizeString(size_t bytes);
|
||||
|
||||
struct MemoryStats
|
||||
{
|
||||
size_t totalAllocated{0}, maxAllocated{0},
|
||||
currentlyAllocated{0}, totalFreed{0};
|
||||
};
|
||||
|
||||
class MemoryProfiler
|
||||
{
|
||||
public:
|
||||
static MemoryStats *stats;
|
||||
static bool debug;
|
||||
};
|
||||
|
||||
#define memString(bytes) std::to_string(bytes) + " (" + sizeString(bytes) + ")"
|
||||
#define profilerDebugPrint \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
std::cout << GridLogDebug << "[Memory debug] Stats " << MemoryProfiler::stats << std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] total : " << memString(s->totalAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] max : " << memString(s->maxAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] current: " << memString(s->currentlyAllocated) \
|
||||
<< std::endl; \
|
||||
std::cout << GridLogDebug << "[Memory debug] freed : " << memString(s->totalFreed) \
|
||||
<< std::endl; \
|
||||
}
|
||||
|
||||
#define profilerAllocate(bytes) \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
s->totalAllocated += (bytes); \
|
||||
s->currentlyAllocated += (bytes); \
|
||||
s->maxAllocated = std::max(s->maxAllocated, s->currentlyAllocated); \
|
||||
} \
|
||||
if (MemoryProfiler::debug) \
|
||||
{ \
|
||||
std::cout << GridLogDebug << "[Memory debug] allocating " << memString(bytes) << std::endl; \
|
||||
profilerDebugPrint; \
|
||||
}
|
||||
|
||||
#define profilerFree(bytes) \
|
||||
if (MemoryProfiler::stats) \
|
||||
{ \
|
||||
auto s = MemoryProfiler::stats; \
|
||||
s->totalFreed += (bytes); \
|
||||
s->currentlyAllocated -= (bytes); \
|
||||
} \
|
||||
if (MemoryProfiler::debug) \
|
||||
{ \
|
||||
std::cout << GridLogDebug << "[Memory debug] freeing " << memString(bytes) << std::endl; \
|
||||
profilerDebugPrint; \
|
||||
}
|
||||
|
||||
void check_huge_pages(void *Buf,uint64_t BYTES);
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -70,8 +70,8 @@ public:
|
||||
Coordinate _istride; // Inner stride i.e. within simd lane
|
||||
int _osites; // _isites*_osites = product(dimensions).
|
||||
int _isites;
|
||||
int64_t _fsites; // _isites*_osites = product(dimensions).
|
||||
int64_t _gsites;
|
||||
int _fsites; // _isites*_osites = product(dimensions).
|
||||
int _gsites;
|
||||
Coordinate _slice_block;// subslice information
|
||||
Coordinate _slice_stride;
|
||||
Coordinate _slice_nblock;
|
||||
@@ -81,8 +81,6 @@ public:
|
||||
|
||||
bool _isCheckerBoarded;
|
||||
int LocallyPeriodic;
|
||||
Coordinate _checker_dim_mask;
|
||||
int _checker_dim;
|
||||
|
||||
public:
|
||||
|
||||
@@ -90,7 +88,7 @@ public:
|
||||
// Checkerboarding interface is virtual and overridden by
|
||||
// GridCartesian / GridRedBlackCartesian
|
||||
////////////////////////////////////////////////////////////////
|
||||
virtual int CheckerBoarded(int dim) =0;
|
||||
virtual int CheckerBoarded(int dim)=0;
|
||||
virtual int CheckerBoard(const Coordinate &site)=0;
|
||||
virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
|
||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
|
||||
@@ -165,7 +163,7 @@ public:
|
||||
//
|
||||
if ( _simd_layout[dimension] > 2 ) {
|
||||
for(int d=0;d<_ndimension;d++){
|
||||
if ( d != dimension ) GRID_ASSERT ( (_simd_layout[d]==1) );
|
||||
if ( d != dimension ) assert ( (_simd_layout[d]==1) );
|
||||
}
|
||||
permute_type = RotateBit; // How to specify distance; this is not just direction.
|
||||
return permute_type;
|
||||
@@ -184,10 +182,10 @@ public:
|
||||
inline int Nsimd(void) const { return _isites; };// Synonymous with iSites
|
||||
inline int oSites(void) const { return _osites; };
|
||||
inline int lSites(void) const { return _isites*_osites; };
|
||||
inline int64_t gSites(void) const { return (int64_t)_isites*(int64_t)_osites*(int64_t)_Nprocessors; };
|
||||
inline int gSites(void) const { return _isites*_osites*_Nprocessors; };
|
||||
inline int Nd (void) const { return _ndimension;};
|
||||
|
||||
inline const Coordinate &LocalStarts(void) { return _lstart; };
|
||||
inline const Coordinate LocalStarts(void) { return _lstart; };
|
||||
inline const Coordinate &FullDimensions(void) { return _fdimensions;};
|
||||
inline const Coordinate &GlobalDimensions(void) { return _gdimensions;};
|
||||
inline const Coordinate &LocalDimensions(void) { return _ldimensions;};
|
||||
@@ -215,15 +213,15 @@ public:
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Global addressing
|
||||
////////////////////////////////////////////////////////////////
|
||||
void GlobalIndexToGlobalCoor(int64_t gidx,Coordinate &gcoor){
|
||||
GRID_ASSERT(gidx< gSites());
|
||||
void GlobalIndexToGlobalCoor(int gidx,Coordinate &gcoor){
|
||||
assert(gidx< gSites());
|
||||
Lexicographic::CoorFromIndex(gcoor,gidx,_gdimensions);
|
||||
}
|
||||
void LocalIndexToLocalCoor(int lidx,Coordinate &lcoor){
|
||||
GRID_ASSERT(lidx<lSites());
|
||||
assert(lidx<lSites());
|
||||
Lexicographic::CoorFromIndex(lcoor,lidx,_ldimensions);
|
||||
}
|
||||
void GlobalCoorToGlobalIndex(const Coordinate & gcoor,int64_t & gidx){
|
||||
void GlobalCoorToGlobalIndex(const Coordinate & gcoor,int & gidx){
|
||||
gidx=0;
|
||||
int mult=1;
|
||||
for(int mu=0;mu<_ndimension;mu++) {
|
||||
|
||||
@@ -38,7 +38,6 @@ class GridCartesian: public GridBase {
|
||||
|
||||
public:
|
||||
int dummy;
|
||||
// Coordinate _checker_dim_mask;
|
||||
virtual int CheckerBoardFromOindexTable (int Oindex) {
|
||||
return 0;
|
||||
}
|
||||
@@ -46,7 +45,7 @@ public:
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoarded(int dim) {
|
||||
virtual int CheckerBoarded(int dim){
|
||||
return 0;
|
||||
}
|
||||
virtual int CheckerBoard(const Coordinate &site){
|
||||
@@ -105,8 +104,6 @@ public:
|
||||
_ldimensions.resize(_ndimension);
|
||||
_rdimensions.resize(_ndimension);
|
||||
_simd_layout.resize(_ndimension);
|
||||
_checker_dim_mask.resize(_ndimension);;
|
||||
_checker_dim = -1;
|
||||
_lstart.resize(_ndimension);
|
||||
_lend.resize(_ndimension);
|
||||
|
||||
@@ -117,8 +114,6 @@ public:
|
||||
|
||||
for (int d = 0; d < _ndimension; d++)
|
||||
{
|
||||
_checker_dim_mask[d]=0;
|
||||
|
||||
_fdimensions[d] = dimensions[d]; // Global dimensions
|
||||
_gdimensions[d] = _fdimensions[d]; // Global dimensions
|
||||
_simd_layout[d] = simd_layout[d];
|
||||
@@ -128,10 +123,10 @@ public:
|
||||
// Use a reduced simd grid
|
||||
_ldimensions[d] = _gdimensions[d] / _processors[d]; //local dimensions
|
||||
//std::cout << _ldimensions[d] << " " << _gdimensions[d] << " " << _processors[d] << std::endl;
|
||||
GRID_ASSERT(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
||||
assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
||||
|
||||
_rdimensions[d] = _ldimensions[d] / _simd_layout[d]; //overdecomposition
|
||||
GRID_ASSERT(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
||||
assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
||||
|
||||
_lstart[d] = _processor_coor[d] * _ldimensions[d];
|
||||
_lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
|
||||
|
||||
@@ -35,39 +35,22 @@ static const int CbRed =0;
|
||||
static const int CbBlack=1;
|
||||
static const int Even =CbRed;
|
||||
static const int Odd =CbBlack;
|
||||
|
||||
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk)
|
||||
{
|
||||
int nd=rdim.size();
|
||||
Coordinate coor(nd);
|
||||
|
||||
Lexicographic::CoorFromIndex(coor,oindex,rdim);
|
||||
|
||||
int linear=0;
|
||||
for(int d=0;d<nd;d++){
|
||||
if(chk_dim_msk[d])
|
||||
linear=linear+coor[d];
|
||||
}
|
||||
return (linear&0x1);
|
||||
}
|
||||
|
||||
|
||||
// Specialise this for red black grids storing half the data like a chess board.
|
||||
class GridRedBlackCartesian : public GridBase
|
||||
{
|
||||
public:
|
||||
// Coordinate _checker_dim_mask;
|
||||
// int _checker_dim;
|
||||
Coordinate _checker_dim_mask;
|
||||
int _checker_dim;
|
||||
std::vector<int> _checker_board;
|
||||
|
||||
virtual int isCheckerBoarded(void) const { return 1; };
|
||||
virtual int CheckerBoarded(int dim){
|
||||
if( dim==_checker_dim) return 1;
|
||||
else return 0;
|
||||
}
|
||||
virtual int CheckerBoard(const Coordinate &site){
|
||||
int linear=0;
|
||||
GRID_ASSERT(site.size()==_ndimension);
|
||||
assert(site.size()==_ndimension);
|
||||
for(int d=0;d<_ndimension;d++){
|
||||
if(_checker_dim_mask[d])
|
||||
linear=linear+site[d];
|
||||
@@ -148,7 +131,7 @@ public:
|
||||
{
|
||||
Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim) ;
|
||||
}
|
||||
|
||||
|
||||
virtual ~GridRedBlackCartesian() = default;
|
||||
|
||||
void Init(const Coordinate &dimensions,
|
||||
@@ -160,11 +143,11 @@ public:
|
||||
|
||||
_isCheckerBoarded = true;
|
||||
_checker_dim = checker_dim;
|
||||
GRID_ASSERT(checker_dim_mask[checker_dim] == 1);
|
||||
assert(checker_dim_mask[checker_dim] == 1);
|
||||
_ndimension = dimensions.size();
|
||||
GRID_ASSERT(checker_dim_mask.size() == _ndimension);
|
||||
GRID_ASSERT(processor_grid.size() == _ndimension);
|
||||
GRID_ASSERT(simd_layout.size() == _ndimension);
|
||||
assert(checker_dim_mask.size() == _ndimension);
|
||||
assert(processor_grid.size() == _ndimension);
|
||||
assert(simd_layout.size() == _ndimension);
|
||||
|
||||
_fdimensions.resize(_ndimension);
|
||||
_gdimensions.resize(_ndimension);
|
||||
@@ -190,20 +173,20 @@ public:
|
||||
|
||||
if (d == _checker_dim)
|
||||
{
|
||||
GRID_ASSERT((_gdimensions[d] & 0x1) == 0);
|
||||
assert((_gdimensions[d] & 0x1) == 0);
|
||||
_gdimensions[d] = _gdimensions[d] / 2; // Remove a checkerboard
|
||||
_gsites /= 2;
|
||||
}
|
||||
_ldimensions[d] = _gdimensions[d] / _processors[d];
|
||||
GRID_ASSERT(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
||||
assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
||||
_lstart[d] = _processor_coor[d] * _ldimensions[d];
|
||||
_lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
|
||||
|
||||
// Use a reduced simd grid
|
||||
_simd_layout[d] = simd_layout[d];
|
||||
_rdimensions[d] = _ldimensions[d] / _simd_layout[d]; // this is not checking if this is integer
|
||||
GRID_ASSERT(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
||||
GRID_ASSERT(_rdimensions[d] > 0);
|
||||
assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
||||
assert(_rdimensions[d] > 0);
|
||||
|
||||
// all elements of a simd vector must have same checkerboard.
|
||||
// If Ls vectorised, this must still be the case; e.g. dwf rb5d
|
||||
|
||||
@@ -33,8 +33,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
bool Stencil_force_mpi = true;
|
||||
|
||||
///////////////////////////////////////////////////////////////
|
||||
// Info that is setup once and indept of cartesian layout
|
||||
///////////////////////////////////////////////////////////////
|
||||
@@ -57,29 +55,18 @@ int CartesianCommunicator::ProcessorCount(void) { return
|
||||
// very VERY rarely (Log, serial RNG) we need world without a grid
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#ifdef USE_GRID_REDUCTION
|
||||
void CartesianCommunicator::GlobalSum(ComplexF &c)
|
||||
{
|
||||
GlobalSumP2P(c);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(ComplexD &c)
|
||||
{
|
||||
GlobalSumP2P(c);
|
||||
}
|
||||
#else
|
||||
void CartesianCommunicator::GlobalSum(ComplexF &c)
|
||||
{
|
||||
GlobalSumVector((float *)&c,2);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(ComplexD &c)
|
||||
{
|
||||
GlobalSumVector((double *)&c,2);
|
||||
}
|
||||
#endif
|
||||
void CartesianCommunicator::GlobalSumVector(ComplexF *c,int N)
|
||||
{
|
||||
GlobalSumVector((float *)c,2*N);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(ComplexD &c)
|
||||
{
|
||||
GlobalSumVector((double *)&c,2);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
|
||||
{
|
||||
GlobalSumVector((double *)c,2*N);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
@@ -33,12 +34,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
///////////////////////////////////
|
||||
#include <Grid/communicator/SharedMemory.h>
|
||||
|
||||
#define NVLINK_GET
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
extern bool Stencil_force_mpi ;
|
||||
|
||||
class CartesianCommunicator : public SharedMemory {
|
||||
|
||||
public:
|
||||
@@ -55,11 +52,10 @@ public:
|
||||
// Communicator should know nothing of the physics grid, only processor grid.
|
||||
////////////////////////////////////////////
|
||||
int _Nprocessors; // How many in all
|
||||
int _processor; // linear processor rank
|
||||
unsigned long _ndimension;
|
||||
Coordinate _shm_processors; // Which dimensions get relayed out over processors lanes.
|
||||
Coordinate _processors; // Which dimensions get relayed out over processors lanes.
|
||||
int _processor; // linear processor rank
|
||||
Coordinate _processor_coor; // linear processor coordinate
|
||||
unsigned long _ndimension;
|
||||
static Grid_MPI_Comm communicator_world;
|
||||
Grid_MPI_Comm communicator;
|
||||
std::vector<Grid_MPI_Comm> communicator_halo;
|
||||
@@ -100,22 +96,18 @@ public:
|
||||
int BossRank(void) ;
|
||||
int ThisRank(void) ;
|
||||
const Coordinate & ThisProcessorCoor(void) ;
|
||||
const Coordinate & ShmGrid(void) { return _shm_processors; } ;
|
||||
const Coordinate & ProcessorGrid(void) ;
|
||||
int ProcessorCount(void) ;
|
||||
int ProcessorCount(void) ;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// very VERY rarely (Log, serial RNG) we need world without a grid
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
static int RankWorld(void) ;
|
||||
static void BroadcastWorld(int root,void* data, uint64_t bytes);
|
||||
static void BarrierWorld(void);
|
||||
static void BroadcastWorld(int root,void* data, int bytes);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Reduction
|
||||
////////////////////////////////////////////////////////////
|
||||
void GlobalMax(RealD &);
|
||||
void GlobalMax(RealF &);
|
||||
void GlobalSum(RealF &);
|
||||
void GlobalSumVector(RealF *,int N);
|
||||
void GlobalSum(RealD &);
|
||||
@@ -129,84 +121,50 @@ public:
|
||||
void GlobalSumVector(ComplexD *c,int N);
|
||||
void GlobalXOR(uint32_t &);
|
||||
void GlobalXOR(uint64_t &);
|
||||
|
||||
template<class obj> void GlobalSumP2P(obj &o)
|
||||
{
|
||||
std::vector<obj> column;
|
||||
obj accum = o;
|
||||
int source,dest;
|
||||
for(int d=0;d<_ndimension;d++){
|
||||
column.resize(_processors[d]);
|
||||
column[0] = accum;
|
||||
std::vector<MpiCommsRequest_t> list;
|
||||
for(int p=1;p<_processors[d];p++){
|
||||
ShiftedRanks(d,p,source,dest);
|
||||
SendToRecvFromBegin(list,
|
||||
&column[0],
|
||||
dest,
|
||||
&column[p],
|
||||
source,
|
||||
sizeof(obj),d*100+p);
|
||||
|
||||
}
|
||||
if (!list.empty()) // avoid triggering GRID_ASSERT in comms == none
|
||||
CommsComplete(list);
|
||||
for(int p=1;p<_processors[d];p++){
|
||||
accum = accum + column[p];
|
||||
}
|
||||
}
|
||||
Broadcast(0,accum);
|
||||
o=accum;
|
||||
}
|
||||
|
||||
|
||||
template<class obj> void GlobalSum(obj &o){
|
||||
typedef typename obj::scalar_type scalar_type;
|
||||
int words = sizeof(obj)/sizeof(scalar_type);
|
||||
scalar_type * ptr = (scalar_type *)& o; // Safe alias
|
||||
scalar_type * ptr = (scalar_type *)& o;
|
||||
GlobalSumVector(ptr,words);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// Face exchange, buffer swap in translational invariant way
|
||||
////////////////////////////////////////////////////////////
|
||||
void CommsComplete(std::vector<MpiCommsRequest_t> &list);
|
||||
void SendToRecvFromBegin(std::vector<MpiCommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
uint64_t bytes,int dir);
|
||||
|
||||
void SendToRecvFrom(void *xmit,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,
|
||||
uint64_t bytes);
|
||||
int bytes);
|
||||
|
||||
int IsOffNode(int rank);
|
||||
void SendRecvPacket(void *xmit,
|
||||
void *recv,
|
||||
int xmit_to_rank,
|
||||
int recv_from_rank,
|
||||
int bytes);
|
||||
|
||||
void SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,
|
||||
int bytes);
|
||||
|
||||
void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
|
||||
|
||||
double StencilSendToRecvFrom(void *xmit,
|
||||
int xmit_to_rank,int do_xmit,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,int do_recv,
|
||||
uint64_t bytes,int dir);
|
||||
|
||||
double StencilSendToRecvFromPrepare(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int xmit_to_rank,int do_xmit,
|
||||
void *recv,
|
||||
int recv_from_rank,int do_recv,
|
||||
uint64_t xbytes,uint64_t rbytes,int dir);
|
||||
|
||||
// Could do a PollHtoD and have a CommsMerge dependence
|
||||
void StencilSendToRecvFromPollDtoH (std::vector<CommsRequest_t> &list);
|
||||
void StencilSendToRecvFromPollIRecv(std::vector<CommsRequest_t> &list);
|
||||
int recv_from_rank,
|
||||
int bytes,int dir);
|
||||
|
||||
double StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,void *xmit_comp,
|
||||
int xmit_to_rank,int do_xmit,
|
||||
void *recv,void *recv_comp,
|
||||
int recv_from_rank,int do_recv,
|
||||
uint64_t xbytes,uint64_t rbytes,int dir);
|
||||
void *xmit,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,
|
||||
int bytes,int dir);
|
||||
|
||||
|
||||
void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int i);
|
||||
@@ -220,20 +178,20 @@ public:
|
||||
////////////////////////////////////////////////////////////
|
||||
// Broadcast a buffer and composite larger
|
||||
////////////////////////////////////////////////////////////
|
||||
void Broadcast(int root,void* data, uint64_t bytes);
|
||||
void Broadcast(int root,void* data, int bytes);
|
||||
|
||||
////////////////////////////////////////////////////////////
|
||||
// All2All down one dimension
|
||||
////////////////////////////////////////////////////////////
|
||||
template<class T> void AllToAll(int dim,std::vector<T> &in, std::vector<T> &out){
|
||||
GRID_ASSERT(dim>=0);
|
||||
GRID_ASSERT(dim<_ndimension);
|
||||
GRID_ASSERT(in.size()==out.size());
|
||||
assert(dim>=0);
|
||||
assert(dim<_ndimension);
|
||||
assert(in.size()==out.size());
|
||||
int numnode = _processors[dim];
|
||||
uint64_t bytes=sizeof(T);
|
||||
uint64_t words=in.size()/numnode;
|
||||
GRID_ASSERT(numnode * words == in.size());
|
||||
GRID_ASSERT(words < (1ULL<<31));
|
||||
assert(numnode * words == in.size());
|
||||
assert(words < (1ULL<<31));
|
||||
AllToAll(dim,(void *)&in[0],(void *)&out[0],words,bytes);
|
||||
}
|
||||
void AllToAll(int dim ,void *in,void *out,uint64_t words,uint64_t bytes);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/*************************************************************************************
|
||||
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
Grid physics library, www.github.com/paboyle/Grid
|
||||
|
||||
Source file: ./lib/communicator/Communicator_mpi.cc
|
||||
|
||||
@@ -28,22 +28,14 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
#include <Grid/GridCore.h>
|
||||
#include <Grid/communicator/SharedMemory.h>
|
||||
|
||||
void GridAbort(void) { MPI_Abort(MPI_COMM_WORLD,SIGABRT); }
|
||||
extern void * Grid_backtrace_buffer[_NBACKTRACE];
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
|
||||
Grid_MPI_Comm CartesianCommunicator::communicator_world;
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
uint64_t checksum_index = 1;
|
||||
#endif
|
||||
|
||||
|
||||
////////////////////////////////////////////
|
||||
// First initialise of comms system
|
||||
////////////////////////////////////////////
|
||||
void CartesianCommunicator::Init(int *argc, char ***argv)
|
||||
void CartesianCommunicator::Init(int *argc, char ***argv)
|
||||
{
|
||||
|
||||
int flag;
|
||||
@@ -51,23 +43,15 @@ void CartesianCommunicator::Init(int *argc, char ***argv)
|
||||
|
||||
MPI_Initialized(&flag); // needed to coexist with other libs apparently
|
||||
if ( !flag ) {
|
||||
|
||||
#ifndef GRID_COMMS_THREADS
|
||||
nCommThreads=1;
|
||||
// wrong results here too
|
||||
// For now: comms-overlap leads to wrong results in Benchmark_wilson even on single node MPI runs
|
||||
// other comms schemes are ok
|
||||
MPI_Init_thread(argc,argv,MPI_THREAD_SERIALIZED,&provided);
|
||||
#else
|
||||
MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided);
|
||||
#endif
|
||||
|
||||
//If only 1 comms thread we require any threading mode other than SINGLE, but for multiple comms threads we need MULTIPLE
|
||||
if( (nCommThreads == 1) && (provided == MPI_THREAD_SINGLE) ) {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
if( (nCommThreads > 1) && (provided != MPI_THREAD_MULTIPLE) ) {
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,33 +72,33 @@ void CartesianCommunicator::Init(int *argc, char ***argv)
|
||||
void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
|
||||
{
|
||||
int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor)
|
||||
{
|
||||
int rank;
|
||||
int ierr=MPI_Cart_rank (communicator, &coor[0], &rank);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
return rank;
|
||||
}
|
||||
void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor)
|
||||
{
|
||||
coor.resize(_ndimension);
|
||||
int ierr=MPI_Cart_coords (communicator, rank, _ndimension,&coor[0]);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Initialises from communicator_world
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
{
|
||||
MPI_Comm optimal_comm;
|
||||
////////////////////////////////////////////////////
|
||||
// Remap using the shared memory optimising routine
|
||||
// The remap creates a comm which must be freed
|
||||
////////////////////////////////////////////////////
|
||||
GlobalSharedMemory::OptimalCommunicator (processors,optimal_comm,_shm_processors);
|
||||
GlobalSharedMemory::OptimalCommunicator (processors,optimal_comm);
|
||||
InitFromMPICommunicator(processors,optimal_comm);
|
||||
SetCommunicator(optimal_comm);
|
||||
///////////////////////////////////////////////////
|
||||
@@ -126,25 +110,24 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
//////////////////////////////////
|
||||
// Try to subdivide communicator
|
||||
//////////////////////////////////
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank)
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank)
|
||||
{
|
||||
_ndimension = processors.size(); GRID_ASSERT(_ndimension>=1);
|
||||
int parent_ndimension = parent._ndimension; GRID_ASSERT(_ndimension >= parent._ndimension);
|
||||
_ndimension = processors.size(); assert(_ndimension>=1);
|
||||
int parent_ndimension = parent._ndimension; assert(_ndimension >= parent._ndimension);
|
||||
Coordinate parent_processor_coor(_ndimension,0);
|
||||
Coordinate parent_processors (_ndimension,1);
|
||||
Coordinate shm_processors (_ndimension,1);
|
||||
|
||||
// Can make 5d grid from 4d etc...
|
||||
int pad = _ndimension-parent_ndimension;
|
||||
for(int d=0;d<parent_ndimension;d++){
|
||||
parent_processor_coor[pad+d]=parent._processor_coor[d];
|
||||
parent_processors [pad+d]=parent._processors[d];
|
||||
shm_processors [pad+d]=parent._shm_processors[d];
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// split the communicator
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// int Nparent = parent._processors ;
|
||||
// int Nparent = parent._processors ;
|
||||
int Nparent;
|
||||
MPI_Comm_size(parent.communicator,&Nparent);
|
||||
|
||||
@@ -153,7 +136,7 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const
|
||||
childsize *= processors[d];
|
||||
}
|
||||
int Nchild = Nparent/childsize;
|
||||
GRID_ASSERT (childsize * Nchild == Nparent);
|
||||
assert (childsize * Nchild == Nparent);
|
||||
|
||||
Coordinate ccoor(_ndimension); // coor within subcommunicator
|
||||
Coordinate scoor(_ndimension); // coor of split within parent
|
||||
@@ -163,28 +146,27 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const
|
||||
ccoor[d] = parent_processor_coor[d] % processors[d];
|
||||
scoor[d] = parent_processor_coor[d] / processors[d];
|
||||
ssize[d] = parent_processors[d] / processors[d];
|
||||
if ( processors[d] < shm_processors[d] ) shm_processors[d] = processors[d]; // subnode splitting.
|
||||
}
|
||||
|
||||
// rank within subcomm ; srank is rank of subcomm within blocks of subcomms
|
||||
int crank;
|
||||
int crank;
|
||||
// Mpi uses the reverse Lexico convention to us; so reversed routines called
|
||||
Lexicographic::IndexFromCoorReversed(ccoor,crank,processors); // processors is the split grid dimensions
|
||||
Lexicographic::IndexFromCoorReversed(scoor,srank,ssize); // ssize is the number of split grids
|
||||
|
||||
MPI_Comm comm_split;
|
||||
if ( Nchild > 1 ) {
|
||||
if ( Nchild > 1 ) {
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Split the communicator
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ierr= MPI_Comm_split(parent.communicator,srank,crank,&comm_split);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
|
||||
} else {
|
||||
srank = 0;
|
||||
int ierr = MPI_Comm_dup (parent.communicator,&comm_split);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -198,18 +180,18 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const
|
||||
SetCommunicator(comm_split);
|
||||
|
||||
///////////////////////////////////////////////
|
||||
// Free the temp communicator
|
||||
// Free the temp communicator
|
||||
///////////////////////////////////////////////
|
||||
MPI_Comm_free(&comm_split);
|
||||
|
||||
if(0){
|
||||
if(0){
|
||||
std::cout << " ndim " <<_ndimension<<" " << parent._ndimension << std::endl;
|
||||
for(int d=0;d<processors.size();d++){
|
||||
std::cout << d<< " " << _processor_coor[d] <<" " << ccoor[d]<<std::endl;
|
||||
}
|
||||
}
|
||||
for(int d=0;d<processors.size();d++){
|
||||
GRID_ASSERT(_processor_coor[d] == ccoor[d] );
|
||||
assert(_processor_coor[d] == ccoor[d] );
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,7 +233,7 @@ void CartesianCommunicator::InitFromMPICommunicator(const Coordinate &processors
|
||||
for(int i=0;i<_ndimension*2;i++){
|
||||
MPI_Comm_dup(communicator,&communicator_halo[i]);
|
||||
}
|
||||
GRID_ASSERT(Size==_Nprocessors);
|
||||
assert(Size==_Nprocessors);
|
||||
}
|
||||
|
||||
CartesianCommunicator::~CartesianCommunicator()
|
||||
@@ -263,463 +245,129 @@ CartesianCommunicator::~CartesianCommunicator()
|
||||
for(int i=0;i<communicator_halo.size();i++){
|
||||
MPI_Comm_free(&communicator_halo[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef USE_GRID_REDUCTION
|
||||
void CartesianCommunicator::GlobalSum(float &f){
|
||||
FlightRecorder::StepLog("GlobalSumP2P");
|
||||
CartesianCommunicator::GlobalSumP2P(f);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(double &d)
|
||||
{
|
||||
FlightRecorder::StepLog("GlobalSumP2P");
|
||||
CartesianCommunicator::GlobalSumP2P(d);
|
||||
}
|
||||
#else
|
||||
void CartesianCommunicator::GlobalSum(float &f){
|
||||
FlightRecorder::StepLog("AllReduce float");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(double &d)
|
||||
{
|
||||
FlightRecorder::StepLog("AllReduce double");
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
}
|
||||
#endif
|
||||
void CartesianCommunicator::GlobalSum(uint32_t &u){
|
||||
FlightRecorder::StepLog("AllReduce uint32_t");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(uint64_t &u){
|
||||
FlightRecorder::StepLog("AllReduce uint64_t");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(uint64_t* u,int N){
|
||||
FlightRecorder::StepLog("AllReduceVector");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,u,N,MPI_UINT64_T,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalXOR(uint32_t &u){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalXOR(uint64_t &u){
|
||||
FlightRecorder::StepLog("GlobalXOR");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalMax(float &f)
|
||||
{
|
||||
FlightRecorder::StepLog("GlobalMax");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_MAX,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalMax(double &d)
|
||||
{
|
||||
FlightRecorder::StepLog("GlobalMax");
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_MAX,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
void CartesianCommunicator::GlobalSum(float &f){
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(float *f,int N)
|
||||
{
|
||||
FlightRecorder::StepLog("GlobalSumVector(float *)");
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSum(double &d)
|
||||
{
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::GlobalSumVector(double *d,int N)
|
||||
{
|
||||
FlightRecorder::StepLog("GlobalSumVector(double *)");
|
||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
void CartesianCommunicator::SendToRecvFromBegin(std::vector<MpiCommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
uint64_t bytes,int dir)
|
||||
{
|
||||
MPI_Request xrq;
|
||||
MPI_Request rrq;
|
||||
|
||||
GRID_ASSERT(dest != _processor);
|
||||
GRID_ASSERT(from != _processor);
|
||||
int tag;
|
||||
|
||||
tag= dir+from*32;
|
||||
int ierr=MPI_Irecv(recv,(int)( bytes/sizeof(int32_t)), MPI_INT32_T,from,tag,communicator,&rrq);
|
||||
GRID_ASSERT(ierr==0);
|
||||
list.push_back(rrq);
|
||||
|
||||
tag= dir+_processor*32;
|
||||
ierr =MPI_Isend(xmit,(int)(bytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator,&xrq);
|
||||
GRID_ASSERT(ierr==0);
|
||||
list.push_back(xrq);
|
||||
}
|
||||
void CartesianCommunicator::CommsComplete(std::vector<MpiCommsRequest_t> &list)
|
||||
{
|
||||
int nreq=list.size();
|
||||
|
||||
if (nreq==0) return;
|
||||
|
||||
std::vector<MPI_Status> status(nreq);
|
||||
int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
|
||||
GRID_ASSERT(ierr==0);
|
||||
list.resize(0);
|
||||
}
|
||||
|
||||
// Basic Halo comms primitive
|
||||
void CartesianCommunicator::SendToRecvFrom(void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
uint64_t bytes)
|
||||
int bytes)
|
||||
{
|
||||
std::vector<CommsRequest_t> reqs(0);
|
||||
// unsigned long xcrc = crc32(0L, Z_NULL, 0);
|
||||
// unsigned long rcrc = crc32(0L, Z_NULL, 0);
|
||||
// xcrc = crc32(xcrc,(unsigned char *)xmit,bytes);
|
||||
SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
|
||||
SendToRecvFromComplete(reqs);
|
||||
// rcrc = crc32(rcrc,(unsigned char *)recv,bytes);
|
||||
// printf("proc %d SendToRecvFrom %d bytes %lx %lx\n",_processor,bytes,xcrc,rcrc);
|
||||
}
|
||||
void CartesianCommunicator::SendRecvPacket(void *xmit,
|
||||
void *recv,
|
||||
int sender,
|
||||
int receiver,
|
||||
int bytes)
|
||||
{
|
||||
MPI_Status stat;
|
||||
assert(sender != receiver);
|
||||
int tag = sender;
|
||||
if ( _processor == sender ) {
|
||||
MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
|
||||
}
|
||||
if ( _processor == receiver ) {
|
||||
MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
|
||||
}
|
||||
}
|
||||
// Basic Halo comms primitive
|
||||
void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
int bytes)
|
||||
{
|
||||
std::vector<MpiCommsRequest_t> reqs(0);
|
||||
|
||||
int myrank = _processor;
|
||||
int ierr;
|
||||
|
||||
// Enforce no UVM in comms, device or host OK
|
||||
GRID_ASSERT(acceleratorIsCommunicable(xmit));
|
||||
GRID_ASSERT(acceleratorIsCommunicable(recv));
|
||||
|
||||
// Give the CPU to MPI immediately; can use threads to overlap optionally
|
||||
// printf("proc %d SendToRecvFrom %d bytes Sendrecv \n",_processor,bytes);
|
||||
ierr=MPI_Sendrecv(xmit,(int)(bytes/sizeof(int32_t)),MPI_INT32_T,dest,myrank,
|
||||
recv,(int)(bytes/sizeof(int32_t)),MPI_INT32_T,from, from,
|
||||
communicator,MPI_STATUS_IGNORE);
|
||||
GRID_ASSERT(ierr==0);
|
||||
if ( CommunicatorPolicy == CommunicatorPolicyConcurrent ) {
|
||||
MPI_Request xrq;
|
||||
MPI_Request rrq;
|
||||
|
||||
ierr =MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
|
||||
ierr|=MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
|
||||
|
||||
assert(ierr==0);
|
||||
list.push_back(xrq);
|
||||
list.push_back(rrq);
|
||||
} else {
|
||||
// Give the CPU to MPI immediately; can use threads to overlap optionally
|
||||
ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank,
|
||||
recv,bytes,MPI_CHAR,from, from,
|
||||
communicator,MPI_STATUS_IGNORE);
|
||||
assert(ierr==0);
|
||||
}
|
||||
}
|
||||
// Basic Halo comms primitive
|
||||
|
||||
double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
|
||||
int dest, int dox,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from, int dor,
|
||||
uint64_t bytes,int dir)
|
||||
int from,
|
||||
int bytes,int dir)
|
||||
{
|
||||
std::vector<CommsRequest_t> list;
|
||||
double offbytes = StencilSendToRecvFromPrepare(list,xmit,dest,dox,recv,from,dor,bytes,bytes,dir);
|
||||
offbytes += StencilSendToRecvFromBegin(list,xmit,xmit,dest,dox,recv,recv,from,dor,bytes,bytes,dir);
|
||||
double offbytes = StencilSendToRecvFromBegin(list,xmit,dest,recv,from,bytes,dir);
|
||||
StencilSendToRecvFromComplete(list,dir);
|
||||
return offbytes;
|
||||
}
|
||||
int CartesianCommunicator::IsOffNode(int rank)
|
||||
{
|
||||
int grank = ShmRanks[rank];
|
||||
if ( grank == MPI_UNDEFINED ) return true;
|
||||
else return false;
|
||||
}
|
||||
|
||||
#ifdef ACCELERATOR_AWARE_MPI
|
||||
void CartesianCommunicator::StencilSendToRecvFromPollIRecv(std::vector<CommsRequest_t> &list) {};
|
||||
void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector<CommsRequest_t> &list) {};
|
||||
double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,int dox,
|
||||
void *recv,
|
||||
int from,int dor,
|
||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
||||
{
|
||||
return 0.0; // Do nothing -- no preparation required
|
||||
}
|
||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,void *xmit_comp,
|
||||
int dest,int dox,
|
||||
void *recv,void *recv_comp,
|
||||
int from,int dor,
|
||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
||||
{
|
||||
int ncomm =communicator_halo.size();
|
||||
int commdir=dir%ncomm;
|
||||
|
||||
MPI_Request xrq;
|
||||
MPI_Request rrq;
|
||||
|
||||
int ierr;
|
||||
int gdest = ShmRanks[dest];
|
||||
int gfrom = ShmRanks[from];
|
||||
int gme = ShmRanks[_processor];
|
||||
|
||||
GRID_ASSERT(dest != _processor);
|
||||
GRID_ASSERT(from != _processor);
|
||||
GRID_ASSERT(gme == ShmRank);
|
||||
double off_node_bytes=0.0;
|
||||
int tag;
|
||||
|
||||
if ( dor ) {
|
||||
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+from*32;
|
||||
// std::cout << " StencilSendToRecvFrom "<<dir<<" MPI_Irecv "<<std::hex<<recv<<std::dec<<std::endl;
|
||||
ierr=MPI_Irecv(recv_comp,(int)(rbytes/sizeof(int32_t)), MPI_INT32_T,from,tag,communicator_halo[commdir],&rrq);
|
||||
GRID_ASSERT(ierr==0);
|
||||
list.push_back(rrq);
|
||||
off_node_bytes+=rbytes;
|
||||
}
|
||||
#ifdef NVLINK_GET
|
||||
else {
|
||||
void *shm = (void *) this->ShmBufferTranslate(from,xmit);
|
||||
GRID_ASSERT(shm!=NULL);
|
||||
// std::cout << " StencilSendToRecvFrom "<<dir<<" CopyDeviceToDevice recv "<<std::hex<<recv<<" remote "<<shm <<std::dec<<std::endl;
|
||||
acceleratorCopyDeviceToDeviceAsynch(shm,recv,rbytes);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
// This is a NVLINK PUT
|
||||
if (dox) {
|
||||
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+_processor*32;
|
||||
ierr =MPI_Isend(xmit_comp,(int)(xbytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator_halo[commdir],&xrq);
|
||||
GRID_ASSERT(ierr==0);
|
||||
list.push_back(xrq);
|
||||
off_node_bytes+=xbytes;
|
||||
} else {
|
||||
#ifndef NVLINK_GET
|
||||
void *shm = (void *) this->ShmBufferTranslate(dest,recv);
|
||||
GRID_ASSERT(shm!=NULL);
|
||||
acceleratorCopyDeviceToDeviceAsynch(xmit,shm,xbytes);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
return off_node_bytes;
|
||||
}
|
||||
|
||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
||||
{
|
||||
int nreq=list.size();
|
||||
/*finishes Get/Put*/
|
||||
acceleratorCopySynchronise();
|
||||
|
||||
if (nreq==0) return;
|
||||
std::vector<MPI_Status> status(nreq);
|
||||
int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
|
||||
GRID_ASSERT(ierr==0);
|
||||
list.resize(0);
|
||||
this->StencilBarrier();
|
||||
}
|
||||
|
||||
#else /* NOT ... ACCELERATOR_AWARE_MPI */
|
||||
///////////////////////////////////////////
|
||||
// Pipeline mode through host memory
|
||||
///////////////////////////////////////////
|
||||
/*
|
||||
* In prepare (phase 1):
|
||||
* PHASE 1: (prepare)
|
||||
* - post MPI receive buffers asynch
|
||||
* - post device - host send buffer transfer asynch
|
||||
* PHASE 2: (Begin)
|
||||
* - complete all copies
|
||||
* - post MPI send asynch
|
||||
* - post device - device transfers
|
||||
* PHASE 3: (Complete)
|
||||
* - MPI_waitall
|
||||
* - host-device transfers
|
||||
*
|
||||
*********************************
|
||||
* NB could split this further:
|
||||
*--------------------------------
|
||||
* PHASE 1: (Prepare)
|
||||
* - post MPI receive buffers asynch
|
||||
* - post device - host send buffer transfer asynch
|
||||
* PHASE 2: (BeginInterNode)
|
||||
* - complete all copies
|
||||
* - post MPI send asynch
|
||||
* PHASE 3: (BeginIntraNode)
|
||||
* - post device - device transfers
|
||||
* PHASE 4: (Complete)
|
||||
* - MPI_waitall
|
||||
* - host-device transfers asynch
|
||||
* - (complete all copies)
|
||||
*/
|
||||
double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,int dox,
|
||||
void *recv,
|
||||
int from,int dor,
|
||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
||||
{
|
||||
/*
|
||||
* Bring sequence from Stencil.h down to lower level.
|
||||
* Assume using XeLink is ok
|
||||
*/
|
||||
int ncomm =communicator_halo.size();
|
||||
int commdir=dir%ncomm;
|
||||
|
||||
MPI_Request xrq;
|
||||
MPI_Request rrq;
|
||||
|
||||
int ierr;
|
||||
int gdest = ShmRanks[dest];
|
||||
int gfrom = ShmRanks[from];
|
||||
int gme = ShmRanks[_processor];
|
||||
|
||||
GRID_ASSERT(dest != _processor);
|
||||
GRID_ASSERT(from != _processor);
|
||||
GRID_ASSERT(gme == ShmRank);
|
||||
double off_node_bytes=0.0;
|
||||
int tag;
|
||||
|
||||
void * host_recv = NULL;
|
||||
void * host_xmit = NULL;
|
||||
|
||||
/*
|
||||
* PHASE 1: (Prepare)
|
||||
* - post MPI receive buffers asynch
|
||||
* - post device - host send buffer transfer asynch
|
||||
*/
|
||||
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
rbytes += 8;
|
||||
xbytes += 8;
|
||||
#endif
|
||||
|
||||
if ( dor ) {
|
||||
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
tag= dir+from*32;
|
||||
host_recv = this->HostBufferMalloc(rbytes);
|
||||
ierr=MPI_Irecv(host_recv,(int)(rbytes/sizeof(int32_t)), MPI_INT32_T,from,tag,communicator_halo[commdir],&rrq);
|
||||
GRID_ASSERT(ierr==0);
|
||||
CommsRequest_t srq;
|
||||
srq.PacketType = InterNodeRecv;
|
||||
srq.bytes = rbytes;
|
||||
srq.req = rrq;
|
||||
srq.host_buf = host_recv;
|
||||
srq.device_buf = recv;
|
||||
srq.tag = tag;
|
||||
list.push_back(srq);
|
||||
off_node_bytes+=rbytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (dox) {
|
||||
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
||||
|
||||
tag= dir+_processor*32;
|
||||
|
||||
host_xmit = this->HostBufferMalloc(xbytes);
|
||||
CommsRequest_t srq;
|
||||
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
uint64_t xbytes_data = xbytes - 8;
|
||||
srq.ev = acceleratorCopyFromDeviceAsynch(xmit, host_xmit,xbytes_data); // Make this Asynch
|
||||
GRID_ASSERT(xbytes % 8 == 0);
|
||||
// flip one bit so that a zero buffer is not consistent
|
||||
uint64_t xsum = checksum_gpu((uint64_t*)xmit, xbytes_data / 8) ^ (checksum_index + 1 + 1000 * tag);
|
||||
*(uint64_t*)(((char*)host_xmit) + xbytes_data) = xsum;
|
||||
#else
|
||||
srq.ev = acceleratorCopyFromDeviceAsynch(xmit, host_xmit,xbytes); // Make this Asynch
|
||||
#endif
|
||||
|
||||
// ierr =MPI_Isend(host_xmit, xbytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
|
||||
// GRID_ASSERT(ierr==0);
|
||||
// off_node_bytes+=xbytes;
|
||||
|
||||
srq.PacketType = InterNodeXmit;
|
||||
srq.bytes = xbytes;
|
||||
// srq.req = xrq;
|
||||
srq.host_buf = host_xmit;
|
||||
srq.device_buf = xmit;
|
||||
srq.tag = tag;
|
||||
srq.dest = dest;
|
||||
srq.commdir = commdir;
|
||||
list.push_back(srq);
|
||||
}
|
||||
}
|
||||
|
||||
return off_node_bytes;
|
||||
}
|
||||
/*
|
||||
* In the interest of better pipelining, poll for completion on each DtoH and
|
||||
* start MPI_ISend in the meantime
|
||||
*/
|
||||
void CartesianCommunicator::StencilSendToRecvFromPollIRecv(std::vector<CommsRequest_t> &list)
|
||||
{
|
||||
int pending = 0;
|
||||
do {
|
||||
|
||||
pending = 0;
|
||||
|
||||
for(int idx = 0; idx<list.size();idx++){
|
||||
|
||||
if ( list[idx].PacketType==InterNodeRecv ) {
|
||||
|
||||
int flag = 0;
|
||||
MPI_Status status;
|
||||
int ierr = MPI_Test(&list[idx].req,&flag,&status);
|
||||
assert(ierr==0);
|
||||
|
||||
if ( flag ) {
|
||||
// std::cout << " PollIrecv "<<idx<<" flag "<<flag<<std::endl;
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
acceleratorCopyToDeviceAsynch(list[idx].host_buf,list[idx].device_buf,list[idx].bytes - 8);
|
||||
#else
|
||||
acceleratorCopyToDeviceAsynch(list[idx].host_buf,list[idx].device_buf,list[idx].bytes);
|
||||
#endif
|
||||
list[idx].PacketType=InterNodeReceiveHtoD;
|
||||
} else {
|
||||
pending ++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// std::cout << " PollIrecv "<<pending<<" pending requests"<<std::endl;
|
||||
} while ( pending );
|
||||
|
||||
}
|
||||
void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector<CommsRequest_t> &list)
|
||||
{
|
||||
int pending = 0;
|
||||
do {
|
||||
|
||||
pending = 0;
|
||||
|
||||
for(int idx = 0; idx<list.size();idx++){
|
||||
|
||||
if ( list[idx].PacketType==InterNodeXmit ) {
|
||||
|
||||
if ( acceleratorEventIsComplete(list[idx].ev) ) {
|
||||
|
||||
void *host_xmit = list[idx].host_buf;
|
||||
uint64_t xbytes = list[idx].bytes;
|
||||
int dest = list[idx].dest;
|
||||
int tag = list[idx].tag;
|
||||
int commdir = list[idx].commdir;
|
||||
///////////////////
|
||||
// Send packet
|
||||
///////////////////
|
||||
|
||||
// std::cout << " DtoH is complete for index "<<idx<<" calling MPI_Isend "<<std::endl;
|
||||
|
||||
MPI_Request xrq;
|
||||
int ierr =MPI_Isend(host_xmit, (int)(xbytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator_halo[commdir],&xrq);
|
||||
GRID_ASSERT(ierr==0);
|
||||
|
||||
list[idx].req = xrq; // Update the MPI request in the list
|
||||
|
||||
list[idx].PacketType=InterNodeXmitISend;
|
||||
|
||||
} else {
|
||||
// not done, so return to polling loop
|
||||
pending++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (pending);
|
||||
}
|
||||
|
||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,void *xmit_comp,
|
||||
int dest,int dox,
|
||||
void *recv,void *recv_comp,
|
||||
int from,int dor,
|
||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
||||
void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
int bytes,int dir)
|
||||
{
|
||||
int ncomm =communicator_halo.size();
|
||||
int ncomm =communicator_halo.size();
|
||||
int commdir=dir%ncomm;
|
||||
|
||||
MPI_Request xrq;
|
||||
@@ -730,194 +378,83 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
|
||||
int gfrom = ShmRanks[from];
|
||||
int gme = ShmRanks[_processor];
|
||||
|
||||
GRID_ASSERT(dest != _processor);
|
||||
GRID_ASSERT(from != _processor);
|
||||
GRID_ASSERT(gme == ShmRank);
|
||||
assert(dest != _processor);
|
||||
assert(from != _processor);
|
||||
assert(gme == ShmRank);
|
||||
double off_node_bytes=0.0;
|
||||
int tag;
|
||||
|
||||
void * host_xmit = NULL;
|
||||
|
||||
////////////////////////////////
|
||||
// Receives already posted
|
||||
// Copies already started
|
||||
////////////////////////////////
|
||||
/*
|
||||
* PHASE 2: (Begin)
|
||||
* - complete all copies
|
||||
* - post MPI send asynch
|
||||
*/
|
||||
#ifdef NVLINK_GET
|
||||
if ( dor ) {
|
||||
|
||||
if ( ! ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) ) {
|
||||
// Intranode
|
||||
void *shm = (void *) this->ShmBufferTranslate(from,xmit);
|
||||
GRID_ASSERT(shm!=NULL);
|
||||
|
||||
CommsRequest_t srq;
|
||||
|
||||
srq.ev = acceleratorCopyDeviceToDeviceAsynch(shm,recv,rbytes);
|
||||
|
||||
srq.PacketType = IntraNodeRecv;
|
||||
srq.bytes = xbytes;
|
||||
// srq.req = xrq;
|
||||
srq.host_buf = NULL;
|
||||
srq.device_buf = xmit;
|
||||
srq.tag = -1;
|
||||
srq.dest = dest;
|
||||
srq.commdir = dir;
|
||||
list.push_back(srq);
|
||||
}
|
||||
}
|
||||
#else
|
||||
if (dox) {
|
||||
|
||||
if ( !( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) ) {
|
||||
// Intranode
|
||||
void *shm = (void *) this->ShmBufferTranslate(dest,recv);
|
||||
GRID_ASSERT(shm!=NULL);
|
||||
|
||||
CommsRequest_t srq;
|
||||
|
||||
srq.ev = acceleratorCopyDeviceToDeviceAsynch(xmit,shm,xbytes);
|
||||
|
||||
srq.PacketType = IntraNodeXmit;
|
||||
srq.bytes = xbytes;
|
||||
// srq.req = xrq;
|
||||
srq.host_buf = NULL;
|
||||
srq.device_buf = xmit;
|
||||
srq.tag = -1;
|
||||
srq.dest = dest;
|
||||
srq.commdir = dir;
|
||||
list.push_back(srq);
|
||||
|
||||
}
|
||||
if ( gfrom ==MPI_UNDEFINED) {
|
||||
ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator_halo[commdir],&rrq);
|
||||
assert(ierr==0);
|
||||
list.push_back(rrq);
|
||||
off_node_bytes+=bytes;
|
||||
}
|
||||
#endif
|
||||
|
||||
if ( gdest == MPI_UNDEFINED ) {
|
||||
ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator_halo[commdir],&xrq);
|
||||
assert(ierr==0);
|
||||
list.push_back(xrq);
|
||||
off_node_bytes+=bytes;
|
||||
}
|
||||
|
||||
if ( CommunicatorPolicy == CommunicatorPolicySequential ) {
|
||||
this->StencilSendToRecvFromComplete(list,dir);
|
||||
}
|
||||
|
||||
return off_node_bytes;
|
||||
}
|
||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
|
||||
{
|
||||
acceleratorCopySynchronise(); // Complete all pending copy transfers D2D
|
||||
|
||||
std::vector<MPI_Status> status;
|
||||
std::vector<MPI_Request> MpiRequests;
|
||||
|
||||
for(int r=0;r<list.size();r++){
|
||||
// Must check each Send buf is clear to reuse
|
||||
if ( list[r].PacketType == InterNodeXmitISend ) MpiRequests.push_back(list[r].req);
|
||||
// if ( list[r].PacketType == InterNodeRecv ) MpiRequests.push_back(list[r].req); // Already "Test" passed
|
||||
}
|
||||
|
||||
int nreq=MpiRequests.size();
|
||||
|
||||
if (nreq>0) {
|
||||
status.resize(MpiRequests.size());
|
||||
int ierr = MPI_Waitall(MpiRequests.size(),&MpiRequests[0],&status[0]); // Sends are guaranteed in order. No harm in not completing.
|
||||
GRID_ASSERT(ierr==0);
|
||||
}
|
||||
|
||||
// for(int r=0;r<nreq;r++){
|
||||
// if ( list[r].PacketType==InterNodeRecv ) {
|
||||
// acceleratorCopyToDeviceAsynch(list[r].host_buf,list[r].device_buf,list[r].bytes);
|
||||
// }
|
||||
// }
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
for(int r=0;r<list.size();r++){
|
||||
if ( list[r].PacketType == InterNodeReceiveHtoD ) {
|
||||
uint64_t rbytes_data = list[r].bytes - 8;
|
||||
uint64_t expected_cs = *(uint64_t*)(((char*)list[r].host_buf) + rbytes_data);
|
||||
uint64_t computed_cs = checksum_gpu((uint64_t*)list[r].device_buf, rbytes_data / 8) ^ (checksum_index + 1 + 1000 * list[r].tag); //
|
||||
if (expected_cs != computed_cs) {
|
||||
// TODO: error message, backtrace, quit
|
||||
|
||||
fprintf(stderr, "GRID_CHECKSUM_COMMS error:\n");
|
||||
fprintf(stderr, " processor = %d\n", (int)_processor);
|
||||
for(int d=0;d<_processors.size();d++)
|
||||
fprintf(stderr, " processor_coord[%d] = %d\n", d, _processor_coor[d]);
|
||||
fprintf(stderr, " hostname: %s\n", GridHostname());
|
||||
fprintf(stderr, " expected_cs: %ld\n", expected_cs);
|
||||
fprintf(stderr, " computed_cs: %ld\n", computed_cs);
|
||||
fprintf(stderr, " dest: %d\n", list[r].dest);
|
||||
fprintf(stderr, " tag: %d\n", list[r].tag);
|
||||
fprintf(stderr, " commdir: %d\n", list[r].commdir);
|
||||
fprintf(stderr, " bytes: %ld\n", (uint64_t)list[r].bytes);
|
||||
|
||||
fflush(stderr);
|
||||
|
||||
// backtrace
|
||||
int symbols = backtrace(Grid_backtrace_buffer,_NBACKTRACE);
|
||||
backtrace_symbols_fd(Grid_backtrace_buffer,symbols, 2);
|
||||
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
checksum_index += 1;
|
||||
#endif
|
||||
|
||||
list.resize(0); // Delete the list
|
||||
this->HostBufferFreeAll(); // Clean up the buffer allocs
|
||||
#ifndef NVLINK_GET
|
||||
this->StencilBarrier(); // if PUT must check our nbrs have filled our receive buffers.
|
||||
#endif
|
||||
SendToRecvFromComplete(waitall);
|
||||
}
|
||||
#endif
|
||||
////////////////////////////////////////////
|
||||
// END PIPELINE MODE / NO CUDA AWARE MPI
|
||||
////////////////////////////////////////////
|
||||
|
||||
void CartesianCommunicator::StencilBarrier(void)
|
||||
{
|
||||
FlightRecorder::StepLog("NodeBarrier");
|
||||
MPI_Barrier (ShmComm);
|
||||
}
|
||||
//void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
|
||||
//{
|
||||
//}
|
||||
void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
|
||||
{
|
||||
int nreq=list.size();
|
||||
|
||||
if (nreq==0) return;
|
||||
|
||||
std::vector<MPI_Status> status(nreq);
|
||||
int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
|
||||
assert(ierr==0);
|
||||
list.resize(0);
|
||||
}
|
||||
void CartesianCommunicator::Barrier(void)
|
||||
{
|
||||
FlightRecorder::StepLog("GridBarrier");
|
||||
int ierr = MPI_Barrier(communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::Broadcast(int root,void* data,uint64_t bytes)
|
||||
void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
|
||||
{
|
||||
FlightRecorder::StepLog("Broadcast");
|
||||
int ierr=MPI_Bcast(data,
|
||||
(int)bytes,
|
||||
bytes,
|
||||
MPI_BYTE,
|
||||
root,
|
||||
communicator);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
int CartesianCommunicator::RankWorld(void){
|
||||
int r;
|
||||
int CartesianCommunicator::RankWorld(void){
|
||||
int r;
|
||||
MPI_Comm_rank(communicator_world,&r);
|
||||
return r;
|
||||
}
|
||||
void CartesianCommunicator::BarrierWorld(void){
|
||||
FlightRecorder::StepLog("BarrierWorld");
|
||||
int ierr = MPI_Barrier(communicator_world);
|
||||
GRID_ASSERT(ierr==0);
|
||||
}
|
||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, uint64_t bytes)
|
||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
|
||||
{
|
||||
FlightRecorder::StepLog("BroadcastWorld");
|
||||
int ierr= MPI_Bcast(data,
|
||||
(int)bytes,
|
||||
bytes,
|
||||
MPI_BYTE,
|
||||
root,
|
||||
communicator_world);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
Coordinate row(_ndimension,1);
|
||||
GRID_ASSERT(dim>=0 && dim<_ndimension);
|
||||
assert(dim>=0 && dim<_ndimension);
|
||||
|
||||
// Split the communicator
|
||||
row[dim] = _processors[dim];
|
||||
@@ -928,18 +465,17 @@ void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,
|
||||
}
|
||||
void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
FlightRecorder::StepLog("AllToAll");
|
||||
// MPI is a pain and uses "int" arguments
|
||||
// 64*64*64*128*16 == 500Million elements of data.
|
||||
// When 24*4 bytes multiples get 50x 10^9 >>> 2x10^9 Y2K bug.
|
||||
// (Turns up on 32^3 x 64 Gparity too)
|
||||
MPI_Datatype object;
|
||||
int iwords;
|
||||
int iwords;
|
||||
int ibytes;
|
||||
iwords = words;
|
||||
ibytes = bytes;
|
||||
GRID_ASSERT(words == iwords); // safe to cast to int ?
|
||||
GRID_ASSERT(bytes == ibytes); // safe to cast to int ?
|
||||
assert(words == iwords); // safe to cast to int ?
|
||||
assert(bytes == ibytes); // safe to cast to int ?
|
||||
MPI_Type_contiguous(ibytes,MPI_BYTE,&object);
|
||||
MPI_Type_commit(&object);
|
||||
MPI_Alltoall(in,iwords,object,out,iwords,object,communicator);
|
||||
@@ -947,3 +483,5 @@ void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
|
||||
@@ -27,8 +27,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
/* END LEGAL */
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
void GridAbort(void) { abort(); }
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -36,7 +34,6 @@ NAMESPACE_BEGIN(Grid);
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Grid_MPI_Comm CartesianCommunicator::communicator_world;
|
||||
|
||||
|
||||
void CartesianCommunicator::Init(int *argc, char *** arv)
|
||||
{
|
||||
GlobalSharedMemory::Init(communicator_world);
|
||||
@@ -48,23 +45,21 @@ void CartesianCommunicator::Init(int *argc, char *** arv)
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank)
|
||||
: CartesianCommunicator(processors)
|
||||
{
|
||||
_shm_processors = Coordinate(processors.size(),1);
|
||||
srank=0;
|
||||
SetCommunicator(communicator_world);
|
||||
}
|
||||
|
||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
{
|
||||
_shm_processors = Coordinate(processors.size(),1);
|
||||
_processors = processors;
|
||||
_ndimension = processors.size(); GRID_ASSERT(_ndimension>=1);
|
||||
_ndimension = processors.size(); assert(_ndimension>=1);
|
||||
_processor_coor.resize(_ndimension);
|
||||
|
||||
// Require 1^N processor grid for fake
|
||||
_Nprocessors=1;
|
||||
_processor = 0;
|
||||
for(int d=0;d<_ndimension;d++) {
|
||||
GRID_ASSERT(_processors[d]==1);
|
||||
assert(_processors[d]==1);
|
||||
_processor_coor[d] = 0;
|
||||
}
|
||||
SetCommunicator(communicator_world);
|
||||
@@ -72,8 +67,6 @@ CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
||||
|
||||
CartesianCommunicator::~CartesianCommunicator(){}
|
||||
|
||||
void CartesianCommunicator::GlobalMax(float &){}
|
||||
void CartesianCommunicator::GlobalMax(double &){}
|
||||
void CartesianCommunicator::GlobalSum(float &){}
|
||||
void CartesianCommunicator::GlobalSumVector(float *,int N){}
|
||||
void CartesianCommunicator::GlobalSum(double &){}
|
||||
@@ -84,27 +77,39 @@ void CartesianCommunicator::GlobalSumVector(uint64_t *,int N){}
|
||||
void CartesianCommunicator::GlobalXOR(uint32_t &){}
|
||||
void CartesianCommunicator::GlobalXOR(uint64_t &){}
|
||||
|
||||
void CartesianCommunicator::SendRecvPacket(void *xmit,
|
||||
void *recv,
|
||||
int xmit_to_rank,
|
||||
int recv_from_rank,
|
||||
int bytes)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
|
||||
|
||||
// Basic Halo comms primitive -- should never call in single node
|
||||
void CartesianCommunicator::SendToRecvFrom(void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
uint64_t bytes)
|
||||
int bytes)
|
||||
{
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
void CartesianCommunicator::CommsComplete(std::vector<CommsRequest_t> &list){ GRID_ASSERT(list.size()==0);}
|
||||
void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int dest,
|
||||
void *recv,
|
||||
int from,
|
||||
uint64_t bytes,int dir)
|
||||
int bytes)
|
||||
{
|
||||
GRID_ASSERT(0);
|
||||
assert(0);
|
||||
}
|
||||
|
||||
void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
|
||||
{
|
||||
assert(0);
|
||||
}
|
||||
void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,uint64_t bytes)
|
||||
{
|
||||
bcopy(in,out,bytes*words);
|
||||
@@ -116,9 +121,8 @@ void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t
|
||||
|
||||
int CartesianCommunicator::RankWorld(void){return 0;}
|
||||
void CartesianCommunicator::Barrier(void){}
|
||||
void CartesianCommunicator::Broadcast(int root,void* data, uint64_t bytes) {}
|
||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, uint64_t bytes) { }
|
||||
void CartesianCommunicator::BarrierWorld(void) { }
|
||||
void CartesianCommunicator::Broadcast(int root,void* data, int bytes) {}
|
||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes) { }
|
||||
int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor) { return 0;}
|
||||
void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor){ coor = _processor_coor; }
|
||||
void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
|
||||
@@ -127,38 +131,32 @@ void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest
|
||||
dest=0;
|
||||
}
|
||||
|
||||
int CartesianCommunicator::IsOffNode(int rank) { return false; }
|
||||
|
||||
double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
|
||||
int xmit_to_rank,int dox,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,int dor,
|
||||
uint64_t bytes, int dir)
|
||||
int recv_from_rank,
|
||||
int bytes, int dir)
|
||||
{
|
||||
std::vector<CommsRequest_t> list;
|
||||
// Discard the "dir"
|
||||
SendToRecvFromBegin (list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
|
||||
SendToRecvFromComplete(list);
|
||||
return 2.0*bytes;
|
||||
}
|
||||
void CartesianCommunicator::StencilSendToRecvFromPollIRecv(std::vector<CommsRequest_t> &list) {};
|
||||
void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector<CommsRequest_t> &list) {};
|
||||
double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector<CommsRequest_t> &list,
|
||||
void *xmit,
|
||||
int xmit_to_rank,int dox,
|
||||
void *recv,
|
||||
int recv_from_rank,int dor,
|
||||
uint64_t xbytes,uint64_t rbytes, int dir)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
||||
void *xmit, void *xmit_comp,
|
||||
int xmit_to_rank,int dox,
|
||||
void *recv, void *recv_comp,
|
||||
int recv_from_rank,int dor,
|
||||
uint64_t xbytes,uint64_t rbytes, int dir)
|
||||
void *xmit,
|
||||
int xmit_to_rank,
|
||||
void *recv,
|
||||
int recv_from_rank,
|
||||
int bytes, int dir)
|
||||
{
|
||||
return xbytes+rbytes;
|
||||
// Discard the "dir"
|
||||
SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
|
||||
return 2.0*bytes;
|
||||
}
|
||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
|
||||
{
|
||||
SendToRecvFromComplete(waitall);
|
||||
}
|
||||
|
||||
void CartesianCommunicator::StencilBarrier(void){};
|
||||
|
||||
@@ -40,9 +40,6 @@ int GlobalSharedMemory::_ShmAlloc;
|
||||
uint64_t GlobalSharedMemory::_ShmAllocBytes;
|
||||
|
||||
std::vector<void *> GlobalSharedMemory::WorldShmCommBufs;
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
void * GlobalSharedMemory::HostCommBuf;
|
||||
#endif
|
||||
|
||||
Grid_MPI_Comm GlobalSharedMemory::WorldShmComm;
|
||||
int GlobalSharedMemory::WorldShmRank;
|
||||
@@ -58,8 +55,8 @@ int GlobalSharedMemory::WorldNode;
|
||||
|
||||
void GlobalSharedMemory::SharedMemoryFree(void)
|
||||
{
|
||||
GRID_ASSERT(_ShmAlloc);
|
||||
GRID_ASSERT(_ShmAllocBytes>0);
|
||||
assert(_ShmAlloc);
|
||||
assert(_ShmAllocBytes>0);
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
munmap(WorldShmCommBufs[r],_ShmAllocBytes);
|
||||
}
|
||||
@@ -69,26 +66,6 @@ void GlobalSharedMemory::SharedMemoryFree(void)
|
||||
/////////////////////////////////
|
||||
// Alloc, free shmem region
|
||||
/////////////////////////////////
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
void *SharedMemory::HostBufferMalloc(size_t bytes){
|
||||
void *ptr = (void *)host_heap_top;
|
||||
host_heap_top += bytes;
|
||||
host_heap_bytes+= bytes;
|
||||
if (host_heap_bytes >= host_heap_size) {
|
||||
std::cout<< " HostBufferMalloc exceeded heap size -- try increasing with --shm <MB> flag" <<std::endl;
|
||||
std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
|
||||
std::cout<< " Current alloc is " << (bytes/(1024*1024)) <<"MB"<<std::endl;
|
||||
std::cout<< " Current bytes is " << (host_heap_bytes/(1024*1024)) <<"MB"<<std::endl;
|
||||
std::cout<< " Current heap is " << (host_heap_size/(1024*1024)) <<"MB"<<std::endl;
|
||||
GRID_ASSERT(host_heap_bytes<host_heap_size);
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
void SharedMemory::HostBufferFreeAll(void) {
|
||||
host_heap_top =(size_t)HostCommBuf;
|
||||
host_heap_bytes=0;
|
||||
}
|
||||
#endif
|
||||
void *SharedMemory::ShmBufferMalloc(size_t bytes){
|
||||
// bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes
|
||||
void *ptr = (void *)heap_top;
|
||||
@@ -100,7 +77,7 @@ void *SharedMemory::ShmBufferMalloc(size_t bytes){
|
||||
std::cout<< " Current alloc is " << (bytes/(1024*1024)) <<"MB"<<std::endl;
|
||||
std::cout<< " Current bytes is " << (heap_bytes/(1024*1024)) <<"MB"<<std::endl;
|
||||
std::cout<< " Current heap is " << (heap_size/(1024*1024)) <<"MB"<<std::endl;
|
||||
GRID_ASSERT(heap_bytes<heap_size);
|
||||
assert(heap_bytes<heap_size);
|
||||
}
|
||||
//std::cerr << "ShmBufferMalloc "<<std::hex<< ptr<<" - "<<((uint64_t)ptr+bytes)<<std::dec<<std::endl;
|
||||
return ptr;
|
||||
@@ -114,59 +91,6 @@ void *SharedMemory::ShmBufferSelf(void)
|
||||
//std::cerr << "ShmBufferSelf "<<ShmRank<<" "<<std::hex<< ShmCommBufs[ShmRank] <<std::dec<<std::endl;
|
||||
return ShmCommBufs[ShmRank];
|
||||
}
|
||||
static inline int divides(int a,int b)
|
||||
{
|
||||
return ( b == ( (b/a)*a ) );
|
||||
}
|
||||
void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Allow user to configure through environment variable
|
||||
////////////////////////////////////////////////////////////////
|
||||
char* str = getenv(("GRID_SHM_DIMS_" + std::to_string(ShmDims.size())).c_str());
|
||||
if ( str ) {
|
||||
std::vector<int> IntShmDims;
|
||||
GridCmdOptionIntVector(std::string(str),IntShmDims);
|
||||
GRID_ASSERT(IntShmDims.size() == WorldDims.size());
|
||||
long ShmSize = 1;
|
||||
for (int dim=0;dim<WorldDims.size();dim++) {
|
||||
ShmSize *= (ShmDims[dim] = IntShmDims[dim]);
|
||||
GRID_ASSERT(divides(ShmDims[dim],WorldDims[dim]));
|
||||
}
|
||||
GRID_ASSERT(ShmSize == WorldShmSize);
|
||||
return;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Powers of 2,3,5 only in prime decomposition for now
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ndimension = WorldDims.size();
|
||||
ShmDims=Coordinate(ndimension,1);
|
||||
|
||||
std::vector<int> primes({2,3,5});
|
||||
|
||||
int dim = 0;
|
||||
int last_dim = ndimension - 1;
|
||||
int AutoShmSize = 1;
|
||||
while(AutoShmSize != WorldShmSize) {
|
||||
int p;
|
||||
for(p=0;p<primes.size();p++) {
|
||||
int prime=primes[p];
|
||||
if ( divides(prime,WorldDims[dim]/ShmDims[dim])
|
||||
&& divides(prime,WorldShmSize/AutoShmSize) ) {
|
||||
AutoShmSize*=prime;
|
||||
ShmDims[dim]*=prime;
|
||||
last_dim = dim;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (p == primes.size() && last_dim == dim) {
|
||||
std::cerr << "GlobalSharedMemory::GetShmDims failed" << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
dim=(dim+1) %ndimension;
|
||||
}
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -46,40 +46,8 @@ NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#if defined (GRID_COMMS_MPI3)
|
||||
typedef MPI_Comm Grid_MPI_Comm;
|
||||
typedef MPI_Request MpiCommsRequest_t;
|
||||
#ifdef ACCELERATOR_AWARE_MPI
|
||||
typedef MPI_Request CommsRequest_t;
|
||||
#else
|
||||
/*
|
||||
* Enable state transitions as each packet flows.
|
||||
*/
|
||||
enum PacketType_t {
|
||||
FaceGather,
|
||||
InterNodeXmit,
|
||||
InterNodeRecv,
|
||||
IntraNodeXmit,
|
||||
IntraNodeRecv,
|
||||
InterNodeXmitISend,
|
||||
InterNodeReceiveHtoD
|
||||
};
|
||||
/*
|
||||
*Package arguments needed for various actions along packet flow
|
||||
*/
|
||||
typedef struct {
|
||||
PacketType_t PacketType;
|
||||
void *host_buf;
|
||||
void *device_buf;
|
||||
int dest;
|
||||
int tag;
|
||||
int commdir;
|
||||
unsigned long bytes;
|
||||
acceleratorEvent_t ev;
|
||||
MpiCommsRequest_t req;
|
||||
} CommsRequest_t;
|
||||
#endif
|
||||
|
||||
#else
|
||||
typedef int MpiCommsRequest_t;
|
||||
typedef int CommsRequest_t;
|
||||
typedef int Grid_MPI_Comm;
|
||||
#endif
|
||||
@@ -107,9 +75,7 @@ public:
|
||||
static int Hugepages;
|
||||
|
||||
static std::vector<void *> WorldShmCommBufs;
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
static void *HostCommBuf;
|
||||
#endif
|
||||
|
||||
static Grid_MPI_Comm WorldComm;
|
||||
static int WorldRank;
|
||||
static int WorldSize;
|
||||
@@ -127,17 +93,16 @@ public:
|
||||
// Create an optimal reordered communicator that makes MPI_Cart_create get it right
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
static void Init(Grid_MPI_Comm comm); // Typically MPI_COMM_WORLD
|
||||
// Turns MPI_COMM_WORLD into right layout for Cartesian
|
||||
static void OptimalCommunicator (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims);
|
||||
static void OptimalCommunicatorHypercube (const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims);
|
||||
static void OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &ShmDims);
|
||||
static void OptimalCommunicator (const Coordinate &processors,Grid_MPI_Comm & optimal_comm); // Turns MPI_COMM_WORLD into right layout for Cartesian
|
||||
static void OptimalCommunicatorHypercube (const Coordinate &processors,Grid_MPI_Comm & optimal_comm); // Turns MPI_COMM_WORLD into right layout for Cartesian
|
||||
static void OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm); // Turns MPI_COMM_WORLD into right layout for Cartesian
|
||||
static void GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims);
|
||||
///////////////////////////////////////////////////
|
||||
// Provide shared memory facilities off comm world
|
||||
///////////////////////////////////////////////////
|
||||
static void SharedMemoryAllocate(uint64_t bytes, int flags);
|
||||
static void SharedMemoryFree(void);
|
||||
// static void SharedMemoryCopy(void *dest,void *src,size_t bytes);
|
||||
static void SharedMemoryCopy(void *dest,const void *src,size_t bytes);
|
||||
static void SharedMemoryZero(void *dest,size_t bytes);
|
||||
|
||||
};
|
||||
@@ -154,13 +119,6 @@ private:
|
||||
size_t heap_bytes;
|
||||
size_t heap_size;
|
||||
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
size_t host_heap_top; // set in free all
|
||||
size_t host_heap_bytes;// set in free all
|
||||
void *HostCommBuf; // set in SetCommunicator
|
||||
size_t host_heap_size; // set in SetCommunicator
|
||||
#endif
|
||||
|
||||
protected:
|
||||
|
||||
Grid_MPI_Comm ShmComm; // for barriers
|
||||
@@ -192,10 +150,7 @@ public:
|
||||
void *ShmBufferTranslate(int rank,void * local_p);
|
||||
void *ShmBufferMalloc(size_t bytes);
|
||||
void ShmBufferFreeAll(void) ;
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
void *HostBufferMalloc(size_t bytes);
|
||||
void HostBufferFreeAll(void);
|
||||
#endif
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Make info on Nodes & ranks and Shared memory available
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
Copyright (C) 2015
|
||||
|
||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
Author: Christoph Lehner <christoph@lhnr.de>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
@@ -27,138 +26,19 @@ Author: Christoph Lehner <christoph@lhnr.de>
|
||||
*************************************************************************************/
|
||||
/* END LEGAL */
|
||||
|
||||
#define Mheader "SharedMemoryMpi: "
|
||||
|
||||
#include <Grid/GridCore.h>
|
||||
#include <pwd.h>
|
||||
|
||||
#ifdef GRID_CUDA
|
||||
#ifdef GRID_NVCC
|
||||
#include <cuda_runtime_api.h>
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
#include <hip/hip_runtime_api.h>
|
||||
#endif
|
||||
#ifdef GRID_SYCL
|
||||
#ifdef ACCELERATOR_AWARE_MPI
|
||||
#define GRID_SYCL_LEVEL_ZERO_IPC
|
||||
#define SHM_SOCKETS
|
||||
#else
|
||||
#endif
|
||||
#include <syscall.h>
|
||||
#endif
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#ifdef SHM_SOCKETS
|
||||
|
||||
/*
|
||||
* Barbaric extra intranode communication route in case we need sockets to pass FDs
|
||||
* Forced by level_zero not being nicely designed
|
||||
*/
|
||||
static int sock;
|
||||
static const char *sock_path_fmt = "/tmp/GridUnixSocket.%d";
|
||||
static char sock_path[256];
|
||||
class UnixSockets {
|
||||
public:
|
||||
static void Open(int rank)
|
||||
{
|
||||
int errnum;
|
||||
|
||||
sock = socket(AF_UNIX, SOCK_DGRAM, 0); GRID_ASSERT(sock>0);
|
||||
|
||||
struct sockaddr_un sa_un = { 0 };
|
||||
sa_un.sun_family = AF_UNIX;
|
||||
snprintf(sa_un.sun_path, sizeof(sa_un.sun_path),sock_path_fmt,rank);
|
||||
unlink(sa_un.sun_path);
|
||||
if (bind(sock, (struct sockaddr *)&sa_un, sizeof(sa_un))) {
|
||||
perror("bind failure");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
static int RecvFileDescriptor(void)
|
||||
{
|
||||
int n;
|
||||
int fd;
|
||||
char buf[1];
|
||||
struct iovec iov;
|
||||
struct msghdr msg;
|
||||
struct cmsghdr *cmsg;
|
||||
char cms[CMSG_SPACE(sizeof(int))];
|
||||
|
||||
iov.iov_base = buf;
|
||||
iov.iov_len = 1;
|
||||
|
||||
memset(&msg, 0, sizeof msg);
|
||||
msg.msg_name = 0;
|
||||
msg.msg_namelen = 0;
|
||||
msg.msg_iov = &iov;
|
||||
msg.msg_iovlen = 1;
|
||||
|
||||
msg.msg_control = (caddr_t)cms;
|
||||
msg.msg_controllen = sizeof cms;
|
||||
|
||||
if((n=recvmsg(sock, &msg, 0)) < 0) {
|
||||
perror("recvmsg failed");
|
||||
return -1;
|
||||
}
|
||||
if(n == 0){
|
||||
perror("recvmsg returned 0");
|
||||
return -1;
|
||||
}
|
||||
cmsg = CMSG_FIRSTHDR(&msg);
|
||||
|
||||
memmove(&fd, CMSG_DATA(cmsg), sizeof(int));
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
static void SendFileDescriptor(int fildes,int xmit_to_rank)
|
||||
{
|
||||
struct msghdr msg;
|
||||
struct iovec iov;
|
||||
struct cmsghdr *cmsg = NULL;
|
||||
char ctrl[CMSG_SPACE(sizeof(int))];
|
||||
char data = ' ';
|
||||
|
||||
memset(&msg, 0, sizeof(struct msghdr));
|
||||
memset(ctrl, 0, CMSG_SPACE(sizeof(int)));
|
||||
iov.iov_base = &data;
|
||||
iov.iov_len = sizeof(data);
|
||||
|
||||
sprintf(sock_path,sock_path_fmt,xmit_to_rank);
|
||||
|
||||
struct sockaddr_un sa_un = { 0 };
|
||||
sa_un.sun_family = AF_UNIX;
|
||||
snprintf(sa_un.sun_path, sizeof(sa_un.sun_path),sock_path_fmt,xmit_to_rank);
|
||||
|
||||
msg.msg_name = (void *)&sa_un;
|
||||
msg.msg_namelen = sizeof(sa_un);
|
||||
msg.msg_iov = &iov;
|
||||
msg.msg_iovlen = 1;
|
||||
msg.msg_controllen = CMSG_SPACE(sizeof(int));
|
||||
msg.msg_control = ctrl;
|
||||
|
||||
cmsg = CMSG_FIRSTHDR(&msg);
|
||||
cmsg->cmsg_level = SOL_SOCKET;
|
||||
cmsg->cmsg_type = SCM_RIGHTS;
|
||||
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
|
||||
|
||||
*((int *) CMSG_DATA(cmsg)) = fildes;
|
||||
|
||||
sendmsg(sock, &msg, 0);
|
||||
};
|
||||
};
|
||||
#endif
|
||||
|
||||
|
||||
#define header "SharedMemoryMpi: "
|
||||
/*Construct from an MPI communicator*/
|
||||
void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
{
|
||||
GRID_ASSERT(_ShmSetup==0);
|
||||
assert(_ShmSetup==0);
|
||||
WorldComm = comm;
|
||||
MPI_Comm_rank(WorldComm,&WorldRank);
|
||||
MPI_Comm_size(WorldComm,&WorldSize);
|
||||
@@ -167,25 +47,19 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Split into groups that can share memory
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
#ifndef GRID_MPI3_SHM_NONE
|
||||
MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&WorldShmComm);
|
||||
#else
|
||||
MPI_Comm_split(comm, WorldRank, 0, &WorldShmComm);
|
||||
#endif
|
||||
|
||||
MPI_Comm_rank(WorldShmComm ,&WorldShmRank);
|
||||
MPI_Comm_size(WorldShmComm ,&WorldShmSize);
|
||||
|
||||
if ( WorldRank == 0) {
|
||||
std::cout << Mheader " World communicator of size " <<WorldSize << std::endl;
|
||||
std::cout << Mheader " Node communicator of size " <<WorldShmSize << std::endl;
|
||||
std::cout << header " World communicator of size " <<WorldSize << std::endl;
|
||||
std::cout << header " Node communicator of size " <<WorldShmSize << std::endl;
|
||||
}
|
||||
// WorldShmComm, WorldShmSize, WorldShmRank
|
||||
|
||||
// WorldNodes
|
||||
WorldNodes = WorldSize/WorldShmSize;
|
||||
GRID_ASSERT( (WorldNodes * WorldShmSize) == WorldSize );
|
||||
|
||||
assert( (WorldNodes * WorldShmSize) == WorldSize );
|
||||
|
||||
// FIXME: Check all WorldShmSize are the same ?
|
||||
|
||||
@@ -209,7 +83,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
MyGroup.resize(WorldShmSize);
|
||||
for(int rank=0;rank<WorldSize;rank++){
|
||||
if(WorldShmRanks[rank]!=MPI_UNDEFINED){
|
||||
GRID_ASSERT(g<WorldShmSize);
|
||||
assert(g<WorldShmSize);
|
||||
MyGroup[g++] = rank;
|
||||
}
|
||||
}
|
||||
@@ -225,7 +99,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
// global sum leaders over comm world
|
||||
///////////////////////////////////////////////////////////////////
|
||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&leaders_1hot[0],WorldSize,MPI_INT,MPI_SUM,WorldComm);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// find the group leaders world rank
|
||||
@@ -246,7 +120,7 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
WorldNode=g;
|
||||
}
|
||||
}
|
||||
GRID_ASSERT(WorldNode!=-1);
|
||||
assert(WorldNode!=-1);
|
||||
_ShmSetup=1;
|
||||
}
|
||||
// Gray encode support
|
||||
@@ -265,7 +139,7 @@ int Log2Size(int TwoToPower,int MAXLOG2)
|
||||
}
|
||||
return log2size;
|
||||
}
|
||||
void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// Look and see if it looks like an HPE 8600 based on hostname conventions
|
||||
@@ -278,17 +152,45 @@ void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_M
|
||||
gethostname(name,namelen);
|
||||
int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ;
|
||||
|
||||
if(nscan==3 && HPEhypercube ) OptimalCommunicatorHypercube(processors,optimal_comm,SHM);
|
||||
else OptimalCommunicatorSharedMemory(processors,optimal_comm,SHM);
|
||||
if(nscan==3 && HPEhypercube ) OptimalCommunicatorHypercube(processors,optimal_comm);
|
||||
else OptimalCommunicatorSharedMemory(processors,optimal_comm);
|
||||
}
|
||||
static inline int divides(int a,int b)
|
||||
{
|
||||
return ( b == ( (b/a)*a ) );
|
||||
}
|
||||
void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Powers of 2,3,5 only in prime decomposition for now
|
||||
////////////////////////////////////////////////////////////////
|
||||
int ndimension = WorldDims.size();
|
||||
ShmDims=Coordinate(ndimension,1);
|
||||
|
||||
void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
std::vector<int> primes({2,3,5});
|
||||
|
||||
int dim = 0;
|
||||
int AutoShmSize = 1;
|
||||
while(AutoShmSize != WorldShmSize) {
|
||||
for(int p=0;p<primes.size();p++) {
|
||||
int prime=primes[p];
|
||||
if ( divides(prime,WorldDims[dim]/ShmDims[dim])
|
||||
&& divides(prime,WorldShmSize/AutoShmSize) ) {
|
||||
AutoShmSize*=prime;
|
||||
ShmDims[dim]*=prime;
|
||||
break;
|
||||
}
|
||||
}
|
||||
dim=(dim+1) %ndimension;
|
||||
}
|
||||
}
|
||||
void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processors,Grid_MPI_Comm & optimal_comm)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Assert power of two shm_size.
|
||||
////////////////////////////////////////////////////////////////
|
||||
int log2size = Log2Size(WorldShmSize,MAXLOG2RANKSPERNODE);
|
||||
GRID_ASSERT(log2size != -1);
|
||||
assert(log2size != -1);
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Identify the hypercube coordinate of this node using hostname
|
||||
@@ -309,7 +211,7 @@ void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processo
|
||||
// Parse ICE-XA hostname to get hypercube location
|
||||
gethostname(name,namelen);
|
||||
int nscan = sscanf(name,"r%di%dn%d",&R,&I,&N) ;
|
||||
GRID_ASSERT(nscan==3);
|
||||
assert(nscan==3);
|
||||
|
||||
int nlo = N%9;
|
||||
int nhi = N/9;
|
||||
@@ -333,8 +235,8 @@ void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processo
|
||||
//////////////////////////////////////////////////////////////////
|
||||
MPI_Bcast(&rootcoor, sizeof(rootcoor), MPI_BYTE, 0, WorldComm);
|
||||
hypercoor=hypercoor-rootcoor;
|
||||
GRID_ASSERT(hypercoor<WorldSize);
|
||||
GRID_ASSERT(hypercoor>=0);
|
||||
assert(hypercoor<WorldSize);
|
||||
assert(hypercoor>=0);
|
||||
|
||||
//////////////////////////////////////
|
||||
// Printing
|
||||
@@ -355,8 +257,7 @@ void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processo
|
||||
Coordinate HyperCoor(ndimension);
|
||||
|
||||
GetShmDims(WorldDims,ShmDims);
|
||||
SHM = ShmDims;
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish torus of processes and nodes with sub-blockings
|
||||
////////////////////////////////////////////////////////////////
|
||||
@@ -382,7 +283,7 @@ void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processo
|
||||
for(int i=0;i<ndimension;i++){
|
||||
Nprocessors*=processors[i];
|
||||
}
|
||||
GRID_ASSERT(WorldSize==Nprocessors);
|
||||
assert(WorldSize==Nprocessors);
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish mapping between lexico physics coord and WorldRank
|
||||
@@ -401,9 +302,9 @@ void GlobalSharedMemory::OptimalCommunicatorHypercube(const Coordinate &processo
|
||||
// Build the new communicator
|
||||
/////////////////////////////////////////////////////////////////
|
||||
int ierr= MPI_Comm_split(WorldComm,0,rank,&optimal_comm);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &processors,Grid_MPI_Comm & optimal_comm)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Identify subblock of ranks on node spreading across dims
|
||||
@@ -415,8 +316,6 @@ void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &proce
|
||||
Coordinate ShmCoor(ndimension); Coordinate NodeCoor(ndimension); Coordinate WorldCoor(ndimension);
|
||||
|
||||
GetShmDims(WorldDims,ShmDims);
|
||||
SHM=ShmDims;
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish torus of processes and nodes with sub-blockings
|
||||
////////////////////////////////////////////////////////////////
|
||||
@@ -431,8 +330,7 @@ void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &proce
|
||||
for(int i=0;i<ndimension;i++){
|
||||
Nprocessors*=processors[i];
|
||||
}
|
||||
// std::cerr << " WorldSize "<<WorldSize << " Nprocessors "<<Nprocessors<<" "<<processors<<std::endl;
|
||||
GRID_ASSERT(WorldSize==Nprocessors);
|
||||
assert(WorldSize==Nprocessors);
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// Establish mapping between lexico physics coord and WorldRank
|
||||
@@ -448,7 +346,7 @@ void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &proce
|
||||
// Build the new communicator
|
||||
/////////////////////////////////////////////////////////////////
|
||||
int ierr= MPI_Comm_split(WorldComm,0,rank,&optimal_comm);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// SHMGET
|
||||
@@ -456,9 +354,9 @@ void GlobalSharedMemory::OptimalCommunicatorSharedMemory(const Coordinate &proce
|
||||
#ifdef GRID_MPI3_SHMGET
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << Mheader "SharedMemoryAllocate "<< bytes<< " shmget implementation "<<std::endl;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " shmget implementation "<<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the shared windows for our group
|
||||
@@ -515,12 +413,12 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbfs mapping intended
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if defined(GRID_CUDA) ||defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
#ifdef GRID_NVCC
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the pointer array for shared windows for our group
|
||||
@@ -535,182 +433,86 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// cudaDeviceGetP2PAttribute(&perfRank, cudaDevP2PAttrPerformanceRank, device1, device2);
|
||||
|
||||
#ifdef GRID_IBM_SUMMIT
|
||||
// IBM Jsrun makes cuda Device numbering screwy and not match rank
|
||||
std::cout << "IBM Summit or similar - NOT setting device to WorldShmRank"<<std::endl;
|
||||
#else
|
||||
std::cout << "setting device to WorldShmRank"<<std::endl;
|
||||
cudaSetDevice(WorldShmRank);
|
||||
#endif
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
// printf("Host buffer allocate for GPU non-aware MPI\n");
|
||||
HostCommBuf= malloc(bytes); /// CHANGE THIS TO malloc_host
|
||||
#endif
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
auto err = cudaMalloc(&ShmCommBuf, bytes);
|
||||
if ( err != cudaSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc cudaMallocManaged failed for " << bytes<<" bytes " <<cudaGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << "SharedMemoryMPI.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
std::cerr << " SharedMemoryMPI.cc cudaMallocManaged failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if ( WorldRank == 0 ){
|
||||
std::cout << Mheader " acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf << " - "<<(bytes-1+(uint64_t)ShmCommBuf) <<std::dec<<" for comms buffers " <<std::endl;
|
||||
std::cout << header " SharedMemoryMPI.cc cudaMalloc "<< bytes << "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
}
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
if ( WorldRank == 0 ){
|
||||
std::cout<< Mheader "Setting up IPC"<<std::endl;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Loop over ranks/gpu's on our node
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#ifdef SHM_SOCKETS
|
||||
UnixSockets::Open(WorldShmRank);
|
||||
#endif
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
MPI_Barrier(WorldShmComm);
|
||||
|
||||
#ifndef GRID_MPI3_SHM_NONE
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// If it is me, pass around the IPC access key
|
||||
//////////////////////////////////////////////////
|
||||
void * thisBuf = ShmCommBuf;
|
||||
if(!Stencil_force_mpi) {
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
typedef struct { int fd; pid_t pid ; ze_ipc_mem_handle_t ze; } clone_mem_t;
|
||||
|
||||
auto zeDevice = sycl::get_native<sycl::backend::ext_oneapi_level_zero>(theGridAccelerator->get_device());
|
||||
auto zeContext = sycl::get_native<sycl::backend::ext_oneapi_level_zero>(theGridAccelerator->get_context());
|
||||
|
||||
ze_ipc_mem_handle_t ihandle;
|
||||
clone_mem_t handle;
|
||||
cudaIpcMemHandle_t handle;
|
||||
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = zeMemGetIpcHandle(zeContext,ShmCommBuf,&ihandle);
|
||||
if ( err != ZE_RESULT_SUCCESS ) {
|
||||
std::cerr << "SharedMemoryMPI.cc zeMemGetIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
memcpy((void *)&handle.fd,(void *)&ihandle,sizeof(int));
|
||||
handle.pid = getpid();
|
||||
memcpy((void *)&handle.ze,(void *)&ihandle,sizeof(ihandle));
|
||||
#ifdef SHM_SOCKETS
|
||||
for(int rr=0;rr<WorldShmSize;rr++){
|
||||
if(rr!=r){
|
||||
UnixSockets::SendFileDescriptor(handle.fd,rr);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_CUDA
|
||||
cudaIpcMemHandle_t handle;
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = cudaIpcGetMemHandle(&handle,ShmCommBuf);
|
||||
err = cudaIpcGetMemHandle(&handle,ShmCommBuf);
|
||||
if ( err != cudaSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc cudaIpcGetMemHandle failed for rank" << r <<" "<<cudaGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
hipIpcMemHandle_t handle;
|
||||
if ( r==WorldShmRank ) {
|
||||
auto err = hipIpcGetMemHandle(&handle,ShmCommBuf);
|
||||
if ( err != hipSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc hipIpcGetMemHandle failed for rank" << r <<" "<<hipGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Share this IPC handle across the Shm Comm
|
||||
//////////////////////////////////////////////////
|
||||
{
|
||||
MPI_Barrier(WorldShmComm);
|
||||
int ierr=MPI_Bcast(&handle,
|
||||
sizeof(handle),
|
||||
MPI_BYTE,
|
||||
r,
|
||||
WorldShmComm);
|
||||
GRID_ASSERT(ierr==0);
|
||||
assert(ierr==0);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////
|
||||
// If I am not the source, overwrite thisBuf with remote buffer
|
||||
///////////////////////////////////////////////////////////////
|
||||
|
||||
#ifdef GRID_SYCL_LEVEL_ZERO_IPC
|
||||
if ( r!=WorldShmRank ) {
|
||||
thisBuf = nullptr;
|
||||
int myfd;
|
||||
#ifdef SHM_SOCKETS
|
||||
myfd=UnixSockets::RecvFileDescriptor();
|
||||
#else
|
||||
// std::cout<<"mapping seeking remote pid/fd "
|
||||
// <<handle.pid<<"/"
|
||||
// <<handle.fd<<std::endl;
|
||||
|
||||
int pidfd = syscall(SYS_pidfd_open,handle.pid,0);
|
||||
// std::cout<<"Using IpcHandle pidfd "<<pidfd<<"\n";
|
||||
// int myfd = syscall(SYS_pidfd_getfd,pidfd,handle.fd,0);
|
||||
myfd = syscall(438,pidfd,handle.fd,0);
|
||||
int err_t = errno;
|
||||
if (myfd < 0) {
|
||||
fprintf(stderr,"pidfd_getfd returned %d errno was %d\n", myfd,err_t); fflush(stderr);
|
||||
perror("pidfd_getfd failed ");
|
||||
assert(0);
|
||||
}
|
||||
#endif
|
||||
// std::cout<<"Using IpcHandle mapped remote pid "<<handle.pid <<" FD "<<handle.fd <<" to myfd "<<myfd<<"\n";
|
||||
memcpy((void *)&ihandle,(void *)&handle.ze,sizeof(ihandle));
|
||||
memcpy((void *)&ihandle,(void *)&myfd,sizeof(int));
|
||||
|
||||
auto err = zeMemOpenIpcHandle(zeContext,zeDevice,ihandle,0,&thisBuf);
|
||||
if ( err != ZE_RESULT_SUCCESS ) {
|
||||
std::cerr << "SharedMemoryMPI.cc "<<zeContext<<" "<<zeDevice<<std::endl;
|
||||
std::cerr << "SharedMemoryMPI.cc zeMemOpenIpcHandle failed for rank "<<r<<" "<<std::hex<<err<<std::dec<<std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
GRID_ASSERT(thisBuf!=nullptr);
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_CUDA
|
||||
void * thisBuf = ShmCommBuf;
|
||||
if ( r!=WorldShmRank ) {
|
||||
auto err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
|
||||
err = cudaIpcOpenMemHandle(&thisBuf,handle,cudaIpcMemLazyEnablePeerAccess);
|
||||
if ( err != cudaSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc cudaIpcOpenMemHandle failed for rank" << r <<" "<<cudaGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#ifdef GRID_HIP
|
||||
if ( r!=WorldShmRank ) {
|
||||
auto err = hipIpcOpenMemHandle(&thisBuf,handle,hipIpcMemLazyEnablePeerAccess);
|
||||
if ( err != hipSuccess) {
|
||||
std::cerr << " SharedMemoryMPI.cc hipIpcOpenMemHandle failed for rank" << r <<" "<<hipGetErrorString(err)<< std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
///////////////////////////////////////////////////////////////
|
||||
// Save a copy of the device buffers
|
||||
///////////////////////////////////////////////////////////////
|
||||
}
|
||||
WorldShmCommBufs[r] = thisBuf;
|
||||
#else
|
||||
WorldShmCommBufs[r] = ShmCommBuf;
|
||||
#endif
|
||||
MPI_Barrier(WorldShmComm);
|
||||
}
|
||||
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
|
||||
#else
|
||||
#ifdef GRID_MPI3_SHMMMAP
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << Mheader "SharedMemoryAllocate "<< bytes<< " MMAP implementation "<< GRID_SHM_PATH <<std::endl;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " MMAP implementation "<< GRID_SHM_PATH <<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -740,14 +542,13 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
void *ptr = (void *) mmap(NULL, bytes, PROT_READ | PROT_WRITE, mmap_flag,fd, 0);
|
||||
if ( ptr == (void *)MAP_FAILED ) {
|
||||
printf("mmap %s failed\n",shm_name);
|
||||
perror("failed mmap"); GRID_ASSERT(0);
|
||||
perror("failed mmap"); assert(0);
|
||||
}
|
||||
GRID_ASSERT(((uint64_t)ptr&0x3F)==0);
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
close(fd);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
// std::cout << Mheader "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
// std::cout << header "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
}
|
||||
std::cout<< Mheader " Intra-node IPC setup is complete "<<std::endl;
|
||||
_ShmAlloc=1;
|
||||
_ShmAllocBytes = bytes;
|
||||
};
|
||||
@@ -756,9 +557,9 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
#ifdef GRID_MPI3_SHM_NONE
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << Mheader "SharedMemoryAllocate "<< bytes<< " MMAP anonymous implementation "<<std::endl;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " MMAP anonymous implementation "<<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// allocate the shared windows for our group
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -769,7 +570,7 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
// Hugetlbf and others map filesystems as mappable huge pages
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
char shm_name [NAME_MAX];
|
||||
GRID_ASSERT(WorldShmSize == 1);
|
||||
assert(WorldShmSize == 1);
|
||||
for(int r=0;r<WorldShmSize;r++){
|
||||
|
||||
int fd=-1;
|
||||
@@ -783,9 +584,9 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
void *ptr = (void *) mmap(NULL, bytes, PROT_READ | PROT_WRITE, mmap_flag,fd, 0);
|
||||
if ( ptr == (void *)MAP_FAILED ) {
|
||||
printf("mmap %s failed\n",shm_name);
|
||||
perror("failed mmap"); GRID_ASSERT(0);
|
||||
perror("failed mmap"); assert(0);
|
||||
}
|
||||
GRID_ASSERT(((uint64_t)ptr&0x3F)==0);
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
close(fd);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
// std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< bytes<< "bytes)"<<std::endl;
|
||||
@@ -803,9 +604,9 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << Mheader "SharedMemoryAllocate "<< bytes<< " SHMOPEN implementation "<<std::endl;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " SHMOPEN implementation "<<std::endl;
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
MPI_Barrier(WorldShmComm);
|
||||
WorldShmCommBufs.resize(WorldShmSize);
|
||||
|
||||
@@ -832,11 +633,12 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
#endif
|
||||
void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0);
|
||||
|
||||
// std::cout << "Set WorldShmCommBufs["<<r<<"]="<<ptr<< "("<< size<< "bytes)"<<std::endl;
|
||||
if ( ptr == (void * )MAP_FAILED ) {
|
||||
perror("failed mmap");
|
||||
assert(0);
|
||||
}
|
||||
GRID_ASSERT(((uint64_t)ptr&0x3F)==0);
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
close(fd);
|
||||
@@ -857,8 +659,8 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
if ( fd<0 ) { perror("failed shm_open"); assert(0); }
|
||||
|
||||
void * ptr = mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
if ( ptr == MAP_FAILED ) { perror("failed mmap"); GRID_ASSERT(0); }
|
||||
GRID_ASSERT(((uint64_t)ptr&0x3F)==0);
|
||||
if ( ptr == MAP_FAILED ) { perror("failed mmap"); assert(0); }
|
||||
assert(((uint64_t)ptr&0x3F)==0);
|
||||
WorldShmCommBufs[r] =ptr;
|
||||
|
||||
close(fd);
|
||||
@@ -875,20 +677,20 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
||||
{
|
||||
#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
acceleratorMemSet(dest,0,bytes);
|
||||
#ifdef GRID_NVCC
|
||||
cudaMemset(dest,0,bytes);
|
||||
#else
|
||||
bzero(dest,bytes);
|
||||
#endif
|
||||
}
|
||||
//void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
||||
//{
|
||||
//#if defined(GRID_CUDA) || defined(GRID_HIP) || defined(GRID_SYCL)
|
||||
// acceleratorCopyToDevice(src,dest,bytes);
|
||||
//#else
|
||||
// bcopy(src,dest,bytes);
|
||||
//#endif
|
||||
//}
|
||||
void GlobalSharedMemory::SharedMemoryCopy(void *dest,const void *src,size_t bytes)
|
||||
{
|
||||
#ifdef GRID_NVCC
|
||||
cudaMemcpy(dest,src,bytes,cudaMemcpyDefault);
|
||||
#else
|
||||
bcopy(src,dest,bytes);
|
||||
#endif
|
||||
}
|
||||
////////////////////////////////////////////////////////
|
||||
// Global shared functionality finished
|
||||
// Now move to per communicator functionality
|
||||
@@ -903,11 +705,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// Split into groups that can share memory
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
#ifndef GRID_MPI3_SHM_NONE
|
||||
MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL,&ShmComm);
|
||||
#else
|
||||
MPI_Comm_split(comm, rank, 0, &ShmComm);
|
||||
#endif
|
||||
MPI_Comm_rank(ShmComm ,&ShmRank);
|
||||
MPI_Comm_size(ShmComm ,&ShmSize);
|
||||
ShmCommBufs.resize(ShmSize);
|
||||
@@ -915,7 +713,7 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// Map ShmRank to WorldShmRank and use the right buffer
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
GRID_ASSERT (GlobalSharedMemory::ShmAlloc()==1);
|
||||
assert (GlobalSharedMemory::ShmAlloc()==1);
|
||||
heap_size = GlobalSharedMemory::ShmAllocBytes();
|
||||
for(int r=0;r<ShmSize;r++){
|
||||
|
||||
@@ -924,16 +722,9 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
MPI_Allreduce(MPI_IN_PLACE,&wsr,1,MPI_UINT32_T,MPI_SUM,ShmComm);
|
||||
|
||||
ShmCommBufs[r] = GlobalSharedMemory::WorldShmCommBufs[wsr];
|
||||
// std::cerr << " SetCommunicator rank "<<r<<" comm "<<ShmCommBufs[r] <<std::endl;
|
||||
}
|
||||
ShmBufferFreeAll();
|
||||
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
host_heap_size = heap_size;
|
||||
HostCommBuf= GlobalSharedMemory::HostCommBuf;
|
||||
HostBufferFreeAll();
|
||||
#endif
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// find comm ranks in our SHM group (i.e. which ranks are on our node)
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
@@ -944,18 +735,25 @@ void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
std::vector<int> ranks(size); for(int r=0;r<size;r++) ranks[r]=r;
|
||||
MPI_Group_translate_ranks (FullGroup,size,&ranks[0],ShmGroup, &ShmRanks[0]);
|
||||
|
||||
#ifdef GRID_SHM_FORCE_MPI
|
||||
// Hide the shared memory path between ranks
|
||||
{
|
||||
#ifdef GRID_IBM_SUMMIT
|
||||
// Hide the shared memory path between sockets
|
||||
// if even number of nodes
|
||||
if ( (ShmSize & 0x1)==0 ) {
|
||||
int SocketSize = ShmSize/2;
|
||||
int mySocket = ShmRank/SocketSize;
|
||||
for(int r=0;r<size;r++){
|
||||
if ( r!=rank ) {
|
||||
ShmRanks[r] = MPI_UNDEFINED;
|
||||
int hisRank=ShmRanks[r];
|
||||
if ( hisRank!= MPI_UNDEFINED ) {
|
||||
int hisSocket=hisRank/SocketSize;
|
||||
if ( hisSocket != mySocket ) {
|
||||
ShmRanks[r] = MPI_UNDEFINED;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// SharedMemoryTest();
|
||||
SharedMemoryTest();
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// On node barrier
|
||||
@@ -977,18 +775,19 @@ void SharedMemory::SharedMemoryTest(void)
|
||||
check[0]=GlobalSharedMemory::WorldNode;
|
||||
check[1]=r;
|
||||
check[2]=magic;
|
||||
acceleratorCopyToDevice(check,ShmCommBufs[r],3*sizeof(uint64_t));
|
||||
GlobalSharedMemory::SharedMemoryCopy( ShmCommBufs[r], check, 3*sizeof(uint64_t));
|
||||
}
|
||||
}
|
||||
ShmBarrier();
|
||||
for(uint64_t r=0;r<ShmSize;r++){
|
||||
acceleratorCopyFromDevice(ShmCommBufs[r],check,3*sizeof(uint64_t));
|
||||
GRID_ASSERT(check[0]==GlobalSharedMemory::WorldNode);
|
||||
GRID_ASSERT(check[1]==r);
|
||||
GRID_ASSERT(check[2]==magic);
|
||||
ShmBarrier();
|
||||
GlobalSharedMemory::SharedMemoryCopy(check,ShmCommBufs[r], 3*sizeof(uint64_t));
|
||||
ShmBarrier();
|
||||
assert(check[0]==GlobalSharedMemory::WorldNode);
|
||||
assert(check[1]==r);
|
||||
assert(check[2]==magic);
|
||||
ShmBarrier();
|
||||
}
|
||||
ShmBarrier();
|
||||
std::cout << GridLogDebug << " SharedMemoryTest has passed "<<std::endl;
|
||||
}
|
||||
|
||||
void *SharedMemory::ShmBuffer(int rank)
|
||||
@@ -1003,14 +802,12 @@ void *SharedMemory::ShmBuffer(int rank)
|
||||
void *SharedMemory::ShmBufferTranslate(int rank,void * local_p)
|
||||
{
|
||||
int gpeer = ShmRanks[rank];
|
||||
GRID_ASSERT(gpeer!=ShmRank); // never send to self
|
||||
// std::cout << "ShmBufferTranslate for rank " << rank<<" peer "<<gpeer<<std::endl;
|
||||
assert(gpeer!=ShmRank); // never send to self
|
||||
if (gpeer == MPI_UNDEFINED){
|
||||
return NULL;
|
||||
} else {
|
||||
uint64_t offset = (uint64_t)local_p - (uint64_t)ShmCommBufs[ShmRank];
|
||||
uint64_t remote = (uint64_t)ShmCommBufs[gpeer]+offset;
|
||||
// std::cout << "ShmBufferTranslate : local,offset,remote "<<std::hex<<local_p<<" "<<offset<<" "<<remote<<std::dec<<std::endl;
|
||||
return (void *) remote;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,12 +29,11 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
#include <Grid/GridCore.h>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
#define header "SharedMemoryNone: "
|
||||
|
||||
/*Construct from an MPI communicator*/
|
||||
void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
{
|
||||
GRID_ASSERT(_ShmSetup==0);
|
||||
assert(_ShmSetup==0);
|
||||
WorldComm = 0;
|
||||
WorldRank = 0;
|
||||
WorldSize = 1;
|
||||
@@ -48,52 +47,19 @@ void GlobalSharedMemory::Init(Grid_MPI_Comm comm)
|
||||
_ShmSetup=1;
|
||||
}
|
||||
|
||||
void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm,Coordinate &SHM)
|
||||
void GlobalSharedMemory::OptimalCommunicator(const Coordinate &processors,Grid_MPI_Comm & optimal_comm)
|
||||
{
|
||||
optimal_comm = WorldComm;
|
||||
SHM = Coordinate(processors.size(),1);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Hugetlbfs mapping intended, use anonymous mmap
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if 1
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
std::cout << header "SharedMemoryAllocate "<< bytes<< " GPU implementation "<<std::endl;
|
||||
void * ShmCommBuf ;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Each MPI rank should allocate our own buffer
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
ShmCommBuf = acceleratorAllocDevice(bytes);
|
||||
|
||||
if (ShmCommBuf == (void *)NULL ) {
|
||||
std::cerr << " SharedMemoryNone.cc acceleratorAllocDevice failed NULL pointer for " << bytes<<" bytes " << std::endl;
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if ( WorldRank == 0 ){
|
||||
std::cout << WorldRank << header " SharedMemoryNone.cc acceleratorAllocDevice "<< bytes
|
||||
<< "bytes at "<< std::hex<< ShmCommBuf <<std::dec<<" for comms buffers " <<std::endl;
|
||||
}
|
||||
SharedMemoryZero(ShmCommBuf,bytes);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Loop over ranks/gpu's on our node
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
WorldShmCommBufs[0] = ShmCommBuf;
|
||||
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
}
|
||||
#else
|
||||
void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
{
|
||||
void * ShmCommBuf ;
|
||||
GRID_ASSERT(_ShmSetup==1);
|
||||
GRID_ASSERT(_ShmAlloc==0);
|
||||
assert(_ShmSetup==1);
|
||||
assert(_ShmAlloc==0);
|
||||
int mmap_flag =0;
|
||||
#ifdef MAP_ANONYMOUS
|
||||
mmap_flag = mmap_flag| MAP_SHARED | MAP_ANONYMOUS;
|
||||
@@ -117,22 +83,14 @@ void GlobalSharedMemory::SharedMemoryAllocate(uint64_t bytes, int flags)
|
||||
_ShmAllocBytes=bytes;
|
||||
_ShmAlloc=1;
|
||||
};
|
||||
#endif
|
||||
void GlobalSharedMemory::SharedMemoryZero(void *dest,size_t bytes)
|
||||
{
|
||||
acceleratorMemSet(dest,0,bytes);
|
||||
}
|
||||
//void GlobalSharedMemory::SharedMemoryCopy(void *dest,void *src,size_t bytes)
|
||||
//{
|
||||
// acceleratorCopyToDevice(src,dest,bytes);
|
||||
//}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
// Global shared functionality finished
|
||||
// Now move to per communicator functionality
|
||||
////////////////////////////////////////////////////////
|
||||
void SharedMemory::SetCommunicator(Grid_MPI_Comm comm)
|
||||
{
|
||||
GRID_ASSERT(GlobalSharedMemory::ShmAlloc()==1);
|
||||
assert(GlobalSharedMemory::ShmAlloc()==1);
|
||||
ShmRanks.resize(1);
|
||||
ShmCommBufs.resize(1);
|
||||
ShmRanks[0] = 0;
|
||||
|
||||
+18
-2
@@ -51,8 +51,24 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
#endif
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
||||
auto Cshift(const Expression &expr,int dim,int shift) -> decltype(closure(expr))
|
||||
|
||||
template<typename Op, typename T1>
|
||||
auto Cshift(const LatticeUnaryExpression<Op,T1> &expr,int dim,int shift)
|
||||
-> Lattice<decltype(expr.op.func(eval(0, expr.arg1)))>
|
||||
{
|
||||
return Cshift(closure(expr),dim,shift);
|
||||
}
|
||||
template <class Op, class T1, class T2>
|
||||
auto Cshift(const LatticeBinaryExpression<Op,T1,T2> &expr,int dim,int shift)
|
||||
-> Lattice<decltype(expr.op.func(eval(0, expr.arg1),eval(0, expr.arg2)))>
|
||||
{
|
||||
return Cshift(closure(expr),dim,shift);
|
||||
}
|
||||
template <class Op, class T1, class T2, class T3>
|
||||
auto Cshift(const LatticeTrinaryExpression<Op,T1,T2,T3> &expr,int dim,int shift)
|
||||
-> Lattice<decltype(expr.op.func(eval(0, expr.arg1),
|
||||
eval(0, expr.arg2),
|
||||
eval(0, expr.arg3)))>
|
||||
{
|
||||
return Cshift(closure(expr),dim,shift);
|
||||
}
|
||||
|
||||
+57
-100
@@ -29,28 +29,11 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
extern std::vector<std::pair<int,int> > Cshift_table;
|
||||
extern deviceVector<std::pair<int,int> > Cshift_table_device;
|
||||
|
||||
inline std::pair<int,int> *MapCshiftTable(void)
|
||||
{
|
||||
// GPU version
|
||||
uint64_t sz=Cshift_table.size();
|
||||
if (Cshift_table_device.size()!=sz ) {
|
||||
Cshift_table_device.resize(sz);
|
||||
}
|
||||
acceleratorCopyToDevice((void *)&Cshift_table[0],
|
||||
(void *)&Cshift_table_device[0],
|
||||
sizeof(Cshift_table[0])*sz);
|
||||
|
||||
return &Cshift_table_device[0];
|
||||
// CPU version use identify map
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// Gather for when there is no need to SIMD split
|
||||
///////////////////////////////////////////////////////////////////
|
||||
template<class vobj> void
|
||||
Gather_plane_simple (const Lattice<vobj> &rhs,deviceVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
|
||||
Gather_plane_simple (const Lattice<vobj> &rhs,commVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
@@ -63,16 +46,16 @@ Gather_plane_simple (const Lattice<vobj> &rhs,deviceVector<vobj> &buffer,int dim
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int ent = 0;
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
static Vector<std::pair<int,int> > table; table.resize(e1*e2);
|
||||
int stride=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
auto rhs_v = rhs.View();
|
||||
if ( cbmask == 0x3 ) {
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o = n*stride;
|
||||
int bo = n*e2;
|
||||
Cshift_table[ent++] = std::pair<int,int>(off+bo+b,so+o+b);
|
||||
table[ent++] = std::pair<int,int>(off+bo+b,so+o+b);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -82,19 +65,14 @@ Gather_plane_simple (const Lattice<vobj> &rhs,deviceVector<vobj> &buffer,int dim
|
||||
int o = n*stride;
|
||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);
|
||||
if ( ocb &cbmask ) {
|
||||
Cshift_table[ent++]=std::pair<int,int> (off+bo++,so+o+b);
|
||||
table[ent++]=std::pair<int,int> (off+bo++,so+o+b);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
auto buffer_p = & buffer[0];
|
||||
auto table = MapCshiftTable();
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
||||
coalescedWrite(buffer_p[table[i].first],coalescedRead(rhs_v[table[i].second]));
|
||||
});
|
||||
}
|
||||
thread_for(i,ent,{
|
||||
buffer[table[i].first]=rhs_v[table[i].second];
|
||||
});
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
@@ -117,48 +95,43 @@ Gather_plane_extract(const Lattice<vobj> &rhs,
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int n1=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
auto rhs_v = rhs.View();
|
||||
if ( cbmask ==0x3){
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
thread_for_collapse(2,n,e1,{
|
||||
for(int b=0;b<e2;b++){
|
||||
|
||||
int o = n*n1;
|
||||
int offset = b+n*e2;
|
||||
|
||||
vobj temp =rhs_v[so+o+b];
|
||||
extract<vobj>(temp,pointers,offset);
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
Coordinate rdim=rhs.Grid()->_rdimensions;
|
||||
Coordinate cdm =rhs.Grid()->_checker_dim_mask;
|
||||
std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
|
||||
Coordinate coor;
|
||||
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
||||
// Test_cshift_red_black code.
|
||||
std::cout << " Dense packed buffer WARNING " <<std::endl;
|
||||
thread_for_collapse(2,n,e1,{
|
||||
for(int b=0;b<e2;b++){
|
||||
|
||||
int o=n*n1;
|
||||
int oindex = o+b;
|
||||
|
||||
int cb = RedBlackCheckerBoardFromOindex(oindex, rdim, cdm);
|
||||
|
||||
int ocb=1<<cb;
|
||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);
|
||||
int offset = b+n*e2;
|
||||
|
||||
if ( ocb & cbmask ) {
|
||||
vobj temp =rhs_v[so+o+b];
|
||||
extract<vobj>(temp,pointers,offset);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
// Scatter for when there is no need to SIMD split
|
||||
//////////////////////////////////////////////////////
|
||||
template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,deviceVector<vobj> &buffer, int dimension,int plane,int cbmask)
|
||||
template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,commVector<vobj> &buffer, int dimension,int plane,int cbmask)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
@@ -172,8 +145,7 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,deviceVector<
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int stride=rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
static std::vector<std::pair<int,int> > table; table.resize(e1*e2);
|
||||
int ent =0;
|
||||
|
||||
if ( cbmask ==0x3 ) {
|
||||
@@ -182,7 +154,7 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,deviceVector<
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*rhs.Grid()->_slice_stride[dimension];
|
||||
int bo =n*rhs.Grid()->_slice_block[dimension];
|
||||
Cshift_table[ent++] = std::pair<int,int>(so+o+b,bo+b);
|
||||
table[ent++] = std::pair<int,int>(so+o+b,bo+b);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,20 +165,16 @@ template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,deviceVector<
|
||||
int o =n*rhs.Grid()->_slice_stride[dimension];
|
||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);// Could easily be a table lookup
|
||||
if ( ocb & cbmask ) {
|
||||
Cshift_table[ent++]=std::pair<int,int> (so+o+b,bo++);
|
||||
table[ent++]=std::pair<int,int> (so+o+b,bo++);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto buffer_p = & buffer[0];
|
||||
auto table = MapCshiftTable();
|
||||
autoView( rhs_v, rhs, AcceleratorWriteDiscard);
|
||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
||||
coalescedWrite(rhs_v[table[i].first],coalescedRead(buffer_p[table[i].second]));
|
||||
});
|
||||
}
|
||||
auto rhs_v = rhs.View();
|
||||
thread_for(i,ent,{
|
||||
rhs_v[table[i].first]=buffer[table[i].second];
|
||||
});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
@@ -226,24 +194,21 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
|
||||
if(cbmask ==0x3 ) {
|
||||
int _slice_stride = rhs.Grid()->_slice_stride[dimension];
|
||||
int _slice_block = rhs.Grid()->_slice_block[dimension];
|
||||
autoView( rhs_v , rhs, AcceleratorWriteDiscard);
|
||||
accelerator_for(nn,e1*e2,1,{
|
||||
int n = nn%e1;
|
||||
int b = nn/e1;
|
||||
int o = n*_slice_stride;
|
||||
int offset = b+n*_slice_block;
|
||||
auto rhs_v = rhs.View();
|
||||
thread_for_collapse(2,n,e1,{
|
||||
for(int b=0;b<e2;b++){
|
||||
int o = n*rhs.Grid()->_slice_stride[dimension];
|
||||
int offset = b+n*rhs.Grid()->_slice_block[dimension];
|
||||
merge(rhs_v[so+o+b],pointers,offset);
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
|
||||
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
||||
// Test_cshift_red_black code.
|
||||
std::cout << "Scatter_plane merge GRID_ASSERT(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
||||
// std::cout << "Scatter_plane merge assert(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
||||
std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
|
||||
GRID_ASSERT(0); // This will fail if hit on GPU
|
||||
autoView( rhs_v, rhs, CpuWrite);
|
||||
auto rhs_v = rhs.View();
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o = n*rhs.Grid()->_slice_stride[dimension];
|
||||
@@ -260,7 +225,6 @@ template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerA
|
||||
//////////////////////////////////////////////////////
|
||||
// local to node block strided copies
|
||||
//////////////////////////////////////////////////////
|
||||
|
||||
template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs, int dimension,int lplane,int rplane,int cbmask)
|
||||
{
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
@@ -275,16 +239,14 @@ template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs
|
||||
int e1=rhs.Grid()->_slice_nblock[dimension]; // clearly loop invariant for icpc
|
||||
int e2=rhs.Grid()->_slice_block[dimension];
|
||||
int stride = rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
static std::vector<std::pair<int,int> > table; table.resize(e1*e2);
|
||||
int ent=0;
|
||||
|
||||
if(cbmask == 0x3 ){
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride+b;
|
||||
Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
||||
table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -293,24 +255,23 @@ template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs
|
||||
int o =n*stride+b;
|
||||
int ocb=1<<lhs.Grid()->CheckerBoardFromOindex(o);
|
||||
if ( ocb&cbmask ) {
|
||||
Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
||||
table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto table = MapCshiftTable();
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
autoView(lhs_v , lhs, AcceleratorWriteDiscard);
|
||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
||||
coalescedWrite(lhs_v[table[i].first],coalescedRead(rhs_v[table[i].second]));
|
||||
});
|
||||
}
|
||||
auto rhs_v = rhs.View();
|
||||
auto lhs_v = lhs.View();
|
||||
thread_for(i,ent,{
|
||||
lhs_v[table[i].first]=rhs_v[table[i].second];
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vobj> &rhs, int dimension,int lplane,int rplane,int cbmask,int permute_type)
|
||||
{
|
||||
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
|
||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
@@ -324,33 +285,29 @@ template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vo
|
||||
int e2=rhs.Grid()->_slice_block [dimension];
|
||||
int stride = rhs.Grid()->_slice_stride[dimension];
|
||||
|
||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
||||
|
||||
static std::vector<std::pair<int,int> > table; table.resize(e1*e2);
|
||||
int ent=0;
|
||||
|
||||
if ( cbmask == 0x3 ) {
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride;
|
||||
Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
||||
table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
||||
}}
|
||||
} else {
|
||||
for(int n=0;n<e1;n++){
|
||||
for(int b=0;b<e2;b++){
|
||||
int o =n*stride;
|
||||
int ocb=1<<lhs.Grid()->CheckerBoardFromOindex(o+b);
|
||||
if ( ocb&cbmask ) Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
||||
if ( ocb&cbmask ) table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
||||
}}
|
||||
}
|
||||
|
||||
{
|
||||
auto table = MapCshiftTable();
|
||||
autoView( rhs_v, rhs, AcceleratorRead);
|
||||
autoView( lhs_v, lhs, AcceleratorWrite);
|
||||
accelerator_for(i,ent,1,{
|
||||
permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
|
||||
});
|
||||
}
|
||||
auto rhs_v = rhs.View();
|
||||
auto lhs_v = lhs.View();
|
||||
thread_for(i,ent,{
|
||||
permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
|
||||
});
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////
|
||||
|
||||
+34
-254
@@ -29,13 +29,9 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
|
||||
#ifndef _GRID_CSHIFT_MPI_H_
|
||||
#define _GRID_CSHIFT_MPI_H_
|
||||
|
||||
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
extern uint64_t checksum_index;
|
||||
#endif
|
||||
|
||||
const int Cshift_verbose=0;
|
||||
template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
@@ -49,20 +45,6 @@ template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension
|
||||
// Map to always positive shift modulo global full dimension.
|
||||
shift = (shift+fd)%fd;
|
||||
|
||||
if( shift ==0 ) {
|
||||
ret = rhs;
|
||||
return ret;
|
||||
}
|
||||
//
|
||||
// Potential easy fast cases:
|
||||
// Shift is a multiple of the local lattice extent.
|
||||
// Then need only to shift whole subvolumes
|
||||
int L = rhs.Grid()->_ldimensions[dimension];
|
||||
if ( (shift%L )==0 && !rhs.Grid()->CheckerBoarded(dimension) ) {
|
||||
Cshift_simple(ret,rhs,dimension,shift);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret.Checkerboard() = rhs.Grid()->CheckerBoardDestination(rhs.Checkerboard(),shift,dimension);
|
||||
|
||||
// the permute type
|
||||
@@ -70,72 +52,20 @@ template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension
|
||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
||||
int splice_dim = rhs.Grid()->_simd_layout[dimension]>1 && (comm_dim);
|
||||
|
||||
RealD t1,t0;
|
||||
t0=usecond();
|
||||
|
||||
if ( !comm_dim ) {
|
||||
// std::cout << "CSHIFT: Cshift_local" <<std::endl;
|
||||
//std::cout << "CSHIFT: Cshift_local" <<std::endl;
|
||||
Cshift_local(ret,rhs,dimension,shift); // Handles checkerboarding
|
||||
} else if ( splice_dim ) {
|
||||
// std::cout << "CSHIFT: Cshift_comms_simd call - splice_dim = " << splice_dim << " shift " << shift << " dimension = " << dimension << std::endl;
|
||||
//std::cout << "CSHIFT: Cshift_comms_simd call - splice_dim = " << splice_dim << " shift " << shift << " dimension = " << dimension << std::endl;
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift);
|
||||
} else {
|
||||
// std::cout << "CSHIFT: Cshift_comms" <<std::endl;
|
||||
//std::cout << "CSHIFT: Cshift_comms" <<std::endl;
|
||||
Cshift_comms(ret,rhs,dimension,shift);
|
||||
}
|
||||
t1=usecond();
|
||||
if(Cshift_verbose) std::cout << GridLogPerformance << "Cshift took "<< (t1-t0)/1e3 << " ms"<<std::endl;
|
||||
return ret;
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_simple(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
GridBase *grid=rhs.Grid();
|
||||
int comm_proc, xmit_to_rank, recv_from_rank;
|
||||
|
||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
||||
int ld = rhs.Grid()->_ldimensions[dimension];
|
||||
int pd = rhs.Grid()->_processors[dimension];
|
||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
||||
|
||||
comm_proc = ((shift)/ld)%pd;
|
||||
|
||||
grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
|
||||
if(comm_dim) {
|
||||
|
||||
int64_t bytes = sizeof(vobj) * grid->oSites();
|
||||
|
||||
autoView(rhs_v , rhs, AcceleratorRead);
|
||||
autoView(ret_v , ret, AcceleratorWrite);
|
||||
void *send_buf = (void *)&rhs_v[0];
|
||||
void *recv_buf = (void *)&ret_v[0];
|
||||
|
||||
#ifdef ACCELERATOR_AWARE_MPI
|
||||
grid->SendToRecvFrom(send_buf,
|
||||
xmit_to_rank,
|
||||
recv_buf,
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
#else
|
||||
static hostVector<vobj> hrhs; hrhs.resize(grid->oSites());
|
||||
static hostVector<vobj> hret; hret.resize(grid->oSites());
|
||||
|
||||
void *hsend_buf = (void *)&hrhs[0];
|
||||
void *hrecv_buf = (void *)&hret[0];
|
||||
|
||||
acceleratorCopyFromDevice(send_buf,hsend_buf,bytes);
|
||||
|
||||
grid->SendToRecvFrom(hsend_buf,
|
||||
xmit_to_rank,
|
||||
hrecv_buf,
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
|
||||
acceleratorCopyToDevice(hrecv_buf,recv_buf,bytes);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
template<class vobj> void Cshift_comms(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
||||
{
|
||||
int sshift[2];
|
||||
@@ -161,16 +91,17 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj>& ret,const Lattice<vob
|
||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
||||
|
||||
// std::cout << "Cshift_comms_simd dim "<<dimension<<"cb "<<rhs.Checkerboard()<<"shift "<<shift<<" sshift " << sshift[0]<<" "<<sshift[1]<<std::endl;
|
||||
//std::cout << "Cshift_comms_simd dim "<<dimension<<"cb "<<rhs.checkerboard<<"shift "<<shift<<" sshift " << sshift[0]<<" "<<sshift[1]<<std::endl;
|
||||
if ( sshift[0] == sshift[1] ) {
|
||||
// std::cout << "Single pass Cshift_comms" <<std::endl;
|
||||
//std::cout << "Single pass Cshift_comms" <<std::endl;
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x3);
|
||||
} else {
|
||||
// std::cout << "Two pass Cshift_comms" <<std::endl;
|
||||
//std::cout << "Two pass Cshift_comms" <<std::endl;
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
{
|
||||
typedef typename vobj::vector_type vector_type;
|
||||
@@ -184,123 +115,52 @@ template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &r
|
||||
int pd = rhs.Grid()->_processors[dimension];
|
||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
||||
GRID_ASSERT(simd_layout==1);
|
||||
GRID_ASSERT(comm_dim==1);
|
||||
GRID_ASSERT(shift>=0);
|
||||
GRID_ASSERT(shift<fd);
|
||||
assert(simd_layout==1);
|
||||
assert(comm_dim==1);
|
||||
assert(shift>=0);
|
||||
assert(shift<fd);
|
||||
|
||||
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
||||
static deviceVector<vobj> send_buf; send_buf.resize(buffer_size);
|
||||
static deviceVector<vobj> recv_buf; recv_buf.resize(buffer_size);
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
int pad = (8 + sizeof(vobj) - 1) / sizeof(vobj);
|
||||
static hostVector<vobj> hsend_buf; hsend_buf.resize(buffer_size+pad);
|
||||
static hostVector<vobj> hrecv_buf; hrecv_buf.resize(buffer_size+pad);
|
||||
#endif
|
||||
|
||||
commVector<vobj> send_buf(buffer_size);
|
||||
commVector<vobj> recv_buf(buffer_size);
|
||||
|
||||
int cb= (cbmask==0x2)? Odd : Even;
|
||||
int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
||||
RealD tcopy=0.0;
|
||||
RealD tgather=0.0;
|
||||
RealD tscatter=0.0;
|
||||
RealD tcomms=0.0;
|
||||
uint64_t xbytes=0;
|
||||
|
||||
for(int x=0;x<rd;x++){
|
||||
|
||||
int sx = (x+sshift)%rd;
|
||||
int comm_proc = ((x+sshift)/rd)%pd;
|
||||
|
||||
if (comm_proc==0) {
|
||||
FlightRecorder::StepLog("Cshift_Copy_plane");
|
||||
tcopy-=usecond();
|
||||
|
||||
Copy_plane(ret,rhs,dimension,x,sx,cbmask);
|
||||
tcopy+=usecond();
|
||||
FlightRecorder::StepLog("Cshift_Copy_plane_complete");
|
||||
|
||||
} else {
|
||||
|
||||
int words = buffer_size;
|
||||
int words = send_buf.size();
|
||||
if (cbmask != 0x3) words=words>>1;
|
||||
|
||||
int bytes = words * sizeof(vobj);
|
||||
|
||||
FlightRecorder::StepLog("Cshift_Gather_plane");
|
||||
tgather-=usecond();
|
||||
Gather_plane_simple (rhs,send_buf,dimension,sx,cbmask);
|
||||
tgather+=usecond();
|
||||
FlightRecorder::StepLog("Cshift_Gather_plane_complete");
|
||||
|
||||
// int rank = grid->_processor;
|
||||
int recv_from_rank;
|
||||
int xmit_to_rank;
|
||||
|
||||
grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
|
||||
|
||||
tcomms-=usecond();
|
||||
grid->Barrier();
|
||||
|
||||
FlightRecorder::StepLog("Cshift_SendRecv");
|
||||
#ifdef ACCELERATOR_AWARE_MPI
|
||||
|
||||
grid->SendToRecvFrom((void *)&send_buf[0],
|
||||
xmit_to_rank,
|
||||
(void *)&recv_buf[0],
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
#else
|
||||
// bouncy bouncy
|
||||
acceleratorCopyFromDevice(&send_buf[0],&hsend_buf[0],bytes);
|
||||
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
GRID_ASSERT(bytes % 8 == 0);
|
||||
checksum_index++;
|
||||
uint64_t xsum = checksum_gpu((uint64_t*)&send_buf[0], bytes / 8) ^ (1 + checksum_index);
|
||||
*(uint64_t*)(((char*)&hsend_buf[0]) + bytes) = xsum;
|
||||
bytes += 8;
|
||||
#endif
|
||||
|
||||
grid->SendToRecvFrom((void *)&hsend_buf[0],
|
||||
xmit_to_rank,
|
||||
(void *)&hrecv_buf[0],
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
bytes -= 8;
|
||||
acceleratorCopyToDevice(&hrecv_buf[0],&recv_buf[0],bytes);
|
||||
uint64_t expected_cs = *(uint64_t*)(((char*)&hrecv_buf[0]) + bytes);
|
||||
uint64_t computed_cs = checksum_gpu((uint64_t*)&recv_buf[0], bytes / 8) ^ (1 + checksum_index);
|
||||
std::cout << GridLogComms<< " Cshift: "
|
||||
<<" dim"<<dimension
|
||||
<<" shift "<<shift
|
||||
<< " rank "<< grid->ThisRank()
|
||||
<<" Coor "<<grid->ThisProcessorCoor()
|
||||
<<" send "<<xsum<<" to "<<xmit_to_rank
|
||||
<<" recv "<<computed_cs<<" from "<<recv_from_rank
|
||||
<<std::endl;
|
||||
GRID_ASSERT(expected_cs == computed_cs);
|
||||
#else
|
||||
acceleratorCopyToDevice(&hrecv_buf[0],&recv_buf[0],bytes);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
FlightRecorder::StepLog("Cshift_SendRecv_complete");
|
||||
|
||||
xbytes+=bytes;
|
||||
grid->Barrier();
|
||||
tcomms+=usecond();
|
||||
FlightRecorder::StepLog("Cshift_barrier_complete");
|
||||
|
||||
tscatter-=usecond();
|
||||
Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
|
||||
tscatter+=usecond();
|
||||
}
|
||||
}
|
||||
if (Cshift_verbose){
|
||||
std::cout << GridLogPerformance << " Cshift copy "<<tcopy/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift gather "<<tgather/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift scatter "<<tscatter/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift comm "<<tcomms/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift BW "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
||||
@@ -318,21 +178,15 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
|
||||
int simd_layout = grid->_simd_layout[dimension];
|
||||
int comm_dim = grid->_processors[dimension] >1 ;
|
||||
|
||||
// std::cout << "Cshift_comms_simd dim "<< dimension << " fd "<<fd<<" rd "<<rd
|
||||
// << " ld "<<ld<<" pd " << pd<<" simd_layout "<<simd_layout
|
||||
// << " comm_dim " << comm_dim << " cbmask " << cbmask <<std::endl;
|
||||
//std::cout << "Cshift_comms_simd dim "<< dimension << " fd "<<fd<<" rd "<<rd
|
||||
// << " ld "<<ld<<" pd " << pd<<" simd_layout "<<simd_layout
|
||||
// << " comm_dim " << comm_dim << " cbmask " << cbmask <<std::endl;
|
||||
|
||||
GRID_ASSERT(comm_dim==1);
|
||||
GRID_ASSERT(simd_layout==2);
|
||||
GRID_ASSERT(shift>=0);
|
||||
GRID_ASSERT(shift<fd);
|
||||
assert(comm_dim==1);
|
||||
assert(simd_layout==2);
|
||||
assert(shift>=0);
|
||||
assert(shift<fd);
|
||||
|
||||
RealD tcopy=0.0;
|
||||
RealD tgather=0.0;
|
||||
RealD tscatter=0.0;
|
||||
RealD tcomms=0.0;
|
||||
uint64_t xbytes=0;
|
||||
|
||||
int permute_type=grid->PermuteType(dimension);
|
||||
|
||||
///////////////////////////////////////////////
|
||||
@@ -341,28 +195,9 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
|
||||
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
||||
// int words = sizeof(vobj)/sizeof(vector_type);
|
||||
|
||||
static std::vector<deviceVector<scalar_object> > send_buf_extract; send_buf_extract.resize(Nsimd);
|
||||
static std::vector<deviceVector<scalar_object> > recv_buf_extract; recv_buf_extract.resize(Nsimd);
|
||||
scalar_object * recv_buf_extract_mpi;
|
||||
scalar_object * send_buf_extract_mpi;
|
||||
std::vector<commVector<scalar_object> > send_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
|
||||
std::vector<commVector<scalar_object> > recv_buf_extract(Nsimd,commVector<scalar_object>(buffer_size) );
|
||||
|
||||
for(int s=0;s<Nsimd;s++){
|
||||
send_buf_extract[s].resize(buffer_size);
|
||||
recv_buf_extract[s].resize(buffer_size);
|
||||
}
|
||||
#ifndef ACCELERATOR_AWARE_MPI
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
buffer_size += (8 + sizeof(vobj) - 1) / sizeof(vobj);
|
||||
#endif
|
||||
|
||||
static hostVector<vobj> hsend_buf; hsend_buf.resize(buffer_size);
|
||||
static hostVector<vobj> hrecv_buf; hrecv_buf.resize(buffer_size);
|
||||
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
buffer_size -= (8 + sizeof(vobj) - 1) / sizeof(vobj);
|
||||
#endif
|
||||
#endif
|
||||
|
||||
int bytes = buffer_size*sizeof(scalar_object);
|
||||
|
||||
ExtractPointerArray<scalar_object> pointers(Nsimd); //
|
||||
@@ -382,9 +217,7 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
|
||||
pointers[i] = &send_buf_extract[i][0];
|
||||
}
|
||||
int sx = (x+sshift)%rd;
|
||||
tgather-=usecond();
|
||||
Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
|
||||
tgather+=usecond();
|
||||
|
||||
for(int i=0;i<Nsimd;i++){
|
||||
|
||||
@@ -404,79 +237,26 @@ template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vo
|
||||
|
||||
if (nbr_ic) nbr_lane|=inner_bit;
|
||||
|
||||
GRID_ASSERT (sx == nbr_ox);
|
||||
assert (sx == nbr_ox);
|
||||
|
||||
if(nbr_proc){
|
||||
grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank);
|
||||
|
||||
tcomms-=usecond();
|
||||
grid->Barrier();
|
||||
|
||||
send_buf_extract_mpi = &send_buf_extract[nbr_lane][0];
|
||||
recv_buf_extract_mpi = &recv_buf_extract[i][0];
|
||||
#ifdef ACCELERATOR_AWARE_MPI
|
||||
grid->SendToRecvFrom((void *)send_buf_extract_mpi,
|
||||
grid->SendToRecvFrom((void *)&send_buf_extract[nbr_lane][0],
|
||||
xmit_to_rank,
|
||||
(void *)recv_buf_extract_mpi,
|
||||
(void *)&recv_buf_extract[i][0],
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
#else
|
||||
// bouncy bouncy
|
||||
acceleratorCopyFromDevice((void *)send_buf_extract_mpi,(void *)&hsend_buf[0],bytes);
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
assert(bytes % 8 == 0);
|
||||
checksum_index++;
|
||||
uint64_t xsum = checksum_gpu((uint64_t*)send_buf_extract_mpi, bytes / 8) ^ (1 + checksum_index);
|
||||
*(uint64_t*)(((char*)&hsend_buf[0]) + bytes) = xsum;
|
||||
bytes += 8;
|
||||
#endif
|
||||
grid->SendToRecvFrom((void *)&hsend_buf[0],
|
||||
xmit_to_rank,
|
||||
(void *)&hrecv_buf[0],
|
||||
recv_from_rank,
|
||||
bytes);
|
||||
#ifdef GRID_CHECKSUM_COMMS
|
||||
bytes -= 8;
|
||||
acceleratorCopyToDevice((void *)&hrecv_buf[0],(void *)recv_buf_extract_mpi,bytes);
|
||||
uint64_t expected_cs = *(uint64_t*)(((char*)&hrecv_buf[0]) + bytes);
|
||||
uint64_t computed_cs = checksum_gpu((uint64_t*)recv_buf_extract_mpi, bytes / 8) ^ (1 + checksum_index);
|
||||
|
||||
std::cout << GridLogComms<< " Cshift_comms_simd: "
|
||||
<<" dim"<<dimension
|
||||
<<" shift "<<shift
|
||||
<< " rank "<< grid->ThisRank()
|
||||
<<" Coor "<<grid->ThisProcessorCoor()
|
||||
<<" send "<<xsum<<" to "<<xmit_to_rank
|
||||
<<" recv "<<computed_cs<<" from "<<recv_from_rank
|
||||
<<std::endl;
|
||||
assert(expected_cs == computed_cs);
|
||||
#else
|
||||
acceleratorCopyToDevice((void *)&hrecv_buf[0],(void *)recv_buf_extract_mpi,bytes);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
xbytes+=bytes;
|
||||
grid->Barrier();
|
||||
tcomms+=usecond();
|
||||
|
||||
rpointers[i] = &recv_buf_extract[i][0];
|
||||
} else {
|
||||
rpointers[i] = &send_buf_extract[nbr_lane][0];
|
||||
}
|
||||
|
||||
}
|
||||
tscatter-=usecond();
|
||||
Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
|
||||
tscatter+=usecond();
|
||||
}
|
||||
if(Cshift_verbose){
|
||||
std::cout << GridLogPerformance << " Cshift (s) copy "<<tcopy/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift (s) gather "<<tgather/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift (s) scatter "<<tscatter/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift (s) comm "<<tcomms/1e3<<" ms"<<std::endl;
|
||||
std::cout << GridLogPerformance << " Cshift BW "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
NAMESPACE_END(Grid);
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
#include <Grid/GridCore.h>
|
||||
NAMESPACE_BEGIN(Grid);
|
||||
std::vector<std::pair<int,int> > Cshift_table;
|
||||
deviceVector<std::pair<int,int> > Cshift_table_device;
|
||||
NAMESPACE_END(Grid);
|
||||
+10531
-13700
File diff suppressed because it is too large.
Load diff
Loaded 100 of 939 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user