mirror of
https://github.com/paboyle/Grid.git
synced 2026-05-15 06:34:31 +01:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Guido Cossu
|
fa5d8702cb | ||
|
Guido Cossu
|
0064685bd7 |
@@ -1,54 +0,0 @@
|
|||||||
name: Bug report
|
|
||||||
description: Report a bug.
|
|
||||||
title: "<insert title>"
|
|
||||||
labels: [bug]
|
|
||||||
|
|
||||||
body:
|
|
||||||
- type: markdown
|
|
||||||
attributes:
|
|
||||||
value: >
|
|
||||||
Thank you for taking the time to file a bug report.
|
|
||||||
Please check that the code is pointing to the HEAD of develop
|
|
||||||
or any commit in master which is tagged with a version number.
|
|
||||||
|
|
||||||
- type: textarea
|
|
||||||
attributes:
|
|
||||||
label: "Describe the issue:"
|
|
||||||
description: >
|
|
||||||
Describe the issue and any previous attempt to solve it.
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
|
|
||||||
- type: textarea
|
|
||||||
attributes:
|
|
||||||
label: "Code example:"
|
|
||||||
description: >
|
|
||||||
If relevant, show how to reproduce the issue using a minimal working
|
|
||||||
example.
|
|
||||||
placeholder: |
|
|
||||||
<< your code here >>
|
|
||||||
render: shell
|
|
||||||
validations:
|
|
||||||
required: false
|
|
||||||
|
|
||||||
- type: textarea
|
|
||||||
attributes:
|
|
||||||
label: "Target platform:"
|
|
||||||
description: >
|
|
||||||
Give a description of the target platform (CPU, network, compiler).
|
|
||||||
Please give the full CPU part description, using for example
|
|
||||||
`cat /proc/cpuinfo | grep 'model name' | uniq` (Linux)
|
|
||||||
or `sysctl machdep.cpu.brand_string` (macOS) and the full output
|
|
||||||
the `--version` option of your compiler.
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
|
|
||||||
- type: textarea
|
|
||||||
attributes:
|
|
||||||
label: "Configure options:"
|
|
||||||
description: >
|
|
||||||
Please give the exact configure command used and attach
|
|
||||||
`config.log`, `grid.config.summary` and the output of `make V=1`.
|
|
||||||
render: shell
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
+21
-11
@@ -1,7 +1,3 @@
|
|||||||
# Doxygen stuff
|
|
||||||
html/*
|
|
||||||
latex/*
|
|
||||||
|
|
||||||
# Compiled Object files #
|
# Compiled Object files #
|
||||||
#########################
|
#########################
|
||||||
*.slo
|
*.slo
|
||||||
@@ -87,12 +83,10 @@ ltmain.sh
|
|||||||
.Trashes
|
.Trashes
|
||||||
ehthumbs.db
|
ehthumbs.db
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
.dirstamp
|
|
||||||
|
|
||||||
# build directory #
|
# build directory #
|
||||||
###################
|
###################
|
||||||
build*/*
|
build*/*
|
||||||
Documentation/_build
|
|
||||||
|
|
||||||
# IDE related files #
|
# IDE related files #
|
||||||
#####################
|
#####################
|
||||||
@@ -103,8 +97,11 @@ build.sh
|
|||||||
|
|
||||||
# Eigen source #
|
# Eigen source #
|
||||||
################
|
################
|
||||||
Grid/Eigen
|
lib/Eigen/*
|
||||||
Eigen/*
|
|
||||||
|
# FFTW source #
|
||||||
|
################
|
||||||
|
lib/fftw/*
|
||||||
|
|
||||||
# libtool macros #
|
# libtool macros #
|
||||||
##################
|
##################
|
||||||
@@ -115,8 +112,21 @@ m4/libtool.m4
|
|||||||
################
|
################
|
||||||
gh-pages/
|
gh-pages/
|
||||||
|
|
||||||
|
# Buck files #
|
||||||
|
##############
|
||||||
|
.buck*
|
||||||
|
buck-out
|
||||||
|
BUCK
|
||||||
|
make-bin-BUCK.sh
|
||||||
|
|
||||||
# generated sources #
|
# generated sources #
|
||||||
#####################
|
#####################
|
||||||
Grid/qcd/spin/gamma-gen/*.h
|
lib/qcd/spin/gamma-gen/*.h
|
||||||
Grid/qcd/spin/gamma-gen/*.cc
|
lib/qcd/spin/gamma-gen/*.cc
|
||||||
Grid/util/Version.h
|
lib/version.h
|
||||||
|
|
||||||
|
# vs code editor files #
|
||||||
|
########################
|
||||||
|
.vscode/
|
||||||
|
.vscode/settings.json
|
||||||
|
settings.json
|
||||||
|
|||||||
+47
@@ -0,0 +1,47 @@
|
|||||||
|
language: cpp
|
||||||
|
|
||||||
|
cache:
|
||||||
|
directories:
|
||||||
|
- clang
|
||||||
|
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- os: osx
|
||||||
|
osx_image: xcode8.3
|
||||||
|
compiler: clang
|
||||||
|
|
||||||
|
before_install:
|
||||||
|
- export GRIDDIR=`pwd`
|
||||||
|
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]] && [ ! -e clang/bin ]; then wget $CLANG_LINK; tar -xf `basename $CLANG_LINK`; mkdir clang; mv clang+*/* clang/; fi
|
||||||
|
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export PATH="${GRIDDIR}/clang/bin:${PATH}"; fi
|
||||||
|
- if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then export LD_LIBRARY_PATH="${GRIDDIR}/clang/lib:${LD_LIBRARY_PATH}"; fi
|
||||||
|
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
|
||||||
|
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install libmpc; fi
|
||||||
|
|
||||||
|
install:
|
||||||
|
- export CC=$CC$VERSION
|
||||||
|
- export CXX=$CXX$VERSION
|
||||||
|
- echo $PATH
|
||||||
|
- which autoconf
|
||||||
|
- autoconf --version
|
||||||
|
- which automake
|
||||||
|
- automake --version
|
||||||
|
- which $CC
|
||||||
|
- $CC --version
|
||||||
|
- which $CXX
|
||||||
|
- $CXX --version
|
||||||
|
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export LDFLAGS='-L/usr/local/lib'; fi
|
||||||
|
|
||||||
|
script:
|
||||||
|
- ./bootstrap.sh
|
||||||
|
- mkdir build
|
||||||
|
- cd build
|
||||||
|
- ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=none
|
||||||
|
- make -j4
|
||||||
|
- ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
|
||||||
|
- echo make clean
|
||||||
|
- ../configure --enable-precision=double --enable-simd=SSE4 --enable-comms=none
|
||||||
|
- make -j4
|
||||||
|
- ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
|
||||||
|
- make check
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
|||||||
|
|
||||||
mpicxx -qmkl=parallel -fsycl BatchBlasBench.cc -o BatchBlasBench -DGRID_SYCL
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
CXX=hipcc
|
|
||||||
MPICXX=mpicxx
|
|
||||||
CXXFLAGS="-fPIC -I{$ROCM_PATH}/include/ -I${MPICH_DIR}/include -L/lib64 -I/opt/cray/pe/mpich/8.1.28/ofi/gnu/12.3/include -DGRID_HIP"
|
|
||||||
LDFLAGS="-L/lib64 -L${MPICH_DIR}/lib -lmpi -L${CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa -lamdhip64 -lhipblas -lrocblas -lmpi_gnu_123"
|
|
||||||
hipcc $CXXFLAGS $LDFLAGS BatchBlasBench.cc -o BatchBlasBench
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
|
|
||||||
mpicxx -qmkl=parallel -fsycl BatchBlasBench.cc -o BatchBlasBench -DGRID_SYCL
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
Version : 0.8.0
|
|
||||||
|
|
||||||
- Clang 3.5 and above, ICPC v16 and above, GCC 6.3 and above recommended
|
|
||||||
- MPI and MPI3 comms optimisations for KNL and OPA finished
|
|
||||||
- Half precision comms
|
|
||||||
|
|||||||
@@ -1,75 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/DisableWarnings.h
|
|
||||||
|
|
||||||
Copyright (C) 2016
|
|
||||||
|
|
||||||
Author: Guido Cossu <guido.cossu@ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#ifndef DISABLE_WARNINGS_H
|
|
||||||
#define DISABLE_WARNINGS_H
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#if defined __GNUC__ && __GNUC__>=6
|
|
||||||
#pragma GCC diagnostic ignored "-Wignored-attributes"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
//disables and intel compiler specific warning (in json.hpp)
|
|
||||||
#ifdef __ICC
|
|
||||||
#pragma warning disable 488
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef __NVCC__
|
|
||||||
//disables nvcc specific warning in json.hpp
|
|
||||||
#pragma clang diagnostic ignored "-Wdeprecated-register"
|
|
||||||
|
|
||||||
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__
|
|
||||||
//disables nvcc specific warning in json.hpp
|
|
||||||
#pragma nv_diag_suppress unsigned_compare_with_zero
|
|
||||||
#pragma nv_diag_suppress cast_to_qualified_type
|
|
||||||
//disables nvcc specific warning in many files
|
|
||||||
#pragma nv_diag_suppress esa_on_defaulted_function_ignored
|
|
||||||
#pragma nv_diag_suppress declared_but_not_referenced
|
|
||||||
#pragma nv_diag_suppress extra_semicolon
|
|
||||||
#else
|
|
||||||
//disables nvcc specific warning in json.hpp
|
|
||||||
#pragma diag_suppress unsigned_compare_with_zero
|
|
||||||
#pragma diag_suppress cast_to_qualified_type
|
|
||||||
#pragma diag_suppress declared_but_not_referenced
|
|
||||||
//disables nvcc specific warning in many files
|
|
||||||
#pragma diag_suppress esa_on_defaulted_function_ignored
|
|
||||||
#pragma diag_suppress extra_semicolon
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// Disable vectorisation in Eigen on the Power8/9 and PowerPC
|
|
||||||
#ifdef __ALTIVEC__
|
|
||||||
#define EIGEN_DONT_VECTORIZE
|
|
||||||
#endif
|
|
||||||
#ifdef __VSX__
|
|
||||||
#define EIGEN_DONT_VECTORIZE
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
#ifndef GRID_STD_H
|
|
||||||
#define GRID_STD_H
|
|
||||||
|
|
||||||
///////////////////
|
|
||||||
// Grid config
|
|
||||||
///////////////////
|
|
||||||
#include "Config.h"
|
|
||||||
|
|
||||||
///////////////////
|
|
||||||
// Std C++ dependencies
|
|
||||||
///////////////////
|
|
||||||
#define _NBACKTRACE (256)
|
|
||||||
extern void * Grid_backtrace_buffer[_NBACKTRACE];
|
|
||||||
|
|
||||||
#include <cassert>
|
|
||||||
#include <complex>
|
|
||||||
#include <memory>
|
|
||||||
#include <vector>
|
|
||||||
#include <array>
|
|
||||||
#include <string>
|
|
||||||
#include <iostream>
|
|
||||||
#include <iomanip>
|
|
||||||
#include <random>
|
|
||||||
#include <functional>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <strings.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <signal.h>
|
|
||||||
#include <ctime>
|
|
||||||
#include <sys/time.h>
|
|
||||||
#include <chrono>
|
|
||||||
#include <zlib.h>
|
|
||||||
#ifdef HAVE_EXECINFO_H
|
|
||||||
#include <execinfo.h>
|
|
||||||
#endif
|
|
||||||
|
|
||||||
void GridAbort(void);
|
|
||||||
|
|
||||||
#define ASSLOG(A) ::write(STDERR_FILENO,A,::strlen(A));
|
|
||||||
#ifdef HAVE_EXECINFO_H
|
|
||||||
#define GRID_ASSERT(b) if(!(b)) { \
|
|
||||||
fflush(stdout); \
|
|
||||||
ASSLOG(" GRID_ASSERT failure: "); \
|
|
||||||
ASSLOG(__FILE__); \
|
|
||||||
ASSLOG(" : "); \
|
|
||||||
ASSLOG(#b); \
|
|
||||||
ASSLOG(" : "); \
|
|
||||||
int symbols = backtrace(Grid_backtrace_buffer,_NBACKTRACE); \
|
|
||||||
backtrace_symbols_fd(Grid_backtrace_buffer,symbols,STDERR_FILENO); \
|
|
||||||
GridAbort(); \
|
|
||||||
};
|
|
||||||
#else
|
|
||||||
#define GRID_ASSERT(b) if(!(b)) { \
|
|
||||||
ASSLOG(" GRID_ASSERT failure: "); \
|
|
||||||
ASSLOG(__FILE__); \
|
|
||||||
ASSLOG(" : "); \
|
|
||||||
ASSLOG(#b); \
|
|
||||||
ASSLOG(" : "); \
|
|
||||||
GridAbort(); \
|
|
||||||
};
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
#ifdef TOFU
|
|
||||||
#undef GRID_COMMS_THREADS
|
|
||||||
#endif
|
|
||||||
#endif /* GRID_STD_H */
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
#include <Grid/GridCore.h>
|
|
||||||
#pragma once
|
|
||||||
// Force Eigen to use MKL if Grid has been configured with --enable-mkl
|
|
||||||
#ifdef USE_MKL
|
|
||||||
#define EIGEN_USE_MKL_ALL
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
#if defined __GNUC__
|
|
||||||
#pragma GCC diagnostic push
|
|
||||||
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* NVCC save and restore compile environment*/
|
|
||||||
#ifdef __NVCC__
|
|
||||||
#pragma push
|
|
||||||
#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__
|
|
||||||
#pragma nv_diag_suppress code_is_unreachable
|
|
||||||
#else
|
|
||||||
#pragma diag_suppress code_is_unreachable
|
|
||||||
#endif
|
|
||||||
#pragma push_macro("__CUDA_ARCH__")
|
|
||||||
#pragma push_macro("__NVCC__")
|
|
||||||
#pragma push_macro("__CUDACC__")
|
|
||||||
#undef __CUDA_ARCH__
|
|
||||||
#undef __NVCC__
|
|
||||||
#undef __CUDACC__
|
|
||||||
#define __NVCC__REDEFINE__
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* SYCL save and restore compile environment*/
|
|
||||||
#ifdef GRID_SYCL
|
|
||||||
#pragma push
|
|
||||||
#pragma push_macro("__SYCL_DEVICE_ONLY__")
|
|
||||||
#undef __SYCL_DEVICE_ONLY__
|
|
||||||
#define EIGEN_DONT_VECTORIZE
|
|
||||||
#undef EIGEN_USE_SYCL
|
|
||||||
#define __SYCL__REDEFINE__
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* HIP save and restore compile environment*/
|
|
||||||
#ifdef GRID_HIP
|
|
||||||
#pragma push
|
|
||||||
#pragma push_macro("__HIP_DEVICE_COMPILE__")
|
|
||||||
#endif
|
|
||||||
#define EIGEN_NO_HIP
|
|
||||||
|
|
||||||
#include <Grid/Eigen/Dense>
|
|
||||||
#include <Grid/Eigen/unsupported/CXX11/Tensor>
|
|
||||||
|
|
||||||
/* NVCC restore */
|
|
||||||
#ifdef __NVCC__REDEFINE__
|
|
||||||
#pragma pop_macro("__CUDACC__")
|
|
||||||
#pragma pop_macro("__NVCC__")
|
|
||||||
#pragma pop_macro("__CUDA_ARCH__")
|
|
||||||
#pragma pop
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*SYCL restore*/
|
|
||||||
#ifdef __SYCL__REDEFINE__
|
|
||||||
#pragma pop_macro("__SYCL_DEVICE_ONLY__")
|
|
||||||
#pragma pop
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/*HIP restore*/
|
|
||||||
#ifdef __HIP__REDEFINE__
|
|
||||||
#pragma pop_macro("__HIP_DEVICE_COMPILE__")
|
|
||||||
#pragma pop
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#if defined __GNUC__
|
|
||||||
#pragma GCC diagnostic pop
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
#include <Grid/Grid_Eigen_Dense.h>
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
extra_sources=
|
|
||||||
extra_headers=
|
|
||||||
|
|
||||||
if BUILD_COMMS_MPI3
|
|
||||||
extra_sources+=communicator/Communicator_mpi3.cc
|
|
||||||
extra_sources+=communicator/Communicator_base.cc
|
|
||||||
extra_sources+=communicator/SharedMemoryMPI.cc
|
|
||||||
extra_sources+=communicator/SharedMemory.cc
|
|
||||||
endif
|
|
||||||
|
|
||||||
if BUILD_COMMS_NONE
|
|
||||||
extra_sources+=communicator/Communicator_none.cc
|
|
||||||
extra_sources+=communicator/Communicator_base.cc
|
|
||||||
extra_sources+=communicator/SharedMemoryNone.cc
|
|
||||||
extra_sources+=communicator/SharedMemory.cc
|
|
||||||
endif
|
|
||||||
|
|
||||||
if BUILD_HDF5
|
|
||||||
extra_sources+=serialisation/Hdf5IO.cc
|
|
||||||
extra_headers+=serialisation/Hdf5IO.h
|
|
||||||
extra_headers+=serialisation/Hdf5Type.h
|
|
||||||
endif
|
|
||||||
|
|
||||||
|
|
||||||
all: version-cache Version.h
|
|
||||||
|
|
||||||
version-cache:
|
|
||||||
@if [ `git status --porcelain | grep -v '??' | wc -l` -gt 0 ]; then\
|
|
||||||
a="uncommited changes";\
|
|
||||||
else\
|
|
||||||
a="clean";\
|
|
||||||
fi;\
|
|
||||||
echo "`git log -n 1 --format=format:"#define GITHASH \\"%H:%d $$a\\"%n" HEAD`" > vertmp;\
|
|
||||||
if [ -e version-cache ]; then\
|
|
||||||
d=`diff vertmp version-cache`;\
|
|
||||||
if [ "$${d}" != "" ]; then\
|
|
||||||
mv vertmp version-cache;\
|
|
||||||
rm -f Version.h;\
|
|
||||||
fi;\
|
|
||||||
else\
|
|
||||||
mv vertmp version-cache;\
|
|
||||||
rm -f Version.h;\
|
|
||||||
fi;\
|
|
||||||
rm -f vertmp
|
|
||||||
|
|
||||||
Version.h: version-cache
|
|
||||||
cp version-cache Version.h
|
|
||||||
|
|
||||||
.PHONY: version-cache
|
|
||||||
|
|
||||||
#
|
|
||||||
# Libraries
|
|
||||||
#
|
|
||||||
include Make.inc
|
|
||||||
include Eigen.inc
|
|
||||||
|
|
||||||
if BUILD_FERMION_INSTANTIATIONS
|
|
||||||
extra_sources+=$(WILS_FERMION_FILES)
|
|
||||||
extra_sources+=$(STAG_FERMION_FILES)
|
|
||||||
if BUILD_ZMOBIUS
|
|
||||||
extra_sources+=$(ZWILS_FERMION_FILES)
|
|
||||||
endif
|
|
||||||
if BUILD_GPARITY
|
|
||||||
extra_sources+=$(GP_FERMION_FILES)
|
|
||||||
endif
|
|
||||||
if BUILD_FERMION_REPS
|
|
||||||
extra_sources+=$(ADJ_FERMION_FILES)
|
|
||||||
extra_sources+=$(TWOIND_FERMION_FILES)
|
|
||||||
endif
|
|
||||||
if BUILD_SP
|
|
||||||
extra_sources+=$(SP_FERMION_FILES)
|
|
||||||
if BUILD_FERMION_REPS
|
|
||||||
extra_sources+=$(SP_TWOIND_FERMION_FILES)
|
|
||||||
endif
|
|
||||||
endif
|
|
||||||
endif
|
|
||||||
|
|
||||||
lib_LIBRARIES = libGrid.a
|
|
||||||
|
|
||||||
CCFILES += $(extra_sources)
|
|
||||||
HFILES += $(extra_headers) Config.h Version.h
|
|
||||||
|
|
||||||
libGrid_a_SOURCES = $(CCFILES)
|
|
||||||
libGrid_adir = $(includedir)/Grid
|
|
||||||
nobase_dist_pkginclude_HEADERS = $(HFILES) $(eigen_files) $(eigen_unsupp_files)
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/Namespace.h
|
|
||||||
|
|
||||||
Copyright (C) 2016
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <type_traits>
|
|
||||||
#include <exception>
|
|
||||||
#include <cassert>
|
|
||||||
|
|
||||||
#define NAMESPACE_BEGIN(A) namespace A {
|
|
||||||
#define NAMESPACE_END(A) }
|
|
||||||
#define GRID_NAMESPACE_BEGIN NAMESPACE_BEGIN(Grid)
|
|
||||||
#define GRID_NAMESPACE_END NAMESPACE_END(Grid)
|
|
||||||
#define NAMESPACE_CHECK(x) struct namespaceTEST##x {}; static_assert(std::is_same<namespaceTEST##x, ::namespaceTEST##x>::value,"Not in :: at" );
|
|
||||||
|
|
||||||
#define EXCEPTION_CHECK_BEGIN(A) try {
|
|
||||||
#define EXCEPTION_CHECK_END(A) } catch ( std::exception e ) { BACKTRACEFP(stderr); std::cerr << __PRETTY_FUNCTION__ << " : " <<__LINE__<< " Caught exception "<<e.what()<<std::endl; throw; }
|
|
||||||
|
|
||||||
@@ -1,498 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/Cshift.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef _GRID_FFT_H_
|
|
||||||
#define _GRID_FFT_H_
|
|
||||||
|
|
||||||
#ifdef GRID_CUDA
|
|
||||||
#include <cufft.h>
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef GRID_HIP
|
|
||||||
#include <hipfft/hipfft.h>
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#if !defined(GRID_CUDA) && !defined(GRID_HIP)
|
|
||||||
#ifdef HAVE_FFTW
|
|
||||||
#if defined(USE_MKL) || defined(GRID_SYCL)
|
|
||||||
#include <fftw/fftw3.h>
|
|
||||||
#else
|
|
||||||
#include <fftw3.h>
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#ifndef FFTW_FORWARD
|
|
||||||
#define FFTW_FORWARD (-1)
|
|
||||||
#define FFTW_BACKWARD (+1)
|
|
||||||
#define FFTW_ESTIMATE (0)
|
|
||||||
#endif
|
|
||||||
|
|
||||||
template<class scalar> struct FFTW {
|
|
||||||
};
|
|
||||||
|
|
||||||
#ifdef GRID_HIP
|
|
||||||
template<> struct FFTW<ComplexD> {
|
|
||||||
public:
|
|
||||||
static const int forward=FFTW_FORWARD;
|
|
||||||
static const int backward=FFTW_BACKWARD;
|
|
||||||
typedef hipfftDoubleComplex FFTW_scalar;
|
|
||||||
typedef hipfftHandle FFTW_plan;
|
|
||||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
|
||||||
FFTW_scalar *in, int *inembed,
|
|
||||||
int istride, int idist,
|
|
||||||
FFTW_scalar *out, int *onembed,
|
|
||||||
int ostride, int odist,
|
|
||||||
int sign, unsigned flags) {
|
|
||||||
FFTW_plan p;
|
|
||||||
auto rv = hipfftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,HIPFFT_Z2Z,howmany);
|
|
||||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
|
||||||
hipfftResult rv;
|
|
||||||
if ( sign == forward ) rv =hipfftExecZ2Z(p,in,out,HIPFFT_FORWARD);
|
|
||||||
else rv =hipfftExecZ2Z(p,in,out,HIPFFT_BACKWARD);
|
|
||||||
accelerator_barrier();
|
|
||||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
|
||||||
}
|
|
||||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
|
||||||
hipfftDestroy(p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<> struct FFTW<ComplexF> {
|
|
||||||
public:
|
|
||||||
static const int forward=FFTW_FORWARD;
|
|
||||||
static const int backward=FFTW_BACKWARD;
|
|
||||||
typedef hipfftComplex FFTW_scalar;
|
|
||||||
typedef hipfftHandle FFTW_plan;
|
|
||||||
|
|
||||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
|
||||||
FFTW_scalar *in, int *inembed,
|
|
||||||
int istride, int idist,
|
|
||||||
FFTW_scalar *out, int *onembed,
|
|
||||||
int ostride, int odist,
|
|
||||||
int sign, unsigned flags) {
|
|
||||||
FFTW_plan p;
|
|
||||||
auto rv = hipfftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,HIPFFT_C2C,howmany);
|
|
||||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
|
||||||
hipfftResult rv;
|
|
||||||
if ( sign == forward ) rv =hipfftExecC2C(p,in,out,HIPFFT_FORWARD);
|
|
||||||
else rv =hipfftExecC2C(p,in,out,HIPFFT_BACKWARD);
|
|
||||||
accelerator_barrier();
|
|
||||||
GRID_ASSERT(rv==HIPFFT_SUCCESS);
|
|
||||||
}
|
|
||||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
|
||||||
hipfftDestroy(p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef GRID_CUDA
|
|
||||||
template<> struct FFTW<ComplexD> {
|
|
||||||
public:
|
|
||||||
static const int forward=FFTW_FORWARD;
|
|
||||||
static const int backward=FFTW_BACKWARD;
|
|
||||||
typedef cufftDoubleComplex FFTW_scalar;
|
|
||||||
typedef cufftHandle FFTW_plan;
|
|
||||||
|
|
||||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
|
||||||
FFTW_scalar *in, int *inembed,
|
|
||||||
int istride, int idist,
|
|
||||||
FFTW_scalar *out, int *onembed,
|
|
||||||
int ostride, int odist,
|
|
||||||
int sign, unsigned flags) {
|
|
||||||
FFTW_plan p;
|
|
||||||
cufftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,CUFFT_Z2Z,howmany);
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
|
||||||
if ( sign == forward ) cufftExecZ2Z(p,in,out,CUFFT_FORWARD);
|
|
||||||
else cufftExecZ2Z(p,in,out,CUFFT_INVERSE);
|
|
||||||
accelerator_barrier();
|
|
||||||
}
|
|
||||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
|
||||||
cufftDestroy(p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<> struct FFTW<ComplexF> {
|
|
||||||
public:
|
|
||||||
static const int forward=FFTW_FORWARD;
|
|
||||||
static const int backward=FFTW_BACKWARD;
|
|
||||||
typedef cufftComplex FFTW_scalar;
|
|
||||||
typedef cufftHandle FFTW_plan;
|
|
||||||
|
|
||||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
|
||||||
FFTW_scalar *in, int *inembed,
|
|
||||||
int istride, int idist,
|
|
||||||
FFTW_scalar *out, int *onembed,
|
|
||||||
int ostride, int odist,
|
|
||||||
int sign, unsigned flags) {
|
|
||||||
FFTW_plan p;
|
|
||||||
cufftPlanMany(&p,rank,n,n,istride,idist,n,ostride,odist,CUFFT_C2C,howmany);
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
|
||||||
if ( sign == forward ) cufftExecC2C(p,in,out,CUFFT_FORWARD);
|
|
||||||
else cufftExecC2C(p,in,out,CUFFT_INVERSE);
|
|
||||||
accelerator_barrier();
|
|
||||||
}
|
|
||||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
|
||||||
cufftDestroy(p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#if !defined(GRID_CUDA) && !defined(GRID_HIP)
|
|
||||||
#ifdef HAVE_FFTW
|
|
||||||
template<> struct FFTW<ComplexD> {
|
|
||||||
public:
|
|
||||||
typedef fftw_complex FFTW_scalar;
|
|
||||||
typedef fftw_plan FFTW_plan;
|
|
||||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
|
||||||
FFTW_scalar *in, int *inembed,
|
|
||||||
int istride, int idist,
|
|
||||||
FFTW_scalar *out, int *onembed,
|
|
||||||
int ostride, int odist,
|
|
||||||
int sign, unsigned flags) {
|
|
||||||
return ::fftw_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
|
||||||
}
|
|
||||||
|
|
||||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
|
||||||
::fftw_execute_dft(p,in,out);
|
|
||||||
}
|
|
||||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
|
||||||
::fftw_destroy_plan(p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<> struct FFTW<ComplexF> {
|
|
||||||
public:
|
|
||||||
typedef fftwf_complex FFTW_scalar;
|
|
||||||
typedef fftwf_plan FFTW_plan;
|
|
||||||
static FFTW_plan fftw_plan_many_dft(int rank, int *n,int howmany,
|
|
||||||
FFTW_scalar *in, int *inembed,
|
|
||||||
int istride, int idist,
|
|
||||||
FFTW_scalar *out, int *onembed,
|
|
||||||
int ostride, int odist,
|
|
||||||
int sign, unsigned flags) {
|
|
||||||
return ::fftwf_plan_many_dft(rank,n,howmany,in,inembed,istride,idist,out,onembed,ostride,odist,sign,flags);
|
|
||||||
}
|
|
||||||
|
|
||||||
inline static void fftw_execute_dft(const FFTW_plan p,FFTW_scalar *in,FFTW_scalar *out, int sign) {
|
|
||||||
::fftwf_execute_dft(p,in,out);
|
|
||||||
}
|
|
||||||
inline static void fftw_destroy_plan(const FFTW_plan p) {
|
|
||||||
::fftwf_destroy_plan(p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
class FFT {
|
|
||||||
private:
|
|
||||||
|
|
||||||
double flops;
|
|
||||||
double flops_call;
|
|
||||||
uint64_t usec;
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
static const int forward=FFTW_FORWARD;
|
|
||||||
static const int backward=FFTW_BACKWARD;
|
|
||||||
|
|
||||||
double Flops(void) {return flops;}
|
|
||||||
double MFlops(void) {return flops/usec;}
|
|
||||||
double USec(void) {return (double)usec;}
|
|
||||||
|
|
||||||
FFT ( GridCartesian * grid )
|
|
||||||
{
|
|
||||||
flops=0;
|
|
||||||
usec =0;
|
|
||||||
};
|
|
||||||
|
|
||||||
~FFT ( void) {
|
|
||||||
// delete sgrid;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
void FFT_dim_mask(Lattice<vobj> &result,const Lattice<vobj> &source,Coordinate mask,int sign){
|
|
||||||
|
|
||||||
// vgrid=result.Grid();
|
|
||||||
// conformable(result.Grid(),vgrid);
|
|
||||||
// conformable(source.Grid(),vgrid);
|
|
||||||
const int Ndim = source.Grid()->Nd();
|
|
||||||
Lattice<vobj> tmp = source;
|
|
||||||
for(int d=0;d<Ndim;d++){
|
|
||||||
if( mask[d] ) {
|
|
||||||
FFT_dim(result,tmp,d,sign);
|
|
||||||
tmp=result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
void FFT_all_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int sign){
|
|
||||||
const int Ndim = source.Grid()->Nd();
|
|
||||||
Coordinate mask(Ndim,1);
|
|
||||||
FFT_dim_mask(result,source,mask,sign);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
void FFT_dim(Lattice<vobj> &result,const Lattice<vobj> &source,int dim, int sign){
|
|
||||||
const int Ndim = source.Grid()->Nd();
|
|
||||||
GridBase *grid = source.Grid();
|
|
||||||
conformable(result.Grid(),source.Grid());
|
|
||||||
|
|
||||||
int L = grid->_ldimensions[dim];
|
|
||||||
int G = grid->_fdimensions[dim];
|
|
||||||
|
|
||||||
Coordinate layout(Ndim,1);
|
|
||||||
|
|
||||||
// Construct pencils
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::scalar_type scalar;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
//std::cout << "CPU view" << std::endl;
|
|
||||||
|
|
||||||
typedef typename FFTW<scalar>::FFTW_scalar FFTW_scalar;
|
|
||||||
typedef typename FFTW<scalar>::FFTW_plan FFTW_plan;
|
|
||||||
|
|
||||||
int Ncomp = sizeof(sobj)/sizeof(scalar);
|
|
||||||
int64_t Nlow = 1;
|
|
||||||
int64_t Nhigh = 1;
|
|
||||||
|
|
||||||
for(int d=0;d<dim;d++){
|
|
||||||
Nlow*=grid->_ldimensions[d];
|
|
||||||
}
|
|
||||||
for(int d=dim+1;d<Ndim;d++){
|
|
||||||
Nhigh*=grid->_ldimensions[d];
|
|
||||||
}
|
|
||||||
int64_t Nperp=Nlow*Nhigh;
|
|
||||||
|
|
||||||
deviceVector<scalar> pgbuf; // Layout is [perp][component][dim]
|
|
||||||
pgbuf.resize(Nperp*Ncomp*G);
|
|
||||||
scalar *pgbuf_v = &pgbuf[0];
|
|
||||||
|
|
||||||
int rank = 1; /* 1d transforms */
|
|
||||||
int n[] = {G}; /* 1d transforms of length G */
|
|
||||||
int howmany = Ncomp * Nperp;
|
|
||||||
int odist,idist,istride,ostride;
|
|
||||||
idist = odist = G; /* Distance between consecutive FT's */
|
|
||||||
istride = ostride = 1; /* Distance between two elements in the same FT */
|
|
||||||
int *inembed = n, *onembed = n;
|
|
||||||
|
|
||||||
scalar div;
|
|
||||||
if ( sign == backward ) div = 1.0/G;
|
|
||||||
else if ( sign == forward ) div = 1.0;
|
|
||||||
else GRID_ASSERT(0);
|
|
||||||
|
|
||||||
double t_pencil=0;
|
|
||||||
double t_fft =0;
|
|
||||||
double t_total =-usecond();
|
|
||||||
// std::cout << GridLogPerformance<<"Making FFTW plan" << std::endl;
|
|
||||||
/*
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
FFTW_plan p;
|
|
||||||
{
|
|
||||||
FFTW_scalar *in = (FFTW_scalar *)&pgbuf_v[0];
|
|
||||||
FFTW_scalar *out= (FFTW_scalar *)&pgbuf_v[0];
|
|
||||||
p = FFTW<scalar>::fftw_plan_many_dft(rank,n,howmany,
|
|
||||||
in,inembed,
|
|
||||||
istride,idist,
|
|
||||||
out,onembed,
|
|
||||||
ostride, odist,
|
|
||||||
sign,FFTW_ESTIMATE);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Barrel shift and collect global pencil
|
|
||||||
// std::cout << GridLogPerformance<<"Making pencil" << std::endl;
|
|
||||||
Coordinate lcoor(Ndim), gcoor(Ndim);
|
|
||||||
double t_copy=0;
|
|
||||||
double t_shift=0;
|
|
||||||
t_pencil = -usecond();
|
|
||||||
result = source;
|
|
||||||
int pc = grid->_processor_coor[dim];
|
|
||||||
|
|
||||||
const Coordinate ldims = grid->_ldimensions;
|
|
||||||
const Coordinate rdims = grid->_rdimensions;
|
|
||||||
const Coordinate sdims = grid->_simd_layout;
|
|
||||||
|
|
||||||
Coordinate processors = grid->_processors;
|
|
||||||
Coordinate pgdims(Ndim);
|
|
||||||
pgdims[0] = G;
|
|
||||||
for(int d=0, dd=1;d<Ndim;d++){
|
|
||||||
if ( d!=dim ) pgdims[dd++] = ldims[d];
|
|
||||||
}
|
|
||||||
int64_t pgvol=1;
|
|
||||||
for(int d=0;d<Ndim;d++) pgvol*=pgdims[d];
|
|
||||||
|
|
||||||
const int Nsimd = vobj::Nsimd();
|
|
||||||
for(int p=0;p<processors[dim];p++) {
|
|
||||||
t_copy-=usecond();
|
|
||||||
autoView(r_v,result,AcceleratorRead);
|
|
||||||
accelerator_for(idx, grid->oSites(), vobj::Nsimd(), {
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<Nsimd;lane++) {
|
|
||||||
#endif
|
|
||||||
Coordinate icoor;
|
|
||||||
Coordinate ocoor;
|
|
||||||
Coordinate pgcoor;
|
|
||||||
|
|
||||||
Lexicographic::CoorFromIndex(icoor,lane,sdims);
|
|
||||||
Lexicographic::CoorFromIndex(ocoor,idx,rdims);
|
|
||||||
|
|
||||||
pgcoor[0] = ocoor[dim] + icoor[dim]*rdims[dim] + ((pc+p)%processors[dim])*L;
|
|
||||||
for(int d=0,dd=1;d<Ndim;d++){
|
|
||||||
if ( d!=dim ) {
|
|
||||||
pgcoor[dd] = ocoor[d] + icoor[d]*rdims[d];
|
|
||||||
dd++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Map coordinates in lattice layout to FFTW index
|
|
||||||
int64_t pgidx;
|
|
||||||
Lexicographic::IndexFromCoor(pgcoor,pgidx,pgdims);
|
|
||||||
|
|
||||||
vector_type *from = (vector_type *)&r_v[idx];
|
|
||||||
scalar_type stmp;
|
|
||||||
for(int w=0;w<Ncomp;w++){
|
|
||||||
int64_t pg_idx = pgidx + w*pgvol;
|
|
||||||
stmp = getlane(from[w], lane);
|
|
||||||
pgbuf_v[pg_idx] = stmp;
|
|
||||||
}
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
|
|
||||||
t_copy+=usecond();
|
|
||||||
if (p != processors[dim] - 1) {
|
|
||||||
Lattice<vobj> temp(grid);
|
|
||||||
t_shift-=usecond();
|
|
||||||
temp = Cshift(result,dim,L); result = temp;
|
|
||||||
t_shift+=usecond();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t_pencil += usecond();
|
|
||||||
|
|
||||||
FFTW_scalar *in = (FFTW_scalar *)pgbuf_v;
|
|
||||||
FFTW_scalar *out= (FFTW_scalar *)pgbuf_v;
|
|
||||||
t_fft = -usecond();
|
|
||||||
FFTW<scalar>::fftw_execute_dft(p,in,out,sign);
|
|
||||||
t_fft += usecond();
|
|
||||||
|
|
||||||
// performance counting
|
|
||||||
flops_call = 5.0*howmany*G*log2(G);
|
|
||||||
usec = t_fft;
|
|
||||||
flops= flops_call;
|
|
||||||
|
|
||||||
result = Zero();
|
|
||||||
|
|
||||||
double t_insert = -usecond();
|
|
||||||
{
|
|
||||||
autoView(r_v,result,AcceleratorWrite);
|
|
||||||
accelerator_for(idx,grid->oSites(),Nsimd,{
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<Nsimd;lane++) {
|
|
||||||
#endif
|
|
||||||
Coordinate icoor(Ndim);
|
|
||||||
Coordinate ocoor(Ndim);
|
|
||||||
Coordinate pgcoor(Ndim);
|
|
||||||
|
|
||||||
Lexicographic::CoorFromIndex(icoor,lane,sdims);
|
|
||||||
Lexicographic::CoorFromIndex(ocoor,idx,rdims);
|
|
||||||
|
|
||||||
pgcoor[0] = ocoor[dim] + icoor[dim]*rdims[dim] + pc*L;
|
|
||||||
for(int d=0,dd=1;d<Ndim;d++){
|
|
||||||
if ( d!=dim ) {
|
|
||||||
pgcoor[dd] = ocoor[d] + icoor[d]*rdims[d];
|
|
||||||
dd++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Map coordinates in lattice layout to FFTW index
|
|
||||||
int64_t pgidx;
|
|
||||||
Lexicographic::IndexFromCoor(pgcoor,pgidx,pgdims);
|
|
||||||
|
|
||||||
vector_type *to = (vector_type *)&r_v[idx];
|
|
||||||
scalar_type stmp;
|
|
||||||
for(int w=0;w<Ncomp;w++){
|
|
||||||
int64_t pg_idx = pgidx + w*pgvol;
|
|
||||||
stmp = pgbuf_v[pg_idx];
|
|
||||||
putlane(to[w], stmp, lane);
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
result = result*div;
|
|
||||||
|
|
||||||
t_insert +=usecond();
|
|
||||||
|
|
||||||
// destroying plan
|
|
||||||
FFTW<scalar>::fftw_destroy_plan(p);
|
|
||||||
|
|
||||||
t_total +=usecond();
|
|
||||||
|
|
||||||
std::cout <<GridLogPerformance<< " FFT took "<<t_total/1.0e6 <<" s" << std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< " FFT pencil "<<t_pencil/1.0e6 <<" s" << std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< " of which copy "<<t_copy/1.0e6 <<" s" << std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< " of which shift"<<t_shift/1.0e6 <<" s" << std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< " FFT kernels "<<t_fft/1.0e6 <<" s" << std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< " FFT insert "<<t_insert/1.0e6 <<" s" << std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,743 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/LinearOperator.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// LinearOperators Take a something and return a something.
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
//
|
|
||||||
// Hopefully linearity is satisfied and the AdjOp is indeed the Hermitian Conjugateugate (transpose if real):
|
|
||||||
//SBase
|
|
||||||
// i) F(a x + b y) = aF(x) + b F(y).
|
|
||||||
// ii) <x|Op|y> = <y|AdjOp|x>^\ast
|
|
||||||
//
|
|
||||||
// Would be fun to have a test linearity & Herm Conj function!
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class LinearOperatorBase {
|
|
||||||
public:
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
virtual void OpDiag (const Field &in, Field &out) = 0; // Abstract base
|
|
||||||
virtual void OpDir (const Field &in, Field &out,int dir,int disp) = 0; // Abstract base
|
|
||||||
virtual void OpDirAll (const Field &in, std::vector<Field> &out) = 0; // Abstract base
|
|
||||||
|
|
||||||
virtual void Op (const Field &in, Field &out) = 0; // Abstract base
|
|
||||||
virtual void AdjOp (const Field &in, Field &out) = 0; // Abstract base
|
|
||||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2)=0;
|
|
||||||
virtual void HermOp(const Field &in, Field &out)=0;
|
|
||||||
virtual ~LinearOperatorBase(){};
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// By sharing the class for Sparse Matrix across multiple operator wrappers, we can share code
|
|
||||||
// between RB and non-RB variants. Sparse matrix is like the fermion action def, and then
|
|
||||||
// the wrappers implement the specialisation of "Op" and "AdjOp" to the cases minimising
|
|
||||||
// replication of code.
|
|
||||||
//
|
|
||||||
// I'm not entirely happy with implementation; to share the Schur code between herm and non-herm
|
|
||||||
// while still having a "OpAndNorm" in the abstract base I had to implement it in both cases
|
|
||||||
// with an GRID_ASSERT trap in the non-herm. This isn't right; there must be a better C++ way to
|
|
||||||
// do it, but I fear it required multiple inheritance and mixed in abstract base classes
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
// Construct herm op from non-herm matrix
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class MdagMLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
Matrix &_Mat;
|
|
||||||
public:
|
|
||||||
MdagMLinearOperator(Matrix &Mat): _Mat(Mat){};
|
|
||||||
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
_Mat.Mdiag(in,out);
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
_Mat.Mdir(in,out,dir,disp);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
_Mat.MdirAll(in,out);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
_Mat.Mdag(in,out);
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
_Mat.MdagM(in,out);
|
|
||||||
ComplexD dot = innerProduct(in,out);
|
|
||||||
n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
_Mat.MdagM(in,out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class MMdagLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
Matrix &_Mat;
|
|
||||||
public:
|
|
||||||
MMdagLinearOperator(Matrix &Mat): _Mat(Mat){};
|
|
||||||
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
_Mat.Mdiag(in,out);
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
_Mat.Mdir(in,out,dir,disp);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
_Mat.MdirAll(in,out);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
_Mat.Mdag(in,out);
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
_Mat.MMdag(in,out);
|
|
||||||
ComplexD dot = innerProduct(in,out);
|
|
||||||
n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
_Mat.MMdag(in,out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
// Construct herm op and shift it for mgrid smoother
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class ShiftedMdagMLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
Matrix &_Mat;
|
|
||||||
RealD _shift;
|
|
||||||
public:
|
|
||||||
ShiftedMdagMLinearOperator(Matrix &Mat,RealD shift): _Mat(Mat), _shift(shift){};
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
_Mat.Mdiag(in,out);
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
_Mat.Mdir(in,out,dir,disp);
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
_Mat.Mdag(in,out);
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
HermOp(in,out);
|
|
||||||
ComplexD dot = innerProduct(in,out);
|
|
||||||
n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
_Mat.MdagM(in,out);
|
|
||||||
out = out + _shift*in;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
// Create a shifted HermOp
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field>
|
|
||||||
class ShiftedHermOpLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
LinearOperatorBase<Field> &_Mat;
|
|
||||||
RealD _shift;
|
|
||||||
public:
|
|
||||||
ShiftedHermOpLinearOperator(LinearOperatorBase<Field> &Mat,RealD shift): _Mat(Mat), _shift(shift){};
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
HermOp(in,out);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
HermOp(in,out);
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
HermOp(in,out);
|
|
||||||
ComplexD dot = innerProduct(in,out);
|
|
||||||
n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
_Mat.HermOp(in,out);
|
|
||||||
out = out + _shift*in;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
// Wrap an already herm matrix
|
|
||||||
////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class HermitianLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
Matrix &_Mat;
|
|
||||||
public:
|
|
||||||
HermitianLinearOperator(Matrix &Mat): _Mat(Mat){};
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
_Mat.Mdiag(in,out);
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
_Mat.Mdir(in,out,dir,disp);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
_Mat.MdirAll(in,out);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
HermOp(in,out);
|
|
||||||
ComplexD dot= innerProduct(in,out); n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class NonHermitianLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
Matrix &_Mat;
|
|
||||||
public:
|
|
||||||
NonHermitianLinearOperator(Matrix &Mat): _Mat(Mat){};
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
_Mat.Mdiag(in,out);
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
_Mat.Mdir(in,out,dir,disp);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
_Mat.MdirAll(in,out);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
_Mat.Mdag(in,out);
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class ShiftedNonHermitianLinearOperator : public LinearOperatorBase<Field> {
|
|
||||||
Matrix &_Mat;
|
|
||||||
RealD shift;
|
|
||||||
public:
|
|
||||||
ShiftedNonHermitianLinearOperator(Matrix &Mat,RealD shft): _Mat(Mat),shift(shft){};
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
_Mat.Mdiag(in,out);
|
|
||||||
out = out + shift*in;
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
_Mat.Mdir(in,out,dir,disp);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
_Mat.MdirAll(in,out);
|
|
||||||
};
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
_Mat.M(in,out);
|
|
||||||
out = out + shift * in;
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
_Mat.Mdag(in,out);
|
|
||||||
out = out + shift * in;
|
|
||||||
}
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
// Even Odd Schur decomp operators; there are several
|
|
||||||
// ways to introduce the even odd checkerboarding
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class SchurOperatorBase : public LinearOperatorBase<Field> {
|
|
||||||
public:
|
|
||||||
virtual void Mpc (const Field &in, Field &out) =0;
|
|
||||||
virtual void MpcDag (const Field &in, Field &out) =0;
|
|
||||||
virtual void MpcDagMpc(const Field &in, Field &out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
tmp.Checkerboard() = in.Checkerboard();
|
|
||||||
Mpc(in,tmp);
|
|
||||||
MpcDag(tmp,out);
|
|
||||||
}
|
|
||||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
MpcDagMpc(in,out);
|
|
||||||
ComplexD dot= innerProduct(in,out);
|
|
||||||
n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
virtual void HermOp(const Field &in, Field &out){
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
MpcDagMpc(in,out);
|
|
||||||
}
|
|
||||||
void Op (const Field &in, Field &out){
|
|
||||||
Mpc(in,out);
|
|
||||||
}
|
|
||||||
void AdjOp (const Field &in, Field &out){
|
|
||||||
MpcDag(in,out);
|
|
||||||
}
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag (const Field &in, Field &out) {
|
|
||||||
GRID_ASSERT(0); // must coarsen the unpreconditioned system
|
|
||||||
}
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class SchurDiagMooeeOperator : public SchurOperatorBase<Field> {
|
|
||||||
public:
|
|
||||||
Matrix &_Mat;
|
|
||||||
SchurDiagMooeeOperator (Matrix &Mat): _Mat(Mat){};
|
|
||||||
virtual void Mpc (const Field &in, Field &out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
tmp.Checkerboard() = !in.Checkerboard();
|
|
||||||
|
|
||||||
_Mat.Meooe(in,tmp);
|
|
||||||
_Mat.MooeeInv(tmp,out);
|
|
||||||
_Mat.Meooe(out,tmp);
|
|
||||||
_Mat.Mooee(in,out);
|
|
||||||
axpy(out,-1.0,tmp,out);
|
|
||||||
}
|
|
||||||
virtual void MpcDag (const Field &in, Field &out){
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MeooeDag(in,tmp);
|
|
||||||
_Mat.MooeeInvDag(tmp,out);
|
|
||||||
_Mat.MeooeDag(out,tmp);
|
|
||||||
_Mat.MooeeDag(in,out);
|
|
||||||
axpy(out,-1.0,tmp,out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class SchurDiagOneOperator : public SchurOperatorBase<Field> {
|
|
||||||
protected:
|
|
||||||
Matrix &_Mat;
|
|
||||||
public:
|
|
||||||
SchurDiagOneOperator (Matrix &Mat): _Mat(Mat){};
|
|
||||||
|
|
||||||
virtual void Mpc (const Field &in, Field &out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.Meooe(in,out);
|
|
||||||
_Mat.MooeeInv(out,tmp);
|
|
||||||
_Mat.Meooe(tmp,out);
|
|
||||||
_Mat.MooeeInv(out,tmp);
|
|
||||||
axpy(out,-1.0,tmp,in);
|
|
||||||
}
|
|
||||||
virtual void MpcDag (const Field &in, Field &out){
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MooeeInvDag(in,out);
|
|
||||||
_Mat.MeooeDag(out,tmp);
|
|
||||||
_Mat.MooeeInvDag(tmp,out);
|
|
||||||
_Mat.MeooeDag(out,tmp);
|
|
||||||
axpy(out,-1.0,tmp,in);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class SchurDiagTwoOperator : public SchurOperatorBase<Field> {
|
|
||||||
protected:
|
|
||||||
Matrix &_Mat;
|
|
||||||
public:
|
|
||||||
SchurDiagTwoOperator (Matrix &Mat): _Mat(Mat){};
|
|
||||||
|
|
||||||
virtual void Mpc (const Field &in, Field &out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MooeeInv(in,out);
|
|
||||||
_Mat.Meooe(out,tmp);
|
|
||||||
_Mat.MooeeInv(tmp,out);
|
|
||||||
_Mat.Meooe(out,tmp);
|
|
||||||
|
|
||||||
axpy(out,-1.0,tmp,in);
|
|
||||||
}
|
|
||||||
virtual void MpcDag (const Field &in, Field &out){
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MeooeDag(in,out);
|
|
||||||
_Mat.MooeeInvDag(out,tmp);
|
|
||||||
_Mat.MeooeDag(tmp,out);
|
|
||||||
_Mat.MooeeInvDag(out,tmp);
|
|
||||||
|
|
||||||
axpy(out,-1.0,tmp,in);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class NonHermitianSchurOperatorBase : public LinearOperatorBase<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
virtual void Mpc (const Field& in, Field& out) = 0;
|
|
||||||
virtual void MpcDag (const Field& in, Field& out) = 0;
|
|
||||||
virtual void MpcDagMpc(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
tmp.Checkerboard() = in.Checkerboard();
|
|
||||||
Mpc(in,tmp);
|
|
||||||
MpcDag(tmp,out);
|
|
||||||
}
|
|
||||||
virtual void HermOpAndNorm(const Field& in, Field& out, RealD& n1, RealD& n2) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
virtual void HermOp(const Field& in, Field& out) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void Op(const Field& in, Field& out) {
|
|
||||||
Mpc(in, out);
|
|
||||||
}
|
|
||||||
void AdjOp(const Field& in, Field& out) {
|
|
||||||
MpcDag(in, out);
|
|
||||||
}
|
|
||||||
// Support for coarsening to a multigrid
|
|
||||||
void OpDiag(const Field& in, Field& out) {
|
|
||||||
GRID_ASSERT(0); // must coarsen the unpreconditioned system
|
|
||||||
}
|
|
||||||
void OpDir(const Field& in, Field& out, int dir, int disp) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
void OpDirAll(const Field& in, std::vector<Field>& out){
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Matrix, class Field>
|
|
||||||
class NonHermitianSchurDiagMooeeOperator : public NonHermitianSchurOperatorBase<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
Matrix& _Mat;
|
|
||||||
NonHermitianSchurDiagMooeeOperator(Matrix& Mat): _Mat(Mat){};
|
|
||||||
virtual void Mpc(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
tmp.Checkerboard() = !in.Checkerboard();
|
|
||||||
|
|
||||||
_Mat.Meooe(in, tmp);
|
|
||||||
_Mat.MooeeInv(tmp, out);
|
|
||||||
_Mat.Meooe(out, tmp);
|
|
||||||
|
|
||||||
_Mat.Mooee(in, out);
|
|
||||||
|
|
||||||
axpy(out, -1.0, tmp, out);
|
|
||||||
}
|
|
||||||
virtual void MpcDag(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MeooeDag(in, tmp);
|
|
||||||
_Mat.MooeeInvDag(tmp, out);
|
|
||||||
_Mat.MeooeDag(out, tmp);
|
|
||||||
|
|
||||||
_Mat.MooeeDag(in, out);
|
|
||||||
|
|
||||||
axpy(out, -1.0, tmp, out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class NonHermitianSchurDiagOneOperator : public NonHermitianSchurOperatorBase<Field>
|
|
||||||
{
|
|
||||||
protected:
|
|
||||||
Matrix &_Mat;
|
|
||||||
|
|
||||||
public:
|
|
||||||
NonHermitianSchurDiagOneOperator (Matrix& Mat): _Mat(Mat){};
|
|
||||||
virtual void Mpc(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.Meooe(in, out);
|
|
||||||
_Mat.MooeeInv(out, tmp);
|
|
||||||
_Mat.Meooe(tmp, out);
|
|
||||||
_Mat.MooeeInv(out, tmp);
|
|
||||||
|
|
||||||
axpy(out, -1.0, tmp, in);
|
|
||||||
}
|
|
||||||
virtual void MpcDag(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MooeeInvDag(in, out);
|
|
||||||
_Mat.MeooeDag(out, tmp);
|
|
||||||
_Mat.MooeeInvDag(tmp, out);
|
|
||||||
_Mat.MeooeDag(out, tmp);
|
|
||||||
|
|
||||||
axpy(out, -1.0, tmp, in);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Matrix, class Field>
|
|
||||||
class NonHermitianSchurDiagTwoOperator : public NonHermitianSchurOperatorBase<Field>
|
|
||||||
{
|
|
||||||
protected:
|
|
||||||
Matrix& _Mat;
|
|
||||||
|
|
||||||
public:
|
|
||||||
NonHermitianSchurDiagTwoOperator(Matrix& Mat): _Mat(Mat){};
|
|
||||||
|
|
||||||
virtual void Mpc(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MooeeInv(in, out);
|
|
||||||
_Mat.Meooe(out, tmp);
|
|
||||||
_Mat.MooeeInv(tmp, out);
|
|
||||||
_Mat.Meooe(out, tmp);
|
|
||||||
|
|
||||||
axpy(out, -1.0, tmp, in);
|
|
||||||
}
|
|
||||||
virtual void MpcDag(const Field& in, Field& out) {
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
_Mat.MeooeDag(in, out);
|
|
||||||
_Mat.MooeeInvDag(out, tmp);
|
|
||||||
_Mat.MeooeDag(tmp, out);
|
|
||||||
_Mat.MooeeInvDag(out, tmp);
|
|
||||||
|
|
||||||
axpy(out, -1.0, tmp, in);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Left handed Moo^-1 ; (Moo - Moe Mee^-1 Meo) psi = eta --> ( 1 - Moo^-1 Moe Mee^-1 Meo ) psi = Moo^-1 eta
|
|
||||||
// Right handed Moo^-1 ; (Moo - Moe Mee^-1 Meo) Moo^-1 Moo psi = eta --> ( 1 - Moe Mee^-1 Meo Moo^-1) phi=eta ; psi = Moo^-1 phi
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Matrix,class Field> using SchurDiagOneRH = SchurDiagTwoOperator<Matrix,Field> ;
|
|
||||||
template<class Matrix,class Field> using SchurDiagOneLH = SchurDiagOneOperator<Matrix,Field> ;
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Staggered use
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Matrix,class Field>
|
|
||||||
class SchurStaggeredOperator : public SchurOperatorBase<Field> {
|
|
||||||
protected:
|
|
||||||
Matrix &_Mat;
|
|
||||||
Field tmp;
|
|
||||||
RealD mass;
|
|
||||||
public:
|
|
||||||
SchurStaggeredOperator (Matrix &Mat): _Mat(Mat), tmp(_Mat.RedBlackGrid())
|
|
||||||
{
|
|
||||||
GRID_ASSERT( _Mat.isTrivialEE() );
|
|
||||||
mass = _Mat.Mass();
|
|
||||||
}
|
|
||||||
virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
Mpc(in,out);
|
|
||||||
ComplexD dot= innerProduct(in,out);
|
|
||||||
n1 = real(dot);
|
|
||||||
n2 =0.0;
|
|
||||||
}
|
|
||||||
virtual void HermOp(const Field &in, Field &out){
|
|
||||||
Mpc(in,out);
|
|
||||||
// _Mat.Meooe(in,out);
|
|
||||||
// _Mat.Meooe(out,tmp);
|
|
||||||
// axpby(out,-1.0,mass*mass,tmp,in);
|
|
||||||
}
|
|
||||||
virtual void Mpc (const Field &in, Field &out)
|
|
||||||
{
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
Field tmp2(in.Grid());
|
|
||||||
|
|
||||||
// _Mat.Mooee(in,out);
|
|
||||||
// _Mat.Mooee(out,tmp);
|
|
||||||
|
|
||||||
_Mat.Meooe(in,out);
|
|
||||||
_Mat.Meooe(out,tmp);
|
|
||||||
axpby(out,-1.0,mass*mass,tmp,in);
|
|
||||||
}
|
|
||||||
virtual void MpcDag (const Field &in, Field &out){
|
|
||||||
Mpc(in,out);
|
|
||||||
}
|
|
||||||
virtual void MpcDagMpc(const Field &in, Field &out) {
|
|
||||||
GRID_ASSERT(0);// Never need with staggered
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for functions of operators
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class OperatorFunction {
|
|
||||||
public:
|
|
||||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) = 0;
|
|
||||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const std::vector<Field> &in,std::vector<Field> &out) {
|
|
||||||
GRID_ASSERT(in.size()==out.size());
|
|
||||||
for(int k=0;k<in.size();k++){
|
|
||||||
(*this)(Linop,in[k],out[k]);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
virtual ~OperatorFunction(){};
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class LinearFunction {
|
|
||||||
public:
|
|
||||||
virtual void operator() (const Field &in, Field &out) = 0;
|
|
||||||
|
|
||||||
virtual void operator() (const std::vector<Field> &in, std::vector<Field> &out)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(in.size() == out.size());
|
|
||||||
|
|
||||||
for (unsigned int i = 0; i < in.size(); ++i)
|
|
||||||
{
|
|
||||||
(*this)(in[i], out[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual ~LinearFunction(){};
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class IdentityLinearFunction : public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
void operator() (const Field &in, Field &out){
|
|
||||||
out = in;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for Multishift solvers for operators
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class OperatorMultiFunction {
|
|
||||||
public:
|
|
||||||
virtual void operator() (LinearOperatorBase<Field> &Linop, const Field &in, std::vector<Field> &out) = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
// FIXME : To think about
|
|
||||||
|
|
||||||
// Chroma functionality list defining LinearOperator
|
|
||||||
/*
|
|
||||||
virtual void operator() (T& chi, const T& psi, enum PlusMinus isign) const = 0;
|
|
||||||
virtual void operator() (T& chi, const T& psi, enum PlusMinus isign, Real epsilon) const
|
|
||||||
virtual const Subset& subset() const = 0;
|
|
||||||
virtual unsigned long nFlops() const { return 0; }
|
|
||||||
virtual void deriv(P& ds_u, const T& chi, const T& psi, enum PlusMinus isign) const
|
|
||||||
class UnprecLinearOperator : public DiffLinearOperator<T,P,Q>
|
|
||||||
const Subset& subset() const {return all;}
|
|
||||||
};
|
|
||||||
*/
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Hermitian operator Linear function and operator function
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field>
|
|
||||||
class HermOpOperatorFunction : public OperatorFunction<Field> {
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
|
||||||
Linop.HermOp(in,out);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
template<typename Field>
|
|
||||||
class PlainHermOp : public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
LinearOperatorBase<Field> &_Linop;
|
|
||||||
|
|
||||||
PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop)
|
|
||||||
{}
|
|
||||||
|
|
||||||
void operator()(const Field& in, Field& out) {
|
|
||||||
_Linop.HermOp(in,out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<typename Field>
|
|
||||||
class FunctionHermOp : public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
OperatorFunction<Field> & _poly;
|
|
||||||
LinearOperatorBase<Field> &_Linop;
|
|
||||||
|
|
||||||
FunctionHermOp(OperatorFunction<Field> & poly,LinearOperatorBase<Field>& linop)
|
|
||||||
: _poly(poly), _Linop(linop) {};
|
|
||||||
|
|
||||||
void operator()(const Field& in, Field& out) {
|
|
||||||
_poly(_Linop,in,out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class Polynomial : public OperatorFunction<Field> {
|
|
||||||
private:
|
|
||||||
std::vector<RealD> Coeffs;
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
Polynomial(std::vector<RealD> &_Coeffs) : Coeffs(_Coeffs) { };
|
|
||||||
|
|
||||||
// Implement the required interface
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
|
||||||
|
|
||||||
Field AtoN(in.Grid());
|
|
||||||
Field Mtmp(in.Grid());
|
|
||||||
AtoN = in;
|
|
||||||
out = AtoN*Coeffs[0];
|
|
||||||
for(int n=1;n<Coeffs.size();n++){
|
|
||||||
Mtmp = AtoN;
|
|
||||||
Linop.HermOp(Mtmp,AtoN);
|
|
||||||
out=out+AtoN*Coeffs[n];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/Preconditioner.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_PRECONDITIONER_H
|
|
||||||
#define GRID_PRECONDITIONER_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class Field> using Preconditioner = LinearFunction<Field> ;
|
|
||||||
|
|
||||||
/*
|
|
||||||
template<class Field> class Preconditioner : public LinearFunction<Field> {
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
virtual void operator()(const Field &src, Field & psi)=0;
|
|
||||||
};
|
|
||||||
*/
|
|
||||||
|
|
||||||
template<class Field> class TrivialPrecon : public Preconditioner<Field> {
|
|
||||||
public:
|
|
||||||
using Preconditioner<Field>::operator();
|
|
||||||
virtual void operator()(const Field &src, Field & psi){
|
|
||||||
psi = src;
|
|
||||||
}
|
|
||||||
TrivialPrecon(void){};
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/SparseMatrix.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_ALGORITHM_SPARSE_MATRIX_H
|
|
||||||
#define GRID_ALGORITHM_SPARSE_MATRIX_H
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Interface defining what I expect of a general sparse matrix, such as a Fermion action
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class SparseMatrixBase {
|
|
||||||
public:
|
|
||||||
virtual GridBase *Grid(void) =0;
|
|
||||||
// Full checkerboar operations
|
|
||||||
virtual void M (const Field &in, Field &out)=0;
|
|
||||||
virtual void Mdag (const Field &in, Field &out)=0;
|
|
||||||
virtual void MdagM(const Field &in, Field &out) {
|
|
||||||
Field tmp (in.Grid());
|
|
||||||
M(in,tmp);
|
|
||||||
Mdag(tmp,out);
|
|
||||||
}
|
|
||||||
virtual void MMdag(const Field &in, Field &out) {
|
|
||||||
Field tmp (in.Grid());
|
|
||||||
Mdag(in,tmp);
|
|
||||||
M(tmp,out);
|
|
||||||
}
|
|
||||||
virtual void Mdiag (const Field &in, Field &out)=0;
|
|
||||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp)=0;
|
|
||||||
virtual void MdirAll (const Field &in, std::vector<Field> &out)=0;
|
|
||||||
virtual ~SparseMatrixBase() {};
|
|
||||||
};
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Interface augmented by a red black sparse matrix, such as a Fermion action
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class CheckerBoardedSparseMatrixBase : public SparseMatrixBase<Field> {
|
|
||||||
public:
|
|
||||||
virtual GridBase *RedBlackGrid(void)=0;
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Query the even even properties to make algorithmic decisions
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
virtual RealD Mass(void) { return 0.0; };
|
|
||||||
virtual int ConstEE(void) { return 1; }; // Disable assumptions unless overridden
|
|
||||||
virtual int isTrivialEE(void) { return 0; }; // by a derived class that knows better
|
|
||||||
|
|
||||||
// half checkerboard operaions
|
|
||||||
virtual void Meooe (const Field &in, Field &out)=0;
|
|
||||||
virtual void Mooee (const Field &in, Field &out)=0;
|
|
||||||
virtual void MooeeInv (const Field &in, Field &out)=0;
|
|
||||||
|
|
||||||
virtual void MeooeDag (const Field &in, Field &out)=0;
|
|
||||||
virtual void MooeeDag (const Field &in, Field &out)=0;
|
|
||||||
virtual void MooeeInvDag (const Field &in, Field &out)=0;
|
|
||||||
virtual ~CheckerBoardedSparseMatrixBase() {};
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,416 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/approx/Chebyshev.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christoph Lehner <clehner@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CHEBYSHEV_H
|
|
||||||
#define GRID_CHEBYSHEV_H
|
|
||||||
|
|
||||||
#include <Grid/algorithms/LinearOperator.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
struct ChebyParams : Serializable {
|
|
||||||
GRID_SERIALIZABLE_CLASS_MEMBERS(ChebyParams,
|
|
||||||
RealD, alpha,
|
|
||||||
RealD, beta,
|
|
||||||
int, Npoly);
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Generic Chebyshev approximations
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field>
|
|
||||||
class Chebyshev : public OperatorFunction<Field> {
|
|
||||||
private:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
std::vector<RealD> Coeffs;
|
|
||||||
int order;
|
|
||||||
RealD hi;
|
|
||||||
RealD lo;
|
|
||||||
|
|
||||||
public:
|
|
||||||
void csv(std::ostream &out){
|
|
||||||
RealD diff = hi-lo;
|
|
||||||
RealD delta = diff*1.0e-9;
|
|
||||||
for (RealD x=lo; x<hi; x+=delta) {
|
|
||||||
delta*=1.02;
|
|
||||||
RealD f = approx(x);
|
|
||||||
out<< x<<" "<<f<<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convenience for plotting the approximation
|
|
||||||
void PlotApprox(std::ostream &out) {
|
|
||||||
out<<"Polynomial approx ["<<lo<<","<<hi<<"]"<<std::endl;
|
|
||||||
for(RealD x=lo;x<hi;x+=(hi-lo)/50.0){
|
|
||||||
out <<x<<"\t"<<approx(x)<<std::endl;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Chebyshev(){};
|
|
||||||
Chebyshev(ChebyParams p){ Init(p.alpha,p.beta,p.Npoly);};
|
|
||||||
Chebyshev(RealD _lo,RealD _hi,int _order, RealD (* func)(RealD) ) {Init(_lo,_hi,_order,func);};
|
|
||||||
Chebyshev(RealD _lo,RealD _hi,int _order) {Init(_lo,_hi,_order);};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// c.f. numerical recipes "chebft"/"chebev". This is sec 5.8 "Chebyshev approximation".
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// CJ: the one we need for Lanczos
|
|
||||||
void Init(RealD _lo,RealD _hi,int _order)
|
|
||||||
{
|
|
||||||
lo=_lo;
|
|
||||||
hi=_hi;
|
|
||||||
order=_order;
|
|
||||||
|
|
||||||
if(order < 2) exit(-1);
|
|
||||||
Coeffs.resize(order,0.0);
|
|
||||||
Coeffs[order-1] = 1.0;
|
|
||||||
};
|
|
||||||
|
|
||||||
// PB - more efficient low pass drops high modes above the low as 1/x uses all Chebyshev's.
|
|
||||||
// Similar kick effect below the threshold as Lanczos filter approach
|
|
||||||
void InitLowPass(RealD _lo,RealD _hi,int _order)
|
|
||||||
{
|
|
||||||
lo=_lo;
|
|
||||||
hi=_hi;
|
|
||||||
order=_order;
|
|
||||||
|
|
||||||
if(order < 2) exit(-1);
|
|
||||||
Coeffs.resize(order);
|
|
||||||
for(int j=0;j<order;j++){
|
|
||||||
RealD k=(order-1.0);
|
|
||||||
RealD s=std::cos( j*M_PI*(k+0.5)/order );
|
|
||||||
Coeffs[j] = s * 2.0/order;
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
void Init(RealD _lo,RealD _hi,int _order, RealD (* func)(RealD))
|
|
||||||
{
|
|
||||||
lo=_lo;
|
|
||||||
hi=_hi;
|
|
||||||
order=_order;
|
|
||||||
|
|
||||||
if(order < 2) exit(-1);
|
|
||||||
Coeffs.resize(order);
|
|
||||||
for(int j=0;j<order;j++){
|
|
||||||
RealD s=0;
|
|
||||||
for(int k=0;k<order;k++){
|
|
||||||
RealD y=std::cos(M_PI*(k+0.5)/order);
|
|
||||||
RealD x=0.5*(y*(hi-lo)+(hi+lo));
|
|
||||||
RealD f=func(x);
|
|
||||||
s=s+f*std::cos( j*M_PI*(k+0.5)/order );
|
|
||||||
}
|
|
||||||
Coeffs[j] = s * 2.0/order;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class functor>
|
|
||||||
void Init(RealD _lo,RealD _hi,int _order, functor & func)
|
|
||||||
{
|
|
||||||
lo=_lo;
|
|
||||||
hi=_hi;
|
|
||||||
order=_order;
|
|
||||||
|
|
||||||
if(order < 2) exit(-1);
|
|
||||||
Coeffs.resize(order);
|
|
||||||
for(int j=0;j<order;j++){
|
|
||||||
RealD s=0;
|
|
||||||
for(int k=0;k<order;k++){
|
|
||||||
RealD y=std::cos(M_PI*(k+0.5)/order);
|
|
||||||
RealD x=0.5*(y*(hi-lo)+(hi+lo));
|
|
||||||
RealD f=func(x);
|
|
||||||
s=s+f*std::cos( j*M_PI*(k+0.5)/order );
|
|
||||||
}
|
|
||||||
Coeffs[j] = s * 2.0/order;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
void JacksonSmooth(void){
|
|
||||||
RealD M=order;
|
|
||||||
RealD alpha = M_PI/(M+2);
|
|
||||||
RealD lmax = std::cos(alpha);
|
|
||||||
RealD sumUsq =0;
|
|
||||||
std::vector<RealD> U(M);
|
|
||||||
std::vector<RealD> a(M);
|
|
||||||
std::vector<RealD> g(M);
|
|
||||||
for(int n=0;n<=M;n++){
|
|
||||||
U[n] = std::sin((n+1)*std::acos(lmax))/std::sin(std::acos(lmax));
|
|
||||||
sumUsq += U[n]*U[n];
|
|
||||||
}
|
|
||||||
sumUsq = std::sqrt(sumUsq);
|
|
||||||
|
|
||||||
for(int i=1;i<=M;i++){
|
|
||||||
a[i] = U[i]/sumUsq;
|
|
||||||
}
|
|
||||||
g[0] = 1.0;
|
|
||||||
for(int m=1;m<=M;m++){
|
|
||||||
g[m] = 0;
|
|
||||||
for(int i=0;i<=M-m;i++){
|
|
||||||
g[m]+= a[i]*a[m+i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for(int m=1;m<=M;m++){
|
|
||||||
Coeffs[m]*=g[m];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
RealD approx(RealD x) // Convenience for plotting the approximation
|
|
||||||
{
|
|
||||||
RealD Tn;
|
|
||||||
RealD Tnm;
|
|
||||||
RealD Tnp;
|
|
||||||
|
|
||||||
RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
|
||||||
|
|
||||||
RealD T0=1;
|
|
||||||
RealD T1=y;
|
|
||||||
|
|
||||||
RealD sum;
|
|
||||||
sum = 0.5*Coeffs[0]*T0;
|
|
||||||
sum+= Coeffs[1]*T1;
|
|
||||||
|
|
||||||
Tn =T1;
|
|
||||||
Tnm=T0;
|
|
||||||
for(int i=2;i<order;i++){
|
|
||||||
Tnp=2*y*Tn-Tnm;
|
|
||||||
Tnm=Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
sum+= Tn*Coeffs[i];
|
|
||||||
}
|
|
||||||
return sum;
|
|
||||||
};
|
|
||||||
|
|
||||||
RealD approxD(RealD x)
|
|
||||||
{
|
|
||||||
RealD Un;
|
|
||||||
RealD Unm;
|
|
||||||
RealD Unp;
|
|
||||||
|
|
||||||
RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
|
||||||
|
|
||||||
RealD U0=1;
|
|
||||||
RealD U1=2*y;
|
|
||||||
|
|
||||||
RealD sum;
|
|
||||||
sum = Coeffs[1]*U0;
|
|
||||||
sum+= Coeffs[2]*U1*2.0;
|
|
||||||
|
|
||||||
Un =U1;
|
|
||||||
Unm=U0;
|
|
||||||
for(int i=2;i<order-1;i++){
|
|
||||||
Unp=2*y*Un-Unm;
|
|
||||||
Unm=Un;
|
|
||||||
Un =Unp;
|
|
||||||
sum+= Un*Coeffs[i+1]*(i+1.0);
|
|
||||||
}
|
|
||||||
return sum/(0.5*(hi-lo));
|
|
||||||
};
|
|
||||||
|
|
||||||
RealD approxInv(RealD z, RealD x0, int maxiter, RealD resid) {
|
|
||||||
RealD x = x0;
|
|
||||||
RealD eps;
|
|
||||||
|
|
||||||
int i;
|
|
||||||
for (i=0;i<maxiter;i++) {
|
|
||||||
eps = approx(x) - z;
|
|
||||||
if (fabs(eps / z) < resid)
|
|
||||||
return x;
|
|
||||||
x = x - eps / approxD(x);
|
|
||||||
}
|
|
||||||
|
|
||||||
return std::numeric_limits<double>::quiet_NaN();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Implement the required interface
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
|
||||||
|
|
||||||
GridBase *grid=in.Grid();
|
|
||||||
|
|
||||||
int vol=grid->gSites();
|
|
||||||
typedef typename Field::vector_type vector_type;
|
|
||||||
|
|
||||||
Field T0(grid); T0 = in;
|
|
||||||
Field T1(grid);
|
|
||||||
Field T2(grid);
|
|
||||||
Field y(grid);
|
|
||||||
|
|
||||||
Field *Tnm = &T0;
|
|
||||||
Field *Tn = &T1;
|
|
||||||
Field *Tnp = &T2;
|
|
||||||
|
|
||||||
// Tn=T1 = (xscale M + mscale)in
|
|
||||||
RealD xscale = 2.0/(hi-lo);
|
|
||||||
RealD mscale = -(hi+lo)/(hi-lo);
|
|
||||||
Linop.HermOp(T0,y);
|
|
||||||
grid->Barrier();
|
|
||||||
axpby(T1,xscale,mscale,y,in);
|
|
||||||
grid->Barrier();
|
|
||||||
|
|
||||||
// sum = .5 c[0] T0 + c[1] T1
|
|
||||||
// out = ()*T0 + Coeffs[1]*T1;
|
|
||||||
axpby(out,0.5*Coeffs[0],Coeffs[1],T0,T1);
|
|
||||||
for(int n=2;n<order;n++){
|
|
||||||
|
|
||||||
Linop.HermOp(*Tn,y);
|
|
||||||
axpby(y,xscale,mscale,y,(*Tn));
|
|
||||||
axpby(*Tnp,2.0,-1.0,y,(*Tnm));
|
|
||||||
if ( Coeffs[n] != 0.0) {
|
|
||||||
axpy(out,Coeffs[n],*Tnp,out);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cycle pointers to avoid copies
|
|
||||||
Field *swizzle = Tnm;
|
|
||||||
Tnm =Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
Tnp =swizzle;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class ChebyshevLanczos : public Chebyshev<Field> {
|
|
||||||
private:
|
|
||||||
std::vector<RealD> Coeffs;
|
|
||||||
int order;
|
|
||||||
RealD alpha;
|
|
||||||
RealD beta;
|
|
||||||
RealD mu;
|
|
||||||
|
|
||||||
public:
|
|
||||||
ChebyshevLanczos(RealD _alpha,RealD _beta,RealD _mu,int _order) :
|
|
||||||
alpha(_alpha),
|
|
||||||
beta(_beta),
|
|
||||||
mu(_mu)
|
|
||||||
{
|
|
||||||
order=_order;
|
|
||||||
Coeffs.resize(order);
|
|
||||||
for(int i=0;i<_order;i++){
|
|
||||||
Coeffs[i] = 0.0;
|
|
||||||
}
|
|
||||||
Coeffs[order-1]=1.0;
|
|
||||||
};
|
|
||||||
|
|
||||||
void csv(std::ostream &out){
|
|
||||||
for (RealD x=-1.2*alpha; x<1.2*alpha; x+=(2.0*alpha)/10000) {
|
|
||||||
RealD f = approx(x);
|
|
||||||
out<< x<<" "<<f<<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD approx(RealD xx) // Convenience for plotting the approximation
|
|
||||||
{
|
|
||||||
RealD Tn;
|
|
||||||
RealD Tnm;
|
|
||||||
RealD Tnp;
|
|
||||||
Real aa = alpha * alpha;
|
|
||||||
Real bb = beta * beta;
|
|
||||||
|
|
||||||
RealD x = ( 2.0 * (xx-mu)*(xx-mu) - (aa+bb) ) / (aa-bb);
|
|
||||||
|
|
||||||
RealD y= x;
|
|
||||||
|
|
||||||
RealD T0=1;
|
|
||||||
RealD T1=y;
|
|
||||||
|
|
||||||
RealD sum;
|
|
||||||
sum = 0.5*Coeffs[0]*T0;
|
|
||||||
sum+= Coeffs[1]*T1;
|
|
||||||
|
|
||||||
Tn =T1;
|
|
||||||
Tnm=T0;
|
|
||||||
for(int i=2;i<order;i++){
|
|
||||||
Tnp=2*y*Tn-Tnm;
|
|
||||||
Tnm=Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
sum+= Tn*Coeffs[i];
|
|
||||||
}
|
|
||||||
return sum;
|
|
||||||
};
|
|
||||||
|
|
||||||
// shift_Multiply in Rudy's code
|
|
||||||
void AminusMuSq(LinearOperatorBase<Field> &Linop, const Field &in, Field &out)
|
|
||||||
{
|
|
||||||
GridBase *grid=in.Grid();
|
|
||||||
Field tmp(grid);
|
|
||||||
|
|
||||||
RealD aa= alpha*alpha;
|
|
||||||
RealD bb= beta * beta;
|
|
||||||
|
|
||||||
Linop.HermOp(in,out);
|
|
||||||
out = out - mu*in;
|
|
||||||
|
|
||||||
Linop.HermOp(out,tmp);
|
|
||||||
tmp = tmp - mu * out;
|
|
||||||
|
|
||||||
out = (2.0/ (aa-bb) ) * tmp - ((aa+bb)/(aa-bb))*in;
|
|
||||||
};
|
|
||||||
// Implement the required interface
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
|
||||||
|
|
||||||
GridBase *grid=in.Grid();
|
|
||||||
|
|
||||||
int vol=grid->gSites();
|
|
||||||
|
|
||||||
Field T0(grid); T0 = in;
|
|
||||||
Field T1(grid);
|
|
||||||
Field T2(grid);
|
|
||||||
Field y(grid);
|
|
||||||
|
|
||||||
Field *Tnm = &T0;
|
|
||||||
Field *Tn = &T1;
|
|
||||||
Field *Tnp = &T2;
|
|
||||||
|
|
||||||
// Tn=T1 = (xscale M )*in
|
|
||||||
AminusMuSq(Linop,T0,T1);
|
|
||||||
|
|
||||||
// sum = .5 c[0] T0 + c[1] T1
|
|
||||||
out = (0.5*Coeffs[0])*T0 + Coeffs[1]*T1;
|
|
||||||
for(int n=2;n<order;n++){
|
|
||||||
|
|
||||||
AminusMuSq(Linop,*Tn,y);
|
|
||||||
|
|
||||||
*Tnp=2.0*y-(*Tnm);
|
|
||||||
|
|
||||||
out=out+Coeffs[n]* (*Tnp);
|
|
||||||
|
|
||||||
// Cycle pointers to avoid copies
|
|
||||||
Field *swizzle = Tnm;
|
|
||||||
Tnm =Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
Tnp =swizzle;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/approx/Forecast.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: David Murphy <dmurphy@phys.columbia.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#ifndef INCLUDED_FORECAST_H
|
|
||||||
#define INCLUDED_FORECAST_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
// Abstract base class.
|
|
||||||
// Takes a matrix (Mat), a source (phi), and a vector of Fields (chi)
|
|
||||||
// and returns a forecasted solution to the system D*psi = phi (psi).
|
|
||||||
template<class Matrix, class Field>
|
|
||||||
class Forecast
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
virtual Field operator()(Matrix &Mat, const Field& phi, const std::vector<Field>& chi) = 0;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Implementation of Brower et al.'s chronological inverter (arXiv:hep-lat/9509012),
|
|
||||||
// used to forecast solutions across poles of the EOFA heatbath.
|
|
||||||
//
|
|
||||||
// Modified from CPS (cps_pp/src/util/dirac_op/d_op_base/comsrc/minresext.C)
|
|
||||||
template<class Matrix, class Field>
|
|
||||||
class ChronoForecast : public Forecast<Matrix,Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
Field operator()(Matrix &Mat, const Field& phi, const std::vector<Field>& prev_solns)
|
|
||||||
{
|
|
||||||
int degree = prev_solns.size();
|
|
||||||
Field chi(phi); // forecasted solution
|
|
||||||
|
|
||||||
// Trivial cases
|
|
||||||
if(degree == 0){ chi = Zero(); return chi; }
|
|
||||||
else if(degree == 1){ return prev_solns[0]; }
|
|
||||||
|
|
||||||
// RealD dot;
|
|
||||||
ComplexD xp;
|
|
||||||
Field r(phi); // residual
|
|
||||||
Field Mv(phi);
|
|
||||||
std::vector<Field> v(prev_solns); // orthonormalized previous solutions
|
|
||||||
std::vector<Field> MdagMv(degree,phi);
|
|
||||||
|
|
||||||
// Array to hold the matrix elements
|
|
||||||
std::vector<std::vector<ComplexD>> G(degree, std::vector<ComplexD>(degree));
|
|
||||||
|
|
||||||
// Solution and source vectors
|
|
||||||
std::vector<ComplexD> a(degree);
|
|
||||||
std::vector<ComplexD> b(degree);
|
|
||||||
|
|
||||||
// Orthonormalize the vector basis
|
|
||||||
for(int i=0; i<degree; i++){
|
|
||||||
v[i] *= 1.0/std::sqrt(norm2(v[i]));
|
|
||||||
for(int j=i+1; j<degree; j++){ v[j] -= innerProduct(v[i],v[j]) * v[i]; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Perform sparse matrix multiplication and construct rhs
|
|
||||||
for(int i=0; i<degree; i++){
|
|
||||||
b[i] = innerProduct(v[i],phi);
|
|
||||||
Mat.M(v[i],Mv);
|
|
||||||
Mat.Mdag(Mv,MdagMv[i]);
|
|
||||||
G[i][i] = innerProduct(v[i],MdagMv[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Construct the matrix
|
|
||||||
for(int j=0; j<degree; j++){
|
|
||||||
for(int k=j+1; k<degree; k++){
|
|
||||||
G[j][k] = innerProduct(v[j],MdagMv[k]);
|
|
||||||
G[k][j] = conjugate(G[j][k]);
|
|
||||||
}}
|
|
||||||
|
|
||||||
// Gauss-Jordan elimination with partial pivoting
|
|
||||||
for(int i=0; i<degree; i++){
|
|
||||||
|
|
||||||
// Perform partial pivoting
|
|
||||||
int k = i;
|
|
||||||
for(int j=i+1; j<degree; j++){ if(abs(G[j][j]) > abs(G[k][k])){ k = j; } }
|
|
||||||
if(k != i){
|
|
||||||
xp = b[k];
|
|
||||||
b[k] = b[i];
|
|
||||||
b[i] = xp;
|
|
||||||
for(int j=0; j<degree; j++){
|
|
||||||
xp = G[k][j];
|
|
||||||
G[k][j] = G[i][j];
|
|
||||||
G[i][j] = xp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert matrix to upper triangular form
|
|
||||||
for(int j=i+1; j<degree; j++){
|
|
||||||
xp = G[j][i]/G[i][i];
|
|
||||||
b[j] -= xp * b[i];
|
|
||||||
for(int k=0; k<degree; k++){ G[j][k] -= xp*G[i][k]; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use Gaussian elimination to solve equations and calculate initial guess
|
|
||||||
chi = Zero();
|
|
||||||
r = phi;
|
|
||||||
for(int i=degree-1; i>=0; i--){
|
|
||||||
a[i] = 0.0;
|
|
||||||
for(int j=i+1; j<degree; j++){ a[i] += G[i][j] * a[j]; }
|
|
||||||
a[i] = (b[i]-a[i])/G[i][i];
|
|
||||||
chi += a[i]*v[i];
|
|
||||||
r -= a[i]*MdagMv[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD true_r(0.0);
|
|
||||||
ComplexD tmp;
|
|
||||||
for(int i=0; i<degree; i++){
|
|
||||||
tmp = -b[i];
|
|
||||||
for(int j=0; j<degree; j++){ tmp += G[i][j]*a[j]; }
|
|
||||||
tmp = conjugate(tmp)*tmp;
|
|
||||||
true_r += std::sqrt(tmp.real());
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD error = std::sqrt(norm2(r)/norm2(phi));
|
|
||||||
std::cout << GridLogMessage << "ChronoForecast: |res|/|src| = " << error << std::endl;
|
|
||||||
|
|
||||||
return chi;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,129 +0,0 @@
|
|||||||
#ifndef GRID_JACOBIPOLYNOMIAL_H
|
|
||||||
#define GRID_JACOBIPOLYNOMIAL_H
|
|
||||||
|
|
||||||
#include <Grid/algorithms/LinearOperator.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class JacobiPolynomial : public OperatorFunction<Field> {
|
|
||||||
private:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
int order;
|
|
||||||
RealD hi;
|
|
||||||
RealD lo;
|
|
||||||
RealD alpha;
|
|
||||||
RealD beta;
|
|
||||||
|
|
||||||
public:
|
|
||||||
void csv(std::ostream &out){
|
|
||||||
csv(out,lo,hi);
|
|
||||||
}
|
|
||||||
void csv(std::ostream &out,RealD llo,RealD hhi){
|
|
||||||
RealD diff = hhi-llo;
|
|
||||||
RealD delta = diff*1.0e-5;
|
|
||||||
for (RealD x=llo-delta; x<=hhi; x+=delta) {
|
|
||||||
RealD f = approx(x);
|
|
||||||
out<< x<<" "<<f <<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
JacobiPolynomial(){};
|
|
||||||
JacobiPolynomial(RealD _lo,RealD _hi,int _order,RealD _alpha, RealD _beta)
|
|
||||||
{
|
|
||||||
lo=_lo;
|
|
||||||
hi=_hi;
|
|
||||||
alpha=_alpha;
|
|
||||||
beta=_beta;
|
|
||||||
order=_order;
|
|
||||||
};
|
|
||||||
|
|
||||||
RealD approx(RealD x) // Convenience for plotting the approximation
|
|
||||||
{
|
|
||||||
RealD Tn;
|
|
||||||
RealD Tnm;
|
|
||||||
RealD Tnp;
|
|
||||||
|
|
||||||
RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
|
||||||
|
|
||||||
RealD T0=1.0;
|
|
||||||
RealD T1=(alpha-beta)*0.5+(alpha+beta+2.0)*0.5*y;
|
|
||||||
|
|
||||||
Tn =T1;
|
|
||||||
Tnm=T0;
|
|
||||||
for(int n=2;n<=order;n++){
|
|
||||||
RealD cnp = 2.0*n*(n+alpha+beta)*(2.0*n-2.0+alpha+beta);
|
|
||||||
RealD cny = (2.0*n-2.0+alpha+beta)*(2.0*n-1.0+alpha+beta)*(2.0*n+alpha+beta);
|
|
||||||
RealD cn1 = (2.0*n+alpha+beta-1.0)*(alpha*alpha-beta*beta);
|
|
||||||
RealD cnm = - 2.0*(n+alpha-1.0)*(n+beta-1.0)*(2.0*n+alpha+beta);
|
|
||||||
Tnp= ( cny * y *Tn + cn1 * Tn + cnm * Tnm )/ cnp;
|
|
||||||
Tnm=Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
}
|
|
||||||
return Tnp;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Implement the required interface
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
|
|
||||||
GridBase *grid=in.Grid();
|
|
||||||
|
|
||||||
int vol=grid->gSites();
|
|
||||||
|
|
||||||
Field T0(grid);
|
|
||||||
Field T1(grid);
|
|
||||||
Field T2(grid);
|
|
||||||
Field y(grid);
|
|
||||||
|
|
||||||
|
|
||||||
Field *Tnm = &T0;
|
|
||||||
Field *Tn = &T1;
|
|
||||||
Field *Tnp = &T2;
|
|
||||||
|
|
||||||
// RealD T0=1.0;
|
|
||||||
T0=in;
|
|
||||||
|
|
||||||
// RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
|
|
||||||
// = x * 2/(hi-lo) - (hi+lo)/(hi-lo)
|
|
||||||
Linop.HermOp(T0,y);
|
|
||||||
RealD xscale = 2.0/(hi-lo);
|
|
||||||
RealD mscale = -(hi+lo)/(hi-lo);
|
|
||||||
Linop.HermOp(T0,y);
|
|
||||||
y=y*xscale+in*mscale;
|
|
||||||
|
|
||||||
// RealD T1=(alpha-beta)*0.5+(alpha+beta+2.0)*0.5*y;
|
|
||||||
RealD halfAmB = (alpha-beta)*0.5;
|
|
||||||
RealD halfApBp2= (alpha+beta+2.0)*0.5;
|
|
||||||
T1 = halfAmB * in + halfApBp2*y;
|
|
||||||
|
|
||||||
for(int n=2;n<=order;n++){
|
|
||||||
|
|
||||||
Linop.HermOp(*Tn,y);
|
|
||||||
y=xscale*y+mscale*(*Tn);
|
|
||||||
|
|
||||||
RealD cnp = 2.0*n*(n+alpha+beta)*(2.0*n-2.0+alpha+beta);
|
|
||||||
RealD cny = (2.0*n-2.0+alpha+beta)*(2.0*n-1.0+alpha+beta)*(2.0*n+alpha+beta);
|
|
||||||
RealD cn1 = (2.0*n+alpha+beta-1.0)*(alpha*alpha-beta*beta);
|
|
||||||
RealD cnm = - 2.0*(n+alpha-1.0)*(n+beta-1.0)*(2.0*n+alpha+beta);
|
|
||||||
|
|
||||||
// Tnp= ( cny * y *Tn + cn1 * Tn + cnm * Tnm )/ cnp;
|
|
||||||
cny=cny/cnp;
|
|
||||||
cn1=cn1/cnp;
|
|
||||||
cn1=cn1/cnp;
|
|
||||||
cnm=cnm/cnp;
|
|
||||||
|
|
||||||
*Tnp=cny*y + cn1 *(*Tn) + cnm * (*Tnm);
|
|
||||||
|
|
||||||
// Cycle pointers to avoid copies
|
|
||||||
Field *swizzle = Tnm;
|
|
||||||
Tnm =Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
Tnp =swizzle;
|
|
||||||
}
|
|
||||||
out=*Tnp;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,473 +0,0 @@
|
|||||||
#include<math.h>
|
|
||||||
#include<stdio.h>
|
|
||||||
#include<stdlib.h>
|
|
||||||
#include<string>
|
|
||||||
#include<iostream>
|
|
||||||
#include<iomanip>
|
|
||||||
#include<cassert>
|
|
||||||
|
|
||||||
#include<Grid/algorithms/approx/RemezGeneral.h>
|
|
||||||
|
|
||||||
|
|
||||||
// Constructor
|
|
||||||
AlgRemezGeneral::AlgRemezGeneral(double lower, double upper, long precision,
|
|
||||||
bigfloat (*f)(bigfloat x, void *data), void *data): f(f),
|
|
||||||
data(data),
|
|
||||||
prec(precision),
|
|
||||||
apstrt(lower), apend(upper), apwidt(upper - lower),
|
|
||||||
n(0), d(0), pow_n(0), pow_d(0)
|
|
||||||
{
|
|
||||||
bigfloat::setDefaultPrecision(prec);
|
|
||||||
|
|
||||||
std::cout<<"Approximation bounds are ["<<apstrt<<","<<apend<<"]\n";
|
|
||||||
std::cout<<"Precision of arithmetic is "<<precision<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
//Determine the properties of the numerator and denominator polynomials
|
|
||||||
void AlgRemezGeneral::setupPolyProperties(int num_degree, int den_degree, PolyType num_type_in, PolyType den_type_in){
|
|
||||||
pow_n = num_degree;
|
|
||||||
pow_d = den_degree;
|
|
||||||
|
|
||||||
if(pow_n % 2 == 0 && num_type_in == PolyType::Odd) GRID_ASSERT(0);
|
|
||||||
if(pow_n % 2 == 1 && num_type_in == PolyType::Even) GRID_ASSERT(0);
|
|
||||||
|
|
||||||
if(pow_d % 2 == 0 && den_type_in == PolyType::Odd) GRID_ASSERT(0);
|
|
||||||
if(pow_d % 2 == 1 && den_type_in == PolyType::Even) GRID_ASSERT(0);
|
|
||||||
|
|
||||||
num_type = num_type_in;
|
|
||||||
den_type = den_type_in;
|
|
||||||
|
|
||||||
num_pows.resize(pow_n+1);
|
|
||||||
den_pows.resize(pow_d+1);
|
|
||||||
|
|
||||||
int n_in = 0;
|
|
||||||
bool odd = num_type == PolyType::Full || num_type == PolyType::Odd;
|
|
||||||
bool even = num_type == PolyType::Full || num_type == PolyType::Even;
|
|
||||||
for(int i=0;i<=pow_n;i++){
|
|
||||||
num_pows[i] = -1;
|
|
||||||
if(i % 2 == 0 && even) num_pows[i] = n_in++;
|
|
||||||
if(i % 2 == 1 && odd) num_pows[i] = n_in++;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << n_in << " terms in numerator" << std::endl;
|
|
||||||
--n_in; //power is 1 less than the number of terms, eg pow=1 a x^1 + b x^0
|
|
||||||
|
|
||||||
int d_in = 0;
|
|
||||||
odd = den_type == PolyType::Full || den_type == PolyType::Odd;
|
|
||||||
even = den_type == PolyType::Full || den_type == PolyType::Even;
|
|
||||||
for(int i=0;i<=pow_d;i++){
|
|
||||||
den_pows[i] = -1;
|
|
||||||
if(i % 2 == 0 && even) den_pows[i] = d_in++;
|
|
||||||
if(i % 2 == 1 && odd) den_pows[i] = d_in++;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << d_in << " terms in denominator" << std::endl;
|
|
||||||
--d_in;
|
|
||||||
|
|
||||||
n = n_in;
|
|
||||||
d = d_in;
|
|
||||||
}
|
|
||||||
|
|
||||||
//Setup algorithm
|
|
||||||
void AlgRemezGeneral::reinitializeAlgorithm(){
|
|
||||||
spread = 1.0e37;
|
|
||||||
iter = 0;
|
|
||||||
|
|
||||||
neq = n + d + 1; //not +2 because highest-power term in denominator is fixed to 1
|
|
||||||
|
|
||||||
param.resize(neq);
|
|
||||||
yy.resize(neq+1);
|
|
||||||
|
|
||||||
//Initialize linear equation temporaries
|
|
||||||
A.resize(neq*neq);
|
|
||||||
B.resize(neq);
|
|
||||||
IPS.resize(neq);
|
|
||||||
|
|
||||||
//Initialize maximum and minimum errors
|
|
||||||
xx.resize(neq+2);
|
|
||||||
mm.resize(neq+1);
|
|
||||||
initialGuess();
|
|
||||||
|
|
||||||
//Initialize search steps
|
|
||||||
step.resize(neq+1);
|
|
||||||
stpini();
|
|
||||||
}
|
|
||||||
|
|
||||||
double AlgRemezGeneral::generateApprox(const int num_degree, const int den_degree,
|
|
||||||
const PolyType num_type_in, const PolyType den_type_in,
|
|
||||||
const double _tolerance, const int report_freq){
|
|
||||||
//Setup the properties of the polynomial
|
|
||||||
setupPolyProperties(num_degree, den_degree, num_type_in, den_type_in);
|
|
||||||
|
|
||||||
//Setup the algorithm
|
|
||||||
reinitializeAlgorithm();
|
|
||||||
|
|
||||||
bigfloat tolerance = _tolerance;
|
|
||||||
|
|
||||||
//Iterate until convergance
|
|
||||||
while (spread > tolerance) {
|
|
||||||
if (iter++ % report_freq==0)
|
|
||||||
std::cout<<"Iteration " <<iter-1<<" spread "<<(double)spread<<" delta "<<(double)delta << std::endl;
|
|
||||||
|
|
||||||
equations();
|
|
||||||
if (delta < tolerance) {
|
|
||||||
std::cout<<"Iteration " << iter-1 << " delta too small (" << delta << "<" << tolerance << "), try increasing precision\n";
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
};
|
|
||||||
GRID_ASSERT( delta>= tolerance );
|
|
||||||
|
|
||||||
search();
|
|
||||||
}
|
|
||||||
|
|
||||||
int sign;
|
|
||||||
double error = (double)getErr(mm[0],&sign);
|
|
||||||
std::cout<<"Converged at "<<iter<<" iterations; error = "<<error<<std::endl;
|
|
||||||
|
|
||||||
// Return the maximum error in the approximation
|
|
||||||
return error;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Initial values of maximal and minimal errors
|
|
||||||
void AlgRemezGeneral::initialGuess(){
|
|
||||||
// Supply initial guesses for solution points
|
|
||||||
long ncheb = neq; // Degree of Chebyshev error estimate
|
|
||||||
|
|
||||||
// Find ncheb+1 extrema of Chebyshev polynomial
|
|
||||||
bigfloat a = ncheb;
|
|
||||||
bigfloat r;
|
|
||||||
|
|
||||||
mm[0] = apstrt;
|
|
||||||
for (long i = 1; i < ncheb; i++) {
|
|
||||||
r = 0.5 * (1 - cos((M_PI * i)/(double) a));
|
|
||||||
//r *= sqrt_bf(r);
|
|
||||||
r = (exp((double)r)-1.0)/(exp(1.0)-1.0);
|
|
||||||
mm[i] = apstrt + r * apwidt;
|
|
||||||
}
|
|
||||||
mm[ncheb] = apend;
|
|
||||||
|
|
||||||
a = 2.0 * ncheb;
|
|
||||||
for (long i = 0; i <= ncheb; i++) {
|
|
||||||
r = 0.5 * (1 - cos(M_PI * (2*i+1)/(double) a));
|
|
||||||
//r *= sqrt_bf(r); // Squeeze to low end of interval
|
|
||||||
r = (exp((double)r)-1.0)/(exp(1.0)-1.0);
|
|
||||||
xx[i] = apstrt + r * apwidt;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialise step sizes
|
|
||||||
void AlgRemezGeneral::stpini(){
|
|
||||||
xx[neq+1] = apend;
|
|
||||||
delta = 0.25;
|
|
||||||
step[0] = xx[0] - apstrt;
|
|
||||||
for (int i = 1; i < neq; i++) step[i] = xx[i] - xx[i-1];
|
|
||||||
step[neq] = step[neq-1];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Search for error maxima and minima
|
|
||||||
void AlgRemezGeneral::search(){
|
|
||||||
bigfloat a, q, xm, ym, xn, yn, xx1;
|
|
||||||
int emsign, ensign, steps;
|
|
||||||
|
|
||||||
int meq = neq + 1;
|
|
||||||
|
|
||||||
bigfloat eclose = 1.0e30;
|
|
||||||
bigfloat farther = 0l;
|
|
||||||
|
|
||||||
bigfloat xx0 = apstrt;
|
|
||||||
|
|
||||||
for (int i = 0; i < meq; i++) {
|
|
||||||
steps = 0;
|
|
||||||
xx1 = xx[i]; // Next zero
|
|
||||||
if (i == meq-1) xx1 = apend;
|
|
||||||
xm = mm[i];
|
|
||||||
ym = getErr(xm,&emsign);
|
|
||||||
q = step[i];
|
|
||||||
xn = xm + q;
|
|
||||||
if (xn < xx0 || xn >= xx1) { // Cannot skip over adjacent boundaries
|
|
||||||
q = -q;
|
|
||||||
xn = xm;
|
|
||||||
yn = ym;
|
|
||||||
ensign = emsign;
|
|
||||||
} else {
|
|
||||||
yn = getErr(xn,&ensign);
|
|
||||||
if (yn < ym) {
|
|
||||||
q = -q;
|
|
||||||
xn = xm;
|
|
||||||
yn = ym;
|
|
||||||
ensign = emsign;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
while(yn >= ym) { // March until error becomes smaller.
|
|
||||||
if (++steps > 10)
|
|
||||||
break;
|
|
||||||
|
|
||||||
ym = yn;
|
|
||||||
xm = xn;
|
|
||||||
emsign = ensign;
|
|
||||||
a = xm + q;
|
|
||||||
if (a == xm || a <= xx0 || a >= xx1)
|
|
||||||
break;// Must not skip over the zeros either side.
|
|
||||||
|
|
||||||
xn = a;
|
|
||||||
yn = getErr(xn,&ensign);
|
|
||||||
}
|
|
||||||
|
|
||||||
mm[i] = xm; // Position of maximum
|
|
||||||
yy[i] = ym; // Value of maximum
|
|
||||||
|
|
||||||
if (eclose > ym) eclose = ym;
|
|
||||||
if (farther < ym) farther = ym;
|
|
||||||
|
|
||||||
xx0 = xx1; // Walk to next zero.
|
|
||||||
} // end of search loop
|
|
||||||
|
|
||||||
q = (farther - eclose); // Decrease step size if error spread increased
|
|
||||||
|
|
||||||
if (eclose != 0.0) q /= eclose; // Relative error spread
|
|
||||||
|
|
||||||
if (q >= spread)
|
|
||||||
delta *= 0.5; // Spread is increasing; decrease step size
|
|
||||||
|
|
||||||
spread = q;
|
|
||||||
|
|
||||||
for (int i = 0; i < neq; i++) {
|
|
||||||
q = yy[i+1];
|
|
||||||
if (q != 0.0) q = yy[i] / q - (bigfloat)1l;
|
|
||||||
else q = 0.0625;
|
|
||||||
if (q > (bigfloat)0.25) q = 0.25;
|
|
||||||
q *= mm[i+1] - mm[i];
|
|
||||||
step[i] = q * delta;
|
|
||||||
}
|
|
||||||
step[neq] = step[neq-1];
|
|
||||||
|
|
||||||
for (int i = 0; i < neq; i++) { // Insert new locations for the zeros.
|
|
||||||
xm = xx[i] - step[i];
|
|
||||||
|
|
||||||
if (xm <= apstrt)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (xm >= apend)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (xm <= mm[i])
|
|
||||||
xm = (bigfloat)0.5 * (mm[i] + xx[i]);
|
|
||||||
|
|
||||||
if (xm >= mm[i+1])
|
|
||||||
xm = (bigfloat)0.5 * (mm[i+1] + xx[i]);
|
|
||||||
|
|
||||||
xx[i] = xm;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Solve the equations
|
|
||||||
void AlgRemezGeneral::equations(){
|
|
||||||
bigfloat x, y, z;
|
|
||||||
bigfloat *aa;
|
|
||||||
|
|
||||||
for (int i = 0; i < neq; i++) { // set up the equations for solution by simq()
|
|
||||||
int ip = neq * i; // offset to 1st element of this row of matrix
|
|
||||||
x = xx[i]; // the guess for this row
|
|
||||||
y = func(x); // right-hand-side vector
|
|
||||||
|
|
||||||
z = (bigfloat)1l;
|
|
||||||
aa = A.data()+ip;
|
|
||||||
int t = 0;
|
|
||||||
for (int j = 0; j <= pow_n; j++) {
|
|
||||||
if(num_pows[j] != -1){ *aa++ = z; t++; }
|
|
||||||
z *= x;
|
|
||||||
}
|
|
||||||
GRID_ASSERT(t == n+1);
|
|
||||||
|
|
||||||
z = (bigfloat)1l;
|
|
||||||
t = 0;
|
|
||||||
for (int j = 0; j < pow_d; j++) {
|
|
||||||
if(den_pows[j] != -1){ *aa++ = -y * z; t++; }
|
|
||||||
z *= x;
|
|
||||||
}
|
|
||||||
GRID_ASSERT(t == d);
|
|
||||||
|
|
||||||
B[i] = y * z; // Right hand side vector
|
|
||||||
}
|
|
||||||
|
|
||||||
// Solve the simultaneous linear equations.
|
|
||||||
if (simq()){
|
|
||||||
std::cout<<"simq failed\n";
|
|
||||||
exit(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Evaluate the rational form P(x)/Q(x) using coefficients
|
|
||||||
// from the solution vector param
|
|
||||||
bigfloat AlgRemezGeneral::approx(const bigfloat x) const{
|
|
||||||
// Work backwards toward the constant term.
|
|
||||||
int c = n;
|
|
||||||
bigfloat yn = param[c--]; // Highest order numerator coefficient
|
|
||||||
for (int i = pow_n-1; i >= 0; i--) yn = x * yn + (num_pows[i] != -1 ? param[c--] : bigfloat(0l));
|
|
||||||
|
|
||||||
c = n+d;
|
|
||||||
bigfloat yd = 1l; //Highest degree coefficient is 1.0
|
|
||||||
for (int i = pow_d-1; i >= 0; i--) yd = x * yd + (den_pows[i] != -1 ? param[c--] : bigfloat(0l));
|
|
||||||
|
|
||||||
return(yn/yd);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute size and sign of the approximation error at x
|
|
||||||
bigfloat AlgRemezGeneral::getErr(bigfloat x, int *sign) const{
|
|
||||||
bigfloat f = func(x);
|
|
||||||
bigfloat e = approx(x) - f;
|
|
||||||
if (f != 0) e /= f;
|
|
||||||
if (e < (bigfloat)0.0) {
|
|
||||||
*sign = -1;
|
|
||||||
e = -e;
|
|
||||||
}
|
|
||||||
else *sign = 1;
|
|
||||||
|
|
||||||
return(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Solve the system AX=B
|
|
||||||
int AlgRemezGeneral::simq(){
|
|
||||||
|
|
||||||
int ip, ipj, ipk, ipn;
|
|
||||||
int idxpiv;
|
|
||||||
int kp, kp1, kpk, kpn;
|
|
||||||
int nip, nkp;
|
|
||||||
bigfloat em, q, rownrm, big, size, pivot, sum;
|
|
||||||
bigfloat *aa;
|
|
||||||
bigfloat *X = param.data();
|
|
||||||
|
|
||||||
int n = neq;
|
|
||||||
int nm1 = n - 1;
|
|
||||||
// Initialize IPS and X
|
|
||||||
|
|
||||||
int ij = 0;
|
|
||||||
for (int i = 0; i < n; i++) {
|
|
||||||
IPS[i] = i;
|
|
||||||
rownrm = 0.0;
|
|
||||||
for(int j = 0; j < n; j++) {
|
|
||||||
q = abs_bf(A[ij]);
|
|
||||||
if(rownrm < q) rownrm = q;
|
|
||||||
++ij;
|
|
||||||
}
|
|
||||||
if (rownrm == (bigfloat)0l) {
|
|
||||||
std::cout<<"simq rownrm=0\n";
|
|
||||||
return(1);
|
|
||||||
}
|
|
||||||
X[i] = (bigfloat)1.0 / rownrm;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int k = 0; k < nm1; k++) {
|
|
||||||
big = 0.0;
|
|
||||||
idxpiv = 0;
|
|
||||||
|
|
||||||
for (int i = k; i < n; i++) {
|
|
||||||
ip = IPS[i];
|
|
||||||
ipk = n*ip + k;
|
|
||||||
size = abs_bf(A[ipk]) * X[ip];
|
|
||||||
if (size > big) {
|
|
||||||
big = size;
|
|
||||||
idxpiv = i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (big == (bigfloat)0l) {
|
|
||||||
std::cout<<"simq big=0\n";
|
|
||||||
return(2);
|
|
||||||
}
|
|
||||||
if (idxpiv != k) {
|
|
||||||
int j = IPS[k];
|
|
||||||
IPS[k] = IPS[idxpiv];
|
|
||||||
IPS[idxpiv] = j;
|
|
||||||
}
|
|
||||||
kp = IPS[k];
|
|
||||||
kpk = n*kp + k;
|
|
||||||
pivot = A[kpk];
|
|
||||||
kp1 = k+1;
|
|
||||||
for (int i = kp1; i < n; i++) {
|
|
||||||
ip = IPS[i];
|
|
||||||
ipk = n*ip + k;
|
|
||||||
em = -A[ipk] / pivot;
|
|
||||||
A[ipk] = -em;
|
|
||||||
nip = n*ip;
|
|
||||||
nkp = n*kp;
|
|
||||||
aa = A.data()+nkp+kp1;
|
|
||||||
for (int j = kp1; j < n; j++) {
|
|
||||||
ipj = nip + j;
|
|
||||||
A[ipj] = A[ipj] + em * *aa++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
kpn = n * IPS[n-1] + n - 1; // last element of IPS[n] th row
|
|
||||||
if (A[kpn] == (bigfloat)0l) {
|
|
||||||
std::cout<<"simq A[kpn]=0\n";
|
|
||||||
return(3);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
ip = IPS[0];
|
|
||||||
X[0] = B[ip];
|
|
||||||
for (int i = 1; i < n; i++) {
|
|
||||||
ip = IPS[i];
|
|
||||||
ipj = n * ip;
|
|
||||||
sum = 0.0;
|
|
||||||
for (int j = 0; j < i; j++) {
|
|
||||||
sum += A[ipj] * X[j];
|
|
||||||
++ipj;
|
|
||||||
}
|
|
||||||
X[i] = B[ip] - sum;
|
|
||||||
}
|
|
||||||
|
|
||||||
ipn = n * IPS[n-1] + n - 1;
|
|
||||||
X[n-1] = X[n-1] / A[ipn];
|
|
||||||
|
|
||||||
for (int iback = 1; iback < n; iback++) {
|
|
||||||
//i goes (n-1),...,1
|
|
||||||
int i = nm1 - iback;
|
|
||||||
ip = IPS[i];
|
|
||||||
nip = n*ip;
|
|
||||||
sum = 0.0;
|
|
||||||
aa = A.data()+nip+i+1;
|
|
||||||
for (int j= i + 1; j < n; j++)
|
|
||||||
sum += *aa++ * X[j];
|
|
||||||
X[i] = (X[i] - sum) / A[nip+i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
void AlgRemezGeneral::csv(std::ostream & os) const{
|
|
||||||
os << "Numerator" << std::endl;
|
|
||||||
for(int i=0;i<=pow_n;i++){
|
|
||||||
os << getCoeffNum(i) << "*x^" << i;
|
|
||||||
if(i!=pow_n) os << " + ";
|
|
||||||
}
|
|
||||||
os << std::endl;
|
|
||||||
|
|
||||||
os << "Denominator" << std::endl;
|
|
||||||
for(int i=0;i<=pow_d;i++){
|
|
||||||
os << getCoeffDen(i) << "*x^" << i;
|
|
||||||
if(i!=pow_d) os << " + ";
|
|
||||||
}
|
|
||||||
os << std::endl;
|
|
||||||
|
|
||||||
//For a true minimax solution the errors should all be equal and the signs should oscillate +-+-+- etc
|
|
||||||
int sign;
|
|
||||||
os << "Errors at maxima: coordinate, error, (sign)" << std::endl;
|
|
||||||
for(int i=0;i<neq+1;i++){
|
|
||||||
os << mm[i] << " " << getErr(mm[i],&sign) << " (" << sign << ")" << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
os << "Scan over range:" << std::endl;
|
|
||||||
int npt = 60;
|
|
||||||
bigfloat dlt = (apend - apstrt)/bigfloat(npt-1);
|
|
||||||
|
|
||||||
for (bigfloat x=apstrt; x<=apend; x = x + dlt) {
|
|
||||||
double f = evaluateFunc(x);
|
|
||||||
double r = evaluateApprox(x);
|
|
||||||
os<< x<<","<<r<<","<<f<<","<<r-f<<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
@@ -1,170 +0,0 @@
|
|||||||
/*
|
|
||||||
C.Kelly Jan 2020 based on implementation by M. Clark May 2005
|
|
||||||
|
|
||||||
AlgRemezGeneral is an implementation of the Remez algorithm for approximating an arbitrary function by a rational polynomial
|
|
||||||
It includes optional restriction to odd/even polynomials for the numerator and/or denominator
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifndef INCLUDED_ALG_REMEZ_GENERAL_H
|
|
||||||
#define INCLUDED_ALG_REMEZ_GENERAL_H
|
|
||||||
|
|
||||||
#include <stddef.h>
|
|
||||||
#include <Grid/GridStd.h>
|
|
||||||
|
|
||||||
#ifdef HAVE_LIBGMP
|
|
||||||
#include "bigfloat.h"
|
|
||||||
#else
|
|
||||||
#include "bigfloat_double.h"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
class AlgRemezGeneral{
|
|
||||||
public:
|
|
||||||
enum PolyType { Even, Odd, Full };
|
|
||||||
|
|
||||||
private:
|
|
||||||
|
|
||||||
// In GSL-style, pass the function as a function pointer. Any data required to evaluate the function is passed in as a void pointer
|
|
||||||
bigfloat (*f)(bigfloat x, void *data);
|
|
||||||
void *data;
|
|
||||||
|
|
||||||
// The approximation parameters
|
|
||||||
std::vector<bigfloat> param;
|
|
||||||
bigfloat norm;
|
|
||||||
|
|
||||||
// The number of non-zero terms in the numerator and denominator
|
|
||||||
int n, d;
|
|
||||||
// The numerator and denominator degree (i.e. the largest power)
|
|
||||||
int pow_n, pow_d;
|
|
||||||
|
|
||||||
// Specify if the numerator and/or denominator are odd/even polynomials
|
|
||||||
PolyType num_type;
|
|
||||||
PolyType den_type;
|
|
||||||
std::vector<int> num_pows; //contains the mapping, with -1 if not present
|
|
||||||
std::vector<int> den_pows;
|
|
||||||
|
|
||||||
// The bounds of the approximation
|
|
||||||
bigfloat apstrt, apwidt, apend;
|
|
||||||
|
|
||||||
// Variables used to calculate the approximation
|
|
||||||
int nd1, iter;
|
|
||||||
std::vector<bigfloat> xx;
|
|
||||||
std::vector<bigfloat> mm;
|
|
||||||
std::vector<bigfloat> step;
|
|
||||||
|
|
||||||
bigfloat delta, spread;
|
|
||||||
|
|
||||||
// Variables used in search
|
|
||||||
std::vector<bigfloat> yy;
|
|
||||||
|
|
||||||
// Variables used in solving linear equations
|
|
||||||
std::vector<bigfloat> A;
|
|
||||||
std::vector<bigfloat> B;
|
|
||||||
std::vector<int> IPS;
|
|
||||||
|
|
||||||
// The number of equations we must solve at each iteration (n+d+1)
|
|
||||||
int neq;
|
|
||||||
|
|
||||||
// The precision of the GNU MP library
|
|
||||||
long prec;
|
|
||||||
|
|
||||||
// Initialize member variables associated with the polynomial's properties
|
|
||||||
void setupPolyProperties(int num_degree, int den_degree, PolyType num_type_in, PolyType den_type_in);
|
|
||||||
|
|
||||||
// Initial values of maximal and minmal errors
|
|
||||||
void initialGuess();
|
|
||||||
|
|
||||||
// Initialise step sizes
|
|
||||||
void stpini();
|
|
||||||
|
|
||||||
// Initialize the algorithm
|
|
||||||
void reinitializeAlgorithm();
|
|
||||||
|
|
||||||
// Solve the equations
|
|
||||||
void equations();
|
|
||||||
|
|
||||||
// Search for error maxima and minima
|
|
||||||
void search();
|
|
||||||
|
|
||||||
// Calculate function required for the approximation
|
|
||||||
inline bigfloat func(bigfloat x) const{
|
|
||||||
return f(x, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute size and sign of the approximation error at x
|
|
||||||
bigfloat getErr(bigfloat x, int *sign) const;
|
|
||||||
|
|
||||||
// Solve the system AX=B where X = param
|
|
||||||
int simq();
|
|
||||||
|
|
||||||
// Evaluate the rational form P(x)/Q(x) using coefficients from the solution vector param
|
|
||||||
bigfloat approx(bigfloat x) const;
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
AlgRemezGeneral(double lower, double upper, long prec,
|
|
||||||
bigfloat (*f)(bigfloat x, void *data), void *data);
|
|
||||||
|
|
||||||
inline int getDegree(void) const{
|
|
||||||
GRID_ASSERT(n==d);
|
|
||||||
return n;
|
|
||||||
}
|
|
||||||
// Reset the bounds of the approximation
|
|
||||||
inline void setBounds(double lower, double upper) {
|
|
||||||
apstrt = lower;
|
|
||||||
apend = upper;
|
|
||||||
apwidt = apend - apstrt;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the bounds of the approximation
|
|
||||||
inline void getBounds(double &lower, double &upper) const{
|
|
||||||
lower=(double)apstrt;
|
|
||||||
upper=(double)apend;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run the algorithm to generate the rational approximation
|
|
||||||
double generateApprox(int num_degree, int den_degree,
|
|
||||||
PolyType num_type, PolyType den_type,
|
|
||||||
const double tolerance = 1e-15, const int report_freq = 1000);
|
|
||||||
|
|
||||||
inline double generateApprox(int num_degree, int den_degree,
|
|
||||||
const double tolerance = 1e-15, const int report_freq = 1000){
|
|
||||||
return generateApprox(num_degree, den_degree, Full, Full, tolerance, report_freq);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evaluate the rational form P(x)/Q(x) using coefficients from the
|
|
||||||
// solution vector param
|
|
||||||
inline double evaluateApprox(double x) const{
|
|
||||||
return (double)approx((bigfloat)x);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evaluate the rational form Q(x)/P(x) using coefficients from the solution vector param
|
|
||||||
inline double evaluateInverseApprox(double x) const{
|
|
||||||
return 1.0/(double)approx((bigfloat)x);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate function required for the approximation
|
|
||||||
inline double evaluateFunc(double x) const{
|
|
||||||
return (double)func((bigfloat)x);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate inverse function required for the approximation
|
|
||||||
inline double evaluateInverseFunc(double x) const{
|
|
||||||
return 1.0/(double)func((bigfloat)x);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dump csv of function, approx and error
|
|
||||||
void csv(std::ostream &os = std::cout) const;
|
|
||||||
|
|
||||||
// Get the coefficient of the term x^i in the numerator
|
|
||||||
inline double getCoeffNum(const int i) const{
|
|
||||||
return num_pows[i] == -1 ? 0. : double(param[num_pows[i]]);
|
|
||||||
}
|
|
||||||
// Get the coefficient of the term x^i in the denominator
|
|
||||||
inline double getCoeffDen(const int i) const{
|
|
||||||
if(i == pow_d) return 1.0;
|
|
||||||
else return den_pows[i] == -1 ? 0. : double(param[den_pows[i]+n+1]);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,183 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/approx/ZMobius.cc
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#include <Grid/algorithms/approx/ZMobius.h>
|
|
||||||
#include <Grid/algorithms/approx/RemezGeneral.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
NAMESPACE_BEGIN(Approx);
|
|
||||||
|
|
||||||
//Compute the tanh approximation
|
|
||||||
inline double epsilonMobius(const double x, const std::vector<ComplexD> &w){
|
|
||||||
int Ls = w.size();
|
|
||||||
|
|
||||||
ComplexD fxp = 1., fmp = 1.;
|
|
||||||
for(int i=0;i<Ls;i++){
|
|
||||||
fxp = fxp * ( w[i] + x );
|
|
||||||
fmp = fmp * ( w[i] - x );
|
|
||||||
}
|
|
||||||
return ((fxp - fmp)/(fxp + fmp)).real();
|
|
||||||
}
|
|
||||||
inline double epsilonMobius(const double x, const std::vector<RealD> &w){
|
|
||||||
int Ls = w.size();
|
|
||||||
|
|
||||||
double fxp = 1., fmp = 1.;
|
|
||||||
for(int i=0;i<Ls;i++){
|
|
||||||
fxp = fxp * ( w[i] + x );
|
|
||||||
fmp = fmp * ( w[i] - x );
|
|
||||||
}
|
|
||||||
return (fxp - fmp)/(fxp + fmp);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
//Compute the tanh approximation in a form suitable for the Remez
|
|
||||||
bigfloat epsilonMobius(bigfloat x, void* data){
|
|
||||||
const std::vector<RealD> &omega = *( (std::vector<RealD> const*)data );
|
|
||||||
bigfloat fxp(1.0);
|
|
||||||
bigfloat fmp(1.0);
|
|
||||||
|
|
||||||
for(int i=0;i<omega.size();i++){
|
|
||||||
fxp = fxp * ( bigfloat(omega[i]) + x);
|
|
||||||
fmp = fmp * ( bigfloat(omega[i]) - x);
|
|
||||||
}
|
|
||||||
return (fxp - fmp)/(fxp + fmp);
|
|
||||||
}
|
|
||||||
|
|
||||||
//Compute the Zmobius Omega parameters suitable for eigenvalue range -lambda_bound <= lambda <= lambda_bound
|
|
||||||
//Note omega_i = 1/(b_i + c_i) where b_i and c_i are the Mobius parameters
|
|
||||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out,
|
|
||||||
const std::vector<RealD> &omega_in, const int Ls_in,
|
|
||||||
const RealD lambda_bound){
|
|
||||||
GRID_ASSERT(omega_in.size() == Ls_in);
|
|
||||||
omega_out.resize(Ls_out);
|
|
||||||
|
|
||||||
//Use the Remez algorithm to generate the appropriate rational polynomial
|
|
||||||
//For odd polynomial, to satisfy Haar condition must take either positive or negative half of range (cf https://arxiv.org/pdf/0803.0439.pdf page 6)
|
|
||||||
AlgRemezGeneral remez(0, lambda_bound, 64, &epsilonMobius, (void*)&omega_in);
|
|
||||||
remez.generateApprox(Ls_out-1, Ls_out,AlgRemezGeneral::Odd, AlgRemezGeneral::Even, 1e-15, 100);
|
|
||||||
remez.csv(std::cout);
|
|
||||||
|
|
||||||
//The rational approximation has the form [ f(x) - f(-x) ] / [ f(x) + f(-x) ] where f(x) = \Prod_{i=0}^{L_s-1} ( \omega_i + x )
|
|
||||||
//cf https://academiccommons.columbia.edu/doi/10.7916/D8T72HD7 pg 102
|
|
||||||
//omega_i are therefore the negative of the complex roots of f(x)
|
|
||||||
|
|
||||||
//We can find the roots by recognizing that the eigenvalues of a matrix A are the roots of the characteristic polynomial
|
|
||||||
// \rho(\lambda) = det( A - \lambda I ) where I is the unit matrix
|
|
||||||
//The matrix whose characteristic polynomial is an arbitrary monic polynomial a0 + a1 x + a2 x^2 + ... x^n is the companion matrix
|
|
||||||
// A = | 0 1 0 0 0 .... 0 |
|
|
||||||
// | 0 0 1 0 0 .... 0 |
|
|
||||||
// | : : : : : : |
|
|
||||||
// | 0 0 0 0 0 1
|
|
||||||
// | -a0 -a1 -a2 ... ... -an|
|
|
||||||
|
|
||||||
|
|
||||||
//Note the Remez defines the largest power to have unit coefficient
|
|
||||||
std::vector<RealD> coeffs(Ls_out+1);
|
|
||||||
for(int i=0;i<Ls_out+1;i+=2) coeffs[i] = coeffs[i] = remez.getCoeffDen(i); //even powers
|
|
||||||
for(int i=1;i<Ls_out+1;i+=2) coeffs[i] = coeffs[i] = remez.getCoeffNum(i); //odd powers
|
|
||||||
|
|
||||||
std::vector<std::complex<RealD> > roots(Ls_out);
|
|
||||||
|
|
||||||
//Form the companion matrix
|
|
||||||
Eigen::MatrixXd compn(Ls_out,Ls_out);
|
|
||||||
for(int i=0;i<Ls_out-1;i++) compn(i,0) = 0.;
|
|
||||||
compn(Ls_out - 1, 0) = -coeffs[0];
|
|
||||||
|
|
||||||
for(int j=1;j<Ls_out;j++){
|
|
||||||
for(int i=0;i<Ls_out-1;i++) compn(i,j) = i == j-1 ? 1. : 0.;
|
|
||||||
compn(Ls_out - 1, j) = -coeffs[j];
|
|
||||||
}
|
|
||||||
|
|
||||||
//Eigensolve
|
|
||||||
Eigen::EigenSolver<Eigen::MatrixXd> slv(compn, false);
|
|
||||||
|
|
||||||
const auto & ev = slv.eigenvalues();
|
|
||||||
for(int i=0;i<Ls_out;i++)
|
|
||||||
omega_out[i] = -ev(i);
|
|
||||||
|
|
||||||
//Sort ascending (smallest at start of vector!)
|
|
||||||
std::sort(omega_out.begin(), omega_out.end(),
|
|
||||||
[&](const ComplexD &a, const ComplexD &b){ return a.real() < b.real() || (a.real() == b.real() && a.imag() < b.imag()); });
|
|
||||||
|
|
||||||
//McGlynn thesis pg 122 suggest improved iteration counts if magnitude of omega diminishes towards the center of the 5th dimension
|
|
||||||
std::vector<ComplexD> omega_tmp = omega_out;
|
|
||||||
int s_low=0, s_high=Ls_out-1, ss=0;
|
|
||||||
for(int s_from = Ls_out-1; s_from >= 0; s_from--){ //loop from largest omega
|
|
||||||
int s_to;
|
|
||||||
if(ss % 2 == 0){
|
|
||||||
s_to = s_low++;
|
|
||||||
}else{
|
|
||||||
s_to = s_high--;
|
|
||||||
}
|
|
||||||
omega_out[s_to] = omega_tmp[s_from];
|
|
||||||
++ss;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << "Resulting omega_i:" << std::endl;
|
|
||||||
for(int i=0;i<Ls_out;i++)
|
|
||||||
std::cout << omega_out[i] << std::endl;
|
|
||||||
|
|
||||||
std::cout << "Test result matches the approximate polynomial found by the Remez" << std::endl;
|
|
||||||
std::cout << "<x> <remez approx> <poly approx> <diff poly approx remez approx> <exact> <diff poly approx exact>\n";
|
|
||||||
|
|
||||||
int npt = 60;
|
|
||||||
double dlt = lambda_bound/double(npt-1);
|
|
||||||
|
|
||||||
for (int i =0; i<npt; i++){
|
|
||||||
double x = i*dlt;
|
|
||||||
double r = remez.evaluateApprox(x);
|
|
||||||
double p = epsilonMobius(x, omega_out);
|
|
||||||
double e = epsilonMobius(x, omega_in);
|
|
||||||
|
|
||||||
std::cout << x<< " " << r << " " << p <<" " <<r-p << " " << e << " " << e-p << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
//mobius_param = b+c with b-c=1
|
|
||||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound){
|
|
||||||
std::vector<RealD> omega_in(Ls_in, 1./mobius_param);
|
|
||||||
computeZmobiusOmega(omega_out, Ls_out, omega_in, Ls_in, lambda_bound);
|
|
||||||
}
|
|
||||||
|
|
||||||
//ZMobius class takes gamma_i = (b+c) omega_i as its input, where b, c are factored out
|
|
||||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out,
|
|
||||||
const RealD mobius_param_out, const int Ls_out,
|
|
||||||
const RealD mobius_param_in, const int Ls_in,
|
|
||||||
const RealD lambda_bound){
|
|
||||||
computeZmobiusOmega(gamma_out, Ls_out, mobius_param_in, Ls_in, lambda_bound);
|
|
||||||
for(int i=0;i<Ls_out;i++) gamma_out[i] = gamma_out[i] * mobius_param_out;
|
|
||||||
}
|
|
||||||
//Assumes mobius_param_out == mobius_param_in
|
|
||||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound){
|
|
||||||
computeZmobiusGamma(gamma_out, mobius_param, Ls_out, mobius_param, Ls_in, lambda_bound);
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Approx);
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/approx/ZMobius.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_ZMOBIUS_APPROX_H
|
|
||||||
#define GRID_ZMOBIUS_APPROX_H
|
|
||||||
|
|
||||||
#include <Grid/GridCore.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
NAMESPACE_BEGIN(Approx);
|
|
||||||
|
|
||||||
//Compute the Zmobius Omega parameters suitable for eigenvalue range -lambda_bound <= lambda <= lambda_bound
|
|
||||||
//Note omega_i = 1/(b_i + c_i) where b_i and c_i are the Mobius parameters
|
|
||||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out,
|
|
||||||
const std::vector<RealD> &omega_in, const int Ls_in,
|
|
||||||
const RealD lambda_bound);
|
|
||||||
|
|
||||||
//mobius_param = b+c with b-c=1
|
|
||||||
void computeZmobiusOmega(std::vector<ComplexD> &omega_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound);
|
|
||||||
|
|
||||||
//ZMobius class takes gamma_i = (b+c) omega_i as its input, where b, c are factored out
|
|
||||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out,
|
|
||||||
const RealD mobius_param_out, const int Ls_out,
|
|
||||||
const RealD mobius_param_in, const int Ls_in,
|
|
||||||
const RealD lambda_bound);
|
|
||||||
|
|
||||||
//Assumes mobius_param_out == mobius_param_in
|
|
||||||
void computeZmobiusGamma(std::vector<ComplexD> &gamma_out, const int Ls_out, const RealD mobius_param, const int Ls_in, const RealD lambda_bound);
|
|
||||||
|
|
||||||
NAMESPACE_END(Approx);
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: BatchedBlas.h
|
|
||||||
|
|
||||||
Copyright (C) 2023
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#include <Grid/GridCore.h>
|
|
||||||
#include <Grid/algorithms/blas/BatchedBlas.h>
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
gridblasHandle_t GridBLAS::gridblasHandle;
|
|
||||||
int GridBLAS::gridblasInit;
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,300 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: MomentumProject.h
|
|
||||||
|
|
||||||
Copyright (C) 2025
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
/*
|
|
||||||
MultiMomProject
|
|
||||||
|
|
||||||
Import vectors -> nxyz x (ncomponent x nt)
|
|
||||||
Import complex phases -> nmom x nxy
|
|
||||||
|
|
||||||
apply = via (possibly batched) GEMM
|
|
||||||
*/
|
|
||||||
template<class Field, class ComplexField>
|
|
||||||
class MomentumProject
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
typedef typename Field::scalar_type scalar;
|
|
||||||
typedef typename Field::scalar_object scalar_object;
|
|
||||||
|
|
||||||
GridBase *grid;
|
|
||||||
uint64_t nmom;
|
|
||||||
uint64_t nxyz;
|
|
||||||
uint64_t nt;
|
|
||||||
uint64_t nbtw;
|
|
||||||
uint64_t words;
|
|
||||||
|
|
||||||
deviceVector<scalar> BLAS_V; //
|
|
||||||
deviceVector<scalar> BLAS_M; //
|
|
||||||
deviceVector<scalar> BLAS_P; //
|
|
||||||
|
|
||||||
MomentumProject(){};
|
|
||||||
~MomentumProject(){ Deallocate(); };
|
|
||||||
|
|
||||||
void Deallocate(void)
|
|
||||||
{
|
|
||||||
grid=nullptr;
|
|
||||||
nmom=0;
|
|
||||||
nxyz=0;
|
|
||||||
nt=0;
|
|
||||||
nbtw=0;
|
|
||||||
words=0;
|
|
||||||
BLAS_V.resize(0);
|
|
||||||
BLAS_M.resize(0);
|
|
||||||
BLAS_P.resize(0);
|
|
||||||
}
|
|
||||||
void Allocate(int _nmom,GridBase *_grid)
|
|
||||||
{
|
|
||||||
grid=_grid;
|
|
||||||
Coordinate ldims = grid->LocalDimensions();
|
|
||||||
|
|
||||||
nmom=_nmom;
|
|
||||||
nt = ldims[grid->Nd()-1];
|
|
||||||
nxyz = grid->lSites()/nt;
|
|
||||||
words = sizeof(scalar_object)/sizeof(scalar);
|
|
||||||
nbtw = nt * words;
|
|
||||||
|
|
||||||
BLAS_V.resize (nxyz * nt * words );
|
|
||||||
BLAS_M.resize (nmom * nxyz );
|
|
||||||
BLAS_P.resize (nmom * nt * words );
|
|
||||||
}
|
|
||||||
void ImportMomenta(const std::vector <ComplexField> &momenta)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(momenta.size()==nmom);
|
|
||||||
// might as well just make the momenta here
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
|
|
||||||
int nd = grid->_ndimension;
|
|
||||||
|
|
||||||
uint64_t sz = BLAS_M.size();
|
|
||||||
|
|
||||||
GRID_ASSERT(momenta.size()==nmom)
|
|
||||||
GRID_ASSERT(momenta[0].Grid()==grid);
|
|
||||||
GRID_ASSERT(sz = nxyz * nmom);
|
|
||||||
|
|
||||||
Coordinate rdimensions = grid->_rdimensions;
|
|
||||||
Coordinate ldims = grid->LocalDimensions();
|
|
||||||
int64_t osites = grid->oSites();
|
|
||||||
Coordinate simd = grid->_simd_layout;
|
|
||||||
const int Nsimd = vobj::Nsimd();
|
|
||||||
uint64_t lwords = words; // local variable for copy in to GPU
|
|
||||||
int64_t Nxyz = nxyz;
|
|
||||||
auto blasData_p = &BLAS_M[0];
|
|
||||||
for(int m=0;m<momenta.size();m++){
|
|
||||||
|
|
||||||
autoView( Data , momenta[m], AcceleratorRead);
|
|
||||||
auto Data_p = &Data[0];
|
|
||||||
|
|
||||||
accelerator_for(xyz,nxyz,1,{
|
|
||||||
//////////////////////////////////////////
|
|
||||||
// isite -- map lane within buffer to lane within lattice
|
|
||||||
////////////////////////////////////////////
|
|
||||||
Coordinate lcoor(nd,0);
|
|
||||||
Lexicographic::CoorFromIndex(lcoor,xyz,ldims);
|
|
||||||
|
|
||||||
Coordinate icoor(nd);
|
|
||||||
Coordinate ocoor(nd);
|
|
||||||
for (int d = 0; d < nd; d++) {
|
|
||||||
icoor[d] = lcoor[d]/rdimensions[d];
|
|
||||||
ocoor[d] = lcoor[d]%rdimensions[d];
|
|
||||||
}
|
|
||||||
int64_t osite;
|
|
||||||
int64_t isite;
|
|
||||||
Lexicographic::IndexFromCoor(ocoor,osite,rdimensions);
|
|
||||||
Lexicographic::IndexFromCoor(icoor,isite,simd);
|
|
||||||
|
|
||||||
// BLAS_M[nmom][slice_vol]
|
|
||||||
// Fortran Column major BLAS layout is M_xyz,mom
|
|
||||||
scalar data = extractLane(isite,Data[osite]);
|
|
||||||
uint64_t idx = xyz+m*Nxyz;
|
|
||||||
blasData_p[idx] = data;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void ImportVector(Field &vec)
|
|
||||||
{
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
|
|
||||||
int nd = grid->_ndimension;
|
|
||||||
|
|
||||||
uint64_t sz = BLAS_V.size();
|
|
||||||
|
|
||||||
GRID_ASSERT(sz = nxyz * words * nt);
|
|
||||||
|
|
||||||
Coordinate rdimensions = grid->_rdimensions;
|
|
||||||
Coordinate ldims= grid->LocalDimensions();
|
|
||||||
int64_t osites = grid->oSites();
|
|
||||||
Coordinate simd = grid->_simd_layout;
|
|
||||||
const int Nsimd = vobj::Nsimd();
|
|
||||||
uint64_t lwords= words; // local variable for copy in to GPU
|
|
||||||
|
|
||||||
auto blasData_p = &BLAS_V[0];
|
|
||||||
autoView( Data , vec, AcceleratorRead);
|
|
||||||
auto Data_p = &Data[0];
|
|
||||||
|
|
||||||
int64_t nwords = words;// for capture
|
|
||||||
int64_t Nt = nt;// for capture
|
|
||||||
|
|
||||||
accelerator_for(sf,osites,Nsimd,{
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<Nsimd;lane++) {
|
|
||||||
#endif
|
|
||||||
//////////////////////////////////////////
|
|
||||||
// isite -- map lane within buffer to lane within lattice
|
|
||||||
////////////////////////////////////////////
|
|
||||||
Coordinate lcoor(nd,0);
|
|
||||||
Coordinate icoor(nd);
|
|
||||||
Coordinate ocoor(nd);
|
|
||||||
|
|
||||||
Lexicographic::CoorFromIndex(icoor,lane,simd);
|
|
||||||
Lexicographic::CoorFromIndex(ocoor,sf,rdimensions);
|
|
||||||
|
|
||||||
|
|
||||||
int64_t l_xyz = 0;
|
|
||||||
for (int d = 0; d < nd; d++) {
|
|
||||||
lcoor[d] = rdimensions[d]*icoor[d] + ocoor[d];
|
|
||||||
}
|
|
||||||
uint64_t l_t = lcoor[nd-1];
|
|
||||||
|
|
||||||
Coordinate xyz_coor = lcoor;
|
|
||||||
xyz_coor[nd-1] =0;
|
|
||||||
Lexicographic::IndexFromCoor(xyz_coor,l_xyz,ldims);
|
|
||||||
|
|
||||||
|
|
||||||
scalar_object data = extractLane(lane,Data[sf]);
|
|
||||||
scalar *data_words = (scalar *) &data;
|
|
||||||
for(int w = 0 ; w < nwords; w++) {
|
|
||||||
// BLAS_V[slice_vol][nt][words]
|
|
||||||
// Fortran Column major BLAS layout is V_(t,w)_xyz
|
|
||||||
uint64_t idx = w+l_t*nwords + l_xyz * nwords * Nt;
|
|
||||||
blasData_p[idx] = data_words[w];
|
|
||||||
}
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
}
|
|
||||||
void ExportMomentumProjection(std::vector<typename Field::scalar_object> &projection)
|
|
||||||
{
|
|
||||||
projection.resize(nmom*nt);
|
|
||||||
acceleratorCopyFromDevice(&BLAS_P[0],(scalar *)&projection[0],BLAS_P.size()*sizeof(scalar));
|
|
||||||
// Could decide on a layout late?
|
|
||||||
}
|
|
||||||
|
|
||||||
// Row major layout "C" order:
|
|
||||||
// BLAS_V[slice_vol][nt][words]
|
|
||||||
// BLAS_M[nmom][slice_vol]
|
|
||||||
// BLAS_P[nmom][nt][words]
|
|
||||||
//
|
|
||||||
// Fortran Column major BLAS layout is V_(w,t)_xyz
|
|
||||||
// Fortran Column major BLAS layout is M_xyz,mom
|
|
||||||
// Fortran Column major BLAS layout is P_(w,t),mom
|
|
||||||
//
|
|
||||||
// Projected
|
|
||||||
//
|
|
||||||
// P = (V * M)_(w,t),mom
|
|
||||||
//
|
|
||||||
void Project(Field &data,std::vector< typename Field::scalar_object > & projected_gdata)
|
|
||||||
{
|
|
||||||
double t_import=0;
|
|
||||||
double t_export=0;
|
|
||||||
double t_gemm =0;
|
|
||||||
double t_allreduce=0;
|
|
||||||
t_import-=usecond();
|
|
||||||
this->ImportVector(data);
|
|
||||||
|
|
||||||
std::vector< typename Field::scalar_object > projected_planes;
|
|
||||||
|
|
||||||
deviceVector<scalar *> Vd(1);
|
|
||||||
deviceVector<scalar *> Md(1);
|
|
||||||
deviceVector<scalar *> Pd(1);
|
|
||||||
|
|
||||||
scalar * Vh = & BLAS_V[0];
|
|
||||||
scalar * Mh = & BLAS_M[0];
|
|
||||||
scalar * Ph = & BLAS_P[0];
|
|
||||||
|
|
||||||
acceleratorPut(Vd[0],Vh);
|
|
||||||
acceleratorPut(Md[0],Mh);
|
|
||||||
acceleratorPut(Pd[0],Ph);
|
|
||||||
t_import+=usecond();
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// P_im = VMmx . Vxi
|
|
||||||
/////////////////////////////////////////
|
|
||||||
t_gemm-=usecond();
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
|
||||||
words*nt,nmom,nxyz,
|
|
||||||
scalar(1.0),
|
|
||||||
Vd,
|
|
||||||
Md,
|
|
||||||
scalar(0.0), // wipe out result
|
|
||||||
Pd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
t_gemm+=usecond();
|
|
||||||
|
|
||||||
t_export-=usecond();
|
|
||||||
ExportMomentumProjection(projected_planes); // resizes
|
|
||||||
t_export+=usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////
|
|
||||||
// Reduce across MPI ranks
|
|
||||||
/////////////////////////////////
|
|
||||||
int nd = grid->Nd();
|
|
||||||
int gt = grid->GlobalDimensions()[nd-1];
|
|
||||||
int lt = grid->LocalDimensions()[nd-1];
|
|
||||||
projected_gdata.resize(gt*nmom);
|
|
||||||
for(int t=0;t<gt*nmom;t++){ // global Nt array with zeroes for stuff not on this node
|
|
||||||
projected_gdata[t]=Zero();
|
|
||||||
}
|
|
||||||
for(int t=0;t<lt;t++){
|
|
||||||
for(int m=0;m<nmom;m++){
|
|
||||||
int st = grid->LocalStarts()[nd-1];
|
|
||||||
projected_gdata[t+st + gt*m] = projected_planes[t+lt*m];
|
|
||||||
}}
|
|
||||||
t_allreduce-=usecond();
|
|
||||||
grid->GlobalSumVector((scalar *)&projected_gdata[0],gt*nmom*words);
|
|
||||||
t_allreduce+=usecond();
|
|
||||||
|
|
||||||
std::cout << GridLogPerformance<<" MomentumProject t_import "<<t_import<<"us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<" MomentumProject t_export "<<t_export<<"us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<" MomentumProject t_gemm "<<t_gemm<<"us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<" MomentumProject t_reduce "<<t_allreduce<<"us"<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_DEFLATION_H
|
|
||||||
#define GRID_DEFLATION_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class ZeroGuesser: public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
virtual void operator()(const Field &src, Field &guess) { guess = Zero(); };
|
|
||||||
};
|
|
||||||
template<class Field>
|
|
||||||
class DoNothingGuesser: public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
virtual void operator()(const Field &src, Field &guess) { };
|
|
||||||
};
|
|
||||||
template<class Field>
|
|
||||||
class SourceGuesser: public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
virtual void operator()(const Field &src, Field &guess) { guess = src; };
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
// Fine grid deflation
|
|
||||||
////////////////////////////////
|
|
||||||
template<class Field>
|
|
||||||
class DeflatedGuesser: public LinearFunction<Field> {
|
|
||||||
private:
|
|
||||||
const std::vector<Field> &evec;
|
|
||||||
const std::vector<RealD> &eval;
|
|
||||||
const unsigned int N;
|
|
||||||
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
|
|
||||||
DeflatedGuesser(const std::vector<Field> & _evec,const std::vector<RealD> & _eval)
|
|
||||||
: DeflatedGuesser(_evec, _eval, _evec.size())
|
|
||||||
{}
|
|
||||||
|
|
||||||
DeflatedGuesser(const std::vector<Field> & _evec, const std::vector<RealD> & _eval, const unsigned int _N)
|
|
||||||
: evec(_evec), eval(_eval), N(_N)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(evec.size()==eval.size());
|
|
||||||
GRID_ASSERT(N <= evec.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void operator()(const Field &src,Field &guess) {
|
|
||||||
guess = Zero();
|
|
||||||
for (int i=0;i<N;i++) {
|
|
||||||
const Field& tmp = evec[i];
|
|
||||||
axpy(guess,TensorRemove(innerProduct(tmp,src)) / eval[i],tmp,guess);
|
|
||||||
}
|
|
||||||
guess.Checkerboard() = src.Checkerboard();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class FineField, class CoarseField>
|
|
||||||
class LocalCoherenceDeflatedGuesser: public LinearFunction<FineField> {
|
|
||||||
private:
|
|
||||||
const std::vector<FineField> &subspace;
|
|
||||||
const std::vector<CoarseField> &evec_coarse;
|
|
||||||
const std::vector<RealD> &eval_coarse;
|
|
||||||
public:
|
|
||||||
|
|
||||||
using LinearFunction<FineField>::operator();
|
|
||||||
LocalCoherenceDeflatedGuesser(const std::vector<FineField> &_subspace,
|
|
||||||
const std::vector<CoarseField> &_evec_coarse,
|
|
||||||
const std::vector<RealD> &_eval_coarse)
|
|
||||||
: subspace(_subspace),
|
|
||||||
evec_coarse(_evec_coarse),
|
|
||||||
eval_coarse(_eval_coarse)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator()(const FineField &src,FineField &guess) {
|
|
||||||
int N = (int)evec_coarse.size();
|
|
||||||
CoarseField src_coarse(evec_coarse[0].Grid());
|
|
||||||
CoarseField guess_coarse(evec_coarse[0].Grid()); guess_coarse = Zero();
|
|
||||||
blockProject(src_coarse,src,subspace);
|
|
||||||
for (int i=0;i<N;i++) {
|
|
||||||
const CoarseField & tmp = evec_coarse[i];
|
|
||||||
axpy(guess_coarse,TensorRemove(innerProduct(tmp,src_coarse)) / eval_coarse[i],tmp,guess_coarse);
|
|
||||||
}
|
|
||||||
blockPromote(guess_coarse,guess,subspace);
|
|
||||||
guess.Checkerboard() = src.Checkerboard();
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator()(const std::vector<FineField> &src,std::vector<FineField> &guess) {
|
|
||||||
int Nevec = (int)evec_coarse.size();
|
|
||||||
int Nsrc = (int)src.size();
|
|
||||||
// make temp variables
|
|
||||||
std::vector<CoarseField> src_coarse(Nsrc,evec_coarse[0].Grid());
|
|
||||||
std::vector<CoarseField> guess_coarse(Nsrc,evec_coarse[0].Grid());
|
|
||||||
//Preporcessing
|
|
||||||
std::cout << GridLogMessage << "Start BlockProject for loop" << std::endl;
|
|
||||||
for (int j=0;j<Nsrc;j++)
|
|
||||||
{
|
|
||||||
guess_coarse[j] = Zero();
|
|
||||||
std::cout << GridLogMessage << "BlockProject iter: " << j << std::endl;
|
|
||||||
blockProject(src_coarse[j],src[j],subspace);
|
|
||||||
}
|
|
||||||
//deflation set up for eigen vector batchsize 1 and source batch size equal number of sources
|
|
||||||
std::cout << GridLogMessage << "Start ProjectAccum for loop" << std::endl;
|
|
||||||
for (int i=0;i<Nevec;i++)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage << "ProjectAccum Nvec: " << i << std::endl;
|
|
||||||
const CoarseField & tmp = evec_coarse[i];
|
|
||||||
for (int j=0;j<Nsrc;j++)
|
|
||||||
{
|
|
||||||
axpy(guess_coarse[j],TensorRemove(innerProduct(tmp,src_coarse[j])) / eval_coarse[i],tmp,guess_coarse[j]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
//postprocessing
|
|
||||||
std::cout << GridLogMessage << "Start BlockPromote for loop" << std::endl;
|
|
||||||
for (int j=0;j<Nsrc;j++) {
|
|
||||||
std::cout << GridLogMessage << "BlockProject iter: " << j << std::endl;
|
|
||||||
blockPromote(guess_coarse[j],guess[j],subspace);
|
|
||||||
guess[j].Checkerboard() = src[j].Checkerboard();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
@@ -1,376 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: MultiRHSBlockCGLinalg.h
|
|
||||||
|
|
||||||
Copyright (C) 2024
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
/* Need helper object for BLAS accelerated mrhs blockCG */
|
|
||||||
template<class Field>
|
|
||||||
class MultiRHSBlockCGLinalg
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
typedef typename Field::scalar_type scalar;
|
|
||||||
typedef typename Field::scalar_object scalar_object;
|
|
||||||
typedef typename Field::vector_object vector_object;
|
|
||||||
|
|
||||||
deviceVector<scalar> BLAS_X; // nrhs x vol -- the sources
|
|
||||||
deviceVector<scalar> BLAS_Y; // nrhs x vol -- the result
|
|
||||||
deviceVector<scalar> BLAS_C; // nrhs x nrhs -- the coefficients
|
|
||||||
deviceVector<scalar> BLAS_Cred; // nrhs x nrhs x oSites -- reduction buffer
|
|
||||||
deviceVector<scalar *> Xdip;
|
|
||||||
deviceVector<scalar *> Ydip;
|
|
||||||
deviceVector<scalar *> Cdip;
|
|
||||||
|
|
||||||
MultiRHSBlockCGLinalg() {};
|
|
||||||
~MultiRHSBlockCGLinalg(){ Deallocate(); };
|
|
||||||
|
|
||||||
void Deallocate(void)
|
|
||||||
{
|
|
||||||
Xdip.resize(0);
|
|
||||||
Ydip.resize(0);
|
|
||||||
Cdip.resize(0);
|
|
||||||
BLAS_Cred.resize(0);
|
|
||||||
BLAS_C.resize(0);
|
|
||||||
BLAS_X.resize(0);
|
|
||||||
BLAS_Y.resize(0);
|
|
||||||
}
|
|
||||||
void MaddMatrix(std::vector<Field> &AP, Eigen::MatrixXcd &m , const std::vector<Field> &X,const std::vector<Field> &Y,RealD scale=1.0)
|
|
||||||
{
|
|
||||||
std::vector<Field> Y_copy(AP.size(),AP[0].Grid());
|
|
||||||
for(int r=0;r<AP.size();r++){
|
|
||||||
Y_copy[r] = Y[r];
|
|
||||||
}
|
|
||||||
MulMatrix(AP,m,X);
|
|
||||||
for(int r=0;r<AP.size();r++){
|
|
||||||
AP[r] = scale*AP[r]+Y_copy[r];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void MulMatrix(std::vector<Field> &Y, Eigen::MatrixXcd &m , const std::vector<Field> &X)
|
|
||||||
{
|
|
||||||
typedef typename Field::scalar_type scomplex;
|
|
||||||
GridBase *grid;
|
|
||||||
uint64_t vol;
|
|
||||||
uint64_t words;
|
|
||||||
|
|
||||||
int nrhs = Y.size();
|
|
||||||
grid = X[0].Grid();
|
|
||||||
vol = grid->lSites();
|
|
||||||
words = sizeof(scalar_object)/sizeof(scalar);
|
|
||||||
int64_t vw = vol * words;
|
|
||||||
|
|
||||||
RealD t0 = usecond();
|
|
||||||
BLAS_X.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_Y.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_C.resize(nrhs * nrhs);// cost free if size doesn't change
|
|
||||||
RealD t1 = usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// Copy in the multi-rhs sources
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int64_t offset = r*vw;
|
|
||||||
autoView(x_v,X[r],AcceleratorRead);
|
|
||||||
acceleratorCopyDeviceToDevice(&x_v[0],&BLAS_X[offset],sizeof(scalar_object)*vol);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assumes Eigen storage contiguous
|
|
||||||
acceleratorCopyToDevice(&m(0,0),&BLAS_C[0],BLAS_C.size()*sizeof(scalar));
|
|
||||||
|
|
||||||
/*
|
|
||||||
* in Fortran column major notation (cuBlas order)
|
|
||||||
*
|
|
||||||
* Xxr = [X1(x)][..][Xn(x)]
|
|
||||||
* Yxr = [Y1(x)][..][Ym(x)]
|
|
||||||
* Y = X . C
|
|
||||||
*/
|
|
||||||
deviceVector<scalar *> Xd(1);
|
|
||||||
deviceVector<scalar *> Yd(1);
|
|
||||||
deviceVector<scalar *> Cd(1);
|
|
||||||
|
|
||||||
scalar * Xh = & BLAS_X[0];
|
|
||||||
scalar * Yh = & BLAS_Y[0];
|
|
||||||
scalar * Ch = & BLAS_C[0];
|
|
||||||
|
|
||||||
acceleratorPut(Xd[0],Xh);
|
|
||||||
acceleratorPut(Yd[0],Yh);
|
|
||||||
acceleratorPut(Cd[0],Ch);
|
|
||||||
|
|
||||||
RealD t2 = usecond();
|
|
||||||
GridBLAS BLAS;
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// Y = X*C (transpose?)
|
|
||||||
/////////////////////////////////////////
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
|
||||||
vw,nrhs,nrhs,
|
|
||||||
scalar(1.0),
|
|
||||||
Xd,
|
|
||||||
Cd,
|
|
||||||
scalar(0.0), // wipe out Y
|
|
||||||
Yd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
RealD t3 = usecond();
|
|
||||||
|
|
||||||
// Copy back Y = m X
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int64_t offset = r*vw;
|
|
||||||
autoView(y_v,Y[r],AcceleratorWrite);
|
|
||||||
acceleratorCopyDeviceToDevice(&BLAS_Y[offset],&y_v[0],sizeof(scalar_object)*vol);
|
|
||||||
}
|
|
||||||
RealD t4 = usecond();
|
|
||||||
std::cout <<GridLogPerformance << "MulMatrix alloc took "<< t1-t0<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "MulMatrix preamble took "<< t2-t1<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "MulMatrix blas took "<< t3-t2<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "MulMatrix copy took "<< t4-t3<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "MulMatrix total "<< t4-t0<<" us"<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
void InnerProductMatrix(Eigen::MatrixXcd &m , const std::vector<Field> &X, const std::vector<Field> &Y)
|
|
||||||
{
|
|
||||||
#if 0
|
|
||||||
int nrhs;
|
|
||||||
GridBase *grid;
|
|
||||||
uint64_t vol;
|
|
||||||
uint64_t words;
|
|
||||||
|
|
||||||
nrhs = X.size();
|
|
||||||
GRID_ASSERT(X.size()==Y.size());
|
|
||||||
conformable(X[0],Y[0]);
|
|
||||||
|
|
||||||
grid = X[0].Grid();
|
|
||||||
vol = grid->lSites();
|
|
||||||
words = sizeof(scalar_object)/sizeof(scalar);
|
|
||||||
int64_t vw = vol * words;
|
|
||||||
|
|
||||||
RealD t0 = usecond();
|
|
||||||
BLAS_X.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_Y.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_C.resize(nrhs * nrhs);// cost free if size doesn't change
|
|
||||||
RealD t1 = usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// Copy in the multi-rhs sources
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int64_t offset = r*vw;
|
|
||||||
autoView(x_v,X[r],AcceleratorRead);
|
|
||||||
acceleratorCopyDeviceToDevice(&x_v[0],&BLAS_X[offset],sizeof(scalar_object)*vol);
|
|
||||||
autoView(y_v,Y[r],AcceleratorRead);
|
|
||||||
acceleratorCopyDeviceToDevice(&y_v[0],&BLAS_Y[offset],sizeof(scalar_object)*vol);
|
|
||||||
}
|
|
||||||
RealD t2 = usecond();
|
|
||||||
|
|
||||||
/*
|
|
||||||
* in Fortran column major notation (cuBlas order)
|
|
||||||
*
|
|
||||||
* Xxr = [X1(x)][..][Xn(x)]
|
|
||||||
*
|
|
||||||
* Yxr = [Y1(x)][..][Ym(x)]
|
|
||||||
*
|
|
||||||
* C_rs = X^dag Y
|
|
||||||
*/
|
|
||||||
deviceVector<scalar *> Xd(1);
|
|
||||||
deviceVector<scalar *> Yd(1);
|
|
||||||
deviceVector<scalar *> Cd(1);
|
|
||||||
|
|
||||||
scalar * Xh = & BLAS_X[0];
|
|
||||||
scalar * Yh = & BLAS_Y[0];
|
|
||||||
scalar * Ch = & BLAS_C[0];
|
|
||||||
|
|
||||||
acceleratorPut(Xd[0],Xh);
|
|
||||||
acceleratorPut(Yd[0],Yh);
|
|
||||||
acceleratorPut(Cd[0],Ch);
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
RealD t3 = usecond();
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// C_rs = X^dag Y
|
|
||||||
/////////////////////////////////////////
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
|
||||||
nrhs,nrhs,vw,
|
|
||||||
ComplexD(1.0),
|
|
||||||
Xd,
|
|
||||||
Yd,
|
|
||||||
ComplexD(0.0), // wipe out C
|
|
||||||
Cd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
RealD t4 = usecond();
|
|
||||||
|
|
||||||
std::vector<scalar> HOST_C(BLAS_C.size()); // nrhs . nrhs -- the coefficients
|
|
||||||
acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
|
|
||||||
grid->GlobalSumVector(&HOST_C[0],nrhs*nrhs);
|
|
||||||
|
|
||||||
RealD t5 = usecond();
|
|
||||||
for(int rr=0;rr<nrhs;rr++){
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int off = r+nrhs*rr;
|
|
||||||
m(r,rr)=HOST_C[off];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
RealD t6 = usecond();
|
|
||||||
uint64_t M=nrhs;
|
|
||||||
uint64_t N=nrhs;
|
|
||||||
uint64_t K=vw;
|
|
||||||
RealD bytes = 1.0*sizeof(ComplexD)*(M*N*2+N*K+M*K);
|
|
||||||
RealD flops = 8.0*M*N*K;
|
|
||||||
flops = flops/(t4-t3)/1.e3;
|
|
||||||
bytes = bytes/(t4-t3)/1.e3;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix m,n,k "<< M<<","<<N<<","<<K<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix alloc t1 "<< t1-t0<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t2 "<< t2-t1<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix setup t3 "<< t3-t2<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas t4 "<< t4-t3<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< flops<<" GF/s"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< bytes<<" GB/s"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix gsum t5 "<< t5-t4<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t6 "<< t6-t5<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix took "<< t6-t0<<" us"<<std::endl;
|
|
||||||
#else
|
|
||||||
int nrhs;
|
|
||||||
GridBase *grid;
|
|
||||||
uint64_t vol;
|
|
||||||
uint64_t words;
|
|
||||||
|
|
||||||
nrhs = X.size();
|
|
||||||
GRID_ASSERT(X.size()==Y.size());
|
|
||||||
conformable(X[0],Y[0]);
|
|
||||||
|
|
||||||
grid = X[0].Grid();
|
|
||||||
int rd0 = grid->_rdimensions[0] * grid->_rdimensions[1];
|
|
||||||
vol = grid->oSites()/rd0;
|
|
||||||
words = rd0*sizeof(vector_object)/sizeof(scalar);
|
|
||||||
int64_t vw = vol * words;
|
|
||||||
GRID_ASSERT(vw == grid->lSites()*sizeof(scalar_object)/sizeof(scalar));
|
|
||||||
|
|
||||||
RealD t0 = usecond();
|
|
||||||
BLAS_X.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_Y.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_Cred.resize(nrhs * nrhs * vol);// cost free if size doesn't change
|
|
||||||
RealD t1 = usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// Copy in the multi-rhs sources -- layout batched BLAS ready
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
autoView(x_v,X[r],AcceleratorRead);
|
|
||||||
autoView(y_v,Y[r],AcceleratorRead);
|
|
||||||
scalar *from_x=(scalar *)&x_v[0];
|
|
||||||
scalar *from_y=(scalar *)&y_v[0];
|
|
||||||
scalar *BX = &BLAS_X[0];
|
|
||||||
scalar *BY = &BLAS_Y[0];
|
|
||||||
accelerator_for(ssw,vw,1,{
|
|
||||||
uint64_t ss=ssw/words;
|
|
||||||
uint64_t w=ssw%words;
|
|
||||||
uint64_t offset = w+r*words+ss*nrhs*words; // [ss][rhs][words]
|
|
||||||
BX[offset] = from_x[ssw];
|
|
||||||
BY[offset] = from_y[ssw];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
RealD t2 = usecond();
|
|
||||||
|
|
||||||
/*
|
|
||||||
* in Fortran column major notation (cuBlas order)
|
|
||||||
*
|
|
||||||
* Xxr = [X1(x)][..][Xn(x)]
|
|
||||||
*
|
|
||||||
* Yxr = [Y1(x)][..][Ym(x)]
|
|
||||||
*
|
|
||||||
* C_rs = X^dag Y
|
|
||||||
*/
|
|
||||||
Xdip.resize(vol);
|
|
||||||
Ydip.resize(vol);
|
|
||||||
Cdip.resize(vol);
|
|
||||||
std::vector<scalar *> Xh(vol);
|
|
||||||
std::vector<scalar *> Yh(vol);
|
|
||||||
std::vector<scalar *> Ch(vol);
|
|
||||||
for(uint64_t ss=0;ss<vol;ss++){
|
|
||||||
|
|
||||||
Xh[ss] = & BLAS_X[ss*nrhs*words];
|
|
||||||
Yh[ss] = & BLAS_Y[ss*nrhs*words];
|
|
||||||
Ch[ss] = & BLAS_Cred[ss*nrhs*nrhs];
|
|
||||||
|
|
||||||
}
|
|
||||||
acceleratorCopyToDevice(&Xh[0],&Xdip[0],vol*sizeof(scalar *));
|
|
||||||
acceleratorCopyToDevice(&Yh[0],&Ydip[0],vol*sizeof(scalar *));
|
|
||||||
acceleratorCopyToDevice(&Ch[0],&Cdip[0],vol*sizeof(scalar *));
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
RealD t3 = usecond();
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// C_rs = X^dag Y
|
|
||||||
/////////////////////////////////////////
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
|
||||||
nrhs,nrhs,words,
|
|
||||||
ComplexD(1.0),
|
|
||||||
Xdip,
|
|
||||||
Ydip,
|
|
||||||
ComplexD(0.0), // wipe out C
|
|
||||||
Cdip);
|
|
||||||
BLAS.synchronise();
|
|
||||||
RealD t4 = usecond();
|
|
||||||
|
|
||||||
std::vector<scalar> HOST_C(BLAS_Cred.size()); // nrhs . nrhs -- the coefficients
|
|
||||||
acceleratorCopyFromDevice(&BLAS_Cred[0],&HOST_C[0],BLAS_Cred.size()*sizeof(scalar));
|
|
||||||
|
|
||||||
RealD t5 = usecond();
|
|
||||||
m = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
for(int ss=0;ss<vol;ss++){
|
|
||||||
Eigen::Map<Eigen::MatrixXcd> eC((std::complex<double> *)&HOST_C[ss*nrhs*nrhs],nrhs,nrhs);
|
|
||||||
m = m + eC;
|
|
||||||
}
|
|
||||||
RealD t6l = usecond();
|
|
||||||
grid->GlobalSumVector((scalar *) &m(0,0),nrhs*nrhs);
|
|
||||||
RealD t6 = usecond();
|
|
||||||
uint64_t M=nrhs;
|
|
||||||
uint64_t N=nrhs;
|
|
||||||
uint64_t K=vw;
|
|
||||||
RealD xybytes = grid->lSites()*sizeof(scalar_object);
|
|
||||||
RealD bytes = 1.0*sizeof(ComplexD)*(M*N*2+N*K+M*K);
|
|
||||||
RealD flops = 8.0*M*N*K;
|
|
||||||
flops = flops/(t4-t3)/1.e3;
|
|
||||||
bytes = bytes/(t4-t3)/1.e3;
|
|
||||||
xybytes = 4*xybytes/(t2-t1)/1.e3;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix m,n,k "<< M<<","<<N<<","<<K<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix alloc t1 "<< t1-t0<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t2 "<< t2-t1<<" us "<<xybytes<<" GB/s"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix setup t3 "<< t3-t2<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas t4 "<< t4-t3<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< flops<<" GF/s"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix blas "<< bytes<<" GB/s"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix cp t5 "<< t5-t4<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix lsum t6l "<< t6l-t5<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix gsum t6 "<< t6-t6l<<" us"<<std::endl;
|
|
||||||
std::cout <<GridLogPerformance<< "InnerProductMatrix took "<< t6-t0<<" us"<<std::endl;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,513 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: MultiRHSDeflation.h
|
|
||||||
|
|
||||||
Copyright (C) 2023
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
MultiRHS block projection
|
|
||||||
|
|
||||||
Import basis -> nblock x nbasis x (block x internal)
|
|
||||||
Import vector of fine lattice objects -> nblock x nrhs x (block x internal)
|
|
||||||
|
|
||||||
=> coarse_(nrhs x nbasis )^block = via batched GEMM
|
|
||||||
|
|
||||||
//template<class vobj,class CComplex,int nbasis,class VLattice>
|
|
||||||
//inline void blockProject(Lattice<iVector<CComplex,nbasis > > &coarseData,
|
|
||||||
// const VLattice &fineData,
|
|
||||||
// const VLattice &Basis)
|
|
||||||
*/
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class MultiRHSBlockProject
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
typedef typename Field::scalar_type scalar;
|
|
||||||
typedef typename Field::scalar_object scalar_object;
|
|
||||||
typedef Field Fermion;
|
|
||||||
|
|
||||||
int nbasis;
|
|
||||||
GridBase *coarse_grid;
|
|
||||||
GridBase *fine_grid;
|
|
||||||
uint64_t block_vol;
|
|
||||||
uint64_t fine_vol;
|
|
||||||
uint64_t coarse_vol;
|
|
||||||
uint64_t words;
|
|
||||||
|
|
||||||
// Row major layout "C" order:
|
|
||||||
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
|
||||||
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
|
||||||
// BLAS_C[coarse_vol][nrhs][nbasis]
|
|
||||||
/*
|
|
||||||
* in Fortran column major notation (cuBlas order)
|
|
||||||
*
|
|
||||||
* Vxb = [v1(x)][..][vn(x)] ... x coarse vol
|
|
||||||
*
|
|
||||||
* Fxr = [r1(x)][..][rm(x)] ... x coarse vol
|
|
||||||
*
|
|
||||||
* Block project:
|
|
||||||
* C_br = V^dag F x coarse vol
|
|
||||||
*
|
|
||||||
* Block promote:
|
|
||||||
* F_xr = Vxb Cbr x coarse_vol
|
|
||||||
*/
|
|
||||||
deviceVector<scalar> BLAS_V; // words * block_vol * nbasis x coarse_vol
|
|
||||||
deviceVector<scalar> BLAS_F; // nrhs x fine_vol * words -- the sources
|
|
||||||
deviceVector<scalar> BLAS_C; // nrhs x coarse_vol * nbasis -- the coarse coeffs
|
|
||||||
|
|
||||||
RealD blasNorm2(deviceVector<scalar> &blas)
|
|
||||||
{
|
|
||||||
scalar ss(0.0);
|
|
||||||
std::vector<scalar> tmp(blas.size());
|
|
||||||
acceleratorCopyFromDevice(&blas[0],&tmp[0],blas.size()*sizeof(scalar));
|
|
||||||
for(int64_t s=0;s<blas.size();s++){
|
|
||||||
ss=ss+tmp[s]*adj(tmp[s]);
|
|
||||||
}
|
|
||||||
coarse_grid->GlobalSum(ss);
|
|
||||||
return real(ss);
|
|
||||||
}
|
|
||||||
|
|
||||||
MultiRHSBlockProject(){};
|
|
||||||
~MultiRHSBlockProject(){ Deallocate(); };
|
|
||||||
|
|
||||||
void Deallocate(void)
|
|
||||||
{
|
|
||||||
nbasis=0;
|
|
||||||
coarse_grid=nullptr;
|
|
||||||
fine_grid=nullptr;
|
|
||||||
fine_vol=0;
|
|
||||||
block_vol=0;
|
|
||||||
coarse_vol=0;
|
|
||||||
words=0;
|
|
||||||
BLAS_V.resize(0);
|
|
||||||
BLAS_F.resize(0);
|
|
||||||
BLAS_C.resize(0);
|
|
||||||
}
|
|
||||||
void Allocate(int _nbasis,GridBase *_fgrid,GridBase *_cgrid)
|
|
||||||
{
|
|
||||||
nbasis=_nbasis;
|
|
||||||
|
|
||||||
fine_grid=_fgrid;
|
|
||||||
coarse_grid=_cgrid;
|
|
||||||
|
|
||||||
fine_vol = fine_grid->lSites();
|
|
||||||
coarse_vol = coarse_grid->lSites();
|
|
||||||
block_vol = fine_vol/coarse_vol;
|
|
||||||
|
|
||||||
words = sizeof(scalar_object)/sizeof(scalar);
|
|
||||||
|
|
||||||
BLAS_V.resize (fine_vol * words * nbasis );
|
|
||||||
}
|
|
||||||
void ImportFineGridVectors(std::vector <Field > &vecs, deviceVector<scalar> &blas)
|
|
||||||
{
|
|
||||||
int nvec = vecs.size();
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
// std::cout << GridLogMessage <<" BlockProjector importing "<<nvec<< " fine grid vectors" <<std::endl;
|
|
||||||
|
|
||||||
GRID_ASSERT(vecs[0].Grid()==fine_grid);
|
|
||||||
|
|
||||||
subdivides(coarse_grid,fine_grid); // require they map
|
|
||||||
|
|
||||||
int _ndimension = coarse_grid->_ndimension;
|
|
||||||
GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites());
|
|
||||||
|
|
||||||
Coordinate block_r (_ndimension);
|
|
||||||
for(int d=0 ; d<_ndimension;d++){
|
|
||||||
block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t sz = blas.size();
|
|
||||||
|
|
||||||
acceleratorMemSet(&blas[0],0,blas.size()*sizeof(scalar));
|
|
||||||
|
|
||||||
Coordinate fine_rdimensions = fine_grid->_rdimensions;
|
|
||||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
|
||||||
int64_t bv= block_vol;
|
|
||||||
for(int v=0;v<vecs.size();v++){
|
|
||||||
|
|
||||||
// std::cout << " BlockProjector importing vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
|
|
||||||
autoView( fineData , vecs[v], AcceleratorRead);
|
|
||||||
|
|
||||||
auto blasData_p = &blas[0];
|
|
||||||
auto fineData_p = &fineData[0];
|
|
||||||
|
|
||||||
int64_t osites = fine_grid->oSites();
|
|
||||||
|
|
||||||
// loop over fine sites
|
|
||||||
const int Nsimd = vobj::Nsimd();
|
|
||||||
// std::cout << "sz "<<sz<<std::endl;
|
|
||||||
// std::cout << "prod "<<Nsimd * coarse_grid->oSites() * block_vol * nvec * words<<std::endl;
|
|
||||||
GRID_ASSERT(sz == Nsimd * coarse_grid->oSites() * block_vol * nvec * words);
|
|
||||||
uint64_t lwords= words; // local variable for copy in to GPU
|
|
||||||
accelerator_for(sf,osites,Nsimd,{
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<Nsimd;lane++) {
|
|
||||||
#endif
|
|
||||||
// One thread per fine site
|
|
||||||
Coordinate coor_f(_ndimension);
|
|
||||||
Coordinate coor_b(_ndimension);
|
|
||||||
Coordinate coor_c(_ndimension);
|
|
||||||
|
|
||||||
// Fine site to fine coor
|
|
||||||
Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
|
|
||||||
|
|
||||||
for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
|
|
||||||
for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
|
|
||||||
|
|
||||||
int sc;// coarse site
|
|
||||||
int sb;// block site
|
|
||||||
Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
|
|
||||||
Lexicographic::IndexFromCoor(coor_b,sb,block_r);
|
|
||||||
|
|
||||||
scalar_object data = extractLane(lane,fineData[sf]);
|
|
||||||
|
|
||||||
// BLAS layout address calculation
|
|
||||||
// words * block_vol * nbasis x coarse_vol
|
|
||||||
// coarse oSite x block vole x lanes
|
|
||||||
int64_t site = (lane*osites + sc*bv)*nvec
|
|
||||||
+ v*bv
|
|
||||||
+ sb;
|
|
||||||
|
|
||||||
// GRID_ASSERT(site*lwords<sz);
|
|
||||||
|
|
||||||
scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
|
|
||||||
|
|
||||||
*ptr = data;
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
// std::cout << " import fine Blas norm "<<blasNorm2(blas)<<std::endl;
|
|
||||||
// std::cout << " BlockProjector imported vector"<<v<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void ExportFineGridVectors(std::vector <Field> &vecs, deviceVector<scalar> &blas)
|
|
||||||
{
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
|
|
||||||
int nvec = vecs.size();
|
|
||||||
|
|
||||||
GRID_ASSERT(vecs[0].Grid()==fine_grid);
|
|
||||||
|
|
||||||
subdivides(coarse_grid,fine_grid); // require they map
|
|
||||||
|
|
||||||
int _ndimension = coarse_grid->_ndimension;
|
|
||||||
GRID_ASSERT(block_vol == fine_grid->oSites() / coarse_grid->oSites());
|
|
||||||
|
|
||||||
Coordinate block_r (_ndimension);
|
|
||||||
for(int d=0 ; d<_ndimension;d++){
|
|
||||||
block_r[d] = fine_grid->_rdimensions[d] / coarse_grid->_rdimensions[d];
|
|
||||||
}
|
|
||||||
Coordinate fine_rdimensions = fine_grid->_rdimensions;
|
|
||||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
|
||||||
|
|
||||||
// std::cout << " export fine Blas norm "<<blasNorm2(blas)<<std::endl;
|
|
||||||
|
|
||||||
int64_t bv= block_vol;
|
|
||||||
for(int v=0;v<vecs.size();v++){
|
|
||||||
|
|
||||||
autoView( fineData , vecs[v], AcceleratorWrite);
|
|
||||||
|
|
||||||
auto blasData_p = &blas[0];
|
|
||||||
auto fineData_p = &fineData[0];
|
|
||||||
|
|
||||||
int64_t osites = fine_grid->oSites();
|
|
||||||
uint64_t lwords = words;
|
|
||||||
// std::cout << " Nsimd is "<<vobj::Nsimd() << std::endl;
|
|
||||||
// std::cout << " lwords is "<<lwords << std::endl;
|
|
||||||
// std::cout << " sizeof(scalar_object) is "<<sizeof(scalar_object) << std::endl;
|
|
||||||
// loop over fine sites
|
|
||||||
accelerator_for(sf,osites,vobj::Nsimd(),{
|
|
||||||
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(vobj::Nsimd()); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<vobj::Nsimd();lane++) {
|
|
||||||
#endif
|
|
||||||
// One thread per fine site
|
|
||||||
Coordinate coor_f(_ndimension);
|
|
||||||
Coordinate coor_b(_ndimension);
|
|
||||||
Coordinate coor_c(_ndimension);
|
|
||||||
|
|
||||||
Lexicographic::CoorFromIndex(coor_f,sf,fine_rdimensions);
|
|
||||||
|
|
||||||
for(int d=0;d<_ndimension;d++) coor_b[d] = coor_f[d]%block_r[d];
|
|
||||||
for(int d=0;d<_ndimension;d++) coor_c[d] = coor_f[d]/block_r[d];
|
|
||||||
|
|
||||||
int sc;
|
|
||||||
int sb;
|
|
||||||
Lexicographic::IndexFromCoor(coor_c,sc,coarse_rdimensions);
|
|
||||||
Lexicographic::IndexFromCoor(coor_b,sb,block_r);
|
|
||||||
|
|
||||||
// BLAS layout address calculation
|
|
||||||
// words * block_vol * nbasis x coarse_vol
|
|
||||||
int64_t site = (lane*osites + sc*bv)*nvec
|
|
||||||
+ v*bv
|
|
||||||
+ sb;
|
|
||||||
|
|
||||||
scalar_object * ptr = (scalar_object *)&blasData_p[site*lwords];
|
|
||||||
|
|
||||||
scalar_object data = *ptr;
|
|
||||||
|
|
||||||
insertLane(lane,fineData[sf],data);
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
template<class vobj>
|
|
||||||
void ImportCoarseGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
|
|
||||||
{
|
|
||||||
int nvec = vecs.size();
|
|
||||||
typedef typename vobj::scalar_object coarse_scalar_object;
|
|
||||||
|
|
||||||
// std::cout << " BlockProjector importing "<<nvec<< " coarse grid vectors" <<std::endl;
|
|
||||||
|
|
||||||
GRID_ASSERT(vecs[0].Grid()==coarse_grid);
|
|
||||||
|
|
||||||
int _ndimension = coarse_grid->_ndimension;
|
|
||||||
|
|
||||||
uint64_t sz = blas.size();
|
|
||||||
|
|
||||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
|
||||||
|
|
||||||
for(int v=0;v<vecs.size();v++){
|
|
||||||
|
|
||||||
// std::cout << " BlockProjector importing coarse vector"<<v<<" "<<norm2(vecs[v])<<std::endl;
|
|
||||||
autoView( coarseData , vecs[v], AcceleratorRead);
|
|
||||||
|
|
||||||
auto blasData_p = &blas[0];
|
|
||||||
auto coarseData_p = &coarseData[0];
|
|
||||||
|
|
||||||
int64_t osites = coarse_grid->oSites();
|
|
||||||
|
|
||||||
// loop over fine sites
|
|
||||||
const int Nsimd = vobj::Nsimd();
|
|
||||||
uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
|
|
||||||
GRID_ASSERT(cwords==nbasis);
|
|
||||||
|
|
||||||
accelerator_for(sc,osites,Nsimd,{
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<Nsimd;lane++) {
|
|
||||||
#endif
|
|
||||||
// C_br per site
|
|
||||||
int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
|
|
||||||
|
|
||||||
coarse_scalar_object data = extractLane(lane,coarseData[sc]);
|
|
||||||
|
|
||||||
coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
|
|
||||||
|
|
||||||
*ptr = data;
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
// std::cout << " import coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
template<class vobj>
|
|
||||||
void ExportCoarseGridVectors(std::vector <Lattice<vobj> > &vecs, deviceVector<scalar> &blas)
|
|
||||||
{
|
|
||||||
int nvec = vecs.size();
|
|
||||||
typedef typename vobj::scalar_object coarse_scalar_object;
|
|
||||||
// std::cout << GridLogMessage<<" BlockProjector exporting "<<nvec<< " coarse grid vectors" <<std::endl;
|
|
||||||
|
|
||||||
GRID_ASSERT(vecs[0].Grid()==coarse_grid);
|
|
||||||
|
|
||||||
int _ndimension = coarse_grid->_ndimension;
|
|
||||||
|
|
||||||
uint64_t sz = blas.size();
|
|
||||||
|
|
||||||
Coordinate coarse_rdimensions = coarse_grid->_rdimensions;
|
|
||||||
|
|
||||||
// std::cout << " export coarsee Blas norm "<<blasNorm2(blas)<<std::endl;
|
|
||||||
for(int v=0;v<vecs.size();v++){
|
|
||||||
|
|
||||||
// std::cout << " BlockProjector exporting coarse vector"<<v<<std::endl;
|
|
||||||
autoView( coarseData , vecs[v], AcceleratorWrite);
|
|
||||||
|
|
||||||
auto blasData_p = &blas[0];
|
|
||||||
auto coarseData_p = &coarseData[0];
|
|
||||||
|
|
||||||
int64_t osites = coarse_grid->oSites();
|
|
||||||
|
|
||||||
// loop over fine sites
|
|
||||||
const int Nsimd = vobj::Nsimd();
|
|
||||||
uint64_t cwords=sizeof(typename vobj::scalar_object)/sizeof(scalar);
|
|
||||||
GRID_ASSERT(cwords==nbasis);
|
|
||||||
|
|
||||||
accelerator_for(sc,osites,Nsimd,{
|
|
||||||
// Wrap in a macro "FOR_ALL_LANES(lane,{ ... });
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
{
|
|
||||||
int lane=acceleratorSIMTlane(Nsimd); // buffer lane
|
|
||||||
#else
|
|
||||||
for(int lane=0;lane<Nsimd;lane++) {
|
|
||||||
#endif
|
|
||||||
int64_t blas_site = (lane*osites + sc)*nvec*cwords + v*cwords;
|
|
||||||
coarse_scalar_object * ptr = (coarse_scalar_object *)&blasData_p[blas_site];
|
|
||||||
coarse_scalar_object data = *ptr;
|
|
||||||
insertLane(lane,coarseData[sc],data);
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void ImportBasis(std::vector < Field > &vecs)
|
|
||||||
{
|
|
||||||
// std::cout << " BlockProjector Import basis size "<<vecs.size()<<std::endl;
|
|
||||||
ImportFineGridVectors(vecs,BLAS_V);
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class cobj>
|
|
||||||
void blockProject(std::vector<Field> &fine,std::vector< Lattice<cobj> > & coarse)
|
|
||||||
{
|
|
||||||
int nrhs=fine.size();
|
|
||||||
int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
|
|
||||||
// std::cout << "blockProject nbasis " <<nbasis<<" " << _nbasis<<std::endl;
|
|
||||||
GRID_ASSERT(nbasis==_nbasis);
|
|
||||||
|
|
||||||
BLAS_F.resize (fine_vol * words * nrhs );
|
|
||||||
BLAS_C.resize (coarse_vol * nbasis * nrhs );
|
|
||||||
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// Copy in the multi-rhs sources to same data layout
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// std::cout << "BlockProject import fine"<<std::endl;
|
|
||||||
ImportFineGridVectors(fine,BLAS_F);
|
|
||||||
|
|
||||||
deviceVector<scalar *> Vd(coarse_vol);
|
|
||||||
deviceVector<scalar *> Fd(coarse_vol);
|
|
||||||
deviceVector<scalar *> Cd(coarse_vol);
|
|
||||||
|
|
||||||
// std::cout << "BlockProject pointers"<<std::endl;
|
|
||||||
for(int c=0;c<coarse_vol;c++){
|
|
||||||
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
|
||||||
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
|
||||||
// BLAS_C[coarse_vol][nrhs][nbasis]
|
|
||||||
scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
|
|
||||||
scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
|
|
||||||
scalar * Ch = & BLAS_C[c*nrhs*nbasis];
|
|
||||||
|
|
||||||
acceleratorPut(Vd[c],Vh);
|
|
||||||
acceleratorPut(Fd[c],Fh);
|
|
||||||
acceleratorPut(Cd[c],Ch);
|
|
||||||
}
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
// std::cout << "BlockProject BLAS"<<std::endl;
|
|
||||||
int64_t vw = block_vol * words;
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// C_br = V^dag R
|
|
||||||
/////////////////////////////////////////
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
|
||||||
nbasis,nrhs,vw,
|
|
||||||
scalar(1.0),
|
|
||||||
Vd,
|
|
||||||
Fd,
|
|
||||||
scalar(0.0), // wipe out C
|
|
||||||
Cd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
// std::cout << "BlockProject done"<<std::endl;
|
|
||||||
ExportCoarseGridVectors(coarse, BLAS_C);
|
|
||||||
// std::cout << "BlockProject done"<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class cobj>
|
|
||||||
void blockPromote(std::vector<Field> &fine,std::vector<Lattice<cobj> > & coarse)
|
|
||||||
{
|
|
||||||
int nrhs=fine.size();
|
|
||||||
int _nbasis = sizeof(typename cobj::scalar_object)/sizeof(scalar);
|
|
||||||
GRID_ASSERT(nbasis==_nbasis);
|
|
||||||
|
|
||||||
BLAS_F.resize (fine_vol * words * nrhs );
|
|
||||||
BLAS_C.resize (coarse_vol * nbasis * nrhs );
|
|
||||||
|
|
||||||
ImportCoarseGridVectors(coarse, BLAS_C);
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
deviceVector<scalar *> Vd(coarse_vol);
|
|
||||||
deviceVector<scalar *> Fd(coarse_vol);
|
|
||||||
deviceVector<scalar *> Cd(coarse_vol);
|
|
||||||
|
|
||||||
for(int c=0;c<coarse_vol;c++){
|
|
||||||
// BLAS_V[coarse_vol][nbasis][block_vol][words]
|
|
||||||
// BLAS_F[coarse_vol][nrhs][block_vol][words]
|
|
||||||
// BLAS_C[coarse_vol][nrhs][nbasis]
|
|
||||||
scalar * Vh = & BLAS_V[c*nbasis*block_vol*words];
|
|
||||||
scalar * Fh = & BLAS_F[c*nrhs*block_vol*words];
|
|
||||||
scalar * Ch = & BLAS_C[c*nrhs*nbasis];
|
|
||||||
acceleratorPut(Vd[c],Vh);
|
|
||||||
acceleratorPut(Fd[c],Fh);
|
|
||||||
acceleratorPut(Cd[c],Ch);
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// Block promote:
|
|
||||||
// F_xr = Vxb Cbr (x coarse_vol)
|
|
||||||
/////////////////////////////////////////
|
|
||||||
|
|
||||||
int64_t vw = block_vol * words;
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
|
||||||
vw,nrhs,nbasis,
|
|
||||||
scalar(1.0),
|
|
||||||
Vd,
|
|
||||||
Cd,
|
|
||||||
scalar(0.0), // wipe out C
|
|
||||||
Fd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
// std::cout << " blas call done"<<std::endl;
|
|
||||||
|
|
||||||
ExportFineGridVectors(fine, BLAS_F);
|
|
||||||
// std::cout << " exported "<<std::endl;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: MultiRHSDeflation.h
|
|
||||||
|
|
||||||
Copyright (C) 2023
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
/* Need helper object for BLAS accelerated mrhs projection
|
|
||||||
|
|
||||||
i) MultiRHS Deflation
|
|
||||||
|
|
||||||
Import Evecs -> nev x vol x internal
|
|
||||||
Import vector of Lattice objects -> nrhs x vol x internal
|
|
||||||
=> Cij (nrhs x Nev) via GEMM.
|
|
||||||
=> Guess (nrhs x vol x internal) = C x evecs (via GEMM)
|
|
||||||
Export
|
|
||||||
|
|
||||||
|
|
||||||
ii) MultiRHS block projection
|
|
||||||
|
|
||||||
Import basis -> nblock x nbasis x (block x internal)
|
|
||||||
Import vector of fine lattice objects -> nblock x nrhs x (block x internal)
|
|
||||||
|
|
||||||
=> coarse_(nrhs x nbasis )^block = via batched GEMM
|
|
||||||
|
|
||||||
iii) Alternate interface:
|
|
||||||
Import higher dim Lattice object-> vol x nrhs layout
|
|
||||||
|
|
||||||
*/
|
|
||||||
template<class Field>
|
|
||||||
class MultiRHSDeflation
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
typedef typename Field::scalar_type scalar;
|
|
||||||
typedef typename Field::scalar_object scalar_object;
|
|
||||||
|
|
||||||
int nev;
|
|
||||||
std::vector<RealD> eval;
|
|
||||||
GridBase *grid;
|
|
||||||
uint64_t vol;
|
|
||||||
uint64_t words;
|
|
||||||
|
|
||||||
deviceVector<scalar> BLAS_E; // nev x vol -- the eigenbasis (up to a 1/sqrt(lambda))
|
|
||||||
deviceVector<scalar> BLAS_R; // nrhs x vol -- the sources
|
|
||||||
deviceVector<scalar> BLAS_G; // nrhs x vol -- the guess
|
|
||||||
deviceVector<scalar> BLAS_C; // nrhs x nev -- the coefficients
|
|
||||||
|
|
||||||
MultiRHSDeflation(){};
|
|
||||||
~MultiRHSDeflation(){ Deallocate(); };
|
|
||||||
|
|
||||||
void Deallocate(void)
|
|
||||||
{
|
|
||||||
nev=0;
|
|
||||||
grid=nullptr;
|
|
||||||
vol=0;
|
|
||||||
words=0;
|
|
||||||
BLAS_E.resize(0);
|
|
||||||
BLAS_R.resize(0);
|
|
||||||
BLAS_C.resize(0);
|
|
||||||
BLAS_G.resize(0);
|
|
||||||
}
|
|
||||||
void Allocate(int _nev,GridBase *_grid)
|
|
||||||
{
|
|
||||||
nev=_nev;
|
|
||||||
grid=_grid;
|
|
||||||
vol = grid->lSites();
|
|
||||||
words = sizeof(scalar_object)/sizeof(scalar);
|
|
||||||
eval.resize(nev);
|
|
||||||
BLAS_E.resize (vol * words * nev );
|
|
||||||
std::cout << GridLogMessage << " Allocate for "<<nev<<" eigenvectors and volume "<<vol<<std::endl;
|
|
||||||
}
|
|
||||||
void ImportEigenVector(Field &evec,RealD &_eval, int ev)
|
|
||||||
{
|
|
||||||
// std::cout << " ev " <<ev<<" eval "<<_eval<< std::endl;
|
|
||||||
GRID_ASSERT(ev<eval.size());
|
|
||||||
eval[ev] = _eval;
|
|
||||||
|
|
||||||
int64_t offset = ev*vol*words;
|
|
||||||
autoView(v,evec,AcceleratorRead);
|
|
||||||
acceleratorCopyDeviceToDevice(&v[0],&BLAS_E[offset],sizeof(scalar_object)*vol);
|
|
||||||
|
|
||||||
}
|
|
||||||
void ImportEigenBasis(std::vector<Field> &evec,std::vector<RealD> &_eval)
|
|
||||||
{
|
|
||||||
ImportEigenBasis(evec,_eval,0,evec.size());
|
|
||||||
}
|
|
||||||
// Could use to import a batch of eigenvectors
|
|
||||||
void ImportEigenBasis(std::vector<Field> &evec,std::vector<RealD> &_eval, int _ev0, int _nev)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(_ev0+_nev<=evec.size());
|
|
||||||
|
|
||||||
Allocate(_nev,evec[0].Grid());
|
|
||||||
|
|
||||||
// Imports a sub-batch of eigenvectors, _ev0, ..., _ev0+_nev-1
|
|
||||||
for(int e=0;e<nev;e++){
|
|
||||||
std::cout << "Importing eigenvector "<<e<<" evalue "<<_eval[_ev0+e]<<std::endl;
|
|
||||||
ImportEigenVector(evec[_ev0+e],_eval[_ev0+e],e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void DeflateSources(std::vector<Field> &source,std::vector<Field> & guess)
|
|
||||||
{
|
|
||||||
int nrhs = source.size();
|
|
||||||
GRID_ASSERT(source.size()==guess.size());
|
|
||||||
GRID_ASSERT(grid == guess[0].Grid());
|
|
||||||
conformable(guess[0],source[0]);
|
|
||||||
|
|
||||||
int64_t vw = vol * words;
|
|
||||||
|
|
||||||
RealD t0 = usecond();
|
|
||||||
BLAS_R.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_G.resize(nrhs * vw); // cost free if size doesn't change
|
|
||||||
BLAS_C.resize(nev * nrhs);// cost free if size doesn't change
|
|
||||||
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// Copy in the multi-rhs sources
|
|
||||||
/////////////////////////////////////////////
|
|
||||||
// for(int r=0;r<nrhs;r++){
|
|
||||||
// std::cout << " source["<<r<<"] = "<<norm2(source[r])<<std::endl;
|
|
||||||
// }
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int64_t offset = r*vw;
|
|
||||||
autoView(v,source[r],AcceleratorRead);
|
|
||||||
acceleratorCopyDeviceToDevice(&v[0],&BLAS_R[offset],sizeof(scalar_object)*vol);
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* in Fortran column major notation (cuBlas order)
|
|
||||||
*
|
|
||||||
* Exe = [e1(x)][..][en(x)]
|
|
||||||
*
|
|
||||||
* Rxr = [r1(x)][..][rm(x)]
|
|
||||||
*
|
|
||||||
* C_er = E^dag R
|
|
||||||
* C_er = C_er / lambda_e
|
|
||||||
* G_xr = Exe Cer
|
|
||||||
*/
|
|
||||||
deviceVector<scalar *> Ed(1);
|
|
||||||
deviceVector<scalar *> Rd(1);
|
|
||||||
deviceVector<scalar *> Cd(1);
|
|
||||||
deviceVector<scalar *> Gd(1);
|
|
||||||
|
|
||||||
scalar * Eh = & BLAS_E[0];
|
|
||||||
scalar * Rh = & BLAS_R[0];
|
|
||||||
scalar * Ch = & BLAS_C[0];
|
|
||||||
scalar * Gh = & BLAS_G[0];
|
|
||||||
|
|
||||||
acceleratorPut(Ed[0],Eh);
|
|
||||||
acceleratorPut(Rd[0],Rh);
|
|
||||||
acceleratorPut(Cd[0],Ch);
|
|
||||||
acceleratorPut(Gd[0],Gh);
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// C_er = E^dag R
|
|
||||||
/////////////////////////////////////////
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_C,GridBLAS_OP_N,
|
|
||||||
nev,nrhs,vw,
|
|
||||||
scalar(1.0),
|
|
||||||
Ed,
|
|
||||||
Rd,
|
|
||||||
scalar(0.0), // wipe out C
|
|
||||||
Cd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
|
|
||||||
GRID_ASSERT(BLAS_C.size()==nev*nrhs);
|
|
||||||
|
|
||||||
std::vector<scalar> HOST_C(BLAS_C.size()); // nrhs . nev -- the coefficients
|
|
||||||
acceleratorCopyFromDevice(&BLAS_C[0],&HOST_C[0],BLAS_C.size()*sizeof(scalar));
|
|
||||||
grid->GlobalSumVector(&HOST_C[0],nev*nrhs);
|
|
||||||
for(int e=0;e<nev;e++){
|
|
||||||
RealD lam(1.0/eval[e]);
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int off = e+nev*r;
|
|
||||||
HOST_C[off]=HOST_C[off] * lam;
|
|
||||||
// std::cout << "C["<<e<<"]["<<r<<"] ="<<HOST_C[off]<< " eval[e] "<<eval[e] <<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
acceleratorCopyToDevice(&HOST_C[0],&BLAS_C[0],BLAS_C.size()*sizeof(scalar));
|
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////////////////////
|
|
||||||
// Guess G_xr = Exe Cer
|
|
||||||
/////////////////////////////////////////
|
|
||||||
BLAS.gemmBatched(GridBLAS_OP_N,GridBLAS_OP_N,
|
|
||||||
vw,nrhs,nev,
|
|
||||||
scalar(1.0),
|
|
||||||
Ed, // x . nev
|
|
||||||
Cd, // nev . nrhs
|
|
||||||
scalar(0.0),
|
|
||||||
Gd);
|
|
||||||
BLAS.synchronise();
|
|
||||||
|
|
||||||
///////////////////////////////////////
|
|
||||||
// Copy out the multirhs
|
|
||||||
///////////////////////////////////////
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
int64_t offset = r*vw;
|
|
||||||
autoView(v,guess[r],AcceleratorWrite);
|
|
||||||
acceleratorCopyDeviceToDevice(&BLAS_G[offset],&v[0],sizeof(scalar_object)*vol);
|
|
||||||
}
|
|
||||||
RealD t1 = usecond();
|
|
||||||
std::cout << GridLogMessage << "MultiRHSDeflation for "<<nrhs<<" sources with "<<nev<<" eigenvectors took " << (t1-t0)/1e3 <<" ms"<<std::endl;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,599 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/AdefGeneric.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_ALGORITHMS_ITERATIVE_GENERIC_PCG
|
|
||||||
#define GRID_ALGORITHMS_ITERATIVE_GENERIC_PCG
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Compared to Tang-2009: P=Pleft. P^T = PRight Q=MssInv.
|
|
||||||
* Script A = SolverMatrix
|
|
||||||
* Script P = Preconditioner
|
|
||||||
*
|
|
||||||
* Implement ADEF-2
|
|
||||||
*
|
|
||||||
* Vstart = P^Tx + Qb
|
|
||||||
* M1 = P^TM + Q
|
|
||||||
* M2=M3=1
|
|
||||||
*/
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class TwoLevelCG : public LinearFunction<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
GridBase *grid;
|
|
||||||
|
|
||||||
// Fine operator, Smoother, CoarseSolver
|
|
||||||
LinearOperatorBase<Field> &_FineLinop;
|
|
||||||
LinearFunction<Field> &_Smoother;
|
|
||||||
|
|
||||||
// more most opertor functions
|
|
||||||
TwoLevelCG(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearOperatorBase<Field> &FineLinop,
|
|
||||||
LinearFunction<Field> &Smoother,
|
|
||||||
GridBase *fine) :
|
|
||||||
Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
_FineLinop(FineLinop),
|
|
||||||
_Smoother(Smoother)
|
|
||||||
{
|
|
||||||
grid = fine;
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual void operator() (const Field &src, Field &x)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg starting single RHS"<<std::endl;
|
|
||||||
RealD f;
|
|
||||||
RealD rtzp,rtz,a,d,b;
|
|
||||||
RealD rptzp;
|
|
||||||
|
|
||||||
/////////////////////////////
|
|
||||||
// Set up history vectors
|
|
||||||
/////////////////////////////
|
|
||||||
int mmax = 5;
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocating"<<std::endl;
|
|
||||||
std::vector<Field> p(mmax,grid);
|
|
||||||
std::vector<Field> mmp(mmax,grid);
|
|
||||||
std::vector<RealD> pAp(mmax);
|
|
||||||
Field z(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
Field mp (grid);
|
|
||||||
Field r (grid);
|
|
||||||
Field mu (grid);
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated"<<std::endl;
|
|
||||||
//Initial residual computation & set up
|
|
||||||
RealD guess = norm2(x);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg guess nrm "<<guess<<std::endl;
|
|
||||||
RealD src_nrm = norm2(src);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg src nrm "<<src_nrm<<std::endl;
|
|
||||||
|
|
||||||
if ( src_nrm == 0.0 ) {
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg given trivial source norm "<<src_nrm<<std::endl;
|
|
||||||
x=Zero();
|
|
||||||
}
|
|
||||||
RealD tn;
|
|
||||||
|
|
||||||
GridStopWatch HDCGTimer;
|
|
||||||
HDCGTimer.Start();
|
|
||||||
//////////////////////////
|
|
||||||
// x0 = Vstart -- possibly modify guess
|
|
||||||
//////////////////////////
|
|
||||||
Vstart(x,src);
|
|
||||||
|
|
||||||
// r0 = b -A x0
|
|
||||||
_FineLinop.HermOp(x,mmp[0]);
|
|
||||||
axpy (r, -1.0,mmp[0], src); // Recomputes r=src-Ax0
|
|
||||||
{
|
|
||||||
double n1 = norm2(x);
|
|
||||||
double n2 = norm2(mmp[0]);
|
|
||||||
double n3 = norm2(r);
|
|
||||||
std::cout<<GridLogMessage<<"x,vstart,r = "<<n1<<" "<<n2<<" "<<n3<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// Compute z = M1 x
|
|
||||||
//////////////////////////////////
|
|
||||||
PcgM1(r,z);
|
|
||||||
rtzp =real(innerProduct(r,z));
|
|
||||||
|
|
||||||
///////////////////////////////////////
|
|
||||||
// Solve for Mss mu = P A z and set p = z-mu
|
|
||||||
// Def2 p = 1 - Q Az = Pright z
|
|
||||||
// Other algos M2 is trivial
|
|
||||||
///////////////////////////////////////
|
|
||||||
PcgM2(z,p[0]);
|
|
||||||
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = ssq*Tolerance*Tolerance;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<"HDCG: k=0 residual "<<rtzp<<" rsq "<<rsq<<"\n";
|
|
||||||
|
|
||||||
Field pp(grid);
|
|
||||||
|
|
||||||
for (int k=0;k<=MaxIterations;k++){
|
|
||||||
|
|
||||||
int peri_k = k % mmax;
|
|
||||||
int peri_kp = (k+1) % mmax;
|
|
||||||
|
|
||||||
rtz=rtzp;
|
|
||||||
d= PcgM3(p[peri_k],mmp[peri_k]);
|
|
||||||
a = rtz/d;
|
|
||||||
|
|
||||||
// Memorise this
|
|
||||||
pAp[peri_k] = d;
|
|
||||||
|
|
||||||
axpy(x,a,p[peri_k],x);
|
|
||||||
RealD rn = axpy_norm(r,-a,mmp[peri_k],r);
|
|
||||||
|
|
||||||
// Compute z = M x
|
|
||||||
PcgM1(r,z);
|
|
||||||
|
|
||||||
{
|
|
||||||
RealD n1,n2;
|
|
||||||
n1=norm2(r);
|
|
||||||
n2=norm2(z);
|
|
||||||
std::cout << GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : vector r,z "<<n1<<" "<<n2<<"\n";
|
|
||||||
}
|
|
||||||
rtzp =real(innerProduct(r,z));
|
|
||||||
std::cout << GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : inner rtzp "<<rtzp<<"\n";
|
|
||||||
|
|
||||||
// PcgM2(z,p[0]);
|
|
||||||
PcgM2(z,mu); // ADEF-2 this is identity. Axpy possible to eliminate
|
|
||||||
|
|
||||||
p[peri_kp]=mu;
|
|
||||||
|
|
||||||
// Standard search direction p -> z + b p
|
|
||||||
b = (rtzp)/rtz;
|
|
||||||
|
|
||||||
int northog;
|
|
||||||
// k=zero <=> peri_kp=1; northog = 1
|
|
||||||
// k=1 <=> peri_kp=2; northog = 2
|
|
||||||
// ... ... ...
|
|
||||||
// k=mmax-2<=> peri_kp=mmax-1; northog = mmax-1
|
|
||||||
// k=mmax-1<=> peri_kp=0; northog = 1
|
|
||||||
|
|
||||||
// northog = (peri_kp==0)?1:peri_kp; // This is the fCG(mmax) algorithm
|
|
||||||
northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : orthogonalising to last "<<northog<<" vectors\n";
|
|
||||||
for(int back=0; back < northog; back++){
|
|
||||||
int peri_back = (k-back)%mmax;
|
|
||||||
RealD pbApk= real(innerProduct(mmp[peri_back],p[peri_kp]));
|
|
||||||
RealD beta = -pbApk/pAp[peri_back];
|
|
||||||
axpy(p[peri_kp],beta,p[peri_back],p[peri_kp]);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD rrn=sqrt(rn/ssq);
|
|
||||||
RealD rtn=sqrt(rtz/ssq);
|
|
||||||
RealD rtnp=sqrt(rtzp/ssq);
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: fPcg k= "<<k<<" residual = "<<rrn<<"\n";
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if ( rn <= rsq ) {
|
|
||||||
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: fPcg converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
|
||||||
|
|
||||||
_FineLinop.HermOp(x,mmp[0]);
|
|
||||||
axpy(tmp,-1.0,src,mmp[0]);
|
|
||||||
|
|
||||||
RealD mmpnorm = sqrt(norm2(mmp[0]));
|
|
||||||
RealD xnorm = sqrt(norm2(x));
|
|
||||||
RealD srcnorm = sqrt(norm2(src));
|
|
||||||
RealD tmpnorm = sqrt(norm2(tmp));
|
|
||||||
RealD true_residual = tmpnorm/srcnorm;
|
|
||||||
std::cout<<GridLogMessage
|
|
||||||
<<"HDCG: true residual is "<<true_residual
|
|
||||||
<<" solution "<<xnorm
|
|
||||||
<<" source "<<srcnorm
|
|
||||||
<<" mmp "<<mmpnorm
|
|
||||||
<<std::endl;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
|
||||||
RealD xnorm = sqrt(norm2(x));
|
|
||||||
RealD srcnorm = sqrt(norm2(src));
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
virtual void operator() (std::vector<Field> &src, std::vector<Field> &x)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<<"HDCG: mrhs fPcg starting"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
int nrhs = src.size();
|
|
||||||
std::vector<RealD> f(nrhs);
|
|
||||||
std::vector<RealD> rtzp(nrhs);
|
|
||||||
std::vector<RealD> rtz(nrhs);
|
|
||||||
std::vector<RealD> a(nrhs);
|
|
||||||
std::vector<RealD> d(nrhs);
|
|
||||||
std::vector<RealD> b(nrhs);
|
|
||||||
std::vector<RealD> rptzp(nrhs);
|
|
||||||
/////////////////////////////
|
|
||||||
// Set up history vectors
|
|
||||||
/////////////////////////////
|
|
||||||
int mmax = 3;
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocating"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
std::vector<std::vector<Field> > p(nrhs); for(int r=0;r<nrhs;r++) p[r].resize(mmax,grid);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated p"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
std::vector<std::vector<Field> > mmp(nrhs); for(int r=0;r<nrhs;r++) mmp[r].resize(mmax,grid);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated mmp"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
std::vector<std::vector<RealD> > pAp(nrhs); for(int r=0;r<nrhs;r++) pAp[r].resize(mmax);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated pAp"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
std::vector<Field> z(nrhs,grid);
|
|
||||||
std::vector<Field> mp (nrhs,grid);
|
|
||||||
std::vector<Field> r (nrhs,grid);
|
|
||||||
std::vector<Field> mu (nrhs,grid);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg allocated z,mp,r,mu"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
|
|
||||||
//Initial residual computation & set up
|
|
||||||
std::vector<RealD> src_nrm(nrhs);
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
src_nrm[rhs]=norm2(src[rhs]);
|
|
||||||
GRID_ASSERT(src_nrm[rhs]!=0.0);
|
|
||||||
}
|
|
||||||
std::vector<RealD> tn(nrhs);
|
|
||||||
|
|
||||||
GridStopWatch HDCGTimer;
|
|
||||||
HDCGTimer.Start();
|
|
||||||
//////////////////////////
|
|
||||||
// x0 = Vstart -- possibly modify guess
|
|
||||||
//////////////////////////
|
|
||||||
Vstart(x,src);
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
// r0 = b -A x0
|
|
||||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
|
||||||
axpy (r[rhs], -1.0,mmp[rhs][0], src[rhs]); // Recomputes r=src-Ax0
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// Compute z = M1 x
|
|
||||||
//////////////////////////////////
|
|
||||||
// This needs a multiRHS version for acceleration
|
|
||||||
PcgM1(r,z);
|
|
||||||
|
|
||||||
std::vector<RealD> ssq(nrhs);
|
|
||||||
std::vector<RealD> rsq(nrhs);
|
|
||||||
std::vector<Field> pp(nrhs,grid);
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
|
||||||
p[rhs][0]=z[rhs];
|
|
||||||
ssq[rhs]=norm2(src[rhs]);
|
|
||||||
rsq[rhs]= ssq[rhs]*Tolerance*Tolerance;
|
|
||||||
std::cout << GridLogMessage<<"mrhs HDCG: "<<rhs<<" k=0 residual "<<rtzp[rhs]<<" rsq "<<rsq[rhs]<<"\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<RealD> rn(nrhs);
|
|
||||||
for (int k=0;k<=MaxIterations;k++){
|
|
||||||
|
|
||||||
int peri_k = k % mmax;
|
|
||||||
int peri_kp = (k+1) % mmax;
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
rtz[rhs]=rtzp[rhs];
|
|
||||||
d[rhs]= PcgM3(p[rhs][peri_k],mmp[rhs][peri_k]);
|
|
||||||
a[rhs] = rtz[rhs]/d[rhs];
|
|
||||||
|
|
||||||
// Memorise this
|
|
||||||
pAp[rhs][peri_k] = d[rhs];
|
|
||||||
|
|
||||||
axpy(x[rhs],a[rhs],p[rhs][peri_k],x[rhs]);
|
|
||||||
rn[rhs] = axpy_norm(r[rhs],-a[rhs],mmp[rhs][peri_k],r[rhs]);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute z = M x (for *all* RHS)
|
|
||||||
PcgM1(r,z);
|
|
||||||
std::cout << GridLogMessage<<"HDCG::fPcg M1 complete"<<std::endl;
|
|
||||||
grid->Barrier();
|
|
||||||
|
|
||||||
RealD max_rn=0.0;
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
|
|
||||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<"HDCG::fPcg rhs"<<rhs<<" iteration "<<k<<" : inner rtzp "<<rtzp[rhs]<<"\n";
|
|
||||||
|
|
||||||
mu[rhs]=z[rhs];
|
|
||||||
|
|
||||||
p[rhs][peri_kp]=mu[rhs];
|
|
||||||
|
|
||||||
// Standard search direction p == z + b p
|
|
||||||
b[rhs] = (rtzp[rhs])/rtz[rhs];
|
|
||||||
|
|
||||||
int northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
|
||||||
std::cout<<GridLogMessage<<"HDCG::fPcg iteration "<<k<<" : orthogonalising to last "<<northog<<" vectors\n";
|
|
||||||
for(int back=0; back < northog; back++){
|
|
||||||
int peri_back = (k-back)%mmax;
|
|
||||||
RealD pbApk= real(innerProduct(mmp[rhs][peri_back],p[rhs][peri_kp]));
|
|
||||||
RealD beta = -pbApk/pAp[rhs][peri_back];
|
|
||||||
axpy(p[rhs][peri_kp],beta,p[rhs][peri_back],p[rhs][peri_kp]);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD rrn=sqrt(rn[rhs]/ssq[rhs]);
|
|
||||||
RealD rtn=sqrt(rtz[rhs]/ssq[rhs]);
|
|
||||||
RealD rtnp=sqrt(rtzp[rhs]/ssq[rhs]);
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: rhs "<<rhs<<"fPcg k= "<<k<<" residual = "<<rrn<<"\n";
|
|
||||||
if ( rrn > max_rn ) max_rn = rrn;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stopping condition based on worst case
|
|
||||||
if ( max_rn <= Tolerance ) {
|
|
||||||
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
|
||||||
Field tmp(grid);
|
|
||||||
axpy(tmp,-1.0,src[rhs],mmp[rhs][0]);
|
|
||||||
|
|
||||||
RealD mmpnorm = sqrt(norm2(mmp[rhs][0]));
|
|
||||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
|
||||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
|
||||||
RealD tmpnorm = sqrt(norm2(tmp));
|
|
||||||
RealD true_residual = tmpnorm/srcnorm;
|
|
||||||
std::cout<<GridLogMessage
|
|
||||||
<<"HDCG: true residual ["<<rhs<<"] is "<<true_residual
|
|
||||||
<<" solution "<<xnorm
|
|
||||||
<<" source "<<srcnorm
|
|
||||||
<<" mmp "<<mmpnorm
|
|
||||||
<<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
|
||||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out)
|
|
||||||
{
|
|
||||||
std::cout << "PcgM1 default (cheat) mrhs version"<<std::endl;
|
|
||||||
for(int rhs=0;rhs<in.size();rhs++){
|
|
||||||
this->PcgM1(in[rhs],out[rhs]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual void PcgM1(Field & in, Field & out) =0;
|
|
||||||
virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src)
|
|
||||||
{
|
|
||||||
std::cout << "Vstart default (cheat) mrhs version"<<std::endl;
|
|
||||||
for(int rhs=0;rhs<x.size();rhs++){
|
|
||||||
this->Vstart(x[rhs],src[rhs]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual void Vstart(Field & x,const Field & src)=0;
|
|
||||||
|
|
||||||
virtual void PcgM2(const Field & in, Field & out) {
|
|
||||||
out=in;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual RealD PcgM3(const Field & p, Field & mmp){
|
|
||||||
RealD dd;
|
|
||||||
_FineLinop.HermOp(p,mmp);
|
|
||||||
ComplexD dot = innerProduct(p,mmp);
|
|
||||||
dd=real(dot);
|
|
||||||
return dd;
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
// Only Def1 has non-trivial Vout.
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field, class CoarseField, class Aggregation>
|
|
||||||
class TwoLevelADEF2 : public TwoLevelCG<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Need something that knows how to get from Coarse to fine and back again
|
|
||||||
// void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
|
|
||||||
// void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////
|
|
||||||
GridBase *coarsegrid;
|
|
||||||
Aggregation &_Aggregates;
|
|
||||||
LinearFunction<CoarseField> &_CoarseSolver;
|
|
||||||
LinearFunction<CoarseField> &_CoarseSolverPrecise;
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// more most opertor functions
|
|
||||||
TwoLevelADEF2(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearOperatorBase<Field> &FineLinop,
|
|
||||||
LinearFunction<Field> &Smoother,
|
|
||||||
LinearFunction<CoarseField> &CoarseSolver,
|
|
||||||
LinearFunction<CoarseField> &CoarseSolverPrecise,
|
|
||||||
Aggregation &Aggregates
|
|
||||||
) :
|
|
||||||
TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,Aggregates.FineGrid),
|
|
||||||
_CoarseSolver(CoarseSolver),
|
|
||||||
_CoarseSolverPrecise(CoarseSolverPrecise),
|
|
||||||
_Aggregates(Aggregates)
|
|
||||||
{
|
|
||||||
coarsegrid = Aggregates.CoarseGrid;
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual void PcgM1(Field & in, Field & out)
|
|
||||||
{
|
|
||||||
GRID_TRACE("MultiGridPreconditioner ");
|
|
||||||
// [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
|
|
||||||
|
|
||||||
Field tmp(this->grid);
|
|
||||||
Field Min(this->grid);
|
|
||||||
CoarseField PleftProj(this->coarsegrid);
|
|
||||||
CoarseField PleftMss_proj(this->coarsegrid);
|
|
||||||
|
|
||||||
GridStopWatch SmootherTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
SmootherTimer.Start();
|
|
||||||
this->_Smoother(in,Min);
|
|
||||||
SmootherTimer.Stop();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
this->_FineLinop.HermOp(Min,out);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
axpy(tmp,-1.0,out,in); // tmp = in - A Min
|
|
||||||
|
|
||||||
GridStopWatch ProjTimer;
|
|
||||||
GridStopWatch CoarseTimer;
|
|
||||||
GridStopWatch PromTimer;
|
|
||||||
ProjTimer.Start();
|
|
||||||
this->_Aggregates.ProjectToSubspace(PleftProj,tmp);
|
|
||||||
ProjTimer.Stop();
|
|
||||||
CoarseTimer.Start();
|
|
||||||
this->_CoarseSolver(PleftProj,PleftMss_proj); // Ass^{-1} [in - A Min]_s
|
|
||||||
CoarseTimer.Stop();
|
|
||||||
PromTimer.Start();
|
|
||||||
this->_Aggregates.PromoteFromSubspace(PleftMss_proj,tmp);// tmp = Q[in - A Min]
|
|
||||||
PromTimer.Stop();
|
|
||||||
std::cout << GridLogPerformance << "PcgM1 breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tSmoother " << SmootherTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tProj " << ProjTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tCoarse " << CoarseTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tProm " << PromTimer.Elapsed() <<std::endl;
|
|
||||||
|
|
||||||
axpy(out,1.0,Min,tmp); // Min+tmp
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void Vstart(Field & x,const Field & src)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart "<<std::endl;
|
|
||||||
///////////////////////////////////
|
|
||||||
// Choose x_0 such that
|
|
||||||
// x_0 = guess + (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
|
|
||||||
// = [1 - Ass_inv A] Guess + Assinv src
|
|
||||||
// = P^T guess + Assinv src
|
|
||||||
// = Vstart [Tang notation]
|
|
||||||
// This gives:
|
|
||||||
// W^T (src - A x_0) = src_s - A guess_s - r_s
|
|
||||||
// = src_s - (A guess)_s - src_s + (A guess)_s
|
|
||||||
// = 0
|
|
||||||
///////////////////////////////////
|
|
||||||
Field r(this->grid);
|
|
||||||
Field mmp(this->grid);
|
|
||||||
CoarseField PleftProj(this->coarsegrid);
|
|
||||||
CoarseField PleftMss_proj(this->coarsegrid);
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart projecting "<<std::endl;
|
|
||||||
this->_Aggregates.ProjectToSubspace(PleftProj,src);
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart coarse solve "<<std::endl;
|
|
||||||
this->_CoarseSolverPrecise(PleftProj,PleftMss_proj); // Ass^{-1} r_s
|
|
||||||
std::cout << GridLogMessage<<"HDCG: fPcg Vstart promote "<<std::endl;
|
|
||||||
this->_Aggregates.PromoteFromSubspace(PleftMss_proj,x);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class TwoLevelADEF1defl : public TwoLevelCG<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
const std::vector<Field> &evec;
|
|
||||||
const std::vector<RealD> &eval;
|
|
||||||
|
|
||||||
TwoLevelADEF1defl(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearOperatorBase<Field> &FineLinop,
|
|
||||||
LinearFunction<Field> &Smoother,
|
|
||||||
std::vector<Field> &_evec,
|
|
||||||
std::vector<RealD> &_eval) :
|
|
||||||
TwoLevelCG<Field>(tol,maxit,FineLinop,Smoother,_evec[0].Grid()),
|
|
||||||
evec(_evec),
|
|
||||||
eval(_eval)
|
|
||||||
{};
|
|
||||||
|
|
||||||
// Can just inherit existing M2
|
|
||||||
// Can just inherit existing M3
|
|
||||||
|
|
||||||
// Simple vstart - do nothing
|
|
||||||
virtual void Vstart(Field & x,const Field & src){
|
|
||||||
x=src; // Could apply Q
|
|
||||||
};
|
|
||||||
|
|
||||||
// Override PcgM1
|
|
||||||
virtual void PcgM1(Field & in, Field & out)
|
|
||||||
{
|
|
||||||
GRID_TRACE("EvecPreconditioner ");
|
|
||||||
int N=evec.size();
|
|
||||||
Field Pin(this->grid);
|
|
||||||
Field Qin(this->grid);
|
|
||||||
|
|
||||||
//MP + Q = M(1-AQ) + Q = M
|
|
||||||
// // If we are eigenvector deflating in coarse space
|
|
||||||
// // Q = Sum_i |phi_i> 1/lambda_i <phi_i|
|
|
||||||
// // A Q = Sum_i |phi_i> <phi_i|
|
|
||||||
// // M(1-AQ) = M(1-proj) + Q
|
|
||||||
Qin.Checkerboard()=in.Checkerboard();
|
|
||||||
Qin = Zero();
|
|
||||||
Pin = in;
|
|
||||||
for (int i=0;i<N;i++) {
|
|
||||||
const Field& tmp = evec[i];
|
|
||||||
auto ip = TensorRemove(innerProduct(tmp,in));
|
|
||||||
axpy(Qin, ip / eval[i],tmp,Qin);
|
|
||||||
axpy(Pin, -ip ,tmp,Pin);
|
|
||||||
}
|
|
||||||
|
|
||||||
this->_Smoother(Pin,out);
|
|
||||||
|
|
||||||
out = out + Qin;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,734 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/AdefGeneric.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Compared to Tang-2009: P=Pleft. P^T = PRight Q=MssInv.
|
|
||||||
* Script A = SolverMatrix
|
|
||||||
* Script P = Preconditioner
|
|
||||||
*
|
|
||||||
* Implement ADEF-2
|
|
||||||
*
|
|
||||||
* Vstart = P^Tx + Qb
|
|
||||||
* M1 = P^TM + Q
|
|
||||||
* M2=M3=1
|
|
||||||
*/
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class TwoLevelCGmrhs
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
GridBase *grid;
|
|
||||||
|
|
||||||
// Fine operator, Smoother, CoarseSolver
|
|
||||||
LinearOperatorBase<Field> &_FineLinop;
|
|
||||||
LinearFunction<Field> &_Smoother;
|
|
||||||
MultiRHSBlockCGLinalg<Field> _BlockCGLinalg;
|
|
||||||
|
|
||||||
GridStopWatch ProjectTimer;
|
|
||||||
GridStopWatch PromoteTimer;
|
|
||||||
GridStopWatch DeflateTimer;
|
|
||||||
GridStopWatch CoarseTimer;
|
|
||||||
GridStopWatch FineTimer;
|
|
||||||
GridStopWatch SmoothTimer;
|
|
||||||
GridStopWatch InsertTimer;
|
|
||||||
|
|
||||||
/*
|
|
||||||
Field rrr;
|
|
||||||
Field sss;
|
|
||||||
Field qqq;
|
|
||||||
Field zzz;
|
|
||||||
*/
|
|
||||||
// more most opertor functions
|
|
||||||
TwoLevelCGmrhs(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearOperatorBase<Field> &FineLinop,
|
|
||||||
LinearFunction<Field> &Smoother,
|
|
||||||
GridBase *fine) :
|
|
||||||
Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
_FineLinop(FineLinop),
|
|
||||||
_Smoother(Smoother)
|
|
||||||
/*
|
|
||||||
rrr(fine),
|
|
||||||
sss(fine),
|
|
||||||
qqq(fine),
|
|
||||||
zzz(fine)
|
|
||||||
*/
|
|
||||||
{
|
|
||||||
grid = fine;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Vector case
|
|
||||||
virtual void operator() (std::vector<Field> &src, std::vector<Field> &x)
|
|
||||||
{
|
|
||||||
SolveSingleSystem(src,x);
|
|
||||||
// SolvePrecBlockCG(src,x);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Thin QR factorisation (google it)
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
//Dimensions
|
|
||||||
// R_{ferm x Nblock} = Q_{ferm x Nblock} x C_{Nblock x Nblock} -> ferm x Nblock
|
|
||||||
//
|
|
||||||
// Rdag R = m_rr = Herm = L L^dag <-- Cholesky decomposition (LLT routine in Eigen)
|
|
||||||
//
|
|
||||||
// Q C = R => Q = R C^{-1}
|
|
||||||
//
|
|
||||||
// Want Ident = Q^dag Q = C^{-dag} R^dag R C^{-1} = C^{-dag} L L^dag C^{-1} = 1_{Nblock x Nblock}
|
|
||||||
//
|
|
||||||
// Set C = L^{dag}, and then Q^dag Q = ident
|
|
||||||
//
|
|
||||||
// Checks:
|
|
||||||
// Cdag C = Rdag R ; passes.
|
|
||||||
// QdagQ = 1 ; passes
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
void ThinQRfact (Eigen::MatrixXcd &m_zz,
|
|
||||||
Eigen::MatrixXcd &C,
|
|
||||||
Eigen::MatrixXcd &Cinv,
|
|
||||||
std::vector<Field> & Q,
|
|
||||||
std::vector<Field> & MQ,
|
|
||||||
const std::vector<Field> & Z,
|
|
||||||
const std::vector<Field> & MZ)
|
|
||||||
{
|
|
||||||
RealD t0=usecond();
|
|
||||||
_BlockCGLinalg.InnerProductMatrix(m_zz,MZ,Z);
|
|
||||||
RealD t1=usecond();
|
|
||||||
|
|
||||||
m_zz = 0.5*(m_zz+m_zz.adjoint());
|
|
||||||
|
|
||||||
Eigen::MatrixXcd L = m_zz.llt().matrixL();
|
|
||||||
|
|
||||||
C = L.adjoint();
|
|
||||||
Cinv = C.inverse();
|
|
||||||
|
|
||||||
RealD t3=usecond();
|
|
||||||
_BlockCGLinalg.MulMatrix( Q,Cinv,Z);
|
|
||||||
_BlockCGLinalg.MulMatrix(MQ,Cinv,MZ);
|
|
||||||
RealD t4=usecond();
|
|
||||||
std::cout << " ThinQRfact IP :"<< t1-t0<<" us"<<std::endl;
|
|
||||||
std::cout << " ThinQRfact Eigen :"<< t3-t1<<" us"<<std::endl;
|
|
||||||
std::cout << " ThinQRfact MulMat:"<< t4-t3<<" us"<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void SolvePrecBlockCG (std::vector<Field> &src, std::vector<Field> &X)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<<"HDCG: mrhs fPrecBlockcg starting"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
int nrhs = src.size();
|
|
||||||
// std::vector<RealD> f(nrhs);
|
|
||||||
// std::vector<RealD> rtzp(nrhs);
|
|
||||||
// std::vector<RealD> rtz(nrhs);
|
|
||||||
// std::vector<RealD> a(nrhs);
|
|
||||||
// std::vector<RealD> d(nrhs);
|
|
||||||
// std::vector<RealD> b(nrhs);
|
|
||||||
// std::vector<RealD> rptzp(nrhs);
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
//Initial residual computation & set up
|
|
||||||
////////////////////////////////////////////
|
|
||||||
std::vector<RealD> ssq(nrhs);
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
ssq[rhs]=norm2(src[rhs]); GRID_ASSERT(ssq[rhs]!=0.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////
|
|
||||||
// Fields -- eliminate duplicates between fPcg and block cg
|
|
||||||
///////////////////////////
|
|
||||||
std::vector<Field> Mtmp(nrhs,grid);
|
|
||||||
std::vector<Field> tmp(nrhs,grid);
|
|
||||||
std::vector<Field> Z(nrhs,grid); // Rename Z to R
|
|
||||||
std::vector<Field> MZ(nrhs,grid); // Rename MZ to Z
|
|
||||||
std::vector<Field> Q(nrhs,grid); //
|
|
||||||
std::vector<Field> MQ(nrhs,grid); // Rename to P
|
|
||||||
std::vector<Field> D(nrhs,grid);
|
|
||||||
std::vector<Field> AD(nrhs,grid);
|
|
||||||
|
|
||||||
/************************************************************************
|
|
||||||
* Preconditioned Block conjugate gradient rQ
|
|
||||||
* Generalise Sebastien Birk Thesis, after Dubrulle 2001.
|
|
||||||
* Introduce preconditioning following Saad Ch9
|
|
||||||
************************************************************************
|
|
||||||
* Dimensions:
|
|
||||||
*
|
|
||||||
* X,B etc... ==(Nferm x nrhs)
|
|
||||||
* Matrix A==(Nferm x Nferm)
|
|
||||||
*
|
|
||||||
* Nferm = Nspin x Ncolour x Ncomplex x Nlattice_site
|
|
||||||
* QC => Thin QR factorisation (google it)
|
|
||||||
*
|
|
||||||
* R = B-AX
|
|
||||||
* Z = Mi R
|
|
||||||
* QC = Z
|
|
||||||
* D = Q
|
|
||||||
* for k:
|
|
||||||
* R = AD
|
|
||||||
* Z = Mi R
|
|
||||||
* M = [D^dag R]^{-1}
|
|
||||||
* X = X + D M C
|
|
||||||
* QS = Q - Z.M
|
|
||||||
* D = Q + D S^dag
|
|
||||||
* C = S C
|
|
||||||
*/
|
|
||||||
Eigen::MatrixXcd m_DZ = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_M = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_zz = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_rr = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
|
|
||||||
Eigen::MatrixXcd m_C = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_Cinv = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_S = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_Sinv = Eigen::MatrixXcd::Zero(nrhs,nrhs);
|
|
||||||
|
|
||||||
Eigen::MatrixXcd m_tmp = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
|
||||||
Eigen::MatrixXcd m_tmp1 = Eigen::MatrixXcd::Identity(nrhs,nrhs);
|
|
||||||
|
|
||||||
GridStopWatch HDCGTimer;
|
|
||||||
|
|
||||||
//////////////////////////
|
|
||||||
// x0 = Vstart -- possibly modify guess
|
|
||||||
//////////////////////////
|
|
||||||
Vstart(X,src);
|
|
||||||
|
|
||||||
//////////////////////////
|
|
||||||
// R = B-AX
|
|
||||||
//////////////////////////
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
// r0 = b -A x0
|
|
||||||
_FineLinop.HermOp(X[rhs],tmp[rhs]);
|
|
||||||
axpy (Z[rhs], -1.0,tmp[rhs], src[rhs]); // Computes R=Z=src - A X0
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// Compute MZ = M1 Z = M1 B - M1 A x0
|
|
||||||
//////////////////////////////////
|
|
||||||
PcgM1(Z,MZ);
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// QC = Z
|
|
||||||
//////////////////////////////////
|
|
||||||
ThinQRfact (m_zz, m_C, m_Cinv, Q, MQ, Z, MZ);
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// D=MQ
|
|
||||||
//////////////////////////////////
|
|
||||||
for(int b=0;b<nrhs;b++) D[b]=MQ[b]; // LLT rotation of the MZ basis of search dirs
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<"PrecBlockCGrQ vec computed initial residual and QR fact " <<std::endl;
|
|
||||||
|
|
||||||
ProjectTimer.Reset();
|
|
||||||
PromoteTimer.Reset();
|
|
||||||
DeflateTimer.Reset();
|
|
||||||
CoarseTimer.Reset();
|
|
||||||
SmoothTimer.Reset();
|
|
||||||
FineTimer.Reset();
|
|
||||||
InsertTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch M1Timer;
|
|
||||||
GridStopWatch M2Timer;
|
|
||||||
GridStopWatch M3Timer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch InnerProdTimer;
|
|
||||||
|
|
||||||
HDCGTimer.Start();
|
|
||||||
|
|
||||||
std::vector<RealD> rn(nrhs);
|
|
||||||
for (int k=0;k<=MaxIterations;k++){
|
|
||||||
|
|
||||||
////////////////////
|
|
||||||
// Z = AD
|
|
||||||
////////////////////
|
|
||||||
M3Timer.Start();
|
|
||||||
for(int b=0;b<nrhs;b++) _FineLinop.HermOp(D[b], Z[b]);
|
|
||||||
M3Timer.Stop();
|
|
||||||
|
|
||||||
////////////////////
|
|
||||||
// MZ = M1 Z <==== the Multigrid preconditioner
|
|
||||||
////////////////////
|
|
||||||
M1Timer.Start();
|
|
||||||
PcgM1(Z,MZ);
|
|
||||||
M1Timer.Stop();
|
|
||||||
|
|
||||||
FineTimer.Start();
|
|
||||||
////////////////////
|
|
||||||
// M = [D^dag Z]^{-1} = (<Ddag MZ>_M)^{-1} inner prod, generalising Saad derivation of Precon CG
|
|
||||||
////////////////////
|
|
||||||
InnerProdTimer.Start();
|
|
||||||
_BlockCGLinalg.InnerProductMatrix(m_DZ,D,Z);
|
|
||||||
InnerProdTimer.Stop();
|
|
||||||
m_M = m_DZ.inverse();
|
|
||||||
|
|
||||||
///////////////////////////
|
|
||||||
// X = X + D MC
|
|
||||||
///////////////////////////
|
|
||||||
m_tmp = m_M * m_C;
|
|
||||||
LinalgTimer.Start();
|
|
||||||
_BlockCGLinalg.MaddMatrix(X,m_tmp, D,X); // D are the search directions and X takes the updates
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
///////////////////////////
|
|
||||||
// QS = Q - M Z
|
|
||||||
// (MQ) S = MQ - M (M1Z)
|
|
||||||
///////////////////////////
|
|
||||||
LinalgTimer.Start();
|
|
||||||
_BlockCGLinalg.MaddMatrix(tmp ,m_M, Z, Q,-1.0);
|
|
||||||
_BlockCGLinalg.MaddMatrix(Mtmp,m_M,MZ,MQ,-1.0);
|
|
||||||
ThinQRfact (m_zz, m_S, m_Sinv, Q, MQ, tmp, Mtmp);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
////////////////////////////
|
|
||||||
// D = MQ + D S^dag
|
|
||||||
////////////////////////////
|
|
||||||
m_tmp = m_S.adjoint();
|
|
||||||
LinalgTimer.Start();
|
|
||||||
_BlockCGLinalg.MaddMatrix(D,m_tmp,D,MQ);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
////////////////////////////
|
|
||||||
// C = S C
|
|
||||||
////////////////////////////
|
|
||||||
m_C = m_S*m_C;
|
|
||||||
|
|
||||||
////////////////////////////
|
|
||||||
// convergence monitor
|
|
||||||
////////////////////////////
|
|
||||||
m_rr = m_C.adjoint() * m_C;
|
|
||||||
|
|
||||||
FineTimer.Stop();
|
|
||||||
|
|
||||||
RealD max_resid=0;
|
|
||||||
RealD rrsum=0;
|
|
||||||
RealD sssum=0;
|
|
||||||
RealD rr;
|
|
||||||
|
|
||||||
for(int b=0;b<nrhs;b++) {
|
|
||||||
rrsum+=real(m_rr(b,b));
|
|
||||||
sssum+=ssq[b];
|
|
||||||
rr = real(m_rr(b,b))/ssq[b];
|
|
||||||
if ( rr > max_resid ) max_resid = rr;
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage <<
|
|
||||||
"\t Prec BlockCGrQ Iteration "<<k<<" ave resid "<< std::sqrt(rrsum/sssum) << " max "<< std::sqrt(max_resid) <<std::endl;
|
|
||||||
|
|
||||||
|
|
||||||
if ( max_resid < Tolerance*Tolerance ) {
|
|
||||||
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Linalg "<<LinalgTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : fine H "<<M3Timer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Project "<<ProjectTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Coarse "<<CoarseTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Fine "<<FineTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Smooth "<<SmoothTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs PrecBlockCGrQ : Insert "<<InsertTimer.Elapsed()<<std::endl;;
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
|
|
||||||
_FineLinop.HermOp(X[rhs],tmp[rhs]);
|
|
||||||
|
|
||||||
Field mytmp(grid);
|
|
||||||
axpy(mytmp,-1.0,src[rhs],tmp[rhs]);
|
|
||||||
|
|
||||||
RealD xnorm = sqrt(norm2(X[rhs]));
|
|
||||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
|
||||||
RealD tmpnorm = sqrt(norm2(mytmp));
|
|
||||||
RealD true_residual = tmpnorm/srcnorm;
|
|
||||||
std::cout<<GridLogMessage
|
|
||||||
<<"HDCG: true residual ["<<rhs<<"] is "<<true_residual
|
|
||||||
<<" solution "<<xnorm
|
|
||||||
<<" source "<<srcnorm
|
|
||||||
<<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: PrecBlockCGrQ not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void SolveSingleSystem (std::vector<Field> &src, std::vector<Field> &x)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<<"HDCG: mrhs fPcg starting"<<std::endl;
|
|
||||||
src[0].Grid()->Barrier();
|
|
||||||
int nrhs = src.size();
|
|
||||||
std::vector<RealD> f(nrhs);
|
|
||||||
std::vector<RealD> rtzp(nrhs);
|
|
||||||
std::vector<RealD> rtz(nrhs);
|
|
||||||
std::vector<RealD> a(nrhs);
|
|
||||||
std::vector<RealD> d(nrhs);
|
|
||||||
std::vector<RealD> b(nrhs);
|
|
||||||
std::vector<RealD> rptzp(nrhs);
|
|
||||||
/////////////////////////////
|
|
||||||
// Set up history vectors
|
|
||||||
/////////////////////////////
|
|
||||||
int mmax = 3;
|
|
||||||
|
|
||||||
std::vector<std::vector<Field> > p(nrhs); for(int r=0;r<nrhs;r++) p[r].resize(mmax,grid);
|
|
||||||
std::vector<std::vector<Field> > mmp(nrhs); for(int r=0;r<nrhs;r++) mmp[r].resize(mmax,grid);
|
|
||||||
std::vector<std::vector<RealD> > pAp(nrhs); for(int r=0;r<nrhs;r++) pAp[r].resize(mmax);
|
|
||||||
|
|
||||||
std::vector<Field> z(nrhs,grid);
|
|
||||||
std::vector<Field> mp (nrhs,grid);
|
|
||||||
std::vector<Field> r (nrhs,grid);
|
|
||||||
std::vector<Field> mu (nrhs,grid);
|
|
||||||
|
|
||||||
//Initial residual computation & set up
|
|
||||||
std::vector<RealD> src_nrm(nrhs);
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
src_nrm[rhs]=norm2(src[rhs]);
|
|
||||||
GRID_ASSERT(src_nrm[rhs]!=0.0);
|
|
||||||
}
|
|
||||||
std::vector<RealD> tn(nrhs);
|
|
||||||
|
|
||||||
GridStopWatch HDCGTimer;
|
|
||||||
//////////////////////////
|
|
||||||
// x0 = Vstart -- possibly modify guess
|
|
||||||
//////////////////////////
|
|
||||||
Vstart(x,src);
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
// r0 = b -A x0
|
|
||||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
|
||||||
axpy (r[rhs], -1.0,mmp[rhs][0], src[rhs]); // Recomputes r=src-Ax0
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// Compute z = M1 x
|
|
||||||
//////////////////////////////////
|
|
||||||
// This needs a multiRHS version for acceleration
|
|
||||||
PcgM1(r,z);
|
|
||||||
|
|
||||||
std::vector<RealD> ssq(nrhs);
|
|
||||||
std::vector<RealD> rsq(nrhs);
|
|
||||||
std::vector<Field> pp(nrhs,grid);
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
|
||||||
p[rhs][0]=z[rhs];
|
|
||||||
ssq[rhs]=norm2(src[rhs]);
|
|
||||||
rsq[rhs]= ssq[rhs]*Tolerance*Tolerance;
|
|
||||||
// std::cout << GridLogMessage<<"mrhs HDCG: "<<rhs<<" k=0 residual "<<rtzp[rhs]<<" rsq "<<rsq[rhs]<<"\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
ProjectTimer.Reset();
|
|
||||||
PromoteTimer.Reset();
|
|
||||||
DeflateTimer.Reset();
|
|
||||||
CoarseTimer.Reset();
|
|
||||||
SmoothTimer.Reset();
|
|
||||||
FineTimer.Reset();
|
|
||||||
InsertTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch M1Timer;
|
|
||||||
GridStopWatch M2Timer;
|
|
||||||
GridStopWatch M3Timer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
|
|
||||||
HDCGTimer.Start();
|
|
||||||
|
|
||||||
std::vector<RealD> rn(nrhs);
|
|
||||||
for (int k=0;k<=MaxIterations;k++){
|
|
||||||
|
|
||||||
int peri_k = k % mmax;
|
|
||||||
int peri_kp = (k+1) % mmax;
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
rtz[rhs]=rtzp[rhs];
|
|
||||||
M3Timer.Start();
|
|
||||||
d[rhs]= PcgM3(p[rhs][peri_k],mmp[rhs][peri_k]);
|
|
||||||
M3Timer.Stop();
|
|
||||||
a[rhs] = rtz[rhs]/d[rhs];
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
// Memorise this
|
|
||||||
pAp[rhs][peri_k] = d[rhs];
|
|
||||||
|
|
||||||
axpy(x[rhs],a[rhs],p[rhs][peri_k],x[rhs]);
|
|
||||||
rn[rhs] = axpy_norm(r[rhs],-a[rhs],mmp[rhs][peri_k],r[rhs]);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute z = M x (for *all* RHS)
|
|
||||||
M1Timer.Start();
|
|
||||||
PcgM1(r,z);
|
|
||||||
M1Timer.Stop();
|
|
||||||
|
|
||||||
RealD max_rn=0.0;
|
|
||||||
LinalgTimer.Start();
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
|
|
||||||
rtzp[rhs] =real(innerProduct(r[rhs],z[rhs]));
|
|
||||||
|
|
||||||
// std::cout << GridLogMessage<<"HDCG::fPcg rhs"<<rhs<<" iteration "<<k<<" : inner rtzp "<<rtzp[rhs]<<"\n";
|
|
||||||
mu[rhs]=z[rhs];
|
|
||||||
|
|
||||||
p[rhs][peri_kp]=mu[rhs];
|
|
||||||
|
|
||||||
// Standard search direction p == z + b p
|
|
||||||
b[rhs] = (rtzp[rhs])/rtz[rhs];
|
|
||||||
|
|
||||||
int northog = (k>mmax-1)?(mmax-1):k; // This is the fCG-Tr(mmax-1) algorithm
|
|
||||||
for(int back=0; back < northog; back++){
|
|
||||||
int peri_back = (k-back)%mmax;
|
|
||||||
RealD pbApk= real(innerProduct(mmp[rhs][peri_back],p[rhs][peri_kp]));
|
|
||||||
RealD beta = -pbApk/pAp[rhs][peri_back];
|
|
||||||
axpy(p[rhs][peri_kp],beta,p[rhs][peri_back],p[rhs][peri_kp]);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD rrn=sqrt(rn[rhs]/ssq[rhs]);
|
|
||||||
RealD rtn=sqrt(rtz[rhs]/ssq[rhs]);
|
|
||||||
RealD rtnp=sqrt(rtzp[rhs]/ssq[rhs]);
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"HDCG:fPcg rhs "<<rhs<<" k= "<<k<<" residual = "<<rrn<<"\n";
|
|
||||||
if ( rrn > max_rn ) max_rn = rrn;
|
|
||||||
}
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
// Stopping condition based on worst case
|
|
||||||
if ( max_rn <= Tolerance ) {
|
|
||||||
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg converged in "<<k<<" iterations and "<<HDCGTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Linalg "<<LinalgTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : fine M3 "<<M3Timer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : prec M1 "<<M1Timer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"**** M1 breakdown:"<<std::endl;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Project "<<ProjectTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Promote "<<PromoteTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Deflate "<<DeflateTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Coarse "<<CoarseTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Fine "<<FineTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Smooth "<<SmoothTimer.Elapsed()<<std::endl;;
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: mrhs fPcg : Insert "<<InsertTimer.Elapsed()<<std::endl;;
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
_FineLinop.HermOp(x[rhs],mmp[rhs][0]);
|
|
||||||
Field tmp(grid);
|
|
||||||
axpy(tmp,-1.0,src[rhs],mmp[rhs][0]);
|
|
||||||
|
|
||||||
RealD mmpnorm = sqrt(norm2(mmp[rhs][0]));
|
|
||||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
|
||||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
|
||||||
RealD tmpnorm = sqrt(norm2(tmp));
|
|
||||||
RealD true_residual = tmpnorm/srcnorm;
|
|
||||||
std::cout<<GridLogMessage
|
|
||||||
<<"HDCG: true residual ["<<rhs<<"] is "<<true_residual
|
|
||||||
<<" solution "<<xnorm
|
|
||||||
<<" source "<<srcnorm
|
|
||||||
<<" mmp "<<mmpnorm
|
|
||||||
<<std::endl;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
HDCGTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: not converged "<<HDCGTimer.Elapsed()<<std::endl;
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++){
|
|
||||||
RealD xnorm = sqrt(norm2(x[rhs]));
|
|
||||||
RealD srcnorm = sqrt(norm2(src[rhs]));
|
|
||||||
std::cout<<GridLogMessage<<"HDCG: non-converged solution "<<xnorm<<" source "<<srcnorm<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out) = 0;
|
|
||||||
virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src) = 0;
|
|
||||||
virtual void PcgM2(const Field & in, Field & out) {
|
|
||||||
out=in;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual RealD PcgM3(const Field & p, Field & mmp){
|
|
||||||
RealD dd;
|
|
||||||
_FineLinop.HermOp(p,mmp);
|
|
||||||
ComplexD dot = innerProduct(p,mmp);
|
|
||||||
dd=real(dot);
|
|
||||||
return dd;
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field, class CoarseField>
|
|
||||||
class TwoLevelADEF2mrhs : public TwoLevelCGmrhs<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
GridBase *coarsegrid;
|
|
||||||
GridBase *coarsegridmrhs;
|
|
||||||
LinearFunction<CoarseField> &_CoarseSolverMrhs;
|
|
||||||
LinearFunction<CoarseField> &_CoarseSolverPreciseMrhs;
|
|
||||||
MultiRHSBlockProject<Field> &_Projector;
|
|
||||||
MultiRHSDeflation<CoarseField> &_Deflator;
|
|
||||||
|
|
||||||
|
|
||||||
TwoLevelADEF2mrhs(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearOperatorBase<Field> &FineLinop,
|
|
||||||
LinearFunction<Field> &Smoother,
|
|
||||||
LinearFunction<CoarseField> &CoarseSolverMrhs,
|
|
||||||
LinearFunction<CoarseField> &CoarseSolverPreciseMrhs,
|
|
||||||
MultiRHSBlockProject<Field> &Projector,
|
|
||||||
MultiRHSDeflation<CoarseField> &Deflator,
|
|
||||||
GridBase *_coarsemrhsgrid) :
|
|
||||||
TwoLevelCGmrhs<Field>(tol, maxit,FineLinop,Smoother,Projector.fine_grid),
|
|
||||||
_CoarseSolverMrhs(CoarseSolverMrhs),
|
|
||||||
_CoarseSolverPreciseMrhs(CoarseSolverPreciseMrhs),
|
|
||||||
_Projector(Projector),
|
|
||||||
_Deflator(Deflator)
|
|
||||||
{
|
|
||||||
coarsegrid = Projector.coarse_grid;
|
|
||||||
coarsegridmrhs = _coarsemrhsgrid;// Thi could be in projector
|
|
||||||
};
|
|
||||||
|
|
||||||
// Override Vstart
|
|
||||||
virtual void Vstart(std::vector<Field> & x,std::vector<Field> & src)
|
|
||||||
{
|
|
||||||
int nrhs=x.size();
|
|
||||||
///////////////////////////////////
|
|
||||||
// Choose x_0 such that
|
|
||||||
// x_0 = guess + (A_ss^inv) r_s = guess + Ass_inv [src -Aguess]
|
|
||||||
// = [1 - Ass_inv A] Guess + Assinv src
|
|
||||||
// = P^T guess + Assinv src
|
|
||||||
// = Vstart [Tang notation]
|
|
||||||
// This gives:
|
|
||||||
// W^T (src - A x_0) = src_s - A guess_s - r_s
|
|
||||||
// = src_s - (A guess)_s - src_s + (A guess)_s
|
|
||||||
// = 0
|
|
||||||
///////////////////////////////////
|
|
||||||
std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
|
|
||||||
std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
|
|
||||||
CoarseField PleftProjMrhs(this->coarsegridmrhs);
|
|
||||||
CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
|
|
||||||
|
|
||||||
this->_Projector.blockProject(src,PleftProj);
|
|
||||||
this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
|
|
||||||
InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
|
|
||||||
}
|
|
||||||
|
|
||||||
this->_CoarseSolverPreciseMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} r_s
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
|
|
||||||
}
|
|
||||||
this->_Projector.blockPromote(x,PleftMss_proj);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void PcgM1(std::vector<Field> & in,std::vector<Field> & out){
|
|
||||||
|
|
||||||
int nrhs=in.size();
|
|
||||||
|
|
||||||
// [PTM+Q] in = [1 - Q A] M in + Q in = Min + Q [ in -A Min]
|
|
||||||
std::vector<Field> tmp(nrhs,this->grid);
|
|
||||||
std::vector<Field> Min(nrhs,this->grid);
|
|
||||||
|
|
||||||
std::vector<CoarseField> PleftProj(nrhs,this->coarsegrid);
|
|
||||||
std::vector<CoarseField> PleftMss_proj(nrhs,this->coarsegrid);
|
|
||||||
|
|
||||||
CoarseField PleftProjMrhs(this->coarsegridmrhs);
|
|
||||||
CoarseField PleftMss_projMrhs(this->coarsegridmrhs);
|
|
||||||
|
|
||||||
// this->rrr=in[0];
|
|
||||||
|
|
||||||
#undef SMOOTHER_BLOCK_SOLVE
|
|
||||||
#if SMOOTHER_BLOCK_SOLVE
|
|
||||||
this->SmoothTimer.Start();
|
|
||||||
this->_Smoother(in,Min);
|
|
||||||
this->SmoothTimer.Stop();
|
|
||||||
#else
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
this->SmoothTimer.Start();
|
|
||||||
this->_Smoother(in[rhs],Min[rhs]);
|
|
||||||
this->SmoothTimer.Stop();
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
// this->sss=Min[0];
|
|
||||||
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
|
|
||||||
this->FineTimer.Start();
|
|
||||||
this->_FineLinop.HermOp(Min[rhs],out[rhs]);
|
|
||||||
axpy(tmp[rhs],-1.0,out[rhs],in[rhs]); // resid = in - A Min
|
|
||||||
this->FineTimer.Stop();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
this->ProjectTimer.Start();
|
|
||||||
this->_Projector.blockProject(tmp,PleftProj);
|
|
||||||
this->ProjectTimer.Stop();
|
|
||||||
this->DeflateTimer.Start();
|
|
||||||
this->_Deflator.DeflateSources(PleftProj,PleftMss_proj);
|
|
||||||
this->DeflateTimer.Stop();
|
|
||||||
this->InsertTimer.Start();
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
InsertSliceFast(PleftProj[rhs],PleftProjMrhs,rhs,0);
|
|
||||||
InsertSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0); // the guess
|
|
||||||
}
|
|
||||||
this->InsertTimer.Stop();
|
|
||||||
|
|
||||||
this->CoarseTimer.Start();
|
|
||||||
this->_CoarseSolverMrhs(PleftProjMrhs,PleftMss_projMrhs); // Ass^{-1} [in - A Min]_s
|
|
||||||
this->CoarseTimer.Stop();
|
|
||||||
|
|
||||||
this->InsertTimer.Start();
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
ExtractSliceFast(PleftMss_proj[rhs],PleftMss_projMrhs,rhs,0);
|
|
||||||
}
|
|
||||||
this->InsertTimer.Stop();
|
|
||||||
this->PromoteTimer.Start();
|
|
||||||
this->_Projector.blockPromote(tmp,PleftMss_proj);// tmp= Q[in - A Min]
|
|
||||||
this->PromoteTimer.Stop();
|
|
||||||
this->FineTimer.Start();
|
|
||||||
// this->qqq=tmp[0];
|
|
||||||
for(int rhs=0;rhs<nrhs;rhs++) {
|
|
||||||
axpy(out[rhs],1.0,Min[rhs],tmp[rhs]); // Min+tmp
|
|
||||||
}
|
|
||||||
// this->zzz=out[0];
|
|
||||||
this->FineTimer.Stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/Arnoldi.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Patrick Oare <poare@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_ARNOLDI_H
|
|
||||||
#define GRID_ARNOLDI_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//Moved to KrylovSchur
|
|
||||||
#if 0
|
|
||||||
/**
|
|
||||||
<<<<<<< HEAD
|
|
||||||
* Options for which Ritz values to keep in implicit restart.
|
|
||||||
*/
|
|
||||||
enum RitzFilter {
|
|
||||||
EvalNormSmall, // Keep evals with smallest norm
|
|
||||||
EvalNormLarge, // Keep evals with largest norm
|
|
||||||
EvalReSmall, // Keep evals with smallest real part
|
|
||||||
EvalReLarge // Keep evals with largest real part
|
|
||||||
};
|
|
||||||
|
|
||||||
// Select comparison function from RitzFilter
|
|
||||||
struct ComplexComparator
|
|
||||||
{
|
|
||||||
RitzFilter f;
|
|
||||||
ComplexComparator (RitzFilter _f) : f(_f) {}
|
|
||||||
bool operator()(std::complex<double> z1, std::complex<double> z2) {
|
|
||||||
switch (f) {
|
|
||||||
RealD tmp1, tmp2;
|
|
||||||
tmp1=std::abs(std::imag(z1));
|
|
||||||
tmp2=std::abs(std::imag(z2));
|
|
||||||
case EvalNormSmall:
|
|
||||||
return std::abs(z1) < std::abs(z2);
|
|
||||||
case EvalNormLarge:
|
|
||||||
return std::abs(z1) > std::abs(z2);
|
|
||||||
// Terrible hack
|
|
||||||
// return std::abs(std::real(z1)) < std::abs(std::real(z2));
|
|
||||||
// if ( std::abs(std::real(z1)) >4.) tmp1 +=1.;
|
|
||||||
// if ( std::abs(std::real(z2)) >4.) tmp2 +=1.;
|
|
||||||
case EvalReSmall:
|
|
||||||
return tmp1 < tmp2;
|
|
||||||
// return std::abs(std::imag(z1)) < std::abs(std::imag(z2));
|
|
||||||
case EvalReLarge:
|
|
||||||
return tmp1 > tmp2;
|
|
||||||
// return std::abs(std::real(z1)) > std::abs(std::real(z2));
|
|
||||||
default:
|
|
||||||
assert(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
=======
|
|
||||||
>>>>>>> 68af1bba67dd62881ead5ab1e54962a5486a0791
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Implementation of the Arnoldi algorithm.
|
|
||||||
*/
|
|
||||||
template<class Field>
|
|
||||||
class Arnoldi {
|
|
||||||
|
|
||||||
private:
|
|
||||||
|
|
||||||
std::string cname = std::string("Arnoldi");
|
|
||||||
int MaxIter; // Max iterations
|
|
||||||
RealD Tolerance;
|
|
||||||
RealD ssq;
|
|
||||||
RealD rtol;
|
|
||||||
int Nm; // Number of basis vectors to track (equals MaxIter if no restart)
|
|
||||||
int Nk; // Number of basis vectors to keep every restart (equals -1 if no restart)
|
|
||||||
int Nstop; // Stop after converging Nstop eigenvectors.
|
|
||||||
|
|
||||||
LinearOperatorBase<Field> &Linop;
|
|
||||||
GridBase *Grid;
|
|
||||||
|
|
||||||
RealD approxLambdaMax;
|
|
||||||
RealD beta_k;
|
|
||||||
Field f;
|
|
||||||
std::vector<Field> basis; // orthonormal Arnoldi basis
|
|
||||||
Eigen::MatrixXcd Hess; // Hessenberg matrix of size Nbasis (after construction)
|
|
||||||
Eigen::MatrixXcd Qt; // Transpose of basis rotation which projects out high modes.
|
|
||||||
|
|
||||||
Eigen::VectorXcd evals; // evals of Hess
|
|
||||||
Eigen::MatrixXcd littleEvecs; // Nm x Nm evecs matrix
|
|
||||||
std::vector<Field> evecs; // Vector of evec fields
|
|
||||||
|
|
||||||
RitzFilter ritzFilter; // how to sort evals
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
Arnoldi(LinearOperatorBase<Field> &_Linop, GridBase *_Grid, RealD _Tolerance, RitzFilter filter = EvalReSmall)
|
|
||||||
: Linop(_Linop), Grid(_Grid), Tolerance(_Tolerance), ritzFilter(filter), f(_Grid), MaxIter(-1), Nm(-1), Nk(-1),
|
|
||||||
Nstop (-1), evals (0), evecs (), ssq (0.0), rtol (0.0), beta_k (0.0), approxLambdaMax (0.0)
|
|
||||||
{
|
|
||||||
f = Zero();
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Runs the Arnoldi loop with(out) implicit restarting. For each iteration:
|
|
||||||
* - Runs an Arnoldi step.
|
|
||||||
* - Computes the eigensystem of the Hessenberg matrix.
|
|
||||||
* - Performs implicit restarting.
|
|
||||||
*/
|
|
||||||
void operator()(const Field& v0, int _maxIter, int _Nm, int _Nk, int _Nstop, bool doubleOrthog = false) {
|
|
||||||
MaxIter = _maxIter;
|
|
||||||
Nm = _Nm; Nk = _Nk;
|
|
||||||
Nstop = _Nstop;
|
|
||||||
|
|
||||||
ssq = norm2(v0);
|
|
||||||
RealD approxLambdaMax = approxMaxEval(v0);
|
|
||||||
rtol = Tolerance * approxLambdaMax;
|
|
||||||
|
|
||||||
ComplexComparator compareComplex (ritzFilter);
|
|
||||||
std::cout << GridLogMessage << "Comparing Ritz values with: " << ritzFilter << std::endl;
|
|
||||||
|
|
||||||
int start = 1;
|
|
||||||
Field startVec = v0;
|
|
||||||
littleEvecs = Eigen::MatrixXcd::Zero(Nm, Nm);
|
|
||||||
for (int i = 0; i < MaxIter; i++) {
|
|
||||||
std::cout << GridLogMessage << "Restart Iteration " << i << std::endl;
|
|
||||||
|
|
||||||
// Perform Arnoldi steps to compute Krylov basis and Rayleigh quotient (Hess)
|
|
||||||
arnoldiIteration(startVec, Nm, start, doubleOrthog);
|
|
||||||
startVec = f;
|
|
||||||
|
|
||||||
// compute eigensystem and sort evals
|
|
||||||
// compute_eigensystem();
|
|
||||||
compute_eigensystem(Hess);
|
|
||||||
std::cout << GridLogMessage << "Eigenvalues after Arnoldi step: " << std::endl << evals << std::endl;
|
|
||||||
|
|
||||||
std::sort(evals.begin(), evals.end(), compareComplex);
|
|
||||||
std::cout << GridLogMessage << "Ritz values after sorting (first Nk preserved): " << std::endl << evals << std::endl;
|
|
||||||
// SU(N)::tepidConfiguration
|
|
||||||
|
|
||||||
// Implicit restart to de-weight unwanted eigenvalues
|
|
||||||
implicitRestart(_Nm, _Nk); // probably can delete _Nm and _Nk from function args
|
|
||||||
start = Nk;
|
|
||||||
|
|
||||||
// check convergence and return if needed.
|
|
||||||
int Nconv = converged();
|
|
||||||
std::cout << GridLogMessage << "Number of evecs converged: " << Nconv << std::endl;
|
|
||||||
if (Nconv >= Nstop || i == MaxIter - 1) {
|
|
||||||
std::cout << GridLogMessage << "Converged with " << Nconv << " / " << Nstop << " eigenvectors on iteration "
|
|
||||||
<< i << "." << std::endl;
|
|
||||||
basisRotate(evecs, Qt, 0, Nk, 0, Nk, Nm);
|
|
||||||
std::cout << GridLogMessage << "Eigenvalues [first " << Nconv << " converged]: " << std::endl << evals << std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Approximates the maximum eigenvalue of Linop.Op to normalize the residual and test for convergence.
|
|
||||||
*
|
|
||||||
* Parameters
|
|
||||||
* ----------
|
|
||||||
* Field& v0
|
|
||||||
* Source field to start with. Must have non-zero norm.
|
|
||||||
* int MAX_ITER (default = 50)
|
|
||||||
* Maximum number of iterations for power approximation.
|
|
||||||
*
|
|
||||||
* Returns
|
|
||||||
* -------
|
|
||||||
* RealD lamApprox
|
|
||||||
* Approximation of largest eigenvalue.
|
|
||||||
*/
|
|
||||||
RealD approxMaxEval(const Field& v0, int MAX_ITER = 50) {
|
|
||||||
assert (norm2(v0) > 1e-8); // must have relatively large source norm to start
|
|
||||||
RealD lamApprox = 0.0;
|
|
||||||
RealD denom = 1.0; RealD num = 1.0;
|
|
||||||
Field v0cp (Grid); Field tmp (Grid);
|
|
||||||
v0cp = v0;
|
|
||||||
denom = std::sqrt(norm2(v0cp));
|
|
||||||
for (int i = 0; i < MAX_ITER; i++) {
|
|
||||||
Linop.Op(v0cp, tmp); // CAREFUL: do not do Op(tmp, tmp)
|
|
||||||
v0cp = tmp;
|
|
||||||
num = std::sqrt(norm2(v0cp)); // num = |A^{n+1} v0|
|
|
||||||
lamApprox = num / denom; // lam = |A^{n+1} v0| / |A^n v0|
|
|
||||||
std::cout << GridLogDebug << "Approx for max eval: " << lamApprox << std::endl;
|
|
||||||
denom = num; // denom = |A^{n} v0|
|
|
||||||
}
|
|
||||||
return lamApprox;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Constructs the Arnoldi basis for the Krylov space K_n(D, src). (TODO make private)
|
|
||||||
*
|
|
||||||
* Parameters
|
|
||||||
* ----------
|
|
||||||
* v0 : Field&
|
|
||||||
* Source to generate Krylov basis.
|
|
||||||
* Nm : int
|
|
||||||
* Final size of the basis desired. If the basis becomes complete before a basis of size Nm is constructed
|
|
||||||
* (determined by relative tolerance Tolerance), stops iteration there.
|
|
||||||
* doubleOrthog : bool (default = false)
|
|
||||||
* Whether to double orthogonalize the basis (for numerical cancellations) or not.
|
|
||||||
* start : int (default = 0)
|
|
||||||
* If non-zero, assumes part of the Arnoldi basis has already been constructed.
|
|
||||||
*/
|
|
||||||
void arnoldiIteration(const Field& v0, int Nm, int start = 1, bool doubleOrthog = false)
|
|
||||||
{
|
|
||||||
|
|
||||||
ComplexD coeff;
|
|
||||||
Field w (Grid); // A acting on last Krylov vector.
|
|
||||||
|
|
||||||
if (start == 1) { // initialize everything that we need.
|
|
||||||
RealD v0Norm = 1 / std::sqrt(ssq);
|
|
||||||
basis.push_back(v0Norm * v0); // normalized source
|
|
||||||
|
|
||||||
Hess = Eigen::MatrixXcd::Zero(Nm, Nm);
|
|
||||||
f = Zero();
|
|
||||||
} else {
|
|
||||||
assert( start == basis.size() ); // should be starting at the end of basis (start = Nk)
|
|
||||||
Eigen::MatrixXcd HessCp = Hess;
|
|
||||||
Hess = Eigen::MatrixXcd::Zero(Nm, Nm);
|
|
||||||
Hess(Eigen::seqN(0, Nk), Eigen::seqN(0, Nk)) = HessCp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Construct next Arnoldi vector by normalizing w_i = Dv_i - \sum_j v_j h_{ji}
|
|
||||||
for (int i = start - 1; i < Nm; i++) {
|
|
||||||
|
|
||||||
Linop.Op(basis.back(), w);
|
|
||||||
for (int j = 0; j < basis.size(); j++) {
|
|
||||||
coeff = innerProduct(basis[j], w); // coeff = h_{ij}. Note that since {vi} is ONB it's OK to subtract it off after.
|
|
||||||
Hess(j, i) = coeff;
|
|
||||||
w -= coeff * basis[j];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (doubleOrthog) {
|
|
||||||
// TODO implement
|
|
||||||
}
|
|
||||||
|
|
||||||
// add w_i to the pile
|
|
||||||
if (i < Nm - 1) {
|
|
||||||
coeff = std::sqrt(norm2(w));
|
|
||||||
Hess(i+1, i) = coeff;
|
|
||||||
basis.push_back(
|
|
||||||
(1.0/coeff) * w
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// after iterations, update f and beta_k = ||f||
|
|
||||||
f = w; // make sure f is not normalized
|
|
||||||
beta_k = std::sqrt(norm2(f)); // beta_k = ||f_k|| determines convergence.
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "|f|^2 after Arnoldi step = " << norm2(f) << std::endl;
|
|
||||||
std::cout << GridLogDebug << "Computed Hessenberg matrix = " << std::endl << Hess << std::endl;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Approximates the eigensystem of the linear operator by computing the eigensystem of
|
|
||||||
* the Hessenberg matrix. Assumes that the Hessenberg matrix has already been constructed (by
|
|
||||||
* calling the operator() function).
|
|
||||||
*
|
|
||||||
* TODO implement in parent class eventually.
|
|
||||||
*
|
|
||||||
* Parameters
|
|
||||||
* ----------
|
|
||||||
* Eigen::MatrixXcd& S
|
|
||||||
* Schur matrix (upper triangular) similar to original Rayleigh quotient.
|
|
||||||
*/
|
|
||||||
void compute_eigensystem(Eigen::MatrixXcd& S)
|
|
||||||
{
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "Computing eigenvalues." << std::endl;
|
|
||||||
|
|
||||||
evecs.clear();
|
|
||||||
|
|
||||||
Eigen::ComplexEigenSolver<Eigen::MatrixXcd> es;
|
|
||||||
es.compute(S);
|
|
||||||
evals = es.eigenvalues();
|
|
||||||
littleEvecs = es.eigenvectors();
|
|
||||||
|
|
||||||
// Convert evecs to lattice fields
|
|
||||||
for (int k = 0; k < evals.size(); k++) {
|
|
||||||
Eigen::VectorXcd vec = littleEvecs.col(k);
|
|
||||||
Field tmp (basis[0].Grid());
|
|
||||||
tmp = Zero();
|
|
||||||
for (int j = 0; j < basis.size(); j++) {
|
|
||||||
tmp = tmp + vec[j] * basis[j];
|
|
||||||
}
|
|
||||||
evecs.push_back(tmp);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "Eigenvalues: " << std::endl << evals << std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Verifies the factorization DV = V^\dag H + f e^\dag with the last-computed
|
|
||||||
* V, H, f.
|
|
||||||
*/
|
|
||||||
// RealD verifyFactorization() {
|
|
||||||
// int k = basis.size(); // number of basis vectors, also the size of H.
|
|
||||||
// std::vector<Field> factorized (k, Zero());
|
|
||||||
// Field tmp (FGrid); tmp = Zero();
|
|
||||||
// for (int i = 0; i < basis.size(); i++) {
|
|
||||||
// Linop.Op(basis[i], tmp);
|
|
||||||
// }
|
|
||||||
// // basisRotate(basis, Q, 0, Nk, 0, Nk, Nm);
|
|
||||||
// // Linop.Op(, )
|
|
||||||
// }
|
|
||||||
|
|
||||||
/* Getters */
|
|
||||||
Eigen::MatrixXcd getHessenbergMat() { return Hess; }
|
|
||||||
Field getF() { return f; }
|
|
||||||
std::vector<Field> getBasis() { return basis; }
|
|
||||||
Eigen::VectorXcd getEvals() { return evals; }
|
|
||||||
std::vector<Field> getEvecs() { return evecs; }
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Implements implicit restarting for Arnoldi. Assumes eigenvalues are sorted.
|
|
||||||
*
|
|
||||||
* Parameters
|
|
||||||
* ----------
|
|
||||||
* int _Nm
|
|
||||||
* Size of basis to keep (Hessenberg is MxM).
|
|
||||||
* int Nk
|
|
||||||
* Number of basis vectors to keep at each restart.
|
|
||||||
*/
|
|
||||||
void implicitRestart(int _Nm, int _Nk) {
|
|
||||||
assert ( _Nk <= _Nm );
|
|
||||||
Nm = _Nm; Nk = _Nk;
|
|
||||||
int Np = Nm - Nk; // keep Nk smallest (or largest, depends on sort function) evecs
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "Computing QR Factorizations." << std::endl;
|
|
||||||
|
|
||||||
Eigen::MatrixXcd Q = Eigen::MatrixXcd::Identity(Nm, Nm);
|
|
||||||
Eigen::MatrixXcd Qi (Nm, Nm);
|
|
||||||
Eigen::MatrixXcd R (Nm, Nm);
|
|
||||||
|
|
||||||
for (int i = Nk; i < Nm; i++) { // keep the first Nk eigenvalues and iterate through the last Np. Should loop Np times
|
|
||||||
|
|
||||||
// Useful debugging output
|
|
||||||
std::cout << GridLogDebug << "Computing QR factorization for i = " << i << std::endl;
|
|
||||||
std::cout << GridLogDebug << "Eval shift = " << evals[i] << std::endl;
|
|
||||||
std::cout << GridLogDebug << "Hess before rotation: " << Hess << std::endl;
|
|
||||||
|
|
||||||
// QR factorize
|
|
||||||
Eigen::HouseholderQR<Eigen::MatrixXcd> QR (Hess - evals[i] * Eigen::MatrixXcd::Identity(Nm, Nm));
|
|
||||||
Qi = QR.householderQ();
|
|
||||||
Q = Q * Qi;
|
|
||||||
Hess = Qi.adjoint() * Hess * Qi;
|
|
||||||
|
|
||||||
std::cout << GridLogDebug << "Qt up to i = " << Q.transpose() << std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogDebug << "Hess after all rotations: " << std::endl << Hess << std::endl;
|
|
||||||
|
|
||||||
// form Arnoldi vector f: f is normal to the basis vectors and its norm \beta is used to determine the Ritz estimate.
|
|
||||||
std::complex<double> beta = Hess(Nk, Nk-1);
|
|
||||||
std::complex<double> sigma = Q(Nm-1, Nk-1);
|
|
||||||
f = basis[Nk] * beta + f * sigma;
|
|
||||||
RealD betak = std::sqrt(norm2(f));
|
|
||||||
std::cout << GridLogMessage << "|f|^2 after implicit restart = " << norm2(f) << std::endl;
|
|
||||||
|
|
||||||
// Rotate basis by Qt
|
|
||||||
Qt = Q.transpose();
|
|
||||||
basisRotate(basis, Qt, 0, Nk + 1, 0, Nm, Nm);
|
|
||||||
|
|
||||||
// rotate
|
|
||||||
basisRotate(evecs, Qt, 0, Nk + 1, 0, Nm, Nm);
|
|
||||||
|
|
||||||
// Truncate the basis and restart
|
|
||||||
basis = std::vector<Field> (basis.begin(), basis.begin() + Nk);
|
|
||||||
// evecs = std::vector<Field> (evecs.begin(), evecs.begin() + Nk);
|
|
||||||
Hess = Hess(Eigen::seqN(0, Nk), Eigen::seqN(0, Nk));
|
|
||||||
|
|
||||||
std::cout << "evecs size: " << evecs.size() << std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Computes the number of Arnoldi eigenvectors that have converged. An eigenvector s is considered converged
|
|
||||||
* for a tolerance epsilon if
|
|
||||||
* r(s) := |\beta e_m^T s| < epsilon
|
|
||||||
* where beta is the norm of f_{m+1}.
|
|
||||||
*
|
|
||||||
* Parameters
|
|
||||||
* ----------
|
|
||||||
*
|
|
||||||
* Returns
|
|
||||||
* -------
|
|
||||||
* int : Number of converged eigenvectors.
|
|
||||||
*/
|
|
||||||
int converged() {
|
|
||||||
int Nconv = 0;
|
|
||||||
for (int k = 0; k < evecs.size(); k++) {
|
|
||||||
RealD emTs = std::abs(littleEvecs(Nm - 1, k)); // e_m^T s
|
|
||||||
RealD ritzEstimate = beta_k * emTs;
|
|
||||||
// TODO should be ritzEstimate < Tolerance * lambda_max
|
|
||||||
std::cout << GridLogMessage << "Ritz estimate for evec " << k << " = " << ritzEstimate << std::endl;
|
|
||||||
if (ritzEstimate < rtol) {
|
|
||||||
Nconv++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return Nconv;
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,234 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/BiCGSTAB.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: juettner <juettner@soton.ac.uk>
|
|
||||||
Author: David Murphy <djmurphy@mit.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#ifndef GRID_BICGSTAB_H
|
|
||||||
#define GRID_BICGSTAB_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for iterative processes based on operators
|
|
||||||
// single input vec, single output vec.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template <class Field>
|
|
||||||
class BiCGSTAB : public OperatorFunction<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
|
||||||
// Defaults true.
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
|
|
||||||
BiCGSTAB(RealD tol, Integer maxit, bool err_on_no_conv = true) :
|
|
||||||
Tolerance(tol), MaxIterations(maxit), ErrorOnNoConverge(err_on_no_conv){};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field>& Linop, const Field& src, Field& psi)
|
|
||||||
{
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD cp(0), rho(1), rho_prev(0), alpha(1), beta(0), omega(1);
|
|
||||||
RealD a(0), bo(0), b(0), ssq(0);
|
|
||||||
|
|
||||||
Field p(src);
|
|
||||||
Field r(src);
|
|
||||||
Field rhat(src);
|
|
||||||
Field v(src);
|
|
||||||
Field s(src);
|
|
||||||
Field t(src);
|
|
||||||
Field h(src);
|
|
||||||
|
|
||||||
v = Zero();
|
|
||||||
p = Zero();
|
|
||||||
|
|
||||||
// Initial residual computation & set up
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
Linop.Op(psi, v);
|
|
||||||
b = norm2(v);
|
|
||||||
|
|
||||||
r = src - v;
|
|
||||||
rhat = r;
|
|
||||||
a = norm2(r);
|
|
||||||
ssq = norm2(src);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: src " << ssq << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: mp " << b << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: r " << a << std::endl;
|
|
||||||
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
// Check if guess is really REALLY good :)
|
|
||||||
if(a <= rsq){ return; }
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "BiCGSTAB: k=0 residual " << a << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch InnerTimer;
|
|
||||||
GridStopWatch AxpyNormTimer;
|
|
||||||
GridStopWatch LinearCombTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
|
|
||||||
SolverTimer.Start();
|
|
||||||
int k;
|
|
||||||
for (k = 1; k <= MaxIterations; k++)
|
|
||||||
{
|
|
||||||
rho_prev = rho;
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
InnerTimer.Start();
|
|
||||||
ComplexD Crho = innerProduct(rhat,r);
|
|
||||||
InnerTimer.Stop();
|
|
||||||
rho = Crho.real();
|
|
||||||
|
|
||||||
beta = (rho / rho_prev) * (alpha / omega);
|
|
||||||
|
|
||||||
LinearCombTimer.Start();
|
|
||||||
bo = beta * omega;
|
|
||||||
{
|
|
||||||
autoView( p_v , p, AcceleratorWrite);
|
|
||||||
autoView( r_v , r, AcceleratorRead);
|
|
||||||
autoView( v_v , v, AcceleratorRead);
|
|
||||||
accelerator_for(ss, p_v.size(), Field::vector_object::Nsimd(),{
|
|
||||||
coalescedWrite(p_v[ss], beta*p_v(ss) - bo*v_v(ss) + r_v(ss));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
LinearCombTimer.Stop();
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop.Op(p,v);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
InnerTimer.Start();
|
|
||||||
ComplexD Calpha = innerProduct(rhat,v);
|
|
||||||
InnerTimer.Stop();
|
|
||||||
alpha = rho / Calpha.real();
|
|
||||||
|
|
||||||
LinearCombTimer.Start();
|
|
||||||
{
|
|
||||||
autoView( p_v , p, AcceleratorRead);
|
|
||||||
autoView( r_v , r, AcceleratorRead);
|
|
||||||
autoView( v_v , v, AcceleratorRead);
|
|
||||||
autoView( psi_v,psi, AcceleratorRead);
|
|
||||||
autoView( h_v , h, AcceleratorWrite);
|
|
||||||
autoView( s_v , s, AcceleratorWrite);
|
|
||||||
accelerator_for(ss, h_v.size(), Field::vector_object::Nsimd(),{
|
|
||||||
coalescedWrite(h_v[ss], alpha*p_v(ss) + psi_v(ss));
|
|
||||||
});
|
|
||||||
accelerator_for(ss, s_v.size(), Field::vector_object::Nsimd(),{
|
|
||||||
coalescedWrite(s_v[ss], -alpha*v_v(ss) + r_v(ss));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
LinearCombTimer.Stop();
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop.Op(s,t);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
InnerTimer.Start();
|
|
||||||
ComplexD Comega = innerProduct(t,s);
|
|
||||||
InnerTimer.Stop();
|
|
||||||
omega = Comega.real() / norm2(t);
|
|
||||||
|
|
||||||
LinearCombTimer.Start();
|
|
||||||
{
|
|
||||||
autoView( psi_v,psi, AcceleratorWrite);
|
|
||||||
autoView( r_v , r, AcceleratorWrite);
|
|
||||||
autoView( h_v , h, AcceleratorRead);
|
|
||||||
autoView( s_v , s, AcceleratorRead);
|
|
||||||
autoView( t_v , t, AcceleratorRead);
|
|
||||||
accelerator_for(ss, psi_v.size(), Field::vector_object::Nsimd(),{
|
|
||||||
coalescedWrite(psi_v[ss], h_v(ss) + omega * s_v(ss));
|
|
||||||
coalescedWrite(r_v[ss], -omega * t_v(ss) + s_v(ss));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
LinearCombTimer.Stop();
|
|
||||||
|
|
||||||
cp = norm2(r);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "BiCGSTAB: Iteration " << k << " residual " << sqrt(cp/ssq) << " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if(cp <= rsq)
|
|
||||||
{
|
|
||||||
SolverTimer.Stop();
|
|
||||||
Linop.Op(psi, v);
|
|
||||||
p = v - src;
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(norm2(src));
|
|
||||||
RealD resnorm = sqrt(norm2(p));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "BiCGSTAB Converged on iteration " << k << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tComputed residual " << sqrt(cp/ssq) << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tTrue residual " << true_residual << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "Time breakdown " << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tLinalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tInner " << InnerTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tAxpyNorm " << AxpyNormTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tLinearComb " << LinearCombTimer.Elapsed() << std::endl;
|
|
||||||
|
|
||||||
if(ErrorOnNoConverge){ GRID_ASSERT(true_residual / Tolerance < 10000.0); }
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "BiCGSTAB did NOT converge" << std::endl;
|
|
||||||
|
|
||||||
if(ErrorOnNoConverge){ GRID_ASSERT(0); }
|
|
||||||
IterationsToComplete = k;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,159 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/BiCGSTABMixedPrec.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
|
||||||
Author: David Murphy <djmurphy@mit.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#ifndef GRID_BICGSTAB_MIXED_PREC_H
|
|
||||||
#define GRID_BICGSTAB_MIXED_PREC_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
// Mixed precision restarted defect correction BiCGSTAB
|
|
||||||
template<class FieldD, class FieldF, typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class MixedPrecisionBiCGSTAB : public LinearFunction<FieldD>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
using LinearFunction<FieldD>::operator();
|
|
||||||
RealD Tolerance;
|
|
||||||
RealD InnerTolerance; // Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
|
||||||
Integer MaxInnerIterations;
|
|
||||||
Integer MaxOuterIterations;
|
|
||||||
GridBase* SinglePrecGrid; // Grid for single-precision fields
|
|
||||||
RealD OuterLoopNormMult; // Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
|
||||||
LinearOperatorBase<FieldF> &Linop_f;
|
|
||||||
LinearOperatorBase<FieldD> &Linop_d;
|
|
||||||
|
|
||||||
Integer TotalInnerIterations; //Number of inner CG iterations
|
|
||||||
Integer TotalOuterIterations; //Number of restarts
|
|
||||||
Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
|
|
||||||
|
|
||||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
|
||||||
LinearFunction<FieldF> *guesser;
|
|
||||||
|
|
||||||
MixedPrecisionBiCGSTAB(RealD tol, Integer maxinnerit, Integer maxouterit, GridBase* _sp_grid,
|
|
||||||
LinearOperatorBase<FieldF>& _Linop_f, LinearOperatorBase<FieldD>& _Linop_d) :
|
|
||||||
Linop_f(_Linop_f), Linop_d(_Linop_d), Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit),
|
|
||||||
MaxOuterIterations(maxouterit), SinglePrecGrid(_sp_grid), OuterLoopNormMult(100.), guesser(NULL) {};
|
|
||||||
|
|
||||||
void useGuesser(LinearFunction<FieldF>& g){
|
|
||||||
guesser = &g;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (const FieldD& src_d_in, FieldD& sol_d)
|
|
||||||
{
|
|
||||||
TotalInnerIterations = 0;
|
|
||||||
|
|
||||||
GridStopWatch TotalTimer;
|
|
||||||
TotalTimer.Start();
|
|
||||||
|
|
||||||
int cb = src_d_in.Checkerboard();
|
|
||||||
sol_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
RealD src_norm = norm2(src_d_in);
|
|
||||||
RealD stop = src_norm * Tolerance*Tolerance;
|
|
||||||
|
|
||||||
GridBase* DoublePrecGrid = src_d_in.Grid();
|
|
||||||
FieldD tmp_d(DoublePrecGrid);
|
|
||||||
tmp_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldD tmp2_d(DoublePrecGrid);
|
|
||||||
tmp2_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldD src_d(DoublePrecGrid);
|
|
||||||
src_d = src_d_in; //source for next inner iteration, computed from residual during operation
|
|
||||||
|
|
||||||
RealD inner_tol = InnerTolerance;
|
|
||||||
|
|
||||||
FieldF src_f(SinglePrecGrid);
|
|
||||||
src_f.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldF sol_f(SinglePrecGrid);
|
|
||||||
sol_f.Checkerboard() = cb;
|
|
||||||
|
|
||||||
BiCGSTAB<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
|
||||||
CG_f.ErrorOnNoConverge = false;
|
|
||||||
|
|
||||||
GridStopWatch InnerCGtimer;
|
|
||||||
|
|
||||||
GridStopWatch PrecChangeTimer;
|
|
||||||
|
|
||||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
|
||||||
|
|
||||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++)
|
|
||||||
{
|
|
||||||
// Compute double precision rsd and also new RHS vector.
|
|
||||||
Linop_d.Op(sol_d, tmp_d);
|
|
||||||
RealD norm = axpy_norm(src_d, -1., tmp_d, src_d_in); //src_d is residual vector
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Outer iteration " << outer_iter << " residual " << norm << " target " << stop << std::endl;
|
|
||||||
|
|
||||||
if(norm < OuterLoopNormMult * stop){
|
|
||||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Outer iteration converged on iteration " << outer_iter << std::endl;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
while(norm * inner_tol * inner_tol < stop){ inner_tol *= 2; } // inner_tol = sqrt(stop/norm) ??
|
|
||||||
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(src_f, src_d);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
sol_f = Zero();
|
|
||||||
|
|
||||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
|
||||||
if(guesser != NULL){ (*guesser)(src_f, sol_f); }
|
|
||||||
|
|
||||||
//Inner CG
|
|
||||||
CG_f.Tolerance = inner_tol;
|
|
||||||
InnerCGtimer.Start();
|
|
||||||
CG_f(Linop_f, src_f, sol_f);
|
|
||||||
InnerCGtimer.Stop();
|
|
||||||
TotalInnerIterations += CG_f.IterationsToComplete;
|
|
||||||
|
|
||||||
//Convert sol back to double and add to double prec solution
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(tmp_d, sol_f);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
axpy(sol_d, 1.0, tmp_d, sol_d);
|
|
||||||
}
|
|
||||||
|
|
||||||
//Final trial CG
|
|
||||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Starting final patch-up double-precision solve" << std::endl;
|
|
||||||
|
|
||||||
BiCGSTAB<FieldD> CG_d(Tolerance, MaxInnerIterations);
|
|
||||||
CG_d(Linop_d, src_d_in, sol_d);
|
|
||||||
TotalFinalStepIterations = CG_d.IterationsToComplete;
|
|
||||||
|
|
||||||
TotalTimer.Stop();
|
|
||||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Inner CG iterations " << TotalInnerIterations << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MixedPrecisionBiCGSTAB: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,248 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/CommunicationAvoidingGeneralisedMinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
#define GRID_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class CommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when CAGMRES fails to converge,
|
|
||||||
// defaults to true
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer RestartLength;
|
|
||||||
Integer MaxNumberOfRestarts;
|
|
||||||
Integer IterationCount; // Number of iterations the CAGMRES took to finish,
|
|
||||||
// filled in upon completion
|
|
||||||
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch QrTimer;
|
|
||||||
GridStopWatch CompSolutionTimer;
|
|
||||||
|
|
||||||
Eigen::MatrixXcd H;
|
|
||||||
|
|
||||||
std::vector<ComplexD> y;
|
|
||||||
std::vector<ComplexD> gamma;
|
|
||||||
std::vector<ComplexD> c;
|
|
||||||
std::vector<ComplexD> s;
|
|
||||||
|
|
||||||
CommunicationAvoidingGeneralisedMinimalResidual(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
Integer restart_length,
|
|
||||||
bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol)
|
|
||||||
, MaxIterations(maxit)
|
|
||||||
, RestartLength(restart_length)
|
|
||||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
|
||||||
, ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
|
||||||
, y(RestartLength + 1, 0.)
|
|
||||||
, gamma(RestartLength + 1, 0.)
|
|
||||||
, c(RestartLength + 1, 0.)
|
|
||||||
, s(RestartLength + 1, 0.) {};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
std::cout << GridLogWarning << "This algorithm currently doesn't differ from regular GMRES" << std::endl;
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
std::cout << std::setprecision(4) << std::scientific;
|
|
||||||
std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: src " << ssq << std::endl;
|
|
||||||
|
|
||||||
MatrixTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
QrTimer.Reset();
|
|
||||||
CompSolutionTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
IterationCount = 0;
|
|
||||||
|
|
||||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
|
||||||
|
|
||||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
LinOp.Op(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(ssq);
|
|
||||||
RealD resnorm = sqrt(norm2(r));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "CommunicationAvoidingGeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
|
||||||
<< " computed residual " << sqrt(cp / ssq)
|
|
||||||
<< " true residual " << true_residual
|
|
||||||
<< " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "CAGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "CommunicationAvoidingGeneralisedMinimalResidual did NOT converge" << std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
|
||||||
|
|
||||||
RealD cp = 0;
|
|
||||||
|
|
||||||
Field w(src.Grid());
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
// this should probably be made a class member so that it is only allocated once, not in every restart
|
|
||||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(psi, w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r = src - w;
|
|
||||||
|
|
||||||
gamma[0] = sqrt(norm2(r));
|
|
||||||
|
|
||||||
ComplexD scale = 1.0/gamma[0];
|
|
||||||
v[0] = scale * r;
|
|
||||||
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
for (int i=0; i<RestartLength; i++) {
|
|
||||||
|
|
||||||
IterationCount++;
|
|
||||||
|
|
||||||
arnoldiStep(LinOp, v, w, i);
|
|
||||||
|
|
||||||
qrUpdate(i);
|
|
||||||
|
|
||||||
cp = norm(gamma[i+1]);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "CommunicationAvoidingGeneralisedMinimalResidual: Iteration " << IterationCount
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
|
||||||
|
|
||||||
computeSolution(v, psi, i);
|
|
||||||
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT(0); // Never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, Field &w, int iter) {
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(v[iter], w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
for (int i = 0; i <= iter; ++i) {
|
|
||||||
H(iter, i) = innerProduct(v[i], w);
|
|
||||||
w = w - ComplexD(H(iter, i)) * v[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
H(iter, iter + 1) = sqrt(norm2(w));
|
|
||||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void qrUpdate(int iter) {
|
|
||||||
|
|
||||||
QrTimer.Start();
|
|
||||||
for (int i = 0; i < iter ; ++i) {
|
|
||||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i + 1) = tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute new Givens Rotation
|
|
||||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
|
||||||
c[iter] = H(iter, iter) / nu;
|
|
||||||
s[iter] = H(iter, iter + 1) / nu;
|
|
||||||
|
|
||||||
// Apply new Givens rotation
|
|
||||||
H(iter, iter) = nu;
|
|
||||||
H(iter, iter + 1) = 0.;
|
|
||||||
|
|
||||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
|
||||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
|
||||||
QrTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void computeSolution(std::vector<Field> const &v, Field &psi, int iter) {
|
|
||||||
|
|
||||||
CompSolutionTimer.Start();
|
|
||||||
for (int i = iter; i >= 0; i--) {
|
|
||||||
y[i] = gamma[i];
|
|
||||||
for (int k = i + 1; k <= iter; k++)
|
|
||||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
|
||||||
y[i] = y[i] / ComplexD(H(i, i));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i <= iter; i++)
|
|
||||||
psi = psi + v[i] * y[i];
|
|
||||||
CompSolutionTimer.Stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
@@ -1,373 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradient.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_GRADIENT_H
|
|
||||||
#define GRID_CONJUGATE_GRADIENT_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for iterative processes based on operators
|
|
||||||
// single input vec, single output vec.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
|
|
||||||
template <class Field>
|
|
||||||
class ConjugateGradient : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
|
||||||
// Defaults true.
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
RealD TrueResidual;
|
|
||||||
|
|
||||||
ConjugateGradient(RealD tol, Integer maxit, bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
{};
|
|
||||||
|
|
||||||
virtual void LogIteration(int k,RealD a,RealD b){
|
|
||||||
// std::cout << "ConjugageGradient::LogIteration() "<<std::endl;
|
|
||||||
};
|
|
||||||
virtual void LogBegin(void){
|
|
||||||
std::cout << "ConjugageGradient::LogBegin() "<<std::endl;
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
this->LogBegin();
|
|
||||||
|
|
||||||
GRID_TRACE("ConjugateGradient");
|
|
||||||
GridStopWatch PreambleTimer;
|
|
||||||
GridStopWatch ConstructTimer;
|
|
||||||
GridStopWatch NormTimer;
|
|
||||||
GridStopWatch AssignTimer;
|
|
||||||
PreambleTimer.Start();
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD cp, c, a, d, b, ssq, qq;
|
|
||||||
//RealD b_pred;
|
|
||||||
|
|
||||||
// Was doing copies
|
|
||||||
ConstructTimer.Start();
|
|
||||||
Field p (src.Grid());
|
|
||||||
Field mmp(src.Grid());
|
|
||||||
Field r (src.Grid());
|
|
||||||
ConstructTimer.Stop();
|
|
||||||
|
|
||||||
// Initial residual computation & set up
|
|
||||||
NormTimer.Start();
|
|
||||||
ssq = norm2(src);
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
NormTimer.Stop();
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
AssignTimer.Start();
|
|
||||||
if ( guess == 0.0 ) {
|
|
||||||
r = src;
|
|
||||||
p = r;
|
|
||||||
a = ssq;
|
|
||||||
} else {
|
|
||||||
Linop.HermOpAndNorm(psi, mmp, d, b);
|
|
||||||
r = src - mmp;
|
|
||||||
p = r;
|
|
||||||
a = norm2(p);
|
|
||||||
}
|
|
||||||
cp = a;
|
|
||||||
AssignTimer.Stop();
|
|
||||||
|
|
||||||
// Handle trivial case of zero src
|
|
||||||
if (ssq == 0.){
|
|
||||||
psi = Zero();
|
|
||||||
IterationsToComplete = 1;
|
|
||||||
TrueResidual = 0.;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: src " << ssq << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: mp " << d << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: mmp " << b << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: cp,r " << cp << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: p " << a << std::endl;
|
|
||||||
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
// Check if guess is really REALLY good :)
|
|
||||||
if (cp <= rsq) {
|
|
||||||
TrueResidual = std::sqrt(a/ssq);
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient guess is converged already " << std::endl;
|
|
||||||
IterationsToComplete = 0;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8)
|
|
||||||
<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
PreambleTimer.Stop();
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch InnerTimer;
|
|
||||||
GridStopWatch AxpyNormTimer;
|
|
||||||
GridStopWatch LinearCombTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
|
|
||||||
RealD usecs = -usecond();
|
|
||||||
SolverTimer.Start();
|
|
||||||
int k;
|
|
||||||
for (k = 1; k <= MaxIterations; k++) {
|
|
||||||
|
|
||||||
GridStopWatch IterationTimer;
|
|
||||||
IterationTimer.Start();
|
|
||||||
c = cp;
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop.HermOp(p, mmp);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
InnerTimer.Start();
|
|
||||||
ComplexD dc = innerProduct(p,mmp);
|
|
||||||
InnerTimer.Stop();
|
|
||||||
d = dc.real();
|
|
||||||
a = c / d;
|
|
||||||
|
|
||||||
AxpyNormTimer.Start();
|
|
||||||
cp = axpy_norm(r, -a, mmp, r);
|
|
||||||
AxpyNormTimer.Stop();
|
|
||||||
b = cp / c;
|
|
||||||
|
|
||||||
LinearCombTimer.Start();
|
|
||||||
{
|
|
||||||
autoView( psi_v , psi, AcceleratorWrite);
|
|
||||||
autoView( p_v , p, AcceleratorWrite);
|
|
||||||
autoView( r_v , r, AcceleratorWrite);
|
|
||||||
accelerator_for(ss,p_v.size(), Field::vector_object::Nsimd(),{
|
|
||||||
coalescedWrite(psi_v[ss], a * p_v(ss) + psi_v(ss));
|
|
||||||
coalescedWrite(p_v[ss] , b * p_v(ss) + r_v (ss));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
LinearCombTimer.Stop();
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
LogIteration(k,a,b);
|
|
||||||
|
|
||||||
IterationTimer.Stop();
|
|
||||||
if ( (k % 500) == 0 ) {
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient: Iteration " << k
|
|
||||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << std::endl;
|
|
||||||
} else {
|
|
||||||
std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
|
|
||||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << " took " << IterationTimer.Elapsed() << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
usecs +=usecond();
|
|
||||||
SolverTimer.Stop();
|
|
||||||
Linop.HermOpAndNorm(psi, mmp, d, qq);
|
|
||||||
p = mmp - src;
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
RealD DwfFlops = (1452. )*grid->gSites()*4*k
|
|
||||||
+ (8+4+8+4+4)*12*grid->gSites()*k; // CG linear algebra
|
|
||||||
RealD srcnorm = std::sqrt(norm2(src));
|
|
||||||
RealD resnorm = std::sqrt(norm2(p));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient Converged on iteration " << k
|
|
||||||
<< "\tComputed residual " << std::sqrt(cp / ssq)
|
|
||||||
<< "\tTrue residual " << true_residual
|
|
||||||
<< "\tTarget " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
// std::cout << GridLogMessage << "\tPreamble " << PreambleTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver Elapsed " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "Time breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tInner " << InnerTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogDebug << "\tMobius flop rate " << DwfFlops/ usecs<< " Gflops " <<std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge) GRID_ASSERT(true_residual / Tolerance < 10000.0);
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
TrueResidual = true_residual;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Failed. Calculate true residual before giving up
|
|
||||||
// Linop.HermOpAndNorm(psi, mmp, d, qq);
|
|
||||||
// p = mmp - src;
|
|
||||||
//TrueResidual = sqrt(norm2(p)/ssq);
|
|
||||||
// TrueResidual = 1;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient did NOT converge "<<k<<" / "<< MaxIterations
|
|
||||||
<<" residual "<< std::sqrt(cp / ssq)<< std::endl;
|
|
||||||
SolverTimer.Stop();
|
|
||||||
std::cout << GridLogMessage << "\tPreamble " << PreambleTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tConstruct " << ConstructTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tNorm " << NormTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tAssign " << AssignTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "Solver breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage<< "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tInner " << InnerTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template <class Field>
|
|
||||||
class ConjugateGradientPolynomial : public ConjugateGradient<Field> {
|
|
||||||
public:
|
|
||||||
// Optionally record the CG polynomial
|
|
||||||
std::vector<double> ak;
|
|
||||||
std::vector<double> bk;
|
|
||||||
std::vector<double> poly_p;
|
|
||||||
std::vector<double> poly_r;
|
|
||||||
std::vector<double> poly_Ap;
|
|
||||||
std::vector<double> polynomial;
|
|
||||||
|
|
||||||
public:
|
|
||||||
ConjugateGradientPolynomial(RealD tol, Integer maxit, bool err_on_no_conv = true)
|
|
||||||
: ConjugateGradient<Field>(tol,maxit,err_on_no_conv)
|
|
||||||
{ };
|
|
||||||
void PolyHermOp(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
|
|
||||||
{
|
|
||||||
Field tmp(src.Grid());
|
|
||||||
Field AtoN(src.Grid());
|
|
||||||
AtoN = src;
|
|
||||||
psi=AtoN*polynomial[0];
|
|
||||||
for(int n=1;n<polynomial.size();n++){
|
|
||||||
tmp = AtoN;
|
|
||||||
Linop.HermOp(tmp,AtoN);
|
|
||||||
psi = psi + polynomial[n]*AtoN;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void CGsequenceHermOp(LinearOperatorBase<Field> &Linop, const Field &src, Field &x)
|
|
||||||
{
|
|
||||||
Field Ap(src.Grid());
|
|
||||||
Field r(src.Grid());
|
|
||||||
Field p(src.Grid());
|
|
||||||
p=src;
|
|
||||||
r=src;
|
|
||||||
x=Zero();
|
|
||||||
x.Checkerboard()=src.Checkerboard();
|
|
||||||
for(int k=0;k<ak.size();k++){
|
|
||||||
x = x + ak[k]*p;
|
|
||||||
Linop.HermOp(p,Ap);
|
|
||||||
r = r - ak[k] * Ap;
|
|
||||||
p = r + bk[k] * p;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void Solve(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
|
|
||||||
{
|
|
||||||
psi=Zero();
|
|
||||||
this->operator ()(Linop,src,psi);
|
|
||||||
}
|
|
||||||
virtual void LogBegin(void)
|
|
||||||
{
|
|
||||||
std::cout << "ConjugageGradientPolynomial::LogBegin() "<<std::endl;
|
|
||||||
ak.resize(0);
|
|
||||||
bk.resize(0);
|
|
||||||
polynomial.resize(0);
|
|
||||||
poly_Ap.resize(0);
|
|
||||||
poly_Ap.resize(0);
|
|
||||||
poly_p.resize(1);
|
|
||||||
poly_r.resize(1);
|
|
||||||
poly_p[0]=1.0;
|
|
||||||
poly_r[0]=1.0;
|
|
||||||
};
|
|
||||||
virtual void LogIteration(int k,RealD a,RealD b)
|
|
||||||
{
|
|
||||||
// With zero guess,
|
|
||||||
// p = r = src
|
|
||||||
//
|
|
||||||
// iterate:
|
|
||||||
// x = x + a p
|
|
||||||
// r = r - a A p
|
|
||||||
// p = r + b p
|
|
||||||
//
|
|
||||||
// [0]
|
|
||||||
// r = x
|
|
||||||
// p = x
|
|
||||||
// Ap=0
|
|
||||||
//
|
|
||||||
// [1]
|
|
||||||
// Ap = A x + 0 ==> shift poly P right by 1 and add 0.
|
|
||||||
// x = x + a p ==> add polynomials term by term
|
|
||||||
// r = r - a A p ==> add polynomials term by term
|
|
||||||
// p = r + b p ==> add polynomials term by term
|
|
||||||
//
|
|
||||||
std::cout << "ConjugageGradientPolynomial::LogIteration() "<<k<<std::endl;
|
|
||||||
ak.push_back(a);
|
|
||||||
bk.push_back(b);
|
|
||||||
// Ap= right_shift(p)
|
|
||||||
poly_Ap.resize(k+1);
|
|
||||||
poly_Ap[0]=0.0;
|
|
||||||
for(int i=0;i<k;i++){
|
|
||||||
poly_Ap[i+1]=poly_p[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
// x = x + a p
|
|
||||||
polynomial.resize(k);
|
|
||||||
polynomial[k-1]=0.0;
|
|
||||||
for(int i=0;i<k;i++){
|
|
||||||
polynomial[i] = polynomial[i] + a * poly_p[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
// r = r - a Ap
|
|
||||||
// p = r + b p
|
|
||||||
poly_r.resize(k+1);
|
|
||||||
poly_p.resize(k+1);
|
|
||||||
poly_r[k] = poly_p[k] = 0.0;
|
|
||||||
for(int i=0;i<k+1;i++){
|
|
||||||
poly_r[i] = poly_r[i] - a * poly_Ap[i];
|
|
||||||
poly_p[i] = poly_r[i] + b * poly_p[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,170 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMixedPrec.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_GRADIENT_MIXED_PREC_H
|
|
||||||
#define GRID_CONJUGATE_GRADIENT_MIXED_PREC_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//Mixed precision restarted defect correction CG
|
|
||||||
template<class FieldD,class FieldF,
|
|
||||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
|
||||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class MixedPrecisionConjugateGradient : public LinearFunction<FieldD> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<FieldD>::operator();
|
|
||||||
RealD Tolerance;
|
|
||||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
|
||||||
Integer MaxInnerIterations;
|
|
||||||
Integer MaxOuterIterations;
|
|
||||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
|
||||||
RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
|
||||||
LinearOperatorBase<FieldF> &Linop_f;
|
|
||||||
LinearOperatorBase<FieldD> &Linop_d;
|
|
||||||
|
|
||||||
Integer TotalInnerIterations; //Number of inner CG iterations
|
|
||||||
Integer TotalOuterIterations; //Number of restarts
|
|
||||||
Integer TotalFinalStepIterations; //Number of CG iterations in final patch-up step
|
|
||||||
RealD TrueResidual;
|
|
||||||
|
|
||||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
|
||||||
LinearFunction<FieldF> *guesser;
|
|
||||||
|
|
||||||
MixedPrecisionConjugateGradient(RealD tol,
|
|
||||||
Integer maxinnerit,
|
|
||||||
Integer maxouterit,
|
|
||||||
GridBase* _sp_grid,
|
|
||||||
LinearOperatorBase<FieldF> &_Linop_f,
|
|
||||||
LinearOperatorBase<FieldD> &_Linop_d) :
|
|
||||||
Linop_f(_Linop_f), Linop_d(_Linop_d),
|
|
||||||
Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), SinglePrecGrid(_sp_grid),
|
|
||||||
OuterLoopNormMult(100.), guesser(NULL){ };
|
|
||||||
|
|
||||||
void useGuesser(LinearFunction<FieldF> &g){
|
|
||||||
guesser = &g;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (const FieldD &src_d_in, FieldD &sol_d){
|
|
||||||
std::cout << GridLogMessage << "MixedPrecisionConjugateGradient: Starting mixed precision CG with outer tolerance " << Tolerance << " and inner tolerance " << InnerTolerance << std::endl;
|
|
||||||
TotalInnerIterations = 0;
|
|
||||||
|
|
||||||
GridStopWatch TotalTimer;
|
|
||||||
TotalTimer.Start();
|
|
||||||
|
|
||||||
int cb = src_d_in.Checkerboard();
|
|
||||||
sol_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
RealD src_norm = norm2(src_d_in);
|
|
||||||
RealD stop = src_norm * Tolerance*Tolerance;
|
|
||||||
|
|
||||||
GridBase* DoublePrecGrid = src_d_in.Grid();
|
|
||||||
FieldD tmp_d(DoublePrecGrid);
|
|
||||||
tmp_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldD tmp2_d(DoublePrecGrid);
|
|
||||||
tmp2_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldD src_d(DoublePrecGrid);
|
|
||||||
src_d = src_d_in; //source for next inner iteration, computed from residual during operation
|
|
||||||
|
|
||||||
RealD inner_tol = InnerTolerance;
|
|
||||||
|
|
||||||
FieldF src_f(SinglePrecGrid);
|
|
||||||
src_f.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldF sol_f(SinglePrecGrid);
|
|
||||||
sol_f.Checkerboard() = cb;
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting initial inner CG with tolerance " << inner_tol << std::endl;
|
|
||||||
ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
|
||||||
CG_f.ErrorOnNoConverge = false;
|
|
||||||
|
|
||||||
GridStopWatch InnerCGtimer;
|
|
||||||
|
|
||||||
GridStopWatch PrecChangeTimer;
|
|
||||||
|
|
||||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
|
||||||
|
|
||||||
precisionChangeWorkspace pc_wk_sp_to_dp(DoublePrecGrid, SinglePrecGrid);
|
|
||||||
precisionChangeWorkspace pc_wk_dp_to_sp(SinglePrecGrid, DoublePrecGrid);
|
|
||||||
|
|
||||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
|
|
||||||
//Compute double precision rsd and also new RHS vector.
|
|
||||||
Linop_d.HermOp(sol_d, tmp_d);
|
|
||||||
RealD norm = axpy_norm(src_d, -1., tmp_d, src_d_in); //src_d is residual vector
|
|
||||||
std::cout<<GridLogMessage<<" rsd norm "<<norm<<std::endl;
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " <<outer_iter<<" residual "<< norm<< " target "<< stop<<std::endl;
|
|
||||||
|
|
||||||
if(norm < OuterLoopNormMult * stop){
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration converged on iteration " <<outer_iter <<std::endl;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
while(norm * inner_tol * inner_tol < stop*1.01) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
|
||||||
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(src_f, src_d, pc_wk_dp_to_sp);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
sol_f = Zero();
|
|
||||||
|
|
||||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
|
||||||
if(guesser != NULL)
|
|
||||||
(*guesser)(src_f, sol_f);
|
|
||||||
|
|
||||||
//Inner CG
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Outer iteration " << outer_iter << " starting inner CG with tolerance " << inner_tol << std::endl;
|
|
||||||
CG_f.Tolerance = inner_tol;
|
|
||||||
InnerCGtimer.Start();
|
|
||||||
CG_f(Linop_f, src_f, sol_f);
|
|
||||||
InnerCGtimer.Stop();
|
|
||||||
TotalInnerIterations += CG_f.IterationsToComplete;
|
|
||||||
|
|
||||||
//Convert sol back to double and add to double prec solution
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(tmp_d, sol_f, pc_wk_sp_to_dp);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
axpy(sol_d, 1.0, tmp_d, sol_d);
|
|
||||||
}
|
|
||||||
|
|
||||||
//Final trial CG
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Starting final patch-up double-precision solve"<<std::endl;
|
|
||||||
|
|
||||||
ConjugateGradient<FieldD> CG_d(Tolerance, MaxInnerIterations);
|
|
||||||
CG_d(Linop_d, src_d_in, sol_d);
|
|
||||||
TotalFinalStepIterations = CG_d.IterationsToComplete;
|
|
||||||
TrueResidual = CG_d.TrueResidual;
|
|
||||||
|
|
||||||
TotalTimer.Stop();
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Inner CG iterations " << TotalInnerIterations << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations << std::endl;
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradient: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,213 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMixedPrecBatched.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Raoul Hodgson <raoul.hodgson@ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_GRADIENT_MIXED_PREC_BATCHED_H
|
|
||||||
#define GRID_CONJUGATE_GRADIENT_MIXED_PREC_BATCHED_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//Mixed precision restarted defect correction CG
|
|
||||||
template<class FieldD,class FieldF,
|
|
||||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
|
||||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class MixedPrecisionConjugateGradientBatched : public LinearFunction<FieldD> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<FieldD>::operator();
|
|
||||||
RealD Tolerance;
|
|
||||||
RealD InnerTolerance; //Initial tolerance for inner CG. Defaults to Tolerance but can be changed
|
|
||||||
Integer MaxInnerIterations;
|
|
||||||
Integer MaxOuterIterations;
|
|
||||||
Integer MaxPatchupIterations;
|
|
||||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
|
||||||
RealD OuterLoopNormMult; //Stop the outer loop and move to a final double prec solve when the residual is OuterLoopNormMult * Tolerance
|
|
||||||
LinearOperatorBase<FieldF> &Linop_f;
|
|
||||||
LinearOperatorBase<FieldD> &Linop_d;
|
|
||||||
|
|
||||||
//Option to speed up *inner single precision* solves using a LinearFunction that produces a guess
|
|
||||||
LinearFunction<FieldF> *guesser;
|
|
||||||
bool updateResidual;
|
|
||||||
|
|
||||||
MixedPrecisionConjugateGradientBatched(RealD tol,
|
|
||||||
Integer maxinnerit,
|
|
||||||
Integer maxouterit,
|
|
||||||
Integer maxpatchit,
|
|
||||||
GridBase* _sp_grid,
|
|
||||||
LinearOperatorBase<FieldF> &_Linop_f,
|
|
||||||
LinearOperatorBase<FieldD> &_Linop_d,
|
|
||||||
bool _updateResidual=true) :
|
|
||||||
Linop_f(_Linop_f), Linop_d(_Linop_d),
|
|
||||||
Tolerance(tol), InnerTolerance(tol), MaxInnerIterations(maxinnerit), MaxOuterIterations(maxouterit), MaxPatchupIterations(maxpatchit), SinglePrecGrid(_sp_grid),
|
|
||||||
OuterLoopNormMult(100.), guesser(NULL), updateResidual(_updateResidual) { };
|
|
||||||
|
|
||||||
void useGuesser(LinearFunction<FieldF> &g){
|
|
||||||
guesser = &g;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (const FieldD &src_d_in, FieldD &sol_d){
|
|
||||||
std::vector<FieldD> srcs_d_in{src_d_in};
|
|
||||||
std::vector<FieldD> sols_d{sol_d};
|
|
||||||
|
|
||||||
(*this)(srcs_d_in,sols_d);
|
|
||||||
|
|
||||||
sol_d = sols_d[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (const std::vector<FieldD> &src_d_in, std::vector<FieldD> &sol_d){
|
|
||||||
GRID_ASSERT(src_d_in.size() == sol_d.size());
|
|
||||||
int NBatch = src_d_in.size();
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "NBatch = " << NBatch << std::endl;
|
|
||||||
|
|
||||||
Integer TotalOuterIterations = 0; //Number of restarts
|
|
||||||
std::vector<Integer> TotalInnerIterations(NBatch,0); //Number of inner CG iterations
|
|
||||||
std::vector<Integer> TotalFinalStepIterations(NBatch,0); //Number of CG iterations in final patch-up step
|
|
||||||
|
|
||||||
GridStopWatch TotalTimer;
|
|
||||||
TotalTimer.Start();
|
|
||||||
|
|
||||||
GridStopWatch InnerCGtimer;
|
|
||||||
GridStopWatch PrecChangeTimer;
|
|
||||||
|
|
||||||
int cb = src_d_in[0].Checkerboard();
|
|
||||||
|
|
||||||
std::vector<RealD> src_norm;
|
|
||||||
std::vector<RealD> norm;
|
|
||||||
std::vector<RealD> stop;
|
|
||||||
|
|
||||||
GridBase* DoublePrecGrid = src_d_in[0].Grid();
|
|
||||||
FieldD tmp_d(DoublePrecGrid);
|
|
||||||
tmp_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
FieldD tmp2_d(DoublePrecGrid);
|
|
||||||
tmp2_d.Checkerboard() = cb;
|
|
||||||
|
|
||||||
std::vector<FieldD> src_d;
|
|
||||||
std::vector<FieldF> src_f;
|
|
||||||
std::vector<FieldF> sol_f;
|
|
||||||
|
|
||||||
for (int i=0; i<NBatch; i++) {
|
|
||||||
sol_d[i].Checkerboard() = cb;
|
|
||||||
|
|
||||||
src_norm.push_back(norm2(src_d_in[i]));
|
|
||||||
norm.push_back(0.);
|
|
||||||
stop.push_back(src_norm[i] * Tolerance*Tolerance);
|
|
||||||
|
|
||||||
src_d.push_back(src_d_in[i]); //source for next inner iteration, computed from residual during operation
|
|
||||||
|
|
||||||
src_f.push_back(SinglePrecGrid);
|
|
||||||
src_f[i].Checkerboard() = cb;
|
|
||||||
|
|
||||||
sol_f.push_back(SinglePrecGrid);
|
|
||||||
sol_f[i].Checkerboard() = cb;
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD inner_tol = InnerTolerance;
|
|
||||||
|
|
||||||
ConjugateGradient<FieldF> CG_f(inner_tol, MaxInnerIterations);
|
|
||||||
CG_f.ErrorOnNoConverge = false;
|
|
||||||
|
|
||||||
Integer &outer_iter = TotalOuterIterations; //so it will be equal to the final iteration count
|
|
||||||
|
|
||||||
for(outer_iter = 0; outer_iter < MaxOuterIterations; outer_iter++){
|
|
||||||
std::cout << GridLogMessage << std::endl;
|
|
||||||
std::cout << GridLogMessage << "Outer iteration " << outer_iter << std::endl;
|
|
||||||
|
|
||||||
bool allConverged = true;
|
|
||||||
|
|
||||||
for (int i=0; i<NBatch; i++) {
|
|
||||||
//Compute double precision rsd and also new RHS vector.
|
|
||||||
Linop_d.HermOp(sol_d[i], tmp_d);
|
|
||||||
norm[i] = axpy_norm(src_d[i], -1., tmp_d, src_d_in[i]); //src_d is residual vector
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Outer iteration " << outer_iter <<" solve " << i << " residual "<< norm[i] << " target "<< stop[i] <<std::endl;
|
|
||||||
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(src_f[i], src_d[i]);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
sol_f[i] = Zero();
|
|
||||||
|
|
||||||
if(norm[i] > OuterLoopNormMult * stop[i]) {
|
|
||||||
allConverged = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (allConverged) break;
|
|
||||||
|
|
||||||
if (updateResidual) {
|
|
||||||
RealD normMax = *std::max_element(std::begin(norm), std::end(norm));
|
|
||||||
RealD stopMax = *std::max_element(std::begin(stop), std::end(stop));
|
|
||||||
while( normMax * inner_tol * inner_tol < stopMax) inner_tol *= 2; // inner_tol = sqrt(stop/norm) ??
|
|
||||||
CG_f.Tolerance = inner_tol;
|
|
||||||
}
|
|
||||||
|
|
||||||
//Optionally improve inner solver guess (eg using known eigenvectors)
|
|
||||||
if(guesser != NULL) {
|
|
||||||
(*guesser)(src_f, sol_f);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i=0; i<NBatch; i++) {
|
|
||||||
//Inner CG
|
|
||||||
InnerCGtimer.Start();
|
|
||||||
CG_f(Linop_f, src_f[i], sol_f[i]);
|
|
||||||
InnerCGtimer.Stop();
|
|
||||||
TotalInnerIterations[i] += CG_f.IterationsToComplete;
|
|
||||||
|
|
||||||
//Convert sol back to double and add to double prec solution
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(tmp_d, sol_f[i]);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
axpy(sol_d[i], 1.0, tmp_d, sol_d[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
//Final trial CG
|
|
||||||
std::cout << GridLogMessage << std::endl;
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Starting final patch-up double-precision solve"<<std::endl;
|
|
||||||
|
|
||||||
for (int i=0; i<NBatch; i++) {
|
|
||||||
ConjugateGradient<FieldD> CG_d(Tolerance, MaxPatchupIterations);
|
|
||||||
CG_d(Linop_d, src_d_in[i], sol_d[i]);
|
|
||||||
TotalFinalStepIterations[i] += CG_d.IterationsToComplete;
|
|
||||||
}
|
|
||||||
|
|
||||||
TotalTimer.Stop();
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << std::endl;
|
|
||||||
for (int i=0; i<NBatch; i++) {
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: solve " << i << " Inner CG iterations " << TotalInnerIterations[i] << " Restarts " << TotalOuterIterations << " Final CG iterations " << TotalFinalStepIterations[i] << std::endl;
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage << std::endl;
|
|
||||||
std::cout<<GridLogMessage<<"MixedPrecisionConjugateGradientBatched: Total time " << TotalTimer.Elapsed() << " Precision change " << PrecChangeTimer.Elapsed() << " Inner CG total " << InnerCGtimer.Elapsed() << std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,346 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_MULTI_SHIFT_GRADIENT_H
|
|
||||||
#define GRID_CONJUGATE_MULTI_SHIFT_GRADIENT_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for iterative processes based on operators
|
|
||||||
// single input vec, single output vec.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class ConjugateGradientMultiShift : public OperatorMultiFunction<Field>,
|
|
||||||
public OperatorFunction<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
// RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
|
||||||
int verbose;
|
|
||||||
MultiShiftFunction shifts;
|
|
||||||
std::vector<RealD> TrueResidualShift;
|
|
||||||
|
|
||||||
ConjugateGradientMultiShift(Integer maxit, const MultiShiftFunction &_shifts) :
|
|
||||||
MaxIterations(maxit),
|
|
||||||
shifts(_shifts)
|
|
||||||
{
|
|
||||||
verbose=1;
|
|
||||||
IterationsToCompleteShift.resize(_shifts.order);
|
|
||||||
TrueResidualShift.resize(_shifts.order);
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, Field &psi)
|
|
||||||
{
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
int nshift = shifts.order;
|
|
||||||
std::vector<Field> results(nshift,grid);
|
|
||||||
(*this)(Linop,src,results,psi);
|
|
||||||
}
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector<Field> &results, Field &psi)
|
|
||||||
{
|
|
||||||
int nshift = shifts.order;
|
|
||||||
|
|
||||||
(*this)(Linop,src,results);
|
|
||||||
|
|
||||||
psi = shifts.norm*src;
|
|
||||||
for(int i=0;i<nshift;i++){
|
|
||||||
psi = psi + shifts.residues[i]*results[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop, const Field &src, std::vector<Field> &psi)
|
|
||||||
{
|
|
||||||
GRID_TRACE("ConjugateGradientMultiShift");
|
|
||||||
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
// Convenience references to the info stored in "MultiShiftFunction"
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
int nshift = shifts.order;
|
|
||||||
|
|
||||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
|
||||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
|
||||||
std::vector<RealD> alpha(nshift,1.0);
|
|
||||||
std::vector<Field> ps(nshift,grid);// Search directions
|
|
||||||
|
|
||||||
GRID_ASSERT(psi.size()==nshift);
|
|
||||||
GRID_ASSERT(mass.size()==nshift);
|
|
||||||
GRID_ASSERT(mresidual.size()==nshift);
|
|
||||||
|
|
||||||
// remove dynamic sized arrays on stack; 2d is a pain with vector
|
|
||||||
std::vector<RealD> bs(nshift);
|
|
||||||
std::vector<RealD> rsq(nshift);
|
|
||||||
std::vector<std::array<RealD,2> > z(nshift);
|
|
||||||
std::vector<int> converged(nshift);
|
|
||||||
|
|
||||||
const int primary =0;
|
|
||||||
|
|
||||||
//Primary shift fields CG iteration
|
|
||||||
RealD a,b,c,d;
|
|
||||||
RealD cp,bp,qq; //prev
|
|
||||||
|
|
||||||
// Matrix mult fields
|
|
||||||
Field r(grid);
|
|
||||||
Field p(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
Field mmp(grid);
|
|
||||||
|
|
||||||
// Check lightest mass
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
GRID_ASSERT( mass[s]>= mass[primary] );
|
|
||||||
converged[s]=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wire guess to zero
|
|
||||||
// Residuals "r" are src
|
|
||||||
// First search direction "p" is also src
|
|
||||||
cp = norm2(src);
|
|
||||||
|
|
||||||
// Handle trivial case of zero src.
|
|
||||||
if( cp == 0. ){
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
psi[s] = Zero();
|
|
||||||
IterationsToCompleteShift[s] = 1;
|
|
||||||
TrueResidualShift[s] = 0.;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShift: shift "<<s
|
|
||||||
<<" target resid^2 "<<rsq[s]<<std::endl;
|
|
||||||
ps[s] = src;
|
|
||||||
}
|
|
||||||
// r and p for primary
|
|
||||||
r=src;
|
|
||||||
p=src;
|
|
||||||
|
|
||||||
//MdagM+m[0]
|
|
||||||
Linop.HermOpAndNorm(p,mmp,d,qq);
|
|
||||||
axpy(mmp,mass[0],p,mmp);
|
|
||||||
RealD rn = norm2(p);
|
|
||||||
d += rn*mass[0];
|
|
||||||
|
|
||||||
// have verified that inner product of
|
|
||||||
// p and mmp is equal to d after this since
|
|
||||||
// the d computation is tricky
|
|
||||||
// qq = real(innerProduct(p,mmp));
|
|
||||||
// std::cout<<GridLogMessage << "debug equal ? qq "<<qq<<" d "<< d<<std::endl;
|
|
||||||
|
|
||||||
b = -cp /d;
|
|
||||||
|
|
||||||
// Set up the various shift variables
|
|
||||||
int iz=0;
|
|
||||||
z[0][1-iz] = 1.0;
|
|
||||||
z[0][iz] = 1.0;
|
|
||||||
bs[0] = b;
|
|
||||||
for(int s=1;s<nshift;s++){
|
|
||||||
z[s][1-iz] = 1.0;
|
|
||||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
|
||||||
bs[s] = b*z[s][iz];
|
|
||||||
}
|
|
||||||
|
|
||||||
// r += b[0] A.p[0]
|
|
||||||
// c= norm(r)
|
|
||||||
c=axpy_norm(r,b,mmp,r);
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++) {
|
|
||||||
axpby(psi[s],0.,-bs[s]*alpha[s],src,src);
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "ConjugateGradientMultiShift: initial rn (|src|^2) =" << rn << " qq (|MdagM src|^2) =" << qq << " d ( dot(src, [MdagM + m_0]src) ) =" << d << " c=" << c << std::endl;
|
|
||||||
|
|
||||||
|
|
||||||
///////////////////////////////////////
|
|
||||||
// Timers
|
|
||||||
///////////////////////////////////////
|
|
||||||
GridStopWatch AXPYTimer;
|
|
||||||
GridStopWatch ShiftTimer;
|
|
||||||
GridStopWatch QRTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
// Iteration loop
|
|
||||||
int k;
|
|
||||||
|
|
||||||
for (k=1;k<=MaxIterations;k++){
|
|
||||||
|
|
||||||
a = c /cp;
|
|
||||||
AXPYTimer.Start();
|
|
||||||
axpy(p,a,p,r);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
// Note to self - direction ps is iterated seperately
|
|
||||||
// for each shift. Does not appear to have any scope
|
|
||||||
// for avoiding linear algebra in "single" case.
|
|
||||||
//
|
|
||||||
// However SAME r is used. Could load "r" and update
|
|
||||||
// ALL ps[s]. 2/3 Bandwidth saving
|
|
||||||
// New Kernel: Load r, vector of coeffs, vector of pointers ps
|
|
||||||
AXPYTimer.Start();
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
if ( ! converged[s] ) {
|
|
||||||
if (s==0){
|
|
||||||
axpy(ps[s],a,ps[s],r);
|
|
||||||
} else{
|
|
||||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
|
||||||
axpby(ps[s],z[s][iz],as,r,ps[s]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
cp=c;
|
|
||||||
MatrixTimer.Start();
|
|
||||||
//Linop.HermOpAndNorm(p,mmp,d,qq); // d is used
|
|
||||||
// The below is faster on KNL
|
|
||||||
Linop.HermOp(p,mmp);
|
|
||||||
d=real(innerProduct(p,mmp));
|
|
||||||
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
AXPYTimer.Start();
|
|
||||||
axpy(mmp,mass[0],p,mmp);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
RealD rn = norm2(p);
|
|
||||||
d += rn*mass[0];
|
|
||||||
|
|
||||||
bp=b;
|
|
||||||
b=-cp/d;
|
|
||||||
|
|
||||||
AXPYTimer.Start();
|
|
||||||
c=axpy_norm(r,b,mmp,r);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
// Toggle the recurrence history
|
|
||||||
bs[0] = b;
|
|
||||||
iz = 1-iz;
|
|
||||||
ShiftTimer.Start();
|
|
||||||
for(int s=1;s<nshift;s++){
|
|
||||||
if((!converged[s])){
|
|
||||||
RealD z0 = z[s][1-iz];
|
|
||||||
RealD z1 = z[s][iz];
|
|
||||||
z[s][iz] = z0*z1*bp
|
|
||||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
|
||||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ShiftTimer.Stop();
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
int ss = s;
|
|
||||||
// Scope for optimisation here in case of "single".
|
|
||||||
// Could load psi[0] and pull all ps[s] in.
|
|
||||||
// if ( single ) ss=primary;
|
|
||||||
// Bandwith saving in single case is Ls * 3 -> 2+Ls, so ~ 3x saving
|
|
||||||
// Pipelined CG gain:
|
|
||||||
//
|
|
||||||
// New Kernel: Load r, vector of coeffs, vector of pointers ps
|
|
||||||
// New Kernel: Load psi[0], vector of coeffs, vector of pointers ps
|
|
||||||
// If can predict the coefficient bs then we can fuse these and avoid write reread cyce
|
|
||||||
// on ps[s].
|
|
||||||
// Before: 3 x npole + 3 x npole
|
|
||||||
// After : 2 x npole (ps[s]) => 3x speed up of multishift CG.
|
|
||||||
|
|
||||||
if( (!converged[s]) ) {
|
|
||||||
axpy(psi[ss],-bs[s]*alpha[s],ps[s],psi[ss]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convergence checks
|
|
||||||
int all_converged = 1;
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
|
|
||||||
if ( (!converged[s]) ){
|
|
||||||
IterationsToCompleteShift[s] = k;
|
|
||||||
|
|
||||||
RealD css = c * z[s][iz]* z[s][iz];
|
|
||||||
|
|
||||||
if(css<rsq[s]){
|
|
||||||
if ( ! converged[s] )
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShift k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
|
||||||
converged[s]=1;
|
|
||||||
} else {
|
|
||||||
all_converged=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( all_converged ){
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<< "CGMultiShift: All shifts have converged iteration "<<k<<std::endl;
|
|
||||||
std::cout<<GridLogMessage<< "CGMultiShift: Checking solutions"<<std::endl;
|
|
||||||
|
|
||||||
// Check answers
|
|
||||||
for(int s=0; s < nshift; s++) {
|
|
||||||
Linop.HermOpAndNorm(psi[s],mmp,d,qq);
|
|
||||||
axpy(tmp,mass[s],psi[s],mmp);
|
|
||||||
axpy(r,-alpha[s],src,tmp);
|
|
||||||
RealD rn = norm2(r);
|
|
||||||
RealD cn = norm2(src);
|
|
||||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
|
||||||
std::cout<<GridLogMessage<<"CGMultiShift: shift["<<s<<"] true residual "<< TrueResidualShift[s] <<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "Time Breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
// ugly hack
|
|
||||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
|
||||||
// GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,373 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christopher Kelly <ckelly@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//CK 2020: A variant of the multi-shift conjugate gradient with the matrix multiplication in single precision.
|
|
||||||
//The residual is stored in single precision, but the search directions and solution are stored in double precision.
|
|
||||||
//Every update_freq iterations the residual is corrected in double precision.
|
|
||||||
//For safety the a final regular CG is applied to clean up if necessary
|
|
||||||
|
|
||||||
//PB Pure single, then double fixup
|
|
||||||
|
|
||||||
template<class FieldD, class FieldF,
|
|
||||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
|
||||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class ConjugateGradientMultiShiftMixedPrecCleanup : public OperatorMultiFunction<FieldD>,
|
|
||||||
public OperatorFunction<FieldD>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
using OperatorFunction<FieldD>::operator();
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterationsMshift;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
|
||||||
int verbose;
|
|
||||||
MultiShiftFunction shifts;
|
|
||||||
std::vector<RealD> TrueResidualShift;
|
|
||||||
|
|
||||||
int ReliableUpdateFreq; //number of iterations between reliable updates
|
|
||||||
|
|
||||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
|
||||||
LinearOperatorBase<FieldF> &Linop_f; //single precision
|
|
||||||
|
|
||||||
ConjugateGradientMultiShiftMixedPrecCleanup(Integer maxit, const MultiShiftFunction &_shifts,
|
|
||||||
GridBase* _SinglePrecGrid, LinearOperatorBase<FieldF> &_Linop_f,
|
|
||||||
int _ReliableUpdateFreq) :
|
|
||||||
MaxIterationsMshift(maxit), shifts(_shifts), SinglePrecGrid(_SinglePrecGrid), Linop_f(_Linop_f), ReliableUpdateFreq(_ReliableUpdateFreq),
|
|
||||||
MaxIterations(20000)
|
|
||||||
{
|
|
||||||
verbose=1;
|
|
||||||
IterationsToCompleteShift.resize(_shifts.order);
|
|
||||||
TrueResidualShift.resize(_shifts.order);
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, FieldD &psi)
|
|
||||||
{
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
int nshift = shifts.order;
|
|
||||||
std::vector<FieldD> results(nshift,grid);
|
|
||||||
(*this)(Linop,src,results,psi);
|
|
||||||
}
|
|
||||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, std::vector<FieldD> &results, FieldD &psi)
|
|
||||||
{
|
|
||||||
int nshift = shifts.order;
|
|
||||||
|
|
||||||
(*this)(Linop,src,results);
|
|
||||||
|
|
||||||
psi = shifts.norm*src;
|
|
||||||
for(int i=0;i<nshift;i++){
|
|
||||||
psi = psi + shifts.residues[i]*results[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<FieldD> &Linop_d, const FieldD &src_d, std::vector<FieldD> &psi_d)
|
|
||||||
{
|
|
||||||
GRID_TRACE("ConjugateGradientMultiShiftMixedPrecCleanup");
|
|
||||||
GridBase *DoublePrecGrid = src_d.Grid();
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
// Convenience references to the info stored in "MultiShiftFunction"
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
int nshift = shifts.order;
|
|
||||||
|
|
||||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
|
||||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
|
||||||
std::vector<RealD> alpha(nshift,1.0);
|
|
||||||
|
|
||||||
//Double precision search directions
|
|
||||||
FieldD p_d(DoublePrecGrid);
|
|
||||||
std::vector<FieldF> ps_f (nshift, SinglePrecGrid);// Search directions (single precision)
|
|
||||||
std::vector<FieldF> psi_f(nshift, SinglePrecGrid);// solutions (single precision)
|
|
||||||
|
|
||||||
FieldD tmp_d(DoublePrecGrid);
|
|
||||||
FieldD r_d(DoublePrecGrid);
|
|
||||||
FieldF r_f(SinglePrecGrid);
|
|
||||||
FieldD mmp_d(DoublePrecGrid);
|
|
||||||
|
|
||||||
GRID_ASSERT(psi_d.size()==nshift);
|
|
||||||
GRID_ASSERT(mass.size()==nshift);
|
|
||||||
GRID_ASSERT(mresidual.size()==nshift);
|
|
||||||
|
|
||||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
|
||||||
std::vector<RealD> bs(nshift);
|
|
||||||
std::vector<RealD> rsq(nshift);
|
|
||||||
std::vector<RealD> rsqf(nshift);
|
|
||||||
std::vector<std::array<RealD,2> > z(nshift);
|
|
||||||
std::vector<int> converged(nshift);
|
|
||||||
|
|
||||||
const int primary =0;
|
|
||||||
|
|
||||||
//Primary shift fields CG iteration
|
|
||||||
RealD a,b,c,d;
|
|
||||||
RealD cp,bp,qq; //prev
|
|
||||||
|
|
||||||
// Matrix mult fields
|
|
||||||
FieldF p_f(SinglePrecGrid);
|
|
||||||
FieldF mmp_f(SinglePrecGrid);
|
|
||||||
|
|
||||||
// Check lightest mass
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
GRID_ASSERT( mass[s]>= mass[primary] );
|
|
||||||
converged[s]=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wire guess to zero
|
|
||||||
// Residuals "r" are src
|
|
||||||
// First search direction "p" is also src
|
|
||||||
cp = norm2(src_d);
|
|
||||||
|
|
||||||
// Handle trivial case of zero src.
|
|
||||||
if( cp == 0. ){
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
psi_d[s] = Zero();
|
|
||||||
psi_f[s] = Zero();
|
|
||||||
IterationsToCompleteShift[s] = 1;
|
|
||||||
TrueResidualShift[s] = 0.;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
|
||||||
rsqf[s] =rsq[s];
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: shift "<< s <<" target resid "<<rsq[s]<<std::endl;
|
|
||||||
// ps_d[s] = src_d;
|
|
||||||
precisionChange(ps_f[s],src_d);
|
|
||||||
}
|
|
||||||
// r and p for primary
|
|
||||||
p_d = src_d; //primary copy --- make this a reference to ps_d to save axpys
|
|
||||||
r_d = p_d;
|
|
||||||
|
|
||||||
//MdagM+m[0]
|
|
||||||
precisionChange(p_f,p_d);
|
|
||||||
Linop_f.HermOpAndNorm(p_f,mmp_f,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
|
||||||
precisionChange(tmp_d,mmp_f);
|
|
||||||
Linop_d.HermOpAndNorm(p_d,mmp_d,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
|
||||||
tmp_d = tmp_d - mmp_d;
|
|
||||||
std::cout << " Testing operators match "<<norm2(mmp_d)<<" f "<<norm2(mmp_f)<<" diff "<< norm2(tmp_d)<<std::endl;
|
|
||||||
// GRID_ASSERT(norm2(tmp_d)< 1.0e-4);
|
|
||||||
|
|
||||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
|
||||||
RealD rn = norm2(p_d);
|
|
||||||
d += rn*mass[0];
|
|
||||||
|
|
||||||
b = -cp /d;
|
|
||||||
|
|
||||||
// Set up the various shift variables
|
|
||||||
int iz=0;
|
|
||||||
z[0][1-iz] = 1.0;
|
|
||||||
z[0][iz] = 1.0;
|
|
||||||
bs[0] = b;
|
|
||||||
for(int s=1;s<nshift;s++){
|
|
||||||
z[s][1-iz] = 1.0;
|
|
||||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
|
||||||
bs[s] = b*z[s][iz];
|
|
||||||
}
|
|
||||||
|
|
||||||
// r += b[0] A.p[0]
|
|
||||||
// c= norm(r)
|
|
||||||
c=axpy_norm(r_d,b,mmp_d,r_d);
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++) {
|
|
||||||
axpby(psi_d[s],0.,-bs[s]*alpha[s],src_d,src_d);
|
|
||||||
precisionChange(psi_f[s],psi_d[s]);
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////
|
|
||||||
// Timers
|
|
||||||
///////////////////////////////////////
|
|
||||||
GridStopWatch AXPYTimer, ShiftTimer, QRTimer, MatrixTimer, SolverTimer, PrecChangeTimer, CleanupTimer;
|
|
||||||
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
// Iteration loop
|
|
||||||
int k;
|
|
||||||
|
|
||||||
for (k=1;k<=MaxIterationsMshift;k++){
|
|
||||||
|
|
||||||
a = c /cp;
|
|
||||||
AXPYTimer.Start();
|
|
||||||
axpy(p_d,a,p_d,r_d);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(r_f, r_d);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
AXPYTimer.Start();
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
if ( ! converged[s] ) {
|
|
||||||
if (s==0){
|
|
||||||
axpy(ps_f[s],a,ps_f[s],r_f);
|
|
||||||
} else{
|
|
||||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
|
||||||
axpby(ps_f[s],z[s][iz],as,r_f,ps_f[s]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
cp=c;
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(p_f, p_d); //get back single prec search direction for linop
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop_f.HermOp(p_f,mmp_f);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(mmp_d, mmp_f); // From Float to Double
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
d=real(innerProduct(p_d,mmp_d));
|
|
||||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
|
||||||
RealD rn = norm2(p_d);
|
|
||||||
d += rn*mass[0];
|
|
||||||
|
|
||||||
bp=b;
|
|
||||||
b=-cp/d;
|
|
||||||
|
|
||||||
// Toggle the recurrence history
|
|
||||||
bs[0] = b;
|
|
||||||
iz = 1-iz;
|
|
||||||
ShiftTimer.Start();
|
|
||||||
for(int s=1;s<nshift;s++){
|
|
||||||
if((!converged[s])){
|
|
||||||
RealD z0 = z[s][1-iz];
|
|
||||||
RealD z1 = z[s][iz];
|
|
||||||
z[s][iz] = z0*z1*bp
|
|
||||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
|
||||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ShiftTimer.Stop();
|
|
||||||
|
|
||||||
//Update single precision solutions
|
|
||||||
AXPYTimer.Start();
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
int ss = s;
|
|
||||||
if( (!converged[s]) ) {
|
|
||||||
axpy(psi_f[ss],-bs[s]*alpha[s],ps_f[s],psi_f[ss]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
c = axpy_norm(r_d,b,mmp_d,r_d);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
// Convergence checks
|
|
||||||
int all_converged = 1;
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
|
|
||||||
if ( (!converged[s]) ){
|
|
||||||
IterationsToCompleteShift[s] = k;
|
|
||||||
|
|
||||||
RealD css = c * z[s][iz]* z[s][iz];
|
|
||||||
|
|
||||||
if(css<rsqf[s]){
|
|
||||||
if ( ! converged[s] )
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
|
||||||
converged[s]=1;
|
|
||||||
} else {
|
|
||||||
all_converged=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( all_converged || k == MaxIterationsMshift-1){
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
precisionChange(psi_d[s],psi_f[s]);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if ( all_converged ){
|
|
||||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: All shifts have converged iteration "<<k<<std::endl;
|
|
||||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: Checking solutions"<<std::endl;
|
|
||||||
} else {
|
|
||||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrecCleanup: Not all shifts have converged iteration "<<k<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check answers
|
|
||||||
for(int s=0; s < nshift; s++) {
|
|
||||||
Linop_d.HermOpAndNorm(psi_d[s],mmp_d,d,qq);
|
|
||||||
axpy(tmp_d,mass[s],psi_d[s],mmp_d);
|
|
||||||
axpy(r_d,-alpha[s],src_d,tmp_d);
|
|
||||||
RealD rn = norm2(r_d);
|
|
||||||
RealD cn = norm2(src_d);
|
|
||||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: shift["<<s<<"] true residual "<< TrueResidualShift[s] << " target " << mresidual[s] << std::endl;
|
|
||||||
|
|
||||||
//If we have not reached the desired tolerance, do a (mixed precision) CG cleanup
|
|
||||||
if(rn >= rsq[s]){
|
|
||||||
CleanupTimer.Start();
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrecCleanup: performing cleanup step for shift " << s << std::endl;
|
|
||||||
|
|
||||||
//Setup linear operators for final cleanup
|
|
||||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldD> Linop_shift_d(Linop_d, mass[s]);
|
|
||||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldF> Linop_shift_f(Linop_f, mass[s]);
|
|
||||||
|
|
||||||
MixedPrecisionConjugateGradient<FieldD,FieldF> cg(mresidual[s], MaxIterations, MaxIterations, SinglePrecGrid, Linop_shift_f, Linop_shift_d);
|
|
||||||
cg(src_d, psi_d[s]);
|
|
||||||
|
|
||||||
TrueResidualShift[s] = cg.TrueResidual;
|
|
||||||
CleanupTimer.Stop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientMultiShiftMixedPrecCleanup: Time Breakdown for body"<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tPrecision Change " << PrecChangeTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tFinal Cleanup " << CleanupTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver+Cleanup " << SolverTimer.Elapsed() + CleanupTimer.Elapsed() << std::endl;
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,416 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientMultiShift.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christopher Kelly <ckelly@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_GRADIENT_MULTI_SHIFT_MIXEDPREC_H
|
|
||||||
#define GRID_CONJUGATE_GRADIENT_MULTI_SHIFT_MIXEDPREC_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//CK 2020: A variant of the multi-shift conjugate gradient with the matrix multiplication in single precision.
|
|
||||||
//The residual is stored in single precision, but the search directions and solution are stored in double precision.
|
|
||||||
//Every update_freq iterations the residual is corrected in double precision.
|
|
||||||
|
|
||||||
//For safety the a final regular CG is applied to clean up if necessary
|
|
||||||
|
|
||||||
//Linop to add shift to input linop, used in cleanup CG
|
|
||||||
namespace ConjugateGradientMultiShiftMixedPrecSupport{
|
|
||||||
template<typename Field>
|
|
||||||
class ShiftedLinop: public LinearOperatorBase<Field>{
|
|
||||||
public:
|
|
||||||
LinearOperatorBase<Field> &linop_base;
|
|
||||||
RealD shift;
|
|
||||||
|
|
||||||
ShiftedLinop(LinearOperatorBase<Field> &_linop_base, RealD _shift): linop_base(_linop_base), shift(_shift){}
|
|
||||||
|
|
||||||
void OpDiag (const Field &in, Field &out){ GRID_ASSERT(0); }
|
|
||||||
void OpDir (const Field &in, Field &out,int dir,int disp){ GRID_ASSERT(0); }
|
|
||||||
void OpDirAll (const Field &in, std::vector<Field> &out){ GRID_ASSERT(0); }
|
|
||||||
|
|
||||||
void Op (const Field &in, Field &out){ GRID_ASSERT(0); }
|
|
||||||
void AdjOp (const Field &in, Field &out){ GRID_ASSERT(0); }
|
|
||||||
|
|
||||||
void HermOp(const Field &in, Field &out){
|
|
||||||
linop_base.HermOp(in, out);
|
|
||||||
axpy(out, shift, in, out);
|
|
||||||
}
|
|
||||||
|
|
||||||
void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
|
|
||||||
HermOp(in,out);
|
|
||||||
ComplexD dot = innerProduct(in,out);
|
|
||||||
n1=real(dot);
|
|
||||||
n2=norm2(out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class FieldD, class FieldF,
|
|
||||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
|
||||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class ConjugateGradientMultiShiftMixedPrec : public OperatorMultiFunction<FieldD>,
|
|
||||||
public OperatorFunction<FieldD>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
using OperatorFunction<FieldD>::operator();
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterationsMshift;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
std::vector<int> IterationsToCompleteShift; // Iterations for this shift
|
|
||||||
int verbose;
|
|
||||||
MultiShiftFunction shifts;
|
|
||||||
std::vector<RealD> TrueResidualShift;
|
|
||||||
|
|
||||||
int ReliableUpdateFreq; //number of iterations between reliable updates
|
|
||||||
|
|
||||||
GridBase* SinglePrecGrid; //Grid for single-precision fields
|
|
||||||
LinearOperatorBase<FieldF> &Linop_f; //single precision
|
|
||||||
|
|
||||||
ConjugateGradientMultiShiftMixedPrec(Integer maxit, const MultiShiftFunction &_shifts,
|
|
||||||
GridBase* _SinglePrecGrid, LinearOperatorBase<FieldF> &_Linop_f,
|
|
||||||
int _ReliableUpdateFreq) :
|
|
||||||
MaxIterationsMshift(maxit), shifts(_shifts), SinglePrecGrid(_SinglePrecGrid), Linop_f(_Linop_f), ReliableUpdateFreq(_ReliableUpdateFreq),
|
|
||||||
MaxIterations(20000)
|
|
||||||
{
|
|
||||||
verbose=1;
|
|
||||||
IterationsToCompleteShift.resize(_shifts.order);
|
|
||||||
TrueResidualShift.resize(_shifts.order);
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, FieldD &psi)
|
|
||||||
{
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
int nshift = shifts.order;
|
|
||||||
std::vector<FieldD> results(nshift,grid);
|
|
||||||
(*this)(Linop,src,results,psi);
|
|
||||||
}
|
|
||||||
void operator() (LinearOperatorBase<FieldD> &Linop, const FieldD &src, std::vector<FieldD> &results, FieldD &psi)
|
|
||||||
{
|
|
||||||
int nshift = shifts.order;
|
|
||||||
|
|
||||||
(*this)(Linop,src,results);
|
|
||||||
|
|
||||||
psi = shifts.norm*src;
|
|
||||||
for(int i=0;i<nshift;i++){
|
|
||||||
psi = psi + shifts.residues[i]*results[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<FieldD> &Linop_d, const FieldD &src_d, std::vector<FieldD> &psi_d)
|
|
||||||
{
|
|
||||||
GRID_TRACE("ConjugateGradientMultiShiftMixedPrec");
|
|
||||||
GridBase *DoublePrecGrid = src_d.Grid();
|
|
||||||
|
|
||||||
precisionChangeWorkspace pc_wk_s_to_d(DoublePrecGrid,SinglePrecGrid);
|
|
||||||
precisionChangeWorkspace pc_wk_d_to_s(SinglePrecGrid,DoublePrecGrid);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
// Convenience references to the info stored in "MultiShiftFunction"
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
int nshift = shifts.order;
|
|
||||||
|
|
||||||
std::vector<RealD> &mass(shifts.poles); // Make references to array in "shifts"
|
|
||||||
std::vector<RealD> &mresidual(shifts.tolerances);
|
|
||||||
std::vector<RealD> alpha(nshift,1.0);
|
|
||||||
|
|
||||||
//Double precision search directions
|
|
||||||
FieldD p_d(DoublePrecGrid);
|
|
||||||
std::vector<FieldD> ps_d(nshift, DoublePrecGrid);// Search directions (double precision)
|
|
||||||
|
|
||||||
FieldD tmp_d(DoublePrecGrid);
|
|
||||||
FieldD r_d(DoublePrecGrid);
|
|
||||||
FieldD mmp_d(DoublePrecGrid);
|
|
||||||
|
|
||||||
GRID_ASSERT(psi_d.size()==nshift);
|
|
||||||
GRID_ASSERT(mass.size()==nshift);
|
|
||||||
GRID_ASSERT(mresidual.size()==nshift);
|
|
||||||
|
|
||||||
// dynamic sized arrays on stack; 2d is a pain with vector
|
|
||||||
std::vector<RealD> bs(nshift);
|
|
||||||
std::vector<RealD> rsq(nshift);
|
|
||||||
std::vector<RealD> rsqf(nshift);
|
|
||||||
std::vector<std::array<RealD,2> > z(nshift);
|
|
||||||
std::vector<int> converged(nshift);
|
|
||||||
|
|
||||||
const int primary =0;
|
|
||||||
|
|
||||||
//Primary shift fields CG iteration
|
|
||||||
RealD a,b,c,d;
|
|
||||||
RealD cp,bp,qq; //prev
|
|
||||||
|
|
||||||
// Matrix mult fields
|
|
||||||
FieldF p_f(SinglePrecGrid);
|
|
||||||
FieldF mmp_f(SinglePrecGrid);
|
|
||||||
|
|
||||||
// Check lightest mass
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
GRID_ASSERT( mass[s]>= mass[primary] );
|
|
||||||
converged[s]=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wire guess to zero
|
|
||||||
// Residuals "r" are src
|
|
||||||
// First search direction "p" is also src
|
|
||||||
cp = norm2(src_d);
|
|
||||||
|
|
||||||
// Handle trivial case of zero src.
|
|
||||||
if( cp == 0. ){
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
psi_d[s] = Zero();
|
|
||||||
IterationsToCompleteShift[s] = 1;
|
|
||||||
TrueResidualShift[s] = 0.;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
rsq[s] = cp * mresidual[s] * mresidual[s];
|
|
||||||
rsqf[s] =rsq[s];
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: shift "<< s <<" target resid "<<rsq[s]<<std::endl;
|
|
||||||
ps_d[s] = src_d;
|
|
||||||
}
|
|
||||||
// r and p for primary
|
|
||||||
p_d = src_d; //primary copy --- make this a reference to ps_d to save axpys
|
|
||||||
r_d = p_d;
|
|
||||||
|
|
||||||
//MdagM+m[0]
|
|
||||||
precisionChange(p_f, p_d, pc_wk_d_to_s);
|
|
||||||
|
|
||||||
Linop_f.HermOpAndNorm(p_f,mmp_f,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
|
||||||
precisionChange(tmp_d, mmp_f, pc_wk_s_to_d);
|
|
||||||
Linop_d.HermOpAndNorm(p_d,mmp_d,d,qq); // mmp = MdagM p d=real(dot(p, mmp)), qq=norm2(mmp)
|
|
||||||
tmp_d = tmp_d - mmp_d;
|
|
||||||
std::cout << " Testing operators match "<<norm2(mmp_d)<<" f "<<norm2(mmp_f)<<" diff "<< norm2(tmp_d)<<std::endl;
|
|
||||||
GRID_ASSERT(norm2(tmp_d)< 1.0);
|
|
||||||
|
|
||||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
|
||||||
RealD rn = norm2(p_d);
|
|
||||||
d += rn*mass[0];
|
|
||||||
|
|
||||||
b = -cp /d;
|
|
||||||
|
|
||||||
// Set up the various shift variables
|
|
||||||
int iz=0;
|
|
||||||
z[0][1-iz] = 1.0;
|
|
||||||
z[0][iz] = 1.0;
|
|
||||||
bs[0] = b;
|
|
||||||
for(int s=1;s<nshift;s++){
|
|
||||||
z[s][1-iz] = 1.0;
|
|
||||||
z[s][iz] = 1.0/( 1.0 - b*(mass[s]-mass[0]));
|
|
||||||
bs[s] = b*z[s][iz];
|
|
||||||
}
|
|
||||||
|
|
||||||
// r += b[0] A.p[0]
|
|
||||||
// c= norm(r)
|
|
||||||
c=axpy_norm(r_d,b,mmp_d,r_d);
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++) {
|
|
||||||
axpby(psi_d[s],0.,-bs[s]*alpha[s],src_d,src_d);
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////
|
|
||||||
// Timers
|
|
||||||
///////////////////////////////////////
|
|
||||||
GridStopWatch AXPYTimer, ShiftTimer, QRTimer, MatrixTimer, SolverTimer, PrecChangeTimer, CleanupTimer;
|
|
||||||
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
// Iteration loop
|
|
||||||
int k;
|
|
||||||
|
|
||||||
for (k=1;k<=MaxIterationsMshift;k++){
|
|
||||||
|
|
||||||
a = c /cp;
|
|
||||||
AXPYTimer.Start();
|
|
||||||
axpy(p_d,a,p_d,r_d);
|
|
||||||
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
if ( ! converged[s] ) {
|
|
||||||
if (s==0){
|
|
||||||
axpy(ps_d[s],a,ps_d[s],r_d);
|
|
||||||
} else{
|
|
||||||
RealD as =a *z[s][iz]*bs[s] /(z[s][1-iz]*b);
|
|
||||||
axpby(ps_d[s],z[s][iz],as,r_d,ps_d[s]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(p_f, p_d, pc_wk_d_to_s); //get back single prec search direction for linop
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
cp=c;
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop_f.HermOp(p_f,mmp_f);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(mmp_d, mmp_f, pc_wk_s_to_d); // From Float to Double
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
|
|
||||||
AXPYTimer.Start();
|
|
||||||
d=real(innerProduct(p_d,mmp_d));
|
|
||||||
axpy(mmp_d,mass[0],p_d,mmp_d);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
RealD rn = norm2(p_d);
|
|
||||||
d += rn*mass[0];
|
|
||||||
|
|
||||||
bp=b;
|
|
||||||
b=-cp/d;
|
|
||||||
|
|
||||||
// Toggle the recurrence history
|
|
||||||
bs[0] = b;
|
|
||||||
iz = 1-iz;
|
|
||||||
ShiftTimer.Start();
|
|
||||||
for(int s=1;s<nshift;s++){
|
|
||||||
if((!converged[s])){
|
|
||||||
RealD z0 = z[s][1-iz];
|
|
||||||
RealD z1 = z[s][iz];
|
|
||||||
z[s][iz] = z0*z1*bp
|
|
||||||
/ (b*a*(z1-z0) + z1*bp*(1- (mass[s]-mass[0])*b));
|
|
||||||
bs[s] = b*z[s][iz]/z0; // NB sign rel to Mike
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ShiftTimer.Stop();
|
|
||||||
|
|
||||||
//Update double precision solutions
|
|
||||||
AXPYTimer.Start();
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
int ss = s;
|
|
||||||
if( (!converged[s]) ) {
|
|
||||||
axpy(psi_d[ss],-bs[s]*alpha[s],ps_d[s],psi_d[ss]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//Perform reliable update if necessary; otherwise update residual from single-prec mmp
|
|
||||||
c = axpy_norm(r_d,b,mmp_d,r_d);
|
|
||||||
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
if(k % ReliableUpdateFreq == 0){
|
|
||||||
RealD c_old = c;
|
|
||||||
//Replace r with true residual
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop_d.HermOp(psi_d[0],mmp_d);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
AXPYTimer.Start();
|
|
||||||
axpy(mmp_d,mass[0],psi_d[0],mmp_d);
|
|
||||||
|
|
||||||
c = axpy_norm(r_d, -1.0, mmp_d, src_d);
|
|
||||||
AXPYTimer.Stop();
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec k="<<k<< ", replaced |r|^2 = "<<c_old <<" with |r|^2 = "<<c<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convergence checks
|
|
||||||
int all_converged = 1;
|
|
||||||
for(int s=0;s<nshift;s++){
|
|
||||||
|
|
||||||
if ( (!converged[s]) ){
|
|
||||||
IterationsToCompleteShift[s] = k;
|
|
||||||
|
|
||||||
RealD css = c * z[s][iz]* z[s][iz];
|
|
||||||
|
|
||||||
if(css<rsqf[s]){
|
|
||||||
if ( ! converged[s] )
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec k="<<k<<" Shift "<<s<<" has converged"<<std::endl;
|
|
||||||
converged[s]=1;
|
|
||||||
} else {
|
|
||||||
all_converged=0;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( all_converged || k == MaxIterationsMshift-1){
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
if ( all_converged ){
|
|
||||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: All shifts have converged iteration "<<k<<std::endl;
|
|
||||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: Checking solutions"<<std::endl;
|
|
||||||
} else {
|
|
||||||
std::cout<<GridLogMessage<< "ConjugateGradientMultiShiftMixedPrec: Not all shifts have converged iteration "<<k<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check answers
|
|
||||||
for(int s=0; s < nshift; s++) {
|
|
||||||
Linop_d.HermOpAndNorm(psi_d[s],mmp_d,d,qq);
|
|
||||||
axpy(tmp_d,mass[s],psi_d[s],mmp_d);
|
|
||||||
axpy(r_d,-alpha[s],src_d,tmp_d);
|
|
||||||
RealD rn = norm2(r_d);
|
|
||||||
RealD cn = norm2(src_d);
|
|
||||||
TrueResidualShift[s] = std::sqrt(rn/cn);
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: shift["<<s<<"] true residual "<< TrueResidualShift[s] << " target " << mresidual[s] << std::endl;
|
|
||||||
|
|
||||||
//If we have not reached the desired tolerance, do a (mixed precision) CG cleanup
|
|
||||||
if(rn >= rsq[s]){
|
|
||||||
CleanupTimer.Start();
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateGradientMultiShiftMixedPrec: performing cleanup step for shift " << s << std::endl;
|
|
||||||
|
|
||||||
//Setup linear operators for final cleanup
|
|
||||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldD> Linop_shift_d(Linop_d, mass[s]);
|
|
||||||
ConjugateGradientMultiShiftMixedPrecSupport::ShiftedLinop<FieldF> Linop_shift_f(Linop_f, mass[s]);
|
|
||||||
|
|
||||||
MixedPrecisionConjugateGradient<FieldD,FieldF> cg(mresidual[s], MaxIterations, MaxIterations, SinglePrecGrid, Linop_shift_f, Linop_shift_d);
|
|
||||||
cg(src_d, psi_d[s]);
|
|
||||||
|
|
||||||
TrueResidualShift[s] = cg.TrueResidual;
|
|
||||||
CleanupTimer.Stop();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientMultiShiftMixedPrec: Time Breakdown for body"<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tAXPY " << AXPYTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tShift " << ShiftTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\t\tPrecision Change " << PrecChangeTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tFinal Cleanup " << CleanupTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver+Cleanup " << SolverTimer.Elapsed() + CleanupTimer.Elapsed() << std::endl;
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
std::cout<<GridLogMessage<<"CG multi shift did not converge"<<std::endl;
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,277 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Christopher Kelly <ckelly@phys.columbia.edu>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_GRADIENT_RELIABLE_UPDATE_H
|
|
||||||
#define GRID_CONJUGATE_GRADIENT_RELIABLE_UPDATE_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class FieldD,class FieldF,
|
|
||||||
typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,
|
|
||||||
typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class ConjugateGradientReliableUpdate : public LinearFunction<FieldD> {
|
|
||||||
public:
|
|
||||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the CG fails to converge.
|
|
||||||
// Defaults true.
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
Integer ReliableUpdatesPerformed;
|
|
||||||
|
|
||||||
bool DoFinalCleanup; //Final DP cleanup, defaults to true
|
|
||||||
Integer IterationsToCleanup; //Final DP cleanup step iterations
|
|
||||||
|
|
||||||
LinearOperatorBase<FieldF> &Linop_f;
|
|
||||||
LinearOperatorBase<FieldD> &Linop_d;
|
|
||||||
GridBase* SinglePrecGrid;
|
|
||||||
RealD Delta; //reliable update parameter. A reliable update is performed when the residual drops by a factor of Delta relative to its value at the last update
|
|
||||||
|
|
||||||
//Optional ability to switch to a different linear operator once the tolerance reaches a certain point. Useful for single/half -> single/single
|
|
||||||
LinearOperatorBase<FieldF> *Linop_fallback;
|
|
||||||
RealD fallback_transition_tol;
|
|
||||||
|
|
||||||
|
|
||||||
ConjugateGradientReliableUpdate(RealD tol, Integer maxit, RealD _delta, GridBase* _sp_grid, LinearOperatorBase<FieldF> &_Linop_f, LinearOperatorBase<FieldD> &_Linop_d, bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
Delta(_delta),
|
|
||||||
Linop_f(_Linop_f),
|
|
||||||
Linop_d(_Linop_d),
|
|
||||||
SinglePrecGrid(_sp_grid),
|
|
||||||
ErrorOnNoConverge(err_on_no_conv),
|
|
||||||
DoFinalCleanup(true),
|
|
||||||
Linop_fallback(NULL)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(Delta > 0. && Delta < 1. && "Expect 0 < Delta < 1");
|
|
||||||
};
|
|
||||||
|
|
||||||
void setFallbackLinop(LinearOperatorBase<FieldF> &_Linop_fallback, const RealD _fallback_transition_tol){
|
|
||||||
Linop_fallback = &_Linop_fallback;
|
|
||||||
fallback_transition_tol = _fallback_transition_tol;
|
|
||||||
}
|
|
||||||
|
|
||||||
void operator()(const FieldD &src, FieldD &psi) {
|
|
||||||
GRID_TRACE("ConjugateGradientReliableUpdate");
|
|
||||||
LinearOperatorBase<FieldF> *Linop_f_use = &Linop_f;
|
|
||||||
bool using_fallback = false;
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD cp, c, a, d, b, ssq, qq, b_pred;
|
|
||||||
|
|
||||||
FieldD p(src);
|
|
||||||
FieldD mmp(src);
|
|
||||||
FieldD r(src);
|
|
||||||
|
|
||||||
// Initial residual computation & set up
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
Linop_d.HermOpAndNorm(psi, mmp, d, b);
|
|
||||||
|
|
||||||
r = src - mmp;
|
|
||||||
p = r;
|
|
||||||
|
|
||||||
a = norm2(p);
|
|
||||||
cp = a;
|
|
||||||
ssq = norm2(src);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: src " << ssq << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: mp " << d << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: mmp " << b << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: cp,r " << cp << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: p " << a << std::endl;
|
|
||||||
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
// Check if guess is really REALLY good :)
|
|
||||||
if (cp <= rsq) {
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate guess was REALLY good\n";
|
|
||||||
std::cout << GridLogMessage << "\tComputed residual " << std::sqrt(cp / ssq)<<std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
//Single prec initialization
|
|
||||||
precisionChangeWorkspace pc_wk_sp_to_dp(src.Grid(), SinglePrecGrid);
|
|
||||||
precisionChangeWorkspace pc_wk_dp_to_sp(SinglePrecGrid, src.Grid());
|
|
||||||
|
|
||||||
FieldF r_f(SinglePrecGrid);
|
|
||||||
r_f.Checkerboard() = r.Checkerboard();
|
|
||||||
precisionChange(r_f, r, pc_wk_dp_to_sp);
|
|
||||||
|
|
||||||
FieldF psi_f(r_f);
|
|
||||||
psi_f = Zero();
|
|
||||||
|
|
||||||
FieldF p_f(r_f);
|
|
||||||
FieldF mmp_f(r_f);
|
|
||||||
|
|
||||||
RealD MaxResidSinceLastRelUp = cp; //initial residual
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(4)
|
|
||||||
<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
GridStopWatch PrecChangeTimer;
|
|
||||||
|
|
||||||
SolverTimer.Start();
|
|
||||||
int k = 0;
|
|
||||||
int l = 0;
|
|
||||||
|
|
||||||
for (k = 1; k <= MaxIterations; k++) {
|
|
||||||
c = cp;
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop_f_use->HermOpAndNorm(p_f, mmp_f, d, qq);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
a = c / d;
|
|
||||||
b_pred = a * (a * qq - d) / c;
|
|
||||||
|
|
||||||
cp = axpy_norm(r_f, -a, mmp_f, r_f);
|
|
||||||
b = cp / c;
|
|
||||||
|
|
||||||
// Fuse these loops ; should be really easy
|
|
||||||
psi_f = a * p_f + psi_f;
|
|
||||||
//p_f = p_f * b + r_f;
|
|
||||||
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "ConjugateGradientReliableUpdate: Iteration " << k
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
std::cout << GridLogDebug << "a = "<< a << " b_pred = "<< b_pred << " b = "<< b << std::endl;
|
|
||||||
std::cout << GridLogDebug << "qq = "<< qq << " d = "<< d << " c = "<< c << std::endl;
|
|
||||||
|
|
||||||
if(cp > MaxResidSinceLastRelUp){
|
|
||||||
std::cout << GridLogIterative << "ConjugateGradientReliableUpdate: updating MaxResidSinceLastRelUp : " << MaxResidSinceLastRelUp << " -> " << cp << std::endl;
|
|
||||||
MaxResidSinceLastRelUp = cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
//Although not written in the paper, I assume that I have to add on the final solution
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(mmp, psi_f, pc_wk_sp_to_dp);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
psi = psi + mmp;
|
|
||||||
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
Linop_d.HermOpAndNorm(psi, mmp, d, qq);
|
|
||||||
p = mmp - src;
|
|
||||||
|
|
||||||
RealD srcnorm = std::sqrt(norm2(src));
|
|
||||||
RealD resnorm = std::sqrt(norm2(p));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate Converged on iteration " << k << " after " << l << " reliable updates" << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tComputed residual " << std::sqrt(cp / ssq)<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tTrue residual " << true_residual<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "Time breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tElapsed " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tPrecChange " << PrecChangeTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tPrecChange avg time " << PrecChangeTimer.Elapsed()/(2*l+1) <<std::endl;
|
|
||||||
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
ReliableUpdatesPerformed = l;
|
|
||||||
|
|
||||||
if(DoFinalCleanup){
|
|
||||||
//Do a final CG to cleanup
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate performing final cleanup.\n";
|
|
||||||
ConjugateGradient<FieldD> CG(Tolerance,MaxIterations);
|
|
||||||
CG.ErrorOnNoConverge = ErrorOnNoConverge;
|
|
||||||
CG(Linop_d,src,psi);
|
|
||||||
IterationsToCleanup = CG.IterationsToComplete;
|
|
||||||
}
|
|
||||||
else if (ErrorOnNoConverge) GRID_ASSERT(true_residual / Tolerance < 10000.0);
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate complete.\n";
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
else if(cp < Delta * MaxResidSinceLastRelUp) { //reliable update
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate "
|
|
||||||
<< cp << "(residual) < " << Delta << "(Delta) * " << MaxResidSinceLastRelUp << "(MaxResidSinceLastRelUp) on iteration " << k << " : performing reliable update\n";
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(mmp, psi_f, pc_wk_sp_to_dp);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
psi = psi + mmp;
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop_d.HermOpAndNorm(psi, mmp, d, qq);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
r = src - mmp;
|
|
||||||
|
|
||||||
psi_f = Zero();
|
|
||||||
PrecChangeTimer.Start();
|
|
||||||
precisionChange(r_f, r, pc_wk_dp_to_sp);
|
|
||||||
PrecChangeTimer.Stop();
|
|
||||||
cp = norm2(r);
|
|
||||||
MaxResidSinceLastRelUp = cp;
|
|
||||||
|
|
||||||
b = cp/c;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate new residual " << cp << std::endl;
|
|
||||||
|
|
||||||
l = l+1;
|
|
||||||
}
|
|
||||||
|
|
||||||
p_f = p_f * b + r_f; //update search vector after reliable update appears to help convergence
|
|
||||||
|
|
||||||
if(!using_fallback && Linop_fallback != NULL && cp < fallback_transition_tol){
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate switching to fallback linear operator on iteration " << k << " at residual " << cp << std::endl;
|
|
||||||
Linop_f_use = Linop_fallback;
|
|
||||||
using_fallback = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradientReliableUpdate did NOT converge"
|
|
||||||
<< std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge) GRID_ASSERT(0);
|
|
||||||
IterationsToComplete = k;
|
|
||||||
ReliableUpdatesPerformed = l;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,277 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateGradientTimeslice.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_GRADIENT_TIMESLICE_H
|
|
||||||
#define GRID_CONJUGATE_GRADIENT_TIMESLICE_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for iterative processes based on operators
|
|
||||||
// single input vec, single output vec.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Simple modification of conjugate gradient that outputs the residual as a function
|
|
||||||
* of time, in order to study the large wavelength behavior of the solver.
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
template <class Field>
|
|
||||||
class ConjugateGradientTimeslice : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // throw an assert when the CG fails to converge.
|
|
||||||
// Defaults true.
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
|
|
||||||
RealD TrueResidual;
|
|
||||||
|
|
||||||
ConjugateGradientTimeslice(RealD tol, Integer maxit, bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
{};
|
|
||||||
|
|
||||||
virtual void LogIteration(int k,RealD a,RealD b){
|
|
||||||
// std::cout << "ConjugageGradient::LogIteration() "<<std::endl;
|
|
||||||
};
|
|
||||||
virtual void LogBegin(void){
|
|
||||||
std::cout << "ConjugageGradient::LogBegin() "<<std::endl;
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
this->LogBegin();
|
|
||||||
|
|
||||||
GRID_TRACE("ConjugateGradientTimeslice");
|
|
||||||
GridStopWatch PreambleTimer;
|
|
||||||
GridStopWatch ConstructTimer;
|
|
||||||
GridStopWatch NormTimer;
|
|
||||||
GridStopWatch AssignTimer;
|
|
||||||
PreambleTimer.Start();
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD cp, c, a, d, b, ssq, qq;
|
|
||||||
//RealD b_pred;
|
|
||||||
|
|
||||||
// Was doing copies
|
|
||||||
ConstructTimer.Start();
|
|
||||||
Field p (src.Grid());
|
|
||||||
Field mmp(src.Grid());
|
|
||||||
Field r (src.Grid());
|
|
||||||
ConstructTimer.Stop();
|
|
||||||
|
|
||||||
// Initial residual computation & set up
|
|
||||||
NormTimer.Start();
|
|
||||||
ssq = norm2(src); // Norm of source vector ||b||^2
|
|
||||||
|
|
||||||
ssqtx = localNorm2(src); // Norm |b(x, t)|^2 as a field
|
|
||||||
std::vector<RealD> ssqt; // Norm of source not summed over time slices, ssq(t) = \sum_x |b(x, t)|^2
|
|
||||||
sliceSum(ssqtx, ssqt, Tdir); // TODO make sure Tdir is globally defined
|
|
||||||
|
|
||||||
RealD guess = norm2(psi); // Norm of initial guess ||psi||^2
|
|
||||||
NormTimer.Stop();
|
|
||||||
assert(std::isnan(guess) == 0);
|
|
||||||
AssignTimer.Start();
|
|
||||||
if ( guess == 0.0 ) {
|
|
||||||
r = src;
|
|
||||||
p = r;
|
|
||||||
a = ssq;
|
|
||||||
} else {
|
|
||||||
Linop.HermOpAndNorm(psi, mmp, d, b); //
|
|
||||||
r = src - mmp; // Initial residual r0 = b - A guess
|
|
||||||
p = r; // initial conj vector p0 = r0
|
|
||||||
a = norm2(p);
|
|
||||||
}
|
|
||||||
cp = a;
|
|
||||||
AssignTimer.Stop();
|
|
||||||
|
|
||||||
// Handle trivial case of zero src
|
|
||||||
if (ssq == 0.){
|
|
||||||
psi = Zero();
|
|
||||||
IterationsToComplete = 1;
|
|
||||||
TrueResidual = 0.;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: src " << ssq << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: mp " << d << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: mmp " << b << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: cp,r " << cp << std::endl;
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8) << "ConjugateGradient: p " << a << std::endl;
|
|
||||||
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
// Check if guess is really REALLY good :)
|
|
||||||
if (cp <= rsq) {
|
|
||||||
TrueResidual = std::sqrt(a/ssq);
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient guess is converged already " << std::endl;
|
|
||||||
IterationsToComplete = 0;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << std::setprecision(8)
|
|
||||||
<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
PreambleTimer.Stop();
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch InnerTimer;
|
|
||||||
GridStopWatch AxpyNormTimer;
|
|
||||||
GridStopWatch LinearCombTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
|
|
||||||
RealD usecs = -usecond();
|
|
||||||
SolverTimer.Start();
|
|
||||||
int k;
|
|
||||||
for (k = 1; k <= MaxIterations; k++) {
|
|
||||||
|
|
||||||
GridStopWatch IterationTimer;
|
|
||||||
IterationTimer.Start();
|
|
||||||
c = cp;
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop.HermOp(p, mmp); // Computes mmp = Ap
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
InnerTimer.Start();
|
|
||||||
ComplexD dc = innerProduct(p,mmp); // p^\dagger A p
|
|
||||||
InnerTimer.Stop();
|
|
||||||
d = dc.real();
|
|
||||||
a = c / d;
|
|
||||||
|
|
||||||
// What is axpy? Some accelerator or something? Check Lattice_arith.h
|
|
||||||
AxpyNormTimer.Start();
|
|
||||||
|
|
||||||
// axpy_norm computes ax+by for vectors x and y compatible with a GPU. Here b is set to 1 (see the function in Lattice_reduction.h).
|
|
||||||
// The first argument passes r by reference, so it stores r --> -a * Ap + 1 * r, i.e. it performs an update on
|
|
||||||
// r_k --> r_{k+1} = r_k - \alpha_k A p_k. The function returns the norm squared of the first variable, i.e. ||r_{k+1}||^2.
|
|
||||||
cp = axpy_norm(r, -a, mmp, r);
|
|
||||||
AxpyNormTimer.Stop();
|
|
||||||
b = cp / c;
|
|
||||||
|
|
||||||
LinearCombTimer.Start();
|
|
||||||
{
|
|
||||||
autoView( psi_v , psi, AcceleratorWrite);
|
|
||||||
autoView( p_v , p, AcceleratorWrite);
|
|
||||||
autoView( r_v , r, AcceleratorWrite);
|
|
||||||
accelerator_for(ss,p_v.size(), Field::vector_object::Nsimd(),{
|
|
||||||
coalescedWrite(psi_v[ss], a * p_v(ss) + psi_v(ss));
|
|
||||||
coalescedWrite(p_v[ss] , b * p_v(ss) + r_v (ss));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
LinearCombTimer.Stop();
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
LogIteration(k,a,b);
|
|
||||||
|
|
||||||
IterationTimer.Stop();
|
|
||||||
if ( (k % 500) == 0 ) {
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient: Iteration " << k
|
|
||||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << std::endl;
|
|
||||||
} else {
|
|
||||||
std::cout << GridLogIterative << "ConjugateGradient: Iteration " << k
|
|
||||||
<< " residual " << sqrt(cp/ssq) << " target " << Tolerance << " took " << IterationTimer.Elapsed() << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
usecs +=usecond();
|
|
||||||
SolverTimer.Stop();
|
|
||||||
Linop.HermOpAndNorm(psi, mmp, d, qq);
|
|
||||||
p = mmp - src;
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
RealD DwfFlops = (1452. )*grid->gSites()*4*k
|
|
||||||
+ (8+4+8+4+4)*12*grid->gSites()*k; // CG linear algebra
|
|
||||||
RealD srcnorm = std::sqrt(norm2(src));
|
|
||||||
RealD resnorm = std::sqrt(norm2(p));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient Converged on iteration " << k
|
|
||||||
<< "\tComputed residual " << std::sqrt(cp / ssq)
|
|
||||||
<< "\tTrue residual " << true_residual
|
|
||||||
<< "\tTarget " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
// GridLogMessage logs the message to the terminal output; GridLogPerformance probably writes to a log file?
|
|
||||||
// std::cout << GridLogMessage << "\tPreamble " << PreambleTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver Elapsed " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "Time breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tInner " << InnerTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogDebug << "\tMobius flop rate " << DwfFlops/ usecs<< " Gflops " <<std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
TrueResidual = true_residual;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Failed. Calculate true residual before giving up
|
|
||||||
// Linop.HermOpAndNorm(psi, mmp, d, qq);
|
|
||||||
// p = mmp - src;
|
|
||||||
//TrueResidual = sqrt(norm2(p)/ssq);
|
|
||||||
// TrueResidual = 1;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "ConjugateGradient did NOT converge "<<k<<" / "<< MaxIterations
|
|
||||||
<<" residual "<< std::sqrt(cp / ssq)<< std::endl;
|
|
||||||
SolverTimer.Stop();
|
|
||||||
std::cout << GridLogMessage << "\tPreamble " << PreambleTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tConstruct " << ConstructTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tNorm " << NormTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tAssign " << AssignTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSolver " << SolverTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage << "Solver breakdown "<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tMatrix " << MatrixTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogMessage<< "\tLinalg " << LinalgTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tInner " << InnerTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tAxpyNorm " << AxpyNormTimer.Elapsed() <<std::endl;
|
|
||||||
std::cout << GridLogPerformance << "\t\tLinearComb " << LinearCombTimer.Elapsed() <<std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge) assert(0);
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ConjugateResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CONJUGATE_RESIDUAL_H
|
|
||||||
#define GRID_CONJUGATE_RESIDUAL_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for iterative processes based on operators
|
|
||||||
// single input vec, single output vec.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class ConjugateResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
int verbose;
|
|
||||||
|
|
||||||
ConjugateResidual(RealD tol,Integer maxit) : Tolerance(tol), MaxIterations(maxit) {
|
|
||||||
verbose=0;
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi){
|
|
||||||
|
|
||||||
RealD a, b; // c, d;
|
|
||||||
RealD cp, ssq,rsq;
|
|
||||||
|
|
||||||
RealD rAr, rAAr, rArp;
|
|
||||||
RealD pAp, pAAp;
|
|
||||||
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
psi=Zero();
|
|
||||||
Field r(grid), p(grid), Ap(grid), Ar(grid);
|
|
||||||
|
|
||||||
r=src;
|
|
||||||
p=src;
|
|
||||||
|
|
||||||
Linop.HermOpAndNorm(p,Ap,pAp,pAAp);
|
|
||||||
Linop.HermOpAndNorm(r,Ar,rAr,rAAr);
|
|
||||||
|
|
||||||
cp =norm2(r);
|
|
||||||
ssq=norm2(src);
|
|
||||||
rsq=Tolerance*Tolerance*ssq;
|
|
||||||
|
|
||||||
if (verbose) std::cout<<GridLogMessage<<"ConjugateResidual: iteration " <<0<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
|
||||||
|
|
||||||
for(int k=1;k<MaxIterations;k++){
|
|
||||||
|
|
||||||
a = rAr/pAAp;
|
|
||||||
|
|
||||||
axpy(psi,a,p,psi);
|
|
||||||
|
|
||||||
cp = axpy_norm(r,-a,Ap,r);
|
|
||||||
|
|
||||||
rArp=rAr;
|
|
||||||
|
|
||||||
Linop.HermOpAndNorm(r,Ar,rAr,rAAr);
|
|
||||||
|
|
||||||
b =rAr/rArp;
|
|
||||||
|
|
||||||
axpy(p,b,p,r);
|
|
||||||
pAAp=axpy_norm(Ap,b,Ap,Ar);
|
|
||||||
|
|
||||||
if(verbose) std::cout<<GridLogMessage<<"ConjugateResidual: iteration " <<k<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
|
||||||
|
|
||||||
if(cp<rsq) {
|
|
||||||
Linop.HermOp(psi,Ap);
|
|
||||||
axpy(r,-1.0,src,Ap);
|
|
||||||
RealD true_resid = norm2(r)/ssq;
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateResidual: Converged on iteration " <<k
|
|
||||||
<< " computed residual "<<std::sqrt(cp/ssq)
|
|
||||||
<< " true residual "<<std::sqrt(true_resid)
|
|
||||||
<< " target " <<Tolerance <<std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"ConjugateResidual did NOT converge"<<std::endl;
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,258 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/FlexibleCommunicationAvoidingGeneralisedMinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_FLEXIBLE_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
#define GRID_FLEXIBLE_COMMUNICATION_AVOIDING_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class FlexibleCommunicationAvoidingGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when FCAGMRES fails to converge,
|
|
||||||
// defaults to true
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer RestartLength;
|
|
||||||
Integer MaxNumberOfRestarts;
|
|
||||||
Integer IterationCount; // Number of iterations the FCAGMRES took to finish,
|
|
||||||
// filled in upon completion
|
|
||||||
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch PrecTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch QrTimer;
|
|
||||||
GridStopWatch CompSolutionTimer;
|
|
||||||
|
|
||||||
Eigen::MatrixXcd H;
|
|
||||||
|
|
||||||
std::vector<ComplexD> y;
|
|
||||||
std::vector<ComplexD> gamma;
|
|
||||||
std::vector<ComplexD> c;
|
|
||||||
std::vector<ComplexD> s;
|
|
||||||
|
|
||||||
LinearFunction<Field> &Preconditioner;
|
|
||||||
|
|
||||||
FlexibleCommunicationAvoidingGeneralisedMinimalResidual(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearFunction<Field> &Prec,
|
|
||||||
Integer restart_length,
|
|
||||||
bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol)
|
|
||||||
, MaxIterations(maxit)
|
|
||||||
, RestartLength(restart_length)
|
|
||||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
|
||||||
, ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
|
||||||
, y(RestartLength + 1, 0.)
|
|
||||||
, gamma(RestartLength + 1, 0.)
|
|
||||||
, c(RestartLength + 1, 0.)
|
|
||||||
, s(RestartLength + 1, 0.)
|
|
||||||
, Preconditioner(Prec) {};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
std::cout << GridLogWarning << "This algorithm currently doesn't differ from regular FGMRES" << std::endl;
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
std::cout << std::setprecision(4) << std::scientific;
|
|
||||||
std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: src " << ssq << std::endl;
|
|
||||||
|
|
||||||
PrecTimer.Reset();
|
|
||||||
MatrixTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
QrTimer.Reset();
|
|
||||||
CompSolutionTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
IterationCount = 0;
|
|
||||||
|
|
||||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
|
||||||
|
|
||||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
LinOp.Op(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(ssq);
|
|
||||||
RealD resnorm = sqrt(norm2(r));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
|
||||||
<< " computed residual " << sqrt(cp / ssq)
|
|
||||||
<< " true residual " << true_residual
|
|
||||||
<< " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Precon " << PrecTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FCAGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual did NOT converge" << std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
|
||||||
|
|
||||||
RealD cp = 0;
|
|
||||||
|
|
||||||
Field w(src.Grid());
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
// these should probably be made class members so that they are only allocated once, not in every restart
|
|
||||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
|
||||||
std::vector<Field> z(RestartLength + 1, src.Grid()); for (auto &elem : z) elem = Zero();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(psi, w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r = src - w;
|
|
||||||
|
|
||||||
gamma[0] = sqrt(norm2(r));
|
|
||||||
|
|
||||||
v[0] = (1. / gamma[0]) * r;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
for (int i=0; i<RestartLength; i++) {
|
|
||||||
|
|
||||||
IterationCount++;
|
|
||||||
|
|
||||||
arnoldiStep(LinOp, v, z, w, i);
|
|
||||||
|
|
||||||
qrUpdate(i);
|
|
||||||
|
|
||||||
cp = norm(gamma[i+1]);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "FlexibleCommunicationAvoidingGeneralisedMinimalResidual: Iteration " << IterationCount
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
|
||||||
|
|
||||||
computeSolution(z, psi, i);
|
|
||||||
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT(0); // Never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, std::vector<Field> &z, Field &w, int iter) {
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(v[iter], z[iter]);
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(z[iter], w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
for (int i = 0; i <= iter; ++i) {
|
|
||||||
H(iter, i) = innerProduct(v[i], w);
|
|
||||||
w = w - ComplexD(H(iter, i)) * v[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
H(iter, iter + 1) = sqrt(norm2(w));
|
|
||||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void qrUpdate(int iter) {
|
|
||||||
|
|
||||||
QrTimer.Start();
|
|
||||||
for (int i = 0; i < iter ; ++i) {
|
|
||||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i + 1) = tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute new Givens Rotation
|
|
||||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
|
||||||
c[iter] = H(iter, iter) / nu;
|
|
||||||
s[iter] = H(iter, iter + 1) / nu;
|
|
||||||
|
|
||||||
// Apply new Givens rotation
|
|
||||||
H(iter, iter) = nu;
|
|
||||||
H(iter, iter + 1) = 0.;
|
|
||||||
|
|
||||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
|
||||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
|
||||||
QrTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void computeSolution(std::vector<Field> const &z, Field &psi, int iter) {
|
|
||||||
|
|
||||||
CompSolutionTimer.Start();
|
|
||||||
for (int i = iter; i >= 0; i--) {
|
|
||||||
y[i] = gamma[i];
|
|
||||||
for (int k = i + 1; k <= iter; k++)
|
|
||||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
|
||||||
y[i] = y[i] / ComplexD(H(i, i));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i <= iter; i++)
|
|
||||||
psi = psi + z[i] * y[i];
|
|
||||||
CompSolutionTimer.Stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
@@ -1,256 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/FlexibleGeneralisedMinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
#define GRID_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class FlexibleGeneralisedMinimalResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when FGMRES fails to converge,
|
|
||||||
// defaults to true
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer RestartLength;
|
|
||||||
Integer MaxNumberOfRestarts;
|
|
||||||
Integer IterationCount; // Number of iterations the FGMRES took to finish,
|
|
||||||
// filled in upon completion
|
|
||||||
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch PrecTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch QrTimer;
|
|
||||||
GridStopWatch CompSolutionTimer;
|
|
||||||
|
|
||||||
Eigen::MatrixXcd H;
|
|
||||||
|
|
||||||
std::vector<ComplexD> y;
|
|
||||||
std::vector<ComplexD> gamma;
|
|
||||||
std::vector<ComplexD> c;
|
|
||||||
std::vector<ComplexD> s;
|
|
||||||
|
|
||||||
LinearFunction<Field> &Preconditioner;
|
|
||||||
|
|
||||||
FlexibleGeneralisedMinimalResidual(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
LinearFunction<Field> &Prec,
|
|
||||||
Integer restart_length,
|
|
||||||
bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol)
|
|
||||||
, MaxIterations(maxit)
|
|
||||||
, RestartLength(restart_length)
|
|
||||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
|
||||||
, ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
|
||||||
, y(RestartLength + 1, 0.)
|
|
||||||
, gamma(RestartLength + 1, 0.)
|
|
||||||
, c(RestartLength + 1, 0.)
|
|
||||||
, s(RestartLength + 1, 0.)
|
|
||||||
, Preconditioner(Prec) {};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
std::cout << std::setprecision(4) << std::scientific;
|
|
||||||
std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: src " << ssq << std::endl;
|
|
||||||
|
|
||||||
PrecTimer.Reset();
|
|
||||||
MatrixTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
QrTimer.Reset();
|
|
||||||
CompSolutionTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
IterationCount = 0;
|
|
||||||
|
|
||||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
|
||||||
|
|
||||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
LinOp.Op(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(ssq);
|
|
||||||
RealD resnorm = sqrt(norm2(r));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "FlexibleGeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
|
||||||
<< " computed residual " << sqrt(cp / ssq)
|
|
||||||
<< " true residual " << true_residual
|
|
||||||
<< " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Precon " << PrecTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "FGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "FlexibleGeneralisedMinimalResidual did NOT converge" << std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
|
||||||
|
|
||||||
RealD cp = 0;
|
|
||||||
|
|
||||||
Field w(src.Grid());
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
// these should probably be made class members so that they are only allocated once, not in every restart
|
|
||||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
|
||||||
std::vector<Field> z(RestartLength + 1, src.Grid()); for (auto &elem : z) elem = Zero();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(psi, w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r = src - w;
|
|
||||||
|
|
||||||
gamma[0] = sqrt(norm2(r));
|
|
||||||
|
|
||||||
v[0] = (1. / gamma[0]) * r;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
for (int i=0; i<RestartLength; i++) {
|
|
||||||
|
|
||||||
IterationCount++;
|
|
||||||
|
|
||||||
arnoldiStep(LinOp, v, z, w, i);
|
|
||||||
|
|
||||||
qrUpdate(i);
|
|
||||||
|
|
||||||
cp = norm(gamma[i+1]);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "FlexibleGeneralisedMinimalResidual: Iteration " << IterationCount
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
|
||||||
|
|
||||||
computeSolution(z, psi, i);
|
|
||||||
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT(0); // Never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, std::vector<Field> &z, Field &w, int iter) {
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(v[iter], z[iter]);
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(z[iter], w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
for (int i = 0; i <= iter; ++i) {
|
|
||||||
H(iter, i) = innerProduct(v[i], w);
|
|
||||||
w = w - ComplexD(H(iter, i)) * v[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
H(iter, iter + 1) = sqrt(norm2(w));
|
|
||||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void qrUpdate(int iter) {
|
|
||||||
|
|
||||||
QrTimer.Start();
|
|
||||||
for (int i = 0; i < iter ; ++i) {
|
|
||||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i + 1) = tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute new Givens Rotation
|
|
||||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
|
||||||
c[iter] = H(iter, iter) / nu;
|
|
||||||
s[iter] = H(iter, iter + 1) / nu;
|
|
||||||
|
|
||||||
// Apply new Givens rotation
|
|
||||||
H(iter, iter) = nu;
|
|
||||||
H(iter, iter + 1) = 0.;
|
|
||||||
|
|
||||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
|
||||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
|
||||||
QrTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void computeSolution(std::vector<Field> const &z, Field &psi, int iter) {
|
|
||||||
|
|
||||||
CompSolutionTimer.Start();
|
|
||||||
for (int i = iter; i >= 0; i--) {
|
|
||||||
y[i] = gamma[i];
|
|
||||||
for (int k = i + 1; k <= iter; k++)
|
|
||||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
|
||||||
y[i] = y[i] / ComplexD(H(i, i));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i <= iter; i++)
|
|
||||||
psi = psi + z[i] * y[i];
|
|
||||||
CompSolutionTimer.Stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
@@ -1,244 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/GeneralisedMinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
#define GRID_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class GeneralisedMinimalResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when GMRES fails to converge,
|
|
||||||
// defaults to true
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer RestartLength;
|
|
||||||
Integer MaxNumberOfRestarts;
|
|
||||||
Integer IterationCount; // Number of iterations the GMRES took to finish,
|
|
||||||
// filled in upon completion
|
|
||||||
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch QrTimer;
|
|
||||||
GridStopWatch CompSolutionTimer;
|
|
||||||
|
|
||||||
Eigen::MatrixXcd H;
|
|
||||||
|
|
||||||
std::vector<ComplexD> y;
|
|
||||||
std::vector<ComplexD> gamma;
|
|
||||||
std::vector<ComplexD> c;
|
|
||||||
std::vector<ComplexD> s;
|
|
||||||
|
|
||||||
GeneralisedMinimalResidual(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
Integer restart_length,
|
|
||||||
bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol)
|
|
||||||
, MaxIterations(maxit)
|
|
||||||
, RestartLength(restart_length)
|
|
||||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
|
||||||
, ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
|
||||||
, y(RestartLength + 1, 0.)
|
|
||||||
, gamma(RestartLength + 1, 0.)
|
|
||||||
, c(RestartLength + 1, 0.)
|
|
||||||
, s(RestartLength + 1, 0.) {};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
std::cout << std::setprecision(4) << std::scientific;
|
|
||||||
std::cout << GridLogIterative << "GeneralisedMinimalResidual: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << "GeneralisedMinimalResidual: src " << ssq << std::endl;
|
|
||||||
|
|
||||||
MatrixTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
QrTimer.Reset();
|
|
||||||
CompSolutionTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
IterationCount = 0;
|
|
||||||
|
|
||||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
|
||||||
|
|
||||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
LinOp.Op(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(ssq);
|
|
||||||
RealD resnorm = sqrt(norm2(r));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "GeneralisedMinimalResidual: Converged on iteration " << IterationCount
|
|
||||||
<< " computed residual " << sqrt(cp / ssq)
|
|
||||||
<< " true residual " << true_residual
|
|
||||||
<< " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "GMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "GMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "GMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "GMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "GMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "GeneralisedMinimalResidual did NOT converge" << std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD outerLoopBody(LinearOperatorBase<Field> &LinOp, const Field &src, Field &psi, RealD rsq) {
|
|
||||||
|
|
||||||
RealD cp = 0;
|
|
||||||
|
|
||||||
Field w(src.Grid());
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
// this should probably be made a class member so that it is only allocated once, not in every restart
|
|
||||||
std::vector<Field> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(psi, w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r = src - w;
|
|
||||||
|
|
||||||
gamma[0] = sqrt(norm2(r));
|
|
||||||
|
|
||||||
v[0] = (1. / gamma[0]) * r;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
for (int i=0; i<RestartLength; i++) {
|
|
||||||
|
|
||||||
IterationCount++;
|
|
||||||
|
|
||||||
arnoldiStep(LinOp, v, w, i);
|
|
||||||
|
|
||||||
qrUpdate(i);
|
|
||||||
|
|
||||||
cp = norm(gamma[i+1]);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "GeneralisedMinimalResidual: Iteration " << IterationCount
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
|
||||||
|
|
||||||
computeSolution(v, psi, i);
|
|
||||||
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT(0); // Never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
void arnoldiStep(LinearOperatorBase<Field> &LinOp, std::vector<Field> &v, Field &w, int iter) {
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(v[iter], w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
for (int i = 0; i <= iter; ++i) {
|
|
||||||
H(iter, i) = innerProduct(v[i], w);
|
|
||||||
w = w - ComplexD(H(iter, i)) * v[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
H(iter, iter + 1) = sqrt(norm2(w));
|
|
||||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void qrUpdate(int iter) {
|
|
||||||
|
|
||||||
QrTimer.Start();
|
|
||||||
for (int i = 0; i < iter ; ++i) {
|
|
||||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i + 1) = tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute new Givens Rotation
|
|
||||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
|
||||||
c[iter] = H(iter, iter) / nu;
|
|
||||||
s[iter] = H(iter, iter + 1) / nu;
|
|
||||||
|
|
||||||
// Apply new Givens rotation
|
|
||||||
H(iter, iter) = nu;
|
|
||||||
H(iter, iter + 1) = 0.;
|
|
||||||
|
|
||||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
|
||||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
|
||||||
QrTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void computeSolution(std::vector<Field> const &v, Field &psi, int iter) {
|
|
||||||
|
|
||||||
CompSolutionTimer.Start();
|
|
||||||
for (int i = iter; i >= 0; i--) {
|
|
||||||
y[i] = gamma[i];
|
|
||||||
for (int k = i + 1; k <= iter; k++)
|
|
||||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
|
||||||
y[i] = y[i] / ComplexD(H(i, i));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i <= iter; i++)
|
|
||||||
psi = psi + v[i] * y[i];
|
|
||||||
CompSolutionTimer.Stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,276 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./Grid/algorithms/iterative/LanczosBidiagonalization.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Chulwoo Jung <chulwoo@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LANCZOS_BIDIAGONALIZATION_H
|
|
||||||
#define GRID_LANCZOS_BIDIAGONALIZATION_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Lanczos Bidiagonalization (Golub-Kahan)
|
|
||||||
*
|
|
||||||
* For a linear operator A with adjoint A^dag, constructs the bidiagonal
|
|
||||||
* decomposition:
|
|
||||||
*
|
|
||||||
* A V_m = U_m B_m
|
|
||||||
* A^dag U_m = V_m B_m^T + beta_{m+1} v_{m+1} e_m^T
|
|
||||||
*
|
|
||||||
* where:
|
|
||||||
* V_m = [v_1, ..., v_m] right Lanczos vectors (orthonormal)
|
|
||||||
* U_m = [u_1, ..., u_m] left Lanczos vectors (orthonormal)
|
|
||||||
* B_m is upper bidiagonal with diag(alpha_1,...,alpha_m) and
|
|
||||||
* superdiag(beta_2,...,beta_m)
|
|
||||||
*
|
|
||||||
* The singular values of A are approximated by those of B_m.
|
|
||||||
* The singular values of B_m are the square roots of the eigenvalues of
|
|
||||||
* the symmetric tridiagonal matrix B_m^T B_m.
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
* LanczosBidiagonalization<Field> lb(Linop, grid);
|
|
||||||
* lb.run(src, Nm, tol);
|
|
||||||
* // Access results via getters.
|
|
||||||
*/
|
|
||||||
template <class Field>
|
|
||||||
class LanczosBidiagonalization {
|
|
||||||
|
|
||||||
public:
|
|
||||||
LinearOperatorBase<Field> &Linop;
|
|
||||||
GridBase *Grid;
|
|
||||||
|
|
||||||
int Nm; // number of Lanczos steps taken
|
|
||||||
RealD Tolerance; // convergence threshold on beta_{k+1} / alpha_k
|
|
||||||
|
|
||||||
std::vector<Field> V; // right Lanczos vectors v_1 ... v_m
|
|
||||||
std::vector<Field> U; // left Lanczos vectors u_1 ... u_m
|
|
||||||
std::vector<RealD> alpha; // diagonal of bidiagonal matrix
|
|
||||||
std::vector<RealD> beta; // super-diagonal (beta[k] couples u_k and v_{k+1})
|
|
||||||
|
|
||||||
// SVD of the bidiagonal matrix (filled after computeSVD())
|
|
||||||
Eigen::VectorXd singularValues;
|
|
||||||
Eigen::MatrixXd leftSVecs; // columns are left singular vectors of B
|
|
||||||
Eigen::MatrixXd rightSVecs; // columns are right singular vectors of B
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
LanczosBidiagonalization(LinearOperatorBase<Field> &_Linop, GridBase *_Grid,
|
|
||||||
RealD _tol = 1.0e-8)
|
|
||||||
: Linop(_Linop), Grid(_Grid), Tolerance(_tol), Nm(0)
|
|
||||||
{}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run the Golub-Kahan Lanczos bidiagonalization.
|
|
||||||
*
|
|
||||||
* Parameters
|
|
||||||
* ----------
|
|
||||||
* src : starting vector (need not be normalised)
|
|
||||||
* Nmax : maximum number of Lanczos steps
|
|
||||||
* reorth : if true, full reorthogonalisation of both V and U bases
|
|
||||||
*/
|
|
||||||
void run(const Field &src, int Nmax, bool reorth = true)
|
|
||||||
{
|
|
||||||
assert(norm2(src) > 0.0);
|
|
||||||
|
|
||||||
V.clear(); U.clear();
|
|
||||||
alpha.clear(); beta.clear();
|
|
||||||
Nm = 0;
|
|
||||||
|
|
||||||
Field p(Grid), r(Grid);
|
|
||||||
|
|
||||||
// --- initialise: v_1 = src / ||src|| ---
|
|
||||||
Field v(Grid);
|
|
||||||
v = src;
|
|
||||||
RealD nrm = std::sqrt(norm2(v));
|
|
||||||
v = (1.0 / nrm) * v;
|
|
||||||
V.push_back(v);
|
|
||||||
|
|
||||||
for (int k = 0; k < Nmax; ++k) {
|
|
||||||
|
|
||||||
// p = A v_k
|
|
||||||
Linop.Op(V[k], p);
|
|
||||||
|
|
||||||
// p = p - beta_k * u_{k-1} (remove previous left vector)
|
|
||||||
if (k > 0) {
|
|
||||||
p = p - beta[k-1] * U[k-1];
|
|
||||||
}
|
|
||||||
|
|
||||||
// alpha_k = ||p||
|
|
||||||
RealD ak = std::sqrt(norm2(p));
|
|
||||||
if (ak < 1.0e-14) {
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "LanczosBidiagonalization: lucky breakdown at step "
|
|
||||||
<< k << " (alpha = " << ak << ")" << std::endl;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
alpha.push_back(ak);
|
|
||||||
|
|
||||||
// u_k = p / alpha_k
|
|
||||||
Field u(Grid);
|
|
||||||
u = (1.0 / ak) * p;
|
|
||||||
|
|
||||||
// full reorthogonalisation of u against previous U
|
|
||||||
if (reorth) {
|
|
||||||
for (int j = 0; j < (int)U.size(); ++j) {
|
|
||||||
ComplexD ip = innerProduct(U[j], u);
|
|
||||||
u = u - ip * U[j];
|
|
||||||
}
|
|
||||||
RealD unrm = std::sqrt(norm2(u));
|
|
||||||
if (unrm > 1.0e-14) u = (1.0 / unrm) * u;
|
|
||||||
}
|
|
||||||
U.push_back(u);
|
|
||||||
|
|
||||||
// r = A^dag u_k - alpha_k * v_k
|
|
||||||
Linop.AdjOp(U[k], r);
|
|
||||||
r = r - ak * V[k];
|
|
||||||
|
|
||||||
// full reorthogonalisation of r against previous V
|
|
||||||
if (reorth) {
|
|
||||||
for (int j = 0; j < (int)V.size(); ++j) {
|
|
||||||
ComplexD ip = innerProduct(V[j], r);
|
|
||||||
r = r - ip * V[j];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// beta_{k+1} = ||r||
|
|
||||||
RealD bk = std::sqrt(norm2(r));
|
|
||||||
beta.push_back(bk);
|
|
||||||
|
|
||||||
Nm = k + 1;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "LanczosBidiagonalization step " << k
|
|
||||||
<< " alpha = " << ak
|
|
||||||
<< " beta = " << bk << std::endl;
|
|
||||||
|
|
||||||
// convergence: residual beta / alpha small enough
|
|
||||||
if (bk / ak < Tolerance) {
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "LanczosBidiagonalization converged at step " << k
|
|
||||||
<< " (beta/alpha = " << bk / ak << ")" << std::endl;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (k == Nmax - 1) break; // no v_{k+2} needed after last step
|
|
||||||
|
|
||||||
// v_{k+1} = r / beta_{k+1}
|
|
||||||
Field vnext(Grid);
|
|
||||||
vnext = (1.0 / bk) * r;
|
|
||||||
V.push_back(vnext);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compute the SVD of the bidiagonal matrix B using Eigen.
|
|
||||||
* Singular values are stored in descending order.
|
|
||||||
*/
|
|
||||||
void computeSVD()
|
|
||||||
{
|
|
||||||
int m = Nm;
|
|
||||||
Eigen::MatrixXd B = Eigen::MatrixXd::Zero(m, m);
|
|
||||||
|
|
||||||
for (int k = 0; k < m; ++k) {
|
|
||||||
B(k, k) = alpha[k];
|
|
||||||
if (k + 1 < m && k < (int)beta.size())
|
|
||||||
B(k, k+1) = beta[k];
|
|
||||||
}
|
|
||||||
|
|
||||||
Eigen::JacobiSVD<Eigen::MatrixXd> svd(B,
|
|
||||||
Eigen::ComputeThinU | Eigen::ComputeThinV);
|
|
||||||
|
|
||||||
singularValues = svd.singularValues(); // already sorted descending
|
|
||||||
leftSVecs = svd.matrixU();
|
|
||||||
rightSVecs = svd.matrixV();
|
|
||||||
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "LanczosBidiagonalization: singular values of B_" << m
|
|
||||||
<< std::endl;
|
|
||||||
for (int k = 0; k < m; ++k)
|
|
||||||
std::cout << GridLogMessage << " sigma[" << k << "] = "
|
|
||||||
<< singularValues(k) << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return the k-th approximate left singular vector of A in the full
|
|
||||||
* lattice space. computeSVD() must have been called first.
|
|
||||||
*/
|
|
||||||
Field leftSingularVector(int k)
|
|
||||||
{
|
|
||||||
assert(k < (int)leftSVecs.cols());
|
|
||||||
Field svec(Grid);
|
|
||||||
svec = Zero();
|
|
||||||
for (int j = 0; j < Nm; ++j)
|
|
||||||
svec = svec + leftSVecs(j, k) * U[j];
|
|
||||||
return svec;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return the k-th approximate right singular vector of A in the full
|
|
||||||
* lattice space. computeSVD() must have been called first.
|
|
||||||
*/
|
|
||||||
Field rightSingularVector(int k)
|
|
||||||
{
|
|
||||||
assert(k < (int)rightSVecs.cols());
|
|
||||||
Field svec(Grid);
|
|
||||||
svec = Zero();
|
|
||||||
for (int j = 0; j < Nm; ++j)
|
|
||||||
svec = svec + rightSVecs(j, k) * V[j];
|
|
||||||
return svec;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Verify the bidiagonalization: returns max residual
|
|
||||||
* max_k || A v_k - alpha_k u_k - beta_k u_{k-1} ||
|
|
||||||
*/
|
|
||||||
RealD verify()
|
|
||||||
{
|
|
||||||
Field tmp(Grid);
|
|
||||||
RealD maxres = 0.0;
|
|
||||||
for (int k = 0; k < Nm; ++k) {
|
|
||||||
Linop.Op(V[k], tmp);
|
|
||||||
tmp = tmp - alpha[k] * U[k];
|
|
||||||
if (k > 0 && k-1 < (int)beta.size())
|
|
||||||
tmp = tmp - beta[k-1] * U[k-1];
|
|
||||||
RealD res = std::sqrt(norm2(tmp));
|
|
||||||
if (res > maxres) maxres = res;
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "LanczosBidiagonalization verify step " << k
|
|
||||||
<< " ||A v_k - alpha_k u_k - beta_{k-1} u_{k-1}|| = "
|
|
||||||
<< res << std::endl;
|
|
||||||
}
|
|
||||||
return maxres;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Getters */
|
|
||||||
int getNm() const { return Nm; }
|
|
||||||
const std::vector<Field>& getV() const { return V; }
|
|
||||||
const std::vector<Field>& getU() const { return U; }
|
|
||||||
const std::vector<RealD>& getAlpha() const { return alpha; }
|
|
||||||
const std::vector<RealD>& getBeta() const { return beta; }
|
|
||||||
Eigen::VectorXd getSingularValues() const { return singularValues; }
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,157 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/MinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_MINIMAL_RESIDUAL_H
|
|
||||||
#define GRID_MINIMAL_RESIDUAL_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class Field> class MinimalResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // throw an GRID_ASSERT when the MR fails to converge.
|
|
||||||
// Defaults true.
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
RealD overRelaxParam;
|
|
||||||
Integer IterationsToComplete; // Number of iterations the MR took to finish.
|
|
||||||
// Filled in upon completion
|
|
||||||
|
|
||||||
MinimalResidual(RealD tol, Integer maxit, Real ovrelparam = 1.0, bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol), MaxIterations(maxit), overRelaxParam(ovrelparam), ErrorOnNoConverge(err_on_no_conv){};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
ComplexD a, c;
|
|
||||||
RealD d;
|
|
||||||
|
|
||||||
Field Mr(src);
|
|
||||||
Field r(src);
|
|
||||||
|
|
||||||
// Initial residual computation & set up
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
Linop.Op(psi, Mr);
|
|
||||||
|
|
||||||
r = src - Mr;
|
|
||||||
|
|
||||||
RealD cp = norm2(r);
|
|
||||||
|
|
||||||
std::cout << std::setprecision(4) << std::scientific;
|
|
||||||
std::cout << GridLogIterative << "MinimalResidual: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << "MinimalResidual: src " << ssq << std::endl;
|
|
||||||
std::cout << GridLogIterative << "MinimalResidual: cp,r " << cp << std::endl;
|
|
||||||
|
|
||||||
if (cp <= rsq) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "MinimalResidual: k=0 residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
|
|
||||||
SolverTimer.Start();
|
|
||||||
int k;
|
|
||||||
for (k = 1; k <= MaxIterations; k++) {
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
Linop.Op(r, Mr);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
c = innerProduct(Mr, r);
|
|
||||||
|
|
||||||
d = norm2(Mr);
|
|
||||||
|
|
||||||
a = c / d;
|
|
||||||
|
|
||||||
a = a * overRelaxParam;
|
|
||||||
|
|
||||||
psi = psi + r * a;
|
|
||||||
|
|
||||||
r = r - Mr * a;
|
|
||||||
|
|
||||||
cp = norm2(r);
|
|
||||||
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "MinimalResidual: Iteration " << k
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
std::cout << GridLogDebug << "a = " << a << " c = " << c << " d = " << d << std::endl;
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
Linop.Op(psi, Mr);
|
|
||||||
r = src - Mr;
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(ssq);
|
|
||||||
RealD resnorm = sqrt(norm2(r));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MinimalResidual Converged on iteration " << k
|
|
||||||
<< " computed residual " << sqrt(cp / ssq)
|
|
||||||
<< " true residual " << true_residual
|
|
||||||
<< " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MR Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MR Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MR Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(true_residual / Tolerance < 10000.0);
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MinimalResidual did NOT converge"
|
|
||||||
<< std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
|
|
||||||
IterationsToComplete = k;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} // namespace Grid
|
|
||||||
#endif
|
|
||||||
@@ -1,276 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/MixedPrecisionFlexibleGeneralisedMinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Daniel Richtmann <daniel.richtmann@ur.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_MIXED_PRECISION_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
#define GRID_MIXED_PRECISION_FLEXIBLE_GENERALISED_MINIMAL_RESIDUAL_H
|
|
||||||
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
template<class FieldD, class FieldF, typename std::enable_if<getPrecision<FieldD>::value == 2, int>::type = 0, typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0>
|
|
||||||
class MixedPrecisionFlexibleGeneralisedMinimalResidual : public OperatorFunction<FieldD> {
|
|
||||||
public:
|
|
||||||
|
|
||||||
using OperatorFunction<FieldD>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge; // Throw an GRID_ASSERT when MPFGMRES fails to converge,
|
|
||||||
// defaults to true
|
|
||||||
|
|
||||||
RealD Tolerance;
|
|
||||||
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer RestartLength;
|
|
||||||
Integer MaxNumberOfRestarts;
|
|
||||||
Integer IterationCount; // Number of iterations the MPFGMRES took to finish,
|
|
||||||
// filled in upon completion
|
|
||||||
|
|
||||||
GridStopWatch MatrixTimer;
|
|
||||||
GridStopWatch PrecTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
GridStopWatch QrTimer;
|
|
||||||
GridStopWatch CompSolutionTimer;
|
|
||||||
GridStopWatch ChangePrecTimer;
|
|
||||||
|
|
||||||
Eigen::MatrixXcd H;
|
|
||||||
|
|
||||||
std::vector<ComplexD> y;
|
|
||||||
std::vector<ComplexD> gamma;
|
|
||||||
std::vector<ComplexD> c;
|
|
||||||
std::vector<ComplexD> s;
|
|
||||||
|
|
||||||
GridBase* SinglePrecGrid;
|
|
||||||
|
|
||||||
LinearFunction<FieldF> &Preconditioner;
|
|
||||||
|
|
||||||
MixedPrecisionFlexibleGeneralisedMinimalResidual(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
GridBase * sp_grid,
|
|
||||||
LinearFunction<FieldF> &Prec,
|
|
||||||
Integer restart_length,
|
|
||||||
bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol)
|
|
||||||
, MaxIterations(maxit)
|
|
||||||
, RestartLength(restart_length)
|
|
||||||
, MaxNumberOfRestarts(MaxIterations/RestartLength + ((MaxIterations%RestartLength == 0) ? 0 : 1))
|
|
||||||
, ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
, H(Eigen::MatrixXcd::Zero(RestartLength, RestartLength + 1)) // sizes taken from DD-αAMG code base
|
|
||||||
, y(RestartLength + 1, 0.)
|
|
||||||
, gamma(RestartLength + 1, 0.)
|
|
||||||
, c(RestartLength + 1, 0.)
|
|
||||||
, s(RestartLength + 1, 0.)
|
|
||||||
, SinglePrecGrid(sp_grid)
|
|
||||||
, Preconditioner(Prec) {};
|
|
||||||
|
|
||||||
void operator()(LinearOperatorBase<FieldD> &LinOp, const FieldD &src, FieldD &psi) {
|
|
||||||
|
|
||||||
psi.Checkerboard() = src.Checkerboard();
|
|
||||||
conformable(psi, src);
|
|
||||||
|
|
||||||
RealD guess = norm2(psi);
|
|
||||||
GRID_ASSERT(std::isnan(guess) == 0);
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
RealD ssq = norm2(src);
|
|
||||||
RealD rsq = Tolerance * Tolerance * ssq;
|
|
||||||
|
|
||||||
FieldD r(src.Grid());
|
|
||||||
|
|
||||||
std::cout << std::setprecision(4) << std::scientific;
|
|
||||||
std::cout << GridLogIterative << "MPFGMRES: guess " << guess << std::endl;
|
|
||||||
std::cout << GridLogIterative << "MPFGMRES: src " << ssq << std::endl;
|
|
||||||
|
|
||||||
PrecTimer.Reset();
|
|
||||||
MatrixTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
QrTimer.Reset();
|
|
||||||
CompSolutionTimer.Reset();
|
|
||||||
ChangePrecTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
IterationCount = 0;
|
|
||||||
|
|
||||||
for (int k=0; k<MaxNumberOfRestarts; k++) {
|
|
||||||
|
|
||||||
cp = outerLoopBody(LinOp, src, psi, rsq);
|
|
||||||
|
|
||||||
// Stopping condition
|
|
||||||
if (cp <= rsq) {
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
LinOp.Op(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
|
|
||||||
RealD srcnorm = sqrt(ssq);
|
|
||||||
RealD resnorm = sqrt(norm2(r));
|
|
||||||
RealD true_residual = resnorm / srcnorm;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES: Converged on iteration " << IterationCount
|
|
||||||
<< " computed residual " << sqrt(cp / ssq)
|
|
||||||
<< " true residual " << true_residual
|
|
||||||
<< " target " << Tolerance << std::endl;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Total " << SolverTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Precon " << PrecTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Matrix " << MatrixTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: Linalg " << LinalgTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: QR " << QrTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: CompSol " << CompSolutionTimer.Elapsed() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES Time elapsed: PrecChange " << ChangePrecTimer.Elapsed() << std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "MPFGMRES did NOT converge" << std::endl;
|
|
||||||
|
|
||||||
if (ErrorOnNoConverge)
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD outerLoopBody(LinearOperatorBase<FieldD> &LinOp, const FieldD &src, FieldD &psi, RealD rsq) {
|
|
||||||
|
|
||||||
RealD cp = 0;
|
|
||||||
|
|
||||||
FieldD w(src.Grid());
|
|
||||||
FieldD r(src.Grid());
|
|
||||||
|
|
||||||
// these should probably be made class members so that they are only allocated once, not in every restart
|
|
||||||
std::vector<FieldD> v(RestartLength + 1, src.Grid()); for (auto &elem : v) elem = Zero();
|
|
||||||
std::vector<FieldD> z(RestartLength + 1, src.Grid()); for (auto &elem : z) elem = Zero();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(psi, w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r = src - w;
|
|
||||||
|
|
||||||
gamma[0] = sqrt(norm2(r));
|
|
||||||
|
|
||||||
v[0] = (1. / gamma[0]) * r;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
for (int i=0; i<RestartLength; i++) {
|
|
||||||
|
|
||||||
IterationCount++;
|
|
||||||
|
|
||||||
arnoldiStep(LinOp, v, z, w, i);
|
|
||||||
|
|
||||||
qrUpdate(i);
|
|
||||||
|
|
||||||
cp = norm(gamma[i+1]);
|
|
||||||
|
|
||||||
std::cout << GridLogIterative << "MPFGMRES: Iteration " << IterationCount
|
|
||||||
<< " residual " << cp << " target " << rsq << std::endl;
|
|
||||||
|
|
||||||
if ((i == RestartLength - 1) || (IterationCount == MaxIterations) || (cp <= rsq)) {
|
|
||||||
|
|
||||||
computeSolution(z, psi, i);
|
|
||||||
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT(0); // Never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
void arnoldiStep(LinearOperatorBase<FieldD> &LinOp, std::vector<FieldD> &v, std::vector<FieldD> &z, FieldD &w, int iter) {
|
|
||||||
|
|
||||||
FieldF v_f(SinglePrecGrid);
|
|
||||||
FieldF z_f(SinglePrecGrid);
|
|
||||||
|
|
||||||
ChangePrecTimer.Start();
|
|
||||||
precisionChange(v_f, v[iter]);
|
|
||||||
precisionChange(z_f, z[iter]);
|
|
||||||
ChangePrecTimer.Stop();
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(v_f, z_f);
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
ChangePrecTimer.Start();
|
|
||||||
precisionChange(z[iter], z_f);
|
|
||||||
ChangePrecTimer.Stop();
|
|
||||||
|
|
||||||
MatrixTimer.Start();
|
|
||||||
LinOp.Op(z[iter], w);
|
|
||||||
MatrixTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
for (int i = 0; i <= iter; ++i) {
|
|
||||||
H(iter, i) = innerProduct(v[i], w);
|
|
||||||
w = w - ComplexD(H(iter, i)) * v[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
H(iter, iter + 1) = sqrt(norm2(w));
|
|
||||||
v[iter + 1] = ComplexD(1. / H(iter, iter + 1)) * w;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void qrUpdate(int iter) {
|
|
||||||
|
|
||||||
QrTimer.Start();
|
|
||||||
for (int i = 0; i < iter ; ++i) {
|
|
||||||
auto tmp = -s[i] * ComplexD(H(iter, i)) + c[i] * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i) = conjugate(c[i]) * ComplexD(H(iter, i)) + conjugate(s[i]) * ComplexD(H(iter, i + 1));
|
|
||||||
H(iter, i + 1) = tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute new Givens Rotation
|
|
||||||
auto nu = sqrt(std::norm(H(iter, iter)) + std::norm(H(iter, iter + 1)));
|
|
||||||
c[iter] = H(iter, iter) / nu;
|
|
||||||
s[iter] = H(iter, iter + 1) / nu;
|
|
||||||
|
|
||||||
// Apply new Givens rotation
|
|
||||||
H(iter, iter) = nu;
|
|
||||||
H(iter, iter + 1) = 0.;
|
|
||||||
|
|
||||||
gamma[iter + 1] = -s[iter] * gamma[iter];
|
|
||||||
gamma[iter] = conjugate(c[iter]) * gamma[iter];
|
|
||||||
QrTimer.Stop();
|
|
||||||
}
|
|
||||||
|
|
||||||
void computeSolution(std::vector<FieldD> const &z, FieldD &psi, int iter) {
|
|
||||||
|
|
||||||
CompSolutionTimer.Start();
|
|
||||||
for (int i = iter; i >= 0; i--) {
|
|
||||||
y[i] = gamma[i];
|
|
||||||
for (int k = i + 1; k <= iter; k++)
|
|
||||||
y[i] = y[i] - ComplexD(H(k, i)) * y[k];
|
|
||||||
y[i] = y[i] / ComplexD(H(i, i));
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i <= iter; i++)
|
|
||||||
psi = psi + z[i] * y[i];
|
|
||||||
CompSolutionTimer.Stop();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/NormalEquations.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_NORMAL_EQUATIONS_H
|
|
||||||
#define GRID_NORMAL_EQUATIONS_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Take a matrix and form an NE solver calling a Herm solver
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class NormalEquations : public LinearFunction<Field>{
|
|
||||||
private:
|
|
||||||
SparseMatrixBase<Field> & _Matrix;
|
|
||||||
OperatorFunction<Field> & _HermitianSolver;
|
|
||||||
LinearFunction<Field> & _Guess;
|
|
||||||
public:
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
NormalEquations(SparseMatrixBase<Field> &Matrix, OperatorFunction<Field> &HermitianSolver,
|
|
||||||
LinearFunction<Field> &Guess)
|
|
||||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
|
||||||
|
|
||||||
void operator() (const Field &in, Field &out){
|
|
||||||
|
|
||||||
Field src(in.Grid());
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
MdagMLinearOperator<SparseMatrixBase<Field>,Field> MdagMOp(_Matrix);
|
|
||||||
_Matrix.Mdag(in,src);
|
|
||||||
_Guess(src,out);
|
|
||||||
_HermitianSolver(MdagMOp,src,out); // Mdag M out = Mdag in
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class NormalResidual : public LinearFunction<Field>{
|
|
||||||
private:
|
|
||||||
SparseMatrixBase<Field> & _Matrix;
|
|
||||||
OperatorFunction<Field> & _HermitianSolver;
|
|
||||||
LinearFunction<Field> & _Guess;
|
|
||||||
public:
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
NormalResidual(SparseMatrixBase<Field> &Matrix, OperatorFunction<Field> &HermitianSolver,
|
|
||||||
LinearFunction<Field> &Guess)
|
|
||||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
|
||||||
|
|
||||||
void operator() (const Field &in, Field &out){
|
|
||||||
|
|
||||||
Field res(in.Grid());
|
|
||||||
Field tmp(in.Grid());
|
|
||||||
|
|
||||||
MMdagLinearOperator<SparseMatrixBase<Field>,Field> MMdagOp(_Matrix);
|
|
||||||
_Guess(in,res);
|
|
||||||
_HermitianSolver(MMdagOp,in,res); // M Mdag res = in ;
|
|
||||||
_Matrix.Mdag(res,out); // out = Mdag res
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class HPDSolver : public LinearFunction<Field> {
|
|
||||||
private:
|
|
||||||
LinearOperatorBase<Field> & _Matrix;
|
|
||||||
OperatorFunction<Field> & _HermitianSolver;
|
|
||||||
LinearFunction<Field> & _Guess;
|
|
||||||
public:
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
HPDSolver(LinearOperatorBase<Field> &Matrix,
|
|
||||||
OperatorFunction<Field> &HermitianSolver,
|
|
||||||
LinearFunction<Field> &Guess)
|
|
||||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
|
||||||
|
|
||||||
void operator() (const Field &in, Field &out){
|
|
||||||
|
|
||||||
_Guess(in,out);
|
|
||||||
_HermitianSolver(_Matrix,in,out); //M out = in
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class Field> class MdagMSolver : public LinearFunction<Field> {
|
|
||||||
private:
|
|
||||||
SparseMatrixBase<Field> & _Matrix;
|
|
||||||
OperatorFunction<Field> & _HermitianSolver;
|
|
||||||
LinearFunction<Field> & _Guess;
|
|
||||||
public:
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
MdagMSolver(SparseMatrixBase<Field> &Matrix, OperatorFunction<Field> &HermitianSolver,
|
|
||||||
LinearFunction<Field> &Guess)
|
|
||||||
: _Matrix(Matrix), _HermitianSolver(HermitianSolver), _Guess(Guess) {};
|
|
||||||
|
|
||||||
void operator() (const Field &in, Field &out){
|
|
||||||
|
|
||||||
MdagMLinearOperator<SparseMatrixBase<Field>,Field> MdagMOp(_Matrix);
|
|
||||||
_Guess(in,out);
|
|
||||||
|
|
||||||
_HermitianSolver(MdagMOp,in,out); // Mdag M out = Mdag in
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
#pragma once
|
|
||||||
namespace Grid {
|
|
||||||
template<class Field> class PowerMethod
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
template<typename T> static RealD normalise(T& v)
|
|
||||||
{
|
|
||||||
RealD nn = norm2(v);
|
|
||||||
nn = sqrt(nn);
|
|
||||||
v = v * (1.0/nn);
|
|
||||||
return nn;
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD operator()(LinearOperatorBase<Field> &HermOp, const Field &src)
|
|
||||||
{
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
|
|
||||||
// quickly get an idea of the largest eigenvalue to more properly normalize the residuum
|
|
||||||
RealD evalMaxApprox = 0.0;
|
|
||||||
auto src_n = src;
|
|
||||||
auto tmp = src;
|
|
||||||
const int _MAX_ITER_EST_ = 200;
|
|
||||||
|
|
||||||
for (int i=0;i<_MAX_ITER_EST_;i++) {
|
|
||||||
|
|
||||||
normalise(src_n);
|
|
||||||
HermOp.HermOp(src_n,tmp);
|
|
||||||
RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
|
|
||||||
RealD vden = norm2(src_n);
|
|
||||||
RealD na = vnum/vden;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "PowerMethod: Current approximation of largest eigenvalue " << na << std::endl;
|
|
||||||
|
|
||||||
// if ( (fabs(evalMaxApprox/na - 1.0) < 0.0001) || (i==_MAX_ITER_EST_-1) ) {
|
|
||||||
// evalMaxApprox = na;
|
|
||||||
// return evalMaxApprox;
|
|
||||||
// }
|
|
||||||
evalMaxApprox = na;
|
|
||||||
src_n = tmp;
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
|
|
||||||
return evalMaxApprox;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,76 +0,0 @@
|
|||||||
#pragma once
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
class Band
|
|
||||||
{
|
|
||||||
RealD lo, hi;
|
|
||||||
public:
|
|
||||||
Band(RealD _lo,RealD _hi)
|
|
||||||
{
|
|
||||||
lo=_lo;
|
|
||||||
hi=_hi;
|
|
||||||
}
|
|
||||||
RealD operator() (RealD x){
|
|
||||||
if ( x>lo && x<hi ){
|
|
||||||
return 1.0;
|
|
||||||
} else {
|
|
||||||
return 0.0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class PowerSpectrum
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
|
|
||||||
template<typename T> static RealD normalise(T& v)
|
|
||||||
{
|
|
||||||
RealD nn = norm2(v);
|
|
||||||
nn = sqrt(nn);
|
|
||||||
v = v * (1.0/nn);
|
|
||||||
return nn;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<RealD> ranges;
|
|
||||||
std::vector<int> order;
|
|
||||||
|
|
||||||
PowerSpectrum( std::vector<RealD> &bins, std::vector<int> &_order ) : ranges(bins), order(_order) { };
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
RealD operator()(LinearOperatorBase<Field> &HermOp, const Field &src)
|
|
||||||
{
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
int N=ranges.size();
|
|
||||||
RealD hi = ranges[N-1];
|
|
||||||
|
|
||||||
RealD lo_band = 0.0;
|
|
||||||
RealD hi_band;
|
|
||||||
RealD nn=norm2(src);
|
|
||||||
RealD ss=0.0;
|
|
||||||
|
|
||||||
Field tmp = src;
|
|
||||||
|
|
||||||
for(int b=0;b<N;b++){
|
|
||||||
hi_band = ranges[b];
|
|
||||||
Band Notch(lo_band,hi_band);
|
|
||||||
|
|
||||||
Chebyshev<Field> polynomial;
|
|
||||||
polynomial.Init(0.0,hi,order[b],Notch);
|
|
||||||
polynomial.JacksonSmooth();
|
|
||||||
|
|
||||||
polynomial(HermOp,src,tmp) ;
|
|
||||||
|
|
||||||
RealD p=norm2(tmp);
|
|
||||||
ss=ss+p;
|
|
||||||
std::cout << GridLogMessage << " PowerSpectrum Band["<<lo_band<<","<<hi_band<<"] power "<<norm2(tmp)/nn<<std::endl;
|
|
||||||
|
|
||||||
lo_band=hi_band;
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage << " PowerSpectrum total power "<<ss/nn<<std::endl;
|
|
||||||
std::cout << GridLogMessage << " PowerSpectrum total power (unnormalised) "<<nn<<std::endl;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,119 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/PrecConjugateResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_PREC_CONJUGATE_RESIDUAL_H
|
|
||||||
#define GRID_PREC_CONJUGATE_RESIDUAL_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Base classes for iterative processes based on operators
|
|
||||||
// single input vec, single output vec.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class PrecConjugateResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
int verbose;
|
|
||||||
LinearFunction<Field> &Preconditioner;
|
|
||||||
|
|
||||||
PrecConjugateResidual(RealD tol,Integer maxit,LinearFunction<Field> &Prec) : Tolerance(tol), MaxIterations(maxit), Preconditioner(Prec)
|
|
||||||
{
|
|
||||||
verbose=1;
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator() (LinearOperatorBase<Field> &Linop,const Field &src, Field &psi){
|
|
||||||
|
|
||||||
RealD a, b, c, d;
|
|
||||||
RealD cp, ssq,rsq;
|
|
||||||
|
|
||||||
RealD rAr, rAAr, rArp;
|
|
||||||
RealD pAp, pAAp;
|
|
||||||
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
Field r(grid), p(grid), Ap(grid), Ar(grid), z(grid);
|
|
||||||
|
|
||||||
psi=zero;
|
|
||||||
r = src;
|
|
||||||
Preconditioner(r,p);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Linop.HermOpAndNorm(p,Ap,pAp,pAAp);
|
|
||||||
Ar=Ap;
|
|
||||||
rAr=pAp;
|
|
||||||
rAAr=pAAp;
|
|
||||||
|
|
||||||
cp =norm2(r);
|
|
||||||
ssq=norm2(src);
|
|
||||||
rsq=Tolerance*Tolerance*ssq;
|
|
||||||
|
|
||||||
if (verbose) std::cout<<GridLogMessage<<"PrecConjugateResidual: iteration " <<0<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
|
||||||
|
|
||||||
for(int k=0;k<MaxIterations;k++){
|
|
||||||
|
|
||||||
|
|
||||||
Preconditioner(Ap,z);
|
|
||||||
RealD rq= real(innerProduct(Ap,z));
|
|
||||||
|
|
||||||
a = rAr/rq;
|
|
||||||
|
|
||||||
axpy(psi,a,p,psi);
|
|
||||||
cp = axpy_norm(r,-a,z,r);
|
|
||||||
|
|
||||||
rArp=rAr;
|
|
||||||
|
|
||||||
Linop.HermOpAndNorm(r,Ar,rAr,rAAr);
|
|
||||||
|
|
||||||
b =rAr/rArp;
|
|
||||||
|
|
||||||
axpy(p,b,p,r);
|
|
||||||
pAAp=axpy_norm(Ap,b,Ap,Ar);
|
|
||||||
|
|
||||||
if(verbose) std::cout<<GridLogMessage<<"PrecConjugateResidual: iteration " <<k<<" residual "<<cp<< " target"<< rsq<<std::endl;
|
|
||||||
|
|
||||||
if(cp<rsq) {
|
|
||||||
Linop.HermOp(psi,Ap);
|
|
||||||
axpy(r,-1.0,src,Ap);
|
|
||||||
RealD true_resid = norm2(r)/ssq;
|
|
||||||
std::cout<<GridLogMessage<<"PrecConjugateResidual: Converged on iteration " <<k
|
|
||||||
<< " computed residual "<<sqrt(cp/ssq)
|
|
||||||
<< " true residual "<<sqrt(true_resid)
|
|
||||||
<< " target " <<Tolerance <<std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<<"PrecConjugateResidual did NOT converge"<<std::endl;
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,239 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/PrecGeneralisedConjugateResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_PREC_GCR_H
|
|
||||||
#define GRID_PREC_GCR_H
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
//VPGCR Abe and Zhang, 2005.
|
|
||||||
//INTERNATIONAL JOURNAL OF NUMERICAL ANALYSIS AND MODELING
|
|
||||||
//Computing and Information Volume 2, Number 2, Pages 147-161
|
|
||||||
//NB. Likely not original reference since they are focussing on a preconditioner variant.
|
|
||||||
// but VPGCR was nicely written up in their paper
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#define GCRLogLevel std::cout << GridLogMessage <<std::string(level,'\t')<< " Level "<<level<<" "
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class PrecGeneralisedConjugateResidual : public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
int verbose;
|
|
||||||
int mmax;
|
|
||||||
int nstep;
|
|
||||||
int steps;
|
|
||||||
int level;
|
|
||||||
GridStopWatch PrecTimer;
|
|
||||||
GridStopWatch MatTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
|
|
||||||
LinearFunction<Field> &Preconditioner;
|
|
||||||
LinearOperatorBase<Field> &Linop;
|
|
||||||
|
|
||||||
void Level(int lv) { level=lv; };
|
|
||||||
|
|
||||||
PrecGeneralisedConjugateResidual(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax,int _nstep) :
|
|
||||||
Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
Linop(_Linop),
|
|
||||||
Preconditioner(Prec),
|
|
||||||
mmax(_mmax),
|
|
||||||
nstep(_nstep)
|
|
||||||
{
|
|
||||||
level=1;
|
|
||||||
verbose=1;
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator() (const Field &src, Field &psi){
|
|
||||||
|
|
||||||
psi=Zero();
|
|
||||||
RealD cp, ssq,rsq;
|
|
||||||
ssq=norm2(src);
|
|
||||||
rsq=Tolerance*Tolerance*ssq;
|
|
||||||
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
PrecTimer.Reset();
|
|
||||||
MatTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
steps=0;
|
|
||||||
for(int k=0;k<MaxIterations;k++){
|
|
||||||
|
|
||||||
cp=GCRnStep(src,psi,rsq);
|
|
||||||
|
|
||||||
GCRLogLevel <<"PGCR("<<mmax<<","<<nstep<<") "<< steps <<" steps cp = "<<cp<<" target "<<rsq <<std::endl;
|
|
||||||
|
|
||||||
if(cp<rsq) {
|
|
||||||
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
Linop.HermOp(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
RealD tr = norm2(r);
|
|
||||||
GCRLogLevel<<"PGCR: Converged on iteration " <<steps
|
|
||||||
<< " computed residual "<<sqrt(cp/ssq)
|
|
||||||
<< " true residual " <<sqrt(tr/ssq)
|
|
||||||
<< " target " <<Tolerance <<std::endl;
|
|
||||||
|
|
||||||
GCRLogLevel<<"PGCR Time elapsed: Total "<< SolverTimer.Elapsed() <<std::endl;
|
|
||||||
/*
|
|
||||||
GCRLogLevel<<"PGCR Time elapsed: Precon "<< PrecTimer.Elapsed() <<std::endl;
|
|
||||||
GCRLogLevel<<"PGCR Time elapsed: Matrix "<< MatTimer.Elapsed() <<std::endl;
|
|
||||||
GCRLogLevel<<"PGCR Time elapsed: Linalg "<< LinalgTimer.Elapsed() <<std::endl;
|
|
||||||
*/
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
|
|
||||||
// GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
RealD a, b;
|
|
||||||
RealD zAz, zAAz;
|
|
||||||
RealD rq;
|
|
||||||
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
|
|
||||||
Field r(grid);
|
|
||||||
Field z(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
Field ttmp(grid);
|
|
||||||
Field Az(grid);
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
// history for flexible orthog
|
|
||||||
////////////////////////////////
|
|
||||||
std::vector<Field> q(mmax,grid);
|
|
||||||
std::vector<Field> p(mmax,grid);
|
|
||||||
std::vector<RealD> qq(mmax);
|
|
||||||
|
|
||||||
GCRLogLevel<< "PGCR nStep("<<nstep<<")"<<std::endl;
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// initial guess x0 is taken as nonzero.
|
|
||||||
// r0=src-A x0 = src
|
|
||||||
//////////////////////////////////
|
|
||||||
MatTimer.Start();
|
|
||||||
Linop.HermOpAndNorm(psi,Az,zAz,zAAz);
|
|
||||||
MatTimer.Stop();
|
|
||||||
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r=src-Az;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
GCRLogLevel<< "PGCR true residual r = src - A psi "<<norm2(r) <<std::endl;
|
|
||||||
|
|
||||||
/////////////////////
|
|
||||||
// p = Prec(r)
|
|
||||||
/////////////////////
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(r,z);
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
MatTimer.Start();
|
|
||||||
Linop.HermOpAndNorm(z,Az,zAz,zAAz);
|
|
||||||
MatTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
//p[0],q[0],qq[0]
|
|
||||||
p[0]= z;
|
|
||||||
q[0]= Az;
|
|
||||||
qq[0]= zAAz;
|
|
||||||
|
|
||||||
cp =norm2(r);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
for(int k=0;k<nstep;k++){
|
|
||||||
|
|
||||||
steps++;
|
|
||||||
|
|
||||||
int kp = k+1;
|
|
||||||
int peri_k = k %mmax;
|
|
||||||
int peri_kp= kp%mmax;
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
rq= real(innerProduct(r,q[peri_k])); // what if rAr not real?
|
|
||||||
a = rq/qq[peri_k];
|
|
||||||
|
|
||||||
axpy(psi,a,p[peri_k],psi);
|
|
||||||
|
|
||||||
cp = axpy_norm(r,-a,q[peri_k],r);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
GCRLogLevel<< "PGCR step["<<steps<<"] resid " << cp << " target " <<rsq<<std::endl;
|
|
||||||
|
|
||||||
if((k==nstep-1)||(cp<rsq)){
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(r,z);// solve Az = r
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
MatTimer.Start();
|
|
||||||
Linop.HermOpAndNorm(z,Az,zAz,zAAz);
|
|
||||||
MatTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
q[peri_kp]=Az;
|
|
||||||
p[peri_kp]=z;
|
|
||||||
|
|
||||||
int northog = ((kp)>(mmax-1))?(mmax-1):(kp); // if more than mmax done, we orthog all mmax history.
|
|
||||||
for(int back=0;back<northog;back++){
|
|
||||||
|
|
||||||
int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
|
|
||||||
|
|
||||||
b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
|
|
||||||
p[peri_kp]=p[peri_kp]+b*p[peri_back];
|
|
||||||
q[peri_kp]=q[peri_kp]+b*q[peri_back];
|
|
||||||
|
|
||||||
}
|
|
||||||
qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
}
|
|
||||||
GRID_ASSERT(0); // never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,450 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/PrecGeneralisedConjugateResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_PREC_GCR_NON_HERM_H
|
|
||||||
#define GRID_PREC_GCR_NON_HERM_H
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
//VPGCR Abe and Zhang, 2005.
|
|
||||||
//INTERNATIONAL JOURNAL OF NUMERICAL ANALYSIS AND MODELING
|
|
||||||
//Computing and Information Volume 2, Number 2, Pages 147-161
|
|
||||||
//NB. Likely not original reference since they are focussing on a preconditioner variant.
|
|
||||||
// but VPGCR was nicely written up in their paper
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#define GCRLogLevel std::cout << GridLogMessage <<std::string(level,'\t')<< " Level "<<level<<" "
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class PrecGeneralisedConjugateResidualNonHermitian : public LinearFunction<Field> {
|
|
||||||
public:
|
|
||||||
using LinearFunction<Field>::operator();
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
int verbose;
|
|
||||||
int mmax;
|
|
||||||
int nstep;
|
|
||||||
int steps;
|
|
||||||
int level;
|
|
||||||
GridStopWatch PrecTimer;
|
|
||||||
GridStopWatch MatTimer;
|
|
||||||
GridStopWatch LinalgTimer;
|
|
||||||
|
|
||||||
LinearFunction<Field> &Preconditioner;
|
|
||||||
LinearOperatorBase<Field> &Linop;
|
|
||||||
|
|
||||||
void Level(int lv) { level=lv; };
|
|
||||||
|
|
||||||
PrecGeneralisedConjugateResidualNonHermitian(RealD tol,Integer maxit,LinearOperatorBase<Field> &_Linop,LinearFunction<Field> &Prec,int _mmax, int _nstep) :
|
|
||||||
Tolerance(tol),
|
|
||||||
MaxIterations(maxit),
|
|
||||||
Linop(_Linop),
|
|
||||||
Preconditioner(Prec),
|
|
||||||
mmax(_mmax),
|
|
||||||
nstep(_nstep) // what is nstep vs mmax? one is the number of inner iterations
|
|
||||||
{
|
|
||||||
level=1;
|
|
||||||
verbose=1;
|
|
||||||
};
|
|
||||||
|
|
||||||
// virtual method stubs for updating GCR polynomial
|
|
||||||
virtual void LogBegin(void){
|
|
||||||
std::cout << "GCR::LogBegin() "<<std::endl;
|
|
||||||
};
|
|
||||||
virtual void LogIteration(int k, ComplexD a, std::vector<ComplexD> betas){
|
|
||||||
std::cout << "GCR::LogIteration() "<<std::endl;
|
|
||||||
};
|
|
||||||
virtual void LogComplete(std::vector<ComplexD>& alphas, std::vector<std::vector<ComplexD>>& betas) {
|
|
||||||
std::cout << "GCR::LogComplete() "<<std::endl;
|
|
||||||
};
|
|
||||||
|
|
||||||
void operator() (const Field &src, Field &psi){
|
|
||||||
|
|
||||||
// psi=Zero();
|
|
||||||
RealD cp, ssq,rsq;
|
|
||||||
ssq=norm2(src);
|
|
||||||
rsq=Tolerance*Tolerance*ssq;
|
|
||||||
|
|
||||||
Field r(src.Grid());
|
|
||||||
|
|
||||||
PrecTimer.Reset();
|
|
||||||
MatTimer.Reset();
|
|
||||||
LinalgTimer.Reset();
|
|
||||||
|
|
||||||
GridStopWatch SolverTimer;
|
|
||||||
SolverTimer.Start();
|
|
||||||
|
|
||||||
steps=0;
|
|
||||||
for(int k=0;k<MaxIterations;k++){
|
|
||||||
|
|
||||||
cp=GCRnStep(src,psi,rsq);
|
|
||||||
|
|
||||||
GCRLogLevel <<"PGCR("<<mmax<<","<<nstep<<") "<< steps <<" steps cp = "<<cp<<" target "<<rsq <<std::endl;
|
|
||||||
|
|
||||||
if(cp<rsq) {
|
|
||||||
SolverTimer.Stop();
|
|
||||||
|
|
||||||
Linop.Op(psi,r);
|
|
||||||
axpy(r,-1.0,src,r);
|
|
||||||
RealD tr = norm2(r);
|
|
||||||
GCRLogLevel<<"PGCR: Converged on iteration " <<steps
|
|
||||||
<< " computed residual "<<sqrt(cp/ssq)
|
|
||||||
<< " true residual " <<sqrt(tr/ssq)
|
|
||||||
<< " target " <<Tolerance <<std::endl;
|
|
||||||
|
|
||||||
GCRLogLevel<<"PGCR Time elapsed: Total "<< SolverTimer.Elapsed() <<std::endl;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
GCRLogLevel<<"Variable Preconditioned GCR did not converge"<<std::endl;
|
|
||||||
// GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
RealD GCRnStep(const Field &src, Field &psi,RealD rsq){
|
|
||||||
|
|
||||||
RealD cp;
|
|
||||||
ComplexD a, b;
|
|
||||||
// ComplexD zAz;
|
|
||||||
RealD zAAz;
|
|
||||||
ComplexD rq;
|
|
||||||
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
|
|
||||||
Field r(grid);
|
|
||||||
Field z(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
Field ttmp(grid);
|
|
||||||
Field Az(grid);
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
// history for flexible orthog
|
|
||||||
////////////////////////////////
|
|
||||||
std::vector<Field> q(mmax,grid); // q = Ap
|
|
||||||
std::vector<Field> p(mmax,grid); // store mmax conjugate momenta
|
|
||||||
std::vector<RealD> qq(mmax); // qq = (Ap)^2 = <p|A^\dagger A |p> (denom of \alpha)
|
|
||||||
|
|
||||||
GCRLogLevel<< "PGCR nStep("<<nstep<<")"<<std::endl;
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// initial guess x0 is taken as nonzero.
|
|
||||||
// r0=src-A x0 = src
|
|
||||||
//////////////////////////////////
|
|
||||||
MatTimer.Start();
|
|
||||||
Linop.Op(psi,Az);
|
|
||||||
// zAz = innerProduct(Az,psi);
|
|
||||||
zAAz= norm2(Az);
|
|
||||||
MatTimer.Stop();
|
|
||||||
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
r=src-Az;
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
GCRLogLevel<< "PGCR true residual r = src - A psi "<< norm2(r) <<std::endl;
|
|
||||||
|
|
||||||
this->LogBegin(); // initialize polynomial GCR if needed (TODO think about placement of this)
|
|
||||||
|
|
||||||
/////////////////////
|
|
||||||
// p = Prec(r)
|
|
||||||
/////////////////////
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(r,z);
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
MatTimer.Start();
|
|
||||||
Linop.Op(z,Az);
|
|
||||||
MatTimer.Stop();
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
// zAz = innerProduct(Az,psi);
|
|
||||||
zAAz= norm2(Az);
|
|
||||||
|
|
||||||
//p[0],q[0],qq[0]
|
|
||||||
p[0]= z;
|
|
||||||
q[0]= Az;
|
|
||||||
qq[0]= zAAz;
|
|
||||||
|
|
||||||
std::cout << "||init p - src||: " << norm2(p[0] - src) << std::endl; // for debugging
|
|
||||||
|
|
||||||
cp =norm2(r);
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
std::vector<ComplexD> all_alphas;
|
|
||||||
std::vector<std::vector<ComplexD>> all_betas;
|
|
||||||
|
|
||||||
for(int k=0;k<nstep;k++){
|
|
||||||
|
|
||||||
steps++;
|
|
||||||
|
|
||||||
int kp = k+1;
|
|
||||||
int peri_k = k %mmax; // only store mmax vectors; just roll around if needed
|
|
||||||
int peri_kp= kp%mmax;
|
|
||||||
|
|
||||||
// std::cout << "peri_kp = " << peri_kp << std::endl;
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
rq= innerProduct(q[peri_k],r); // what if rAr not real?
|
|
||||||
a = rq/qq[peri_k]; // compute alpha_j
|
|
||||||
|
|
||||||
all_alphas.push_back(a);
|
|
||||||
|
|
||||||
axpy(psi,a,p[peri_k],psi); // update psi --> psi + \alpha p
|
|
||||||
|
|
||||||
cp = axpy_norm(r,-a,q[peri_k],r); // update r --> r - \alpha D p. Note q = Dp
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
// LogIterationA(k + 1, a);
|
|
||||||
|
|
||||||
GCRLogLevel<< "GCR step["<<steps<<"] resid " << cp << " target " <<rsq<<std::endl;
|
|
||||||
|
|
||||||
// moving this to end of loop so that it doesn't exit beforehand
|
|
||||||
// TODO if I want to uncomment this, I have to split the LogIteration again and put LogIterationA() beforehand
|
|
||||||
// if((k==nstep-1)||(cp<rsq)){
|
|
||||||
// return cp;
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|
||||||
PrecTimer.Start();
|
|
||||||
Preconditioner(r,z);// solve Az = r
|
|
||||||
PrecTimer.Stop();
|
|
||||||
|
|
||||||
MatTimer.Start();
|
|
||||||
Linop.Op(z,Az);
|
|
||||||
MatTimer.Stop();
|
|
||||||
// zAz = innerProduct(Az,psi);
|
|
||||||
zAAz= norm2(Az);
|
|
||||||
|
|
||||||
LinalgTimer.Start();
|
|
||||||
|
|
||||||
q[peri_kp]=Az;
|
|
||||||
p[peri_kp]=z;
|
|
||||||
|
|
||||||
// Field Dsrc (grid);
|
|
||||||
// Linop.Op(src, Dsrc);
|
|
||||||
// std::cout << "||q[peri_kp] - D(src)||: " << norm2(q[peri_kp] - Dsrc) << std::endl; // for debugging
|
|
||||||
|
|
||||||
// // delete after testing
|
|
||||||
// std::cout << "Testing Dsq on one for GCR: " << std::endl;
|
|
||||||
// Field myField (grid);
|
|
||||||
// myField = 1.0;
|
|
||||||
// Field out1 (grid); Field out2 (grid);
|
|
||||||
// Linop.HermOp(myField, out1);
|
|
||||||
// Linop.Op(myField, out2);
|
|
||||||
// std::cout << "Dsq.Hermop(ones) has norm " << norm2(out1) << std::endl;
|
|
||||||
// std::cout << "Dsq.Op(ones) has norm " << norm2(out2) << std::endl;
|
|
||||||
|
|
||||||
// basically northog = k+1 if mmax is large
|
|
||||||
int northog = ((kp)>(mmax-1))?(mmax-1):(kp); // if more than mmax done, we orthog all mmax history.
|
|
||||||
// std::cout << "northog: " << northog << std::endl;
|
|
||||||
std::vector<ComplexD> betas (northog);
|
|
||||||
// std::cout << "peri_kp: " << peri_kp << std::endl;
|
|
||||||
// we iterate backwards counting down from the current k+1 index (peri_kp) because we
|
|
||||||
for(int back=0;back<northog;back++){
|
|
||||||
|
|
||||||
int peri_back=(k-back)%mmax; GRID_ASSERT((k-back)>=0);
|
|
||||||
|
|
||||||
// b=-real(innerProduct(q[peri_back],Az))/qq[peri_back];
|
|
||||||
b=-(innerProduct(q[peri_back],Az))/qq[peri_back]; // TODO try complex beta
|
|
||||||
p[peri_kp]=p[peri_kp]+b*p[peri_back];
|
|
||||||
q[peri_kp]=q[peri_kp]+b*q[peri_back];
|
|
||||||
|
|
||||||
// LogIterationB(peri_back, b);
|
|
||||||
// betas[back] = b; // may need to change the indexing if I ever do it with restarts
|
|
||||||
// std::cout << "[DEBUG] pushing beta for back = " << back << ", peri_back = " << peri_back << std::endl;
|
|
||||||
|
|
||||||
betas[peri_back] = b; // may need to change the indexing if I ever do it with restarts
|
|
||||||
|
|
||||||
}
|
|
||||||
qq[peri_kp]=norm2(q[peri_kp]); // could use axpy_norm
|
|
||||||
LinalgTimer.Stop();
|
|
||||||
|
|
||||||
// log iteration and update GCR polynomial if necessary.
|
|
||||||
all_betas.push_back(betas);
|
|
||||||
LogIteration(k + 1, a, betas);
|
|
||||||
|
|
||||||
// finish if necessary
|
|
||||||
if((k==nstep-1)||(cp<rsq)){
|
|
||||||
std::cout << "All alphas: " << std::endl << all_alphas << std::endl;
|
|
||||||
std::cout << "All betas: " << std::endl << all_betas << std::endl;
|
|
||||||
LogComplete(all_alphas, all_betas);
|
|
||||||
std::cout << "Exiting GCR." << std::endl;
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
GRID_ASSERT(0); // never reached
|
|
||||||
return cp;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
class PolynomialFile: Serializable {
|
|
||||||
public:
|
|
||||||
GRID_SERIALIZABLE_CLASS_MEMBERS(PolynomialFile,
|
|
||||||
std::vector<std::vector<std::complex<double>>>, data,
|
|
||||||
std::vector<std::vector<std::complex<double>>>, betas,
|
|
||||||
std::vector<std::complex<double>>, alphas
|
|
||||||
);
|
|
||||||
};
|
|
||||||
|
|
||||||
// Optionally record the GCR polynomial. [PO]: TODO
|
|
||||||
template <class Field>
|
|
||||||
class PGCRPolynomial : public PrecGeneralisedConjugateResidualNonHermitian<Field> {
|
|
||||||
public:
|
|
||||||
std::vector<ComplexD> ak;
|
|
||||||
std::vector<std::vector<ComplexD>> bk;
|
|
||||||
// std::vector<ComplexD> poly_p;
|
|
||||||
std::vector<std::vector<ComplexD>> poly_p;
|
|
||||||
std::vector<ComplexD> poly_Ap; // polynomial in Ap_j (only store it for last p)
|
|
||||||
std::vector<ComplexD> poly_r;
|
|
||||||
std::vector<ComplexD> polynomial;
|
|
||||||
|
|
||||||
PolynomialFile& PF;
|
|
||||||
|
|
||||||
public:
|
|
||||||
PGCRPolynomial(RealD tol, Integer maxit,LinearOperatorBase<Field> &_Linop, LinearFunction<Field> &Prec, int _mmax, int _nstep, PolynomialFile& _PF)
|
|
||||||
: PrecGeneralisedConjugateResidualNonHermitian<Field>(tol, maxit, _Linop, Prec, _mmax, _nstep), PF(_PF)
|
|
||||||
{};
|
|
||||||
|
|
||||||
// think this applies the polynomial in A = Linop to a field src. The coeffs are
|
|
||||||
// stored in the vector `polynomial`.
|
|
||||||
void PolyOp(const Field &src, Field &psi)
|
|
||||||
{
|
|
||||||
Field tmp(src.Grid());
|
|
||||||
Field AtoN(src.Grid());
|
|
||||||
AtoN = src;
|
|
||||||
psi=AtoN*polynomial[0];
|
|
||||||
for(int n=1;n<polynomial.size();n++){
|
|
||||||
tmp = AtoN;
|
|
||||||
this->Linop.Op(tmp,AtoN); // iterate A^n
|
|
||||||
psi = psi + polynomial[n]*AtoN; // psi += poly_n A^n src
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// [PO TODO] debug this
|
|
||||||
void PGCRsequence(const Field &src, Field &x)
|
|
||||||
{
|
|
||||||
Field Ap(src.Grid());
|
|
||||||
Field r(src.Grid());
|
|
||||||
// Field p(src.Grid());
|
|
||||||
// p=src;
|
|
||||||
std::vector<Field> p;
|
|
||||||
p.push_back(src);
|
|
||||||
r=src;
|
|
||||||
x=Zero();
|
|
||||||
x.Checkerboard()=src.Checkerboard();
|
|
||||||
for(int k=0;k<ak.size();k++){
|
|
||||||
x = x + ak[k]*p[k];
|
|
||||||
this->Linop.Op(p[k], Ap);
|
|
||||||
r = r - ak[k] * Ap;
|
|
||||||
// p[k] = r;
|
|
||||||
p.push_back(r);
|
|
||||||
for (int i = 0; i < k; i++) { // [PO TODO] check indices
|
|
||||||
p[k+1] += bk[i, k+1] * p[i];
|
|
||||||
}
|
|
||||||
// p = r + bk[k] * p;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void Solve(const Field &src, Field &psi)
|
|
||||||
{
|
|
||||||
psi=Zero();
|
|
||||||
this->operator()(src, psi);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void LogBegin(void)
|
|
||||||
{
|
|
||||||
std::cout << "PGCR::LogBegin() "<<std::endl;
|
|
||||||
ak.resize(0);
|
|
||||||
bk.resize(0);
|
|
||||||
polynomial.resize(0);
|
|
||||||
poly_Ap.push_back(0.0); // start with (0.0); during first iteration should change to (0.0, 1.0)
|
|
||||||
std::vector<ComplexD> p0_tmp;
|
|
||||||
p0_tmp.push_back(1.0);
|
|
||||||
poly_p.push_back(p0_tmp);
|
|
||||||
poly_r.push_back(1.0);
|
|
||||||
};
|
|
||||||
|
|
||||||
// Updates vector psi and r and initializes vector p[k+1]
|
|
||||||
virtual void LogIteration(int k, ComplexD a, std::vector<ComplexD> betas){
|
|
||||||
std::cout << "PGCR::LogIteration(k = " << k << ")" << std::endl;
|
|
||||||
ak.push_back(a);
|
|
||||||
bk.push_back(betas);
|
|
||||||
|
|
||||||
// update Ap by pushing p[k] to the right
|
|
||||||
poly_Ap.push_back(0.0); // need to pad the end with an element
|
|
||||||
poly_Ap[0] = 0.0; // technically this should be unnecessary, as the first component is never set
|
|
||||||
for(int i = 0; i < k; i++){
|
|
||||||
poly_Ap[i+1]=poly_p[k-1][i]; // A\vec{p} = (0, \vec{p}) bc A shifts components of p to the right
|
|
||||||
}
|
|
||||||
|
|
||||||
// update psi_{k+1} --> psi_k + a_k p_k
|
|
||||||
polynomial.push_back(0.0);
|
|
||||||
for(int i = 0; i < k; i++) {
|
|
||||||
polynomial[i] += a * poly_p[k-1][i];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
std::vector<std::complex<double>> poly_stdcmplx(polynomial.begin(), polynomial.end());
|
|
||||||
PF.data.push_back(poly_stdcmplx);
|
|
||||||
}
|
|
||||||
|
|
||||||
// r_{k+1} --> r_k - a_k A p_k
|
|
||||||
// p_{k+1} --> r_k + \sum_{i=0}^k \beta_{ik} p_i, input betas = (\beta_{ik})_i
|
|
||||||
poly_r.push_back(0.0); // should be of size k+1 if we start with k = 1
|
|
||||||
std::vector<ComplexD> p_next (k + 1, ComplexD(0.0)); // p_{k+1} = same size as r_{k+1}
|
|
||||||
for(int i = 0; i < k + 1; i++){
|
|
||||||
poly_r[i] = poly_r[i] - a * poly_Ap[i]; // update r_{k+1} --> r_k - \alpha_k A p_k
|
|
||||||
p_next[i] = poly_r[i]; // init new vector as r_{k+1}
|
|
||||||
}
|
|
||||||
|
|
||||||
// p_{k+1} --> p_{k+1} + \sum_i \beta_{ij} p_i
|
|
||||||
int nbeta = betas.size();
|
|
||||||
std::cout << "Betas: " << betas << std::endl;
|
|
||||||
for (int j = 0; j < nbeta; j++) {
|
|
||||||
for (int i = 0; i < j+1; i++) {
|
|
||||||
p_next[i] += betas[j] * poly_p[j][i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
poly_p.push_back(p_next); // add p_{k+1} to the list of p's
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void LogComplete(std::vector<ComplexD>& alphas, std::vector<std::vector<ComplexD>>& betas) {
|
|
||||||
/** Logs all alphas and betas to complete the iterations. */
|
|
||||||
std::cout << "PGCR::LogComplete() "<<std::endl;
|
|
||||||
for (int i = 0; i < alphas.size(); i++) {
|
|
||||||
PF.alphas.push_back(std::complex<double>(alphas[i].real(), alphas[i].imag()));
|
|
||||||
std::vector<std::complex<double>> beta_stdcmplx(betas[i].begin(), betas[i].end());
|
|
||||||
PF.betas.push_back(beta_stdcmplx);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,371 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithmsf/iterative/QuasiMinimalResidual.h
|
|
||||||
|
|
||||||
Copyright (C) 2019
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
RealD innerG5ProductReal(Field &l, Field &r)
|
|
||||||
{
|
|
||||||
Gamma G5(Gamma::Algebra::Gamma5);
|
|
||||||
Field tmp(l.Grid());
|
|
||||||
// tmp = G5*r;
|
|
||||||
G5R5(tmp,r);
|
|
||||||
ComplexD ip =innerProduct(l,tmp);
|
|
||||||
std::cout << "innerProductRealG5R5 "<<ip<<std::endl;
|
|
||||||
return ip.real();
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
class QuasiMinimalResidual : public OperatorFunction<Field> {
|
|
||||||
public:
|
|
||||||
using OperatorFunction<Field>::operator();
|
|
||||||
|
|
||||||
bool ErrorOnNoConverge;
|
|
||||||
RealD Tolerance;
|
|
||||||
Integer MaxIterations;
|
|
||||||
Integer IterationCount;
|
|
||||||
|
|
||||||
QuasiMinimalResidual(RealD tol,
|
|
||||||
Integer maxit,
|
|
||||||
bool err_on_no_conv = true)
|
|
||||||
: Tolerance(tol)
|
|
||||||
, MaxIterations(maxit)
|
|
||||||
, ErrorOnNoConverge(err_on_no_conv)
|
|
||||||
{};
|
|
||||||
|
|
||||||
#if 1
|
|
||||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &b, Field &x)
|
|
||||||
{
|
|
||||||
RealD resid;
|
|
||||||
IterationCount=0;
|
|
||||||
|
|
||||||
RealD rho, rho_1, xi, gamma, gamma_1, theta, theta_1;
|
|
||||||
RealD eta, delta, ep, beta;
|
|
||||||
|
|
||||||
GridBase *Grid = b.Grid();
|
|
||||||
Field r(Grid), d(Grid), s(Grid);
|
|
||||||
Field v(Grid), w(Grid), y(Grid), z(Grid);
|
|
||||||
Field v_tld(Grid), w_tld(Grid), y_tld(Grid), z_tld(Grid);
|
|
||||||
Field p(Grid), q(Grid), p_tld(Grid);
|
|
||||||
|
|
||||||
Real normb = norm2(b);
|
|
||||||
|
|
||||||
LinOp.Op(x,r); r = b - r;
|
|
||||||
|
|
||||||
GRID_ASSERT(normb> 0.0);
|
|
||||||
|
|
||||||
resid = norm2(r)/normb;
|
|
||||||
if (resid <= Tolerance) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
v_tld = r;
|
|
||||||
y = v_tld;
|
|
||||||
rho = norm2(y);
|
|
||||||
|
|
||||||
// Take Gamma5 conjugate
|
|
||||||
// Gamma G5(Gamma::Algebra::Gamma5);
|
|
||||||
// G5R5(w_tld,r);
|
|
||||||
// w_tld = G5* v_tld;
|
|
||||||
w_tld=v_tld;
|
|
||||||
z = w_tld;
|
|
||||||
xi = norm2(z);
|
|
||||||
|
|
||||||
gamma = 1.0;
|
|
||||||
eta = -1.0;
|
|
||||||
theta = 0.0;
|
|
||||||
|
|
||||||
for (int i = 1; i <= MaxIterations; i++) {
|
|
||||||
|
|
||||||
// Breakdown tests
|
|
||||||
GRID_ASSERT( rho != 0.0);
|
|
||||||
GRID_ASSERT( xi != 0.0);
|
|
||||||
|
|
||||||
v = (1. / rho) * v_tld;
|
|
||||||
y = (1. / rho) * y;
|
|
||||||
|
|
||||||
w = (1. / xi) * w_tld;
|
|
||||||
z = (1. / xi) * z;
|
|
||||||
|
|
||||||
ComplexD Zdelta = innerProduct(z, y); // Complex?
|
|
||||||
std::cout << "Zdelta "<<Zdelta<<std::endl;
|
|
||||||
delta = Zdelta.real();
|
|
||||||
|
|
||||||
y_tld = y;
|
|
||||||
z_tld = z;
|
|
||||||
|
|
||||||
if (i > 1) {
|
|
||||||
p = y_tld - (xi * delta / ep) * p;
|
|
||||||
q = z_tld - (rho * delta / ep) * q;
|
|
||||||
} else {
|
|
||||||
p = y_tld;
|
|
||||||
q = z_tld;
|
|
||||||
}
|
|
||||||
|
|
||||||
LinOp.Op(p,p_tld); // p_tld = A * p;
|
|
||||||
ComplexD Zep = innerProduct(q, p_tld);
|
|
||||||
ep=Zep.real();
|
|
||||||
std::cout << "Zep "<<Zep <<std::endl;
|
|
||||||
// Complex Audit
|
|
||||||
GRID_ASSERT(abs(ep)>0);
|
|
||||||
|
|
||||||
beta = ep / delta;
|
|
||||||
GRID_ASSERT(abs(beta)>0);
|
|
||||||
|
|
||||||
v_tld = p_tld - beta * v;
|
|
||||||
y = v_tld;
|
|
||||||
|
|
||||||
rho_1 = rho;
|
|
||||||
rho = norm2(y);
|
|
||||||
LinOp.AdjOp(q,w_tld);
|
|
||||||
w_tld = w_tld - beta * w;
|
|
||||||
z = w_tld;
|
|
||||||
|
|
||||||
xi = norm2(z);
|
|
||||||
|
|
||||||
gamma_1 = gamma;
|
|
||||||
theta_1 = theta;
|
|
||||||
|
|
||||||
theta = rho / (gamma_1 * beta);
|
|
||||||
gamma = 1.0 / sqrt(1.0 + theta * theta);
|
|
||||||
std::cout << "theta "<<theta<<std::endl;
|
|
||||||
std::cout << "gamma "<<gamma<<std::endl;
|
|
||||||
|
|
||||||
GRID_ASSERT(abs(gamma)> 0.0);
|
|
||||||
|
|
||||||
eta = -eta * rho_1 * gamma* gamma / (beta * gamma_1 * gamma_1);
|
|
||||||
|
|
||||||
if (i > 1) {
|
|
||||||
d = eta * p + (theta_1 * theta_1 * gamma * gamma) * d;
|
|
||||||
s = eta * p_tld + (theta_1 * theta_1 * gamma * gamma) * s;
|
|
||||||
} else {
|
|
||||||
d = eta * p;
|
|
||||||
s = eta * p_tld;
|
|
||||||
}
|
|
||||||
|
|
||||||
x =x+d; // update approximation vector
|
|
||||||
r =r-s; // compute residual
|
|
||||||
|
|
||||||
if ((resid = norm2(r) / normb) <= Tolerance) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
std::cout << "Iteration "<<i<<" resid " << resid<<std::endl;
|
|
||||||
}
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
return; // no convergence
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
// QMRg5 SMP thesis
|
|
||||||
void operator()(LinearOperatorBase<Field> &LinOp, const Field &b, Field &x)
|
|
||||||
{
|
|
||||||
// Real scalars
|
|
||||||
GridBase *grid = b.Grid();
|
|
||||||
|
|
||||||
Field r(grid);
|
|
||||||
Field p_m(grid), p_m_minus_1(grid), p_m_minus_2(grid);
|
|
||||||
Field v_m(grid), v_m_minus_1(grid), v_m_plus_1(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
|
|
||||||
RealD w;
|
|
||||||
RealD z1, z2;
|
|
||||||
RealD delta_m, delta_m_minus_1;
|
|
||||||
RealD c_m_plus_1, c_m, c_m_minus_1;
|
|
||||||
RealD s_m_plus_1, s_m, s_m_minus_1;
|
|
||||||
RealD alpha, beta, gamma, epsilon;
|
|
||||||
RealD mu, nu, rho, theta, xi, chi;
|
|
||||||
RealD mod2r, mod2b;
|
|
||||||
RealD tau2, target2;
|
|
||||||
|
|
||||||
mod2b=norm2(b);
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// Initial residual
|
|
||||||
/////////////////////////
|
|
||||||
LinOp.Op(x,tmp);
|
|
||||||
r = b - tmp;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// \mu = \rho = |r_0|
|
|
||||||
/////////////////////////
|
|
||||||
mod2r = norm2(r);
|
|
||||||
rho = sqrt( mod2r);
|
|
||||||
mu=rho;
|
|
||||||
|
|
||||||
std::cout << "QuasiMinimalResidual rho "<< rho<<std::endl;
|
|
||||||
/////////////////////////
|
|
||||||
// Zero negative history
|
|
||||||
/////////////////////////
|
|
||||||
v_m_plus_1 = Zero();
|
|
||||||
v_m_minus_1 = Zero();
|
|
||||||
p_m_minus_1 = Zero();
|
|
||||||
p_m_minus_2 = Zero();
|
|
||||||
|
|
||||||
// v0
|
|
||||||
v_m = (1.0/rho)*r;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// Initial coeffs
|
|
||||||
/////////////////////////
|
|
||||||
delta_m_minus_1 = 1.0;
|
|
||||||
c_m_minus_1 = 1.0;
|
|
||||||
c_m = 1.0;
|
|
||||||
s_m_minus_1 = 0.0;
|
|
||||||
s_m = 0.0;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// Set up convergence check
|
|
||||||
/////////////////////////
|
|
||||||
tau2 = mod2r;
|
|
||||||
target2 = mod2b * Tolerance*Tolerance;
|
|
||||||
|
|
||||||
for(int iter = 0 ; iter < MaxIterations; iter++){
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// \delta_m = (v_m, \gamma_5 v_m)
|
|
||||||
/////////////////////////
|
|
||||||
delta_m = innerG5ProductReal(v_m,v_m);
|
|
||||||
std::cout << "QuasiMinimalResidual delta_m "<< delta_m<<std::endl;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// tmp = A v_m
|
|
||||||
/////////////////////////
|
|
||||||
LinOp.Op(v_m,tmp);
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// \alpha = (v_m, \gamma_5 temp) / \delta_m
|
|
||||||
/////////////////////////
|
|
||||||
alpha = innerG5ProductReal(v_m,tmp);
|
|
||||||
alpha = alpha/delta_m ;
|
|
||||||
std::cout << "QuasiMinimalResidual alpha "<< alpha<<std::endl;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// \beta = \rho \delta_m / \delta_{m-1}
|
|
||||||
/////////////////////////
|
|
||||||
beta = rho * delta_m / delta_m_minus_1;
|
|
||||||
std::cout << "QuasiMinimalResidual beta "<< beta<<std::endl;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// \tilde{v}_{m+1} = temp - \alpha v_m - \beta v_{m-1}
|
|
||||||
/////////////////////////
|
|
||||||
v_m_plus_1 = tmp - alpha*v_m - beta*v_m_minus_1;
|
|
||||||
|
|
||||||
///////////////////////////////
|
|
||||||
// \rho = || \tilde{v}_{m+1} ||
|
|
||||||
///////////////////////////////
|
|
||||||
rho = sqrt( norm2(v_m_plus_1) );
|
|
||||||
std::cout << "QuasiMinimalResidual rho "<< rho<<std::endl;
|
|
||||||
|
|
||||||
///////////////////////////////
|
|
||||||
// v_{m+1} = \tilde{v}_{m+1}
|
|
||||||
///////////////////////////////
|
|
||||||
v_m_plus_1 = (1.0 / rho) * v_m_plus_1;
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
// QMR recurrence coefficients.
|
|
||||||
////////////////////////////////
|
|
||||||
theta = s_m_minus_1 * beta;
|
|
||||||
gamma = c_m_minus_1 * beta;
|
|
||||||
epsilon = c_m * gamma + s_m * alpha;
|
|
||||||
xi = -s_m * gamma + c_m * alpha;
|
|
||||||
nu = sqrt( xi*xi + rho*rho );
|
|
||||||
c_m_plus_1 = fabs(xi) / nu;
|
|
||||||
if ( xi == 0.0 ) {
|
|
||||||
s_m_plus_1 = 1.0;
|
|
||||||
} else {
|
|
||||||
s_m_plus_1 = c_m_plus_1 * rho / xi;
|
|
||||||
}
|
|
||||||
chi = c_m_plus_1 * xi + s_m_plus_1 * rho;
|
|
||||||
|
|
||||||
std::cout << "QuasiMinimalResidual coeffs "<< theta <<" "<<gamma<<" "<< epsilon<<" "<< xi<<" "<< nu<<std::endl;
|
|
||||||
std::cout << "QuasiMinimalResidual coeffs "<< chi <<std::endl;
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
//p_m=(v_m - \epsilon p_{m-1} - \theta p_{m-2}) / \chi
|
|
||||||
////////////////////////////////
|
|
||||||
p_m = (1.0/chi) * v_m - (epsilon/chi) * p_m_minus_1 - (theta/chi) * p_m_minus_2;
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// \psi = \psi + c_{m+1} \mu p_m
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
x = x + ( c_m_plus_1 * mu ) * p_m;
|
|
||||||
|
|
||||||
////////////////////////////////////////
|
|
||||||
//
|
|
||||||
////////////////////////////////////////
|
|
||||||
mu = -s_m_plus_1 * mu;
|
|
||||||
delta_m_minus_1 = delta_m;
|
|
||||||
c_m_minus_1 = c_m;
|
|
||||||
c_m = c_m_plus_1;
|
|
||||||
s_m_minus_1 = s_m;
|
|
||||||
s_m = s_m_plus_1;
|
|
||||||
|
|
||||||
////////////////////////////////////
|
|
||||||
// Could use pointer swizzle games.
|
|
||||||
////////////////////////////////////
|
|
||||||
v_m_minus_1 = v_m;
|
|
||||||
v_m = v_m_plus_1;
|
|
||||||
p_m_minus_2 = p_m_minus_1;
|
|
||||||
p_m_minus_1 = p_m;
|
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////////////////
|
|
||||||
// Convergence checks
|
|
||||||
/////////////////////////////////////
|
|
||||||
z1 = RealD(iter+1.0);
|
|
||||||
z2 = z1 + 1.0;
|
|
||||||
tau2 = tau2 *( z2 / z1 ) * s_m * s_m;
|
|
||||||
std::cout << " QuasiMinimumResidual iteration "<< iter<<std::endl;
|
|
||||||
std::cout << " QuasiMinimumResidual tau bound "<< tau2<<std::endl;
|
|
||||||
|
|
||||||
// Compute true residual
|
|
||||||
mod2r = tau2;
|
|
||||||
if ( 1 || (tau2 < (100.0 * target2)) ) {
|
|
||||||
LinOp.Op(x,tmp);
|
|
||||||
r = b - tmp;
|
|
||||||
mod2r = norm2(r);
|
|
||||||
std::cout << " QuasiMinimumResidual true residual is "<< mod2r<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if ( mod2r < target2 ) {
|
|
||||||
|
|
||||||
std::cout << " QuasiMinimumResidual has converged"<<std::endl;
|
|
||||||
return;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,753 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./Grid/algorithms/iterative/RestartedLanczosBidiagonalization.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Chulwoo Jung <chulwoo@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_RESTARTED_LANCZOS_BIDIAGONALIZATION_H
|
|
||||||
#define GRID_RESTARTED_LANCZOS_BIDIAGONALIZATION_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Implicitly Restarted Lanczos Bidiagonalization (IRLBA)
|
|
||||||
*
|
|
||||||
* Computes the p largest (or p smallest) singular triplets of a linear
|
|
||||||
* operator A using the Golub-Kahan-Lanczos bidiagonalization with implicit
|
|
||||||
* restart via thick-restart / QR shifts.
|
|
||||||
*
|
|
||||||
* Algorithm (Baglama & Reichel, SIAM J. Sci. Comput. 27(1):19-42, 2005):
|
|
||||||
*
|
|
||||||
* Outer loop:
|
|
||||||
* 1. Extend the p-step (or seed) bidiagonalization to k steps:
|
|
||||||
* A V_k = U_k B_k
|
|
||||||
* A^dag U_k = V_k B_k^T + beta_{k+1} v_{k+1} e_k^T
|
|
||||||
* 2. Compute SVD: B_k = X Sigma Y^T
|
|
||||||
* 3. Check convergence of the p desired singular values via
|
|
||||||
* |beta_{k+1} * y_{k,i}| < tol * sigma_i
|
|
||||||
* where y_{k,i} is the last component of the i-th right singular vector.
|
|
||||||
* 4. Apply k-p implicit QR shifts to implicitly compress the basis
|
|
||||||
* to p steps (Sorensen-Lehoucq thick restart):
|
|
||||||
* B_p^+ = X_p^T B_k Y_p (upper bidiagonal, p x p)
|
|
||||||
* and update the lattice vectors:
|
|
||||||
* V_p^+ = V_k Y_p
|
|
||||||
* U_p^+ = U_k X_p
|
|
||||||
* The new residual coupling is
|
|
||||||
* beta_p^+ v_{p+1}^+ = beta_{k+1} v_{k+1} * (e_k^T Y_p)_p
|
|
||||||
* + B_k(p,p+1) * (orthogonal tail from QR)
|
|
||||||
* 5. Go to step 1.
|
|
||||||
*
|
|
||||||
* Template parameter
|
|
||||||
* ------------------
|
|
||||||
* Field : lattice field type (must support Grid algebra operations)
|
|
||||||
*
|
|
||||||
* Usage
|
|
||||||
* -----
|
|
||||||
* RestartedLanczosBidiagonalization<Field> irlba(Linop, grid, p, k, tol, maxIter);
|
|
||||||
* irlba.run(src);
|
|
||||||
* // Results available via getters.
|
|
||||||
*/
|
|
||||||
template <class Field>
|
|
||||||
class RestartedLanczosBidiagonalization {
|
|
||||||
|
|
||||||
public:
|
|
||||||
LinearOperatorBase<Field> &Linop;
|
|
||||||
GridBase *Grid;
|
|
||||||
|
|
||||||
int Nk; // number of desired singular triplets
|
|
||||||
int Nm; // Lanczos basis size (Nm > Nk)
|
|
||||||
RealD Tolerance;
|
|
||||||
int MaxIter;
|
|
||||||
bool largest; // if true, target largest singular values; otherwise smallest
|
|
||||||
|
|
||||||
// Converged singular triplets (filled after run())
|
|
||||||
std::vector<RealD> singularValues; // sigma_0 >= sigma_1 >= ...
|
|
||||||
std::vector<Field> leftVectors; // approximate left singular vectors
|
|
||||||
std::vector<Field> rightVectors; // approximate right singular vectors
|
|
||||||
|
|
||||||
private:
|
|
||||||
// Working bases (size up to Nm+1)
|
|
||||||
std::vector<Field> V; // right Lanczos vectors
|
|
||||||
std::vector<Field> U; // left Lanczos vectors
|
|
||||||
std::vector<RealD> alpha;
|
|
||||||
std::vector<RealD> beta;
|
|
||||||
|
|
||||||
// After a thick restart, the column at index restart_col of U^dag A V
|
|
||||||
// has extra non-zero entries (rows 0..restart_col-2) beyond what the
|
|
||||||
// upper bidiagonal captures. fvec[j] = <U[j] | A V[restart_col]> for
|
|
||||||
// j = 0..restart_col-1. (fvec[restart_col-1] == beta[restart_col-1].)
|
|
||||||
// reset_col == -1 means no restart has occurred yet (pure bidiagonal).
|
|
||||||
std::vector<RealD> fvec;
|
|
||||||
int restart_col;
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
RestartedLanczosBidiagonalization(LinearOperatorBase<Field> &_Linop,
|
|
||||||
GridBase *_Grid,
|
|
||||||
int _Nk, int _Nm,
|
|
||||||
RealD _tol = 1.0e-8,
|
|
||||||
int _maxIt = 300,
|
|
||||||
bool _largest = true)
|
|
||||||
: Linop(_Linop), Grid(_Grid),
|
|
||||||
Nk(_Nk), Nm(_Nm),
|
|
||||||
Tolerance(_tol), MaxIter(_maxIt),
|
|
||||||
largest(_largest)
|
|
||||||
{
|
|
||||||
assert(Nm > Nk);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Run IRLBA starting from src.
|
|
||||||
* On exit, singularValues, leftVectors, rightVectors are filled with
|
|
||||||
* the Nk converged singular triplets.
|
|
||||||
*/
|
|
||||||
void run(const Field &src)
|
|
||||||
{
|
|
||||||
assert(norm2(src) > 0.0);
|
|
||||||
|
|
||||||
singularValues.clear();
|
|
||||||
leftVectors.clear();
|
|
||||||
rightVectors.clear();
|
|
||||||
|
|
||||||
// Allocate working bases
|
|
||||||
V.clear(); U.clear();
|
|
||||||
alpha.clear(); beta.clear();
|
|
||||||
fvec.clear(); restart_col = -1;
|
|
||||||
V.reserve(Nm + 1);
|
|
||||||
U.reserve(Nm);
|
|
||||||
|
|
||||||
// Seed: v_0 = src / ||src||
|
|
||||||
Field vtmp(Grid);
|
|
||||||
vtmp = src;
|
|
||||||
RealD nrm = std::sqrt(norm2(vtmp));
|
|
||||||
vtmp = (1.0 / nrm) * vtmp;
|
|
||||||
V.push_back(vtmp);
|
|
||||||
|
|
||||||
int pStart = 0; // current basis size at start of extension
|
|
||||||
RealD betaRestart = 0.0; // coupling from previous restart
|
|
||||||
|
|
||||||
for (int iter = 0; iter < MaxIter; ++iter) {
|
|
||||||
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
// Step 1: extend from pStart steps to Nm steps
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
extendBasis(pStart, Nm, betaRestart);
|
|
||||||
// verify();
|
|
||||||
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
// Step 2: SVD of the Nm x Nm B matrix.
|
|
||||||
// iter=0 (pStart==0): B is exactly bidiagonal — use buildBidiagonal.
|
|
||||||
// iter>0 (pStart==Nk): after a thick restart, column restart_col of
|
|
||||||
// U^dag A V has extra off-diagonal entries captured by fvec; use
|
|
||||||
// buildFullB so the Ritz values and restart vectors are computed from
|
|
||||||
// the exact projected matrix A V = U B_full.
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
Eigen::MatrixXd B = (pStart == 0) ? buildBidiagonal(Nm) : buildFullB(Nm);
|
|
||||||
Eigen::JacobiSVD<Eigen::MatrixXd> svd(B,
|
|
||||||
Eigen::ComputeThinU | Eigen::ComputeThinV);
|
|
||||||
|
|
||||||
Eigen::VectorXd sigma = svd.singularValues(); // descending
|
|
||||||
Eigen::MatrixXd X = svd.matrixU(); // Nm x Nm left SVecs of B
|
|
||||||
Eigen::MatrixXd Y = svd.matrixV(); // Nm x Nm right SVecs of B
|
|
||||||
|
|
||||||
// If targeting smallest, reorder so desired ones come first
|
|
||||||
Eigen::VectorXi order = sortOrder(sigma);
|
|
||||||
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
// Step 3: check convergence of the Nk desired singular values
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
RealD betaK = beta.back(); // beta_{k+1}
|
|
||||||
// In our convention A V = U B (exact), the residual is in the A^dag
|
|
||||||
// direction: A^dag u_j - sigma_j v_j = betaK * X[Nm-1,j] * V[Nm].
|
|
||||||
// Convergence criterion: |betaK * X[Nm-1, idx]| < tol * sigma_idx.
|
|
||||||
int nconv = 0;
|
|
||||||
for (int i = 0; i < Nk; ++i) {
|
|
||||||
int idx = order(i);
|
|
||||||
RealD res = std::abs(betaK * X(Nm - 1, idx));
|
|
||||||
RealD thr = Tolerance * std::max(sigma(idx), 1.0e-14);
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA iter " << iter
|
|
||||||
<< " sigma[" << i << "] = " << sigma(idx)
|
|
||||||
<< " res = " << res
|
|
||||||
<< " thr = " << thr << std::endl;
|
|
||||||
if (res < thr) ++nconv;
|
|
||||||
else break; // residuals not strictly ordered but break is conservative
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nconv >= Nk) {
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA converged: " << nconv << " singular values after "
|
|
||||||
<< iter + 1 << " iterations." << std::endl;
|
|
||||||
// Collect converged triplets
|
|
||||||
extractTriplets(Nm, sigma, X, Y, order, Nk);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
// Step 4: implicit restart — compress to Nk steps
|
|
||||||
// ----------------------------------------------------------------
|
|
||||||
implicitRestart(Nm, Nk, sigma, X, Y, order, betaK, betaRestart);
|
|
||||||
// verify();
|
|
||||||
|
|
||||||
// Lucky breakdown: exact invariant subspace found; convergence is exact.
|
|
||||||
// B_p^+ = diag(alpha[0..Nk-1]); extract directly from restart basis.
|
|
||||||
if (betaRestart < 1.0e-14) {
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA: lucky breakdown after restart (betaRestart = 0)."
|
|
||||||
<< " Extracting " << Nk << " exact Ritz triplets." << std::endl;
|
|
||||||
// Re-run SVD on the p-step diagonal B^+ to get sorted Ritz triplets.
|
|
||||||
Eigen::MatrixXd Bp = buildBidiagonal(Nk);
|
|
||||||
Eigen::JacobiSVD<Eigen::MatrixXd> svdp(Bp,
|
|
||||||
Eigen::ComputeThinU | Eigen::ComputeThinV);
|
|
||||||
Eigen::VectorXi ordp = sortOrder(svdp.singularValues());
|
|
||||||
extractTriplets(Nk, svdp.singularValues(), svdp.matrixU(),
|
|
||||||
svdp.matrixV(), ordp, Nk);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
pStart = Nk;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA: did not converge in " << MaxIter
|
|
||||||
<< " iterations. Returning best approximations." << std::endl;
|
|
||||||
|
|
||||||
// Return best available approximations
|
|
||||||
Eigen::MatrixXd B = buildFullB((int)alpha.size());
|
|
||||||
Eigen::JacobiSVD<Eigen::MatrixXd> svd(B,
|
|
||||||
Eigen::ComputeThinU | Eigen::ComputeThinV);
|
|
||||||
Eigen::VectorXd sigma = svd.singularValues();
|
|
||||||
Eigen::MatrixXd X = svd.matrixU();
|
|
||||||
Eigen::MatrixXd Y = svd.matrixV();
|
|
||||||
Eigen::VectorXi order = sortOrder(sigma);
|
|
||||||
int nout = std::min(Nk, (int)alpha.size());
|
|
||||||
extractTriplets((int)alpha.size(), sigma, X, Y, order, nout);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Getters */
|
|
||||||
int getNk() const { return (int)singularValues.size(); }
|
|
||||||
const std::vector<RealD>& getSingularValues() const { return singularValues; }
|
|
||||||
const std::vector<Field>& getLeftVectors() const { return leftVectors; }
|
|
||||||
const std::vector<Field>& getRightVectors() const { return rightVectors; }
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Print B_k and U^dag A V to verify the bidiagonalization relation
|
|
||||||
* A V_m = U_m B_m (exact in our GK convention)
|
|
||||||
* On the first call (pStart=0), max|B - U^dag A V| should be ~machine epsilon.
|
|
||||||
* After a restart and extension, the column p of U^dag A V deviates from B
|
|
||||||
* by O(betaK): this is expected because the thick restart breaks the Krylov
|
|
||||||
* structure at column p, introducing off-diagonal terms proportional to betaK.
|
|
||||||
* These terms vanish as betaK -> 0 (convergence), so the algorithm is correct.
|
|
||||||
*/
|
|
||||||
void verify()
|
|
||||||
{
|
|
||||||
int m = (int)alpha.size();
|
|
||||||
int nU = (int)U.size();
|
|
||||||
int nV = (int)V.size();
|
|
||||||
if (m == 0) { std::cout << GridLogMessage << "IRLBA verify: empty basis" << std::endl; return; }
|
|
||||||
|
|
||||||
// Build reference matrix Bref (nU x nV):
|
|
||||||
// Columns 0..m-1 : buildFullB(m) (bidiagonal + fvec column at restart_col)
|
|
||||||
// Column m : residual column, two cases:
|
|
||||||
// (a) restart_col == m (right after implicitRestart, before extendBasis):
|
|
||||||
// V[m] = sgn*V_old[Nm], so <U[i]|A|V[m]> = fvec[i] for all i
|
|
||||||
// (b) otherwise (pure GK or after extendBasis):
|
|
||||||
// only entry (m-1, m) = beta[m-1] (GK recurrence residual)
|
|
||||||
Eigen::MatrixXd Bref = Eigen::MatrixXd::Zero(nU, nV);
|
|
||||||
{
|
|
||||||
Eigen::MatrixXd Bfull = buildFullB(m);
|
|
||||||
int cols = std::min(m, nV);
|
|
||||||
Bref.block(0, 0, m, cols) = Bfull.block(0, 0, m, cols);
|
|
||||||
}
|
|
||||||
if (nV > m && m > 0) {
|
|
||||||
if (restart_col == m && (int)fvec.size() == m) {
|
|
||||||
// Case (a): right after implicitRestart
|
|
||||||
for (int i = 0; i < m; ++i) Bref(i, m) = fvec[i];
|
|
||||||
} else if ((int)beta.size() >= m) {
|
|
||||||
// Case (b): standard GK residual column
|
|
||||||
Bref(m - 1, m) = beta[m - 1];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute M[i,j] = <U[i] | A | V[j]>
|
|
||||||
Eigen::MatrixXd M = Eigen::MatrixXd::Zero(nU, nV);
|
|
||||||
Field Avj(Grid);
|
|
||||||
for (int j = 0; j < nV; ++j) {
|
|
||||||
Linop.Op(V[j], Avj);
|
|
||||||
for (int i = 0; i < nU; ++i) {
|
|
||||||
ComplexD ip = innerProduct(U[i], Avj);
|
|
||||||
M(i, j) = ip.real();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Print Bref
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA verify: Bref (" << nU << "x" << nV << "):" << std::endl;
|
|
||||||
for (int i = 0; i < nU; ++i) {
|
|
||||||
std::cout << GridLogMessage << " row " << i << ": ";
|
|
||||||
for (int j = 0; j < nV; ++j) std::cout << Bref(i,j) << " ";
|
|
||||||
std::cout << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Print U^dag A V
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA verify: U^dag A V (" << nU << "x" << nV << "):" << std::endl;
|
|
||||||
for (int i = 0; i < nU; ++i) {
|
|
||||||
std::cout << GridLogMessage << " row " << i << ": ";
|
|
||||||
for (int j = 0; j < nV; ++j) std::cout << M(i,j) << " ";
|
|
||||||
std::cout << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Max deviation over the full nU x nV matrix
|
|
||||||
RealD maxdev = (Bref - M).cwiseAbs().maxCoeff();
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA verify: max|Bref - U^dag A V| = " << maxdev << std::endl;
|
|
||||||
|
|
||||||
// Beta
|
|
||||||
std::cout << GridLogMessage << "IRLBA verify: beta[0.." << (int)beta.size()-1 << "] = ";
|
|
||||||
for (auto b : beta) std::cout << b << " ";
|
|
||||||
std::cout << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Build the m x m upper-bidiagonal matrix from alpha[0..m-1], beta[0..m-2]
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
Eigen::MatrixXd buildBidiagonal(int m) const
|
|
||||||
{
|
|
||||||
Eigen::MatrixXd B = Eigen::MatrixXd::Zero(m, m);
|
|
||||||
for (int k = 0; k < m; ++k) {
|
|
||||||
B(k, k) = alpha[k];
|
|
||||||
if (k + 1 < m && k < (int)beta.size())
|
|
||||||
B(k, k + 1) = beta[k];
|
|
||||||
}
|
|
||||||
return B;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Build the full m x m B matrix, including the non-bidiagonal column
|
|
||||||
// at restart_col that arises after a thick restart.
|
|
||||||
//
|
|
||||||
// After restart, A V[restart_col] has projections onto all U[0..restart_col-1]
|
|
||||||
// (not just U[restart_col-1]). These are stored in fvec[0..restart_col-1]
|
|
||||||
// and make column restart_col of U^dag A V non-bidiagonal.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
Eigen::MatrixXd buildFullB(int m) const
|
|
||||||
{
|
|
||||||
Eigen::MatrixXd B = buildBidiagonal(m);
|
|
||||||
if (restart_col >= 0 && restart_col < m && (int)fvec.size() > 0) {
|
|
||||||
for (int j = 0; j < restart_col && j < (int)fvec.size(); ++j){
|
|
||||||
B(j, restart_col) = fvec[j];
|
|
||||||
std::cout << GridLogDebug << "buildFullB: B " <<j<<" "<<restart_col<<B(j, restart_col)<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return B;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Return a permutation vector that puts the desired Nk singular values
|
|
||||||
// first (largest first if largest==true, smallest first otherwise).
|
|
||||||
// Eigen's JacobiSVD already returns sigma in descending order, so for
|
|
||||||
// largest we just return 0,1,...,m-1; for smallest we reverse.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
Eigen::VectorXi sortOrder(const Eigen::VectorXd &sigma) const
|
|
||||||
{
|
|
||||||
int m = (int)sigma.size();
|
|
||||||
Eigen::VectorXi ord(m);
|
|
||||||
if (largest) {
|
|
||||||
for (int i = 0; i < m; ++i) ord(i) = i;
|
|
||||||
} else {
|
|
||||||
for (int i = 0; i < m; ++i) ord(i) = m - 1 - i;
|
|
||||||
}
|
|
||||||
return ord;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Extend the Lanczos bidiagonalization from pStart to kEnd steps.
|
|
||||||
// On first call pStart==0 (V[0] already set).
|
|
||||||
// On restart calls V[0..pStart], U[0..pStart-1], alpha[0..pStart-1],
|
|
||||||
// beta[0..pStart-1] are already set; betaRestart is the coupling
|
|
||||||
// beta_{pStart} that drives the first new U step.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
void extendBasis(int pStart, int kEnd, RealD betaRestart)
|
|
||||||
{
|
|
||||||
// Truncate containers to pStart (Lattice has no default constructor)
|
|
||||||
if ((int)V.size() > pStart + 1) V.erase(V.begin() + pStart + 1, V.end());
|
|
||||||
if ((int)U.size() > pStart) U.erase(U.begin() + pStart, U.end());
|
|
||||||
alpha.resize(pStart);
|
|
||||||
beta.resize(pStart);
|
|
||||||
|
|
||||||
Field p(Grid), r(Grid);
|
|
||||||
|
|
||||||
for (int k = pStart; k < kEnd; ++k) {
|
|
||||||
|
|
||||||
// p = A v_k
|
|
||||||
Linop.Op(V[k], p);
|
|
||||||
|
|
||||||
// Remove previous left vector coupling
|
|
||||||
if (k > 0) {
|
|
||||||
p = p - beta[k - 1] * U[k - 1];
|
|
||||||
}
|
|
||||||
// On the first step after a restart, beta[pStart-1] was already set;
|
|
||||||
// but V[pStart] was already constructed including the beta correction,
|
|
||||||
// so no extra subtraction needed here beyond the standard recurrence.
|
|
||||||
|
|
||||||
// Reorthogonalize p against U, then alpha_k = ||p||, u_k = p/alpha_k
|
|
||||||
reorthogonalize(p, U);
|
|
||||||
RealD ak = std::sqrt(norm2(p));
|
|
||||||
if (ak < 1.0e-14) {
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA extendBasis: lucky breakdown at step " << k
|
|
||||||
<< " (alpha = " << ak << ")" << std::endl;
|
|
||||||
alpha.push_back(ak);
|
|
||||||
Field zero(Grid); zero = Zero();
|
|
||||||
U.push_back(zero);
|
|
||||||
beta.push_back(0.0);
|
|
||||||
V.push_back(zero);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
alpha.push_back(ak);
|
|
||||||
|
|
||||||
Field u(Grid);
|
|
||||||
u = (1.0 / ak) * p;
|
|
||||||
U.push_back(u);
|
|
||||||
|
|
||||||
// r = A^dag u_k - alpha_k v_k, reorthogonalize, then beta_{k+1} = ||r||
|
|
||||||
Linop.AdjOp(U[k], r);
|
|
||||||
r = r - ak * V[k];
|
|
||||||
reorthogonalize(r, V);
|
|
||||||
|
|
||||||
RealD bk = std::sqrt(norm2(r));
|
|
||||||
beta.push_back(bk);
|
|
||||||
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA extend step " << k
|
|
||||||
<< " alpha = " << ak
|
|
||||||
<< " beta = " << bk << std::endl;
|
|
||||||
|
|
||||||
// Always push v_{k+1} (needed as residual direction for restart)
|
|
||||||
if (bk < 1.0e-14) {
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA extendBasis: lucky breakdown (beta = 0) at step "
|
|
||||||
<< k << std::endl;
|
|
||||||
Field zero(Grid); zero = Zero();
|
|
||||||
V.push_back(zero);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Field vnext(Grid);
|
|
||||||
vnext = (1.0 / bk) * r;
|
|
||||||
V.push_back(vnext);
|
|
||||||
|
|
||||||
if (k == kEnd - 1) break; // v_{k+1} pushed above; stop here
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public:
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Block reorthogonalization helpers.
|
|
||||||
// Declared public because CUDA extended lambdas cannot live inside
|
|
||||||
// private/protected member functions.
|
|
||||||
//
|
|
||||||
// batchInnerProducts: computes c[j] = <basis[j], vec> for all j
|
|
||||||
// in a single GPU pass (one accelerator_barrier instead of n).
|
|
||||||
// Queues n pairs of (per-site kernel, reduceKernel) to computeStream
|
|
||||||
// without intermediate CPU syncs, then syncs once at the end.
|
|
||||||
//
|
|
||||||
// batchUpdate: computes vec -= sum_j c[j]*basis[j] in one GPU kernel.
|
|
||||||
//
|
|
||||||
// reorthogonalize: two-pass Classical Gram-Schmidt (CGS2) using the
|
|
||||||
// two helpers above. Each pass costs 2 GPU syncs (1 IP + 1 update)
|
|
||||||
// instead of 2n syncs per pass in the old sequential MGS.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
|
|
||||||
void batchInnerProducts(const Field &vec,
|
|
||||||
const std::vector<Field> &basis,
|
|
||||||
std::vector<ComplexD> &c)
|
|
||||||
{
|
|
||||||
int n = (int)basis.size();
|
|
||||||
c.resize(n);
|
|
||||||
if (n == 0) return;
|
|
||||||
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
typedef decltype(innerProduct(vobj(), vobj())) inner_t;
|
|
||||||
typedef decltype(basis[0].View(AcceleratorRead)) View;
|
|
||||||
|
|
||||||
GridBase *grid = vec.Grid();
|
|
||||||
uint64_t oSites = grid->oSites();
|
|
||||||
uint64_t nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
// all_ip[j * oSites + ss] = per-site inner product of basis[j] and vec at site ss.
|
|
||||||
// Layout: n contiguous blocks of oSites each.
|
|
||||||
deviceVector<inner_t> all_ip((uint64_t)n * oSites);
|
|
||||||
inner_t *all_ip_p = &all_ip[0];
|
|
||||||
|
|
||||||
hostVector<View> h_basis_v(n);
|
|
||||||
deviceVector<View> d_basis_v(n);
|
|
||||||
for (int j = 0; j < n; ++j) {
|
|
||||||
h_basis_v[j] = basis[j].View(AcceleratorRead);
|
|
||||||
acceleratorPut(d_basis_v[j], h_basis_v[j]);
|
|
||||||
}
|
|
||||||
View *basis_vp = &d_basis_v[0];
|
|
||||||
|
|
||||||
// Queue n per-site kernels to the accelerator stream — no intermediate barriers.
|
|
||||||
autoView(vec_v, vec, AcceleratorRead);
|
|
||||||
for (int j = 0; j < n; ++j) {
|
|
||||||
int jj = j;
|
|
||||||
uint64_t oSites_ = oSites;
|
|
||||||
accelerator_for(ss, oSites, nsimd, {
|
|
||||||
auto x = coalescedRead(basis_vp[jj][ss]);
|
|
||||||
auto y = coalescedRead(vec_v[ss]);
|
|
||||||
coalescedWrite(all_ip_p[jj * oSites_ + ss], innerProduct(x, y));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// ONE sync after all n kernels
|
|
||||||
accelerator_barrier();
|
|
||||||
|
|
||||||
// Copy all per-site results to host
|
|
||||||
hostVector<inner_t> all_ip_h((uint64_t)n * oSites);
|
|
||||||
acceleratorCopyFromDevice(all_ip_p, &all_ip_h[0], (uint64_t)n * oSites * sizeof(inner_t));
|
|
||||||
|
|
||||||
// Reduce on host: sum over oSites, then collapse SIMD lanes via Reduce(TensorRemove(...))
|
|
||||||
// TensorRemove strips the iSinglet tensor wrapper to expose the SIMD scalar type.
|
|
||||||
// Reduce sums all nsimd lanes and returns a plain scalar (RealD or ComplexD).
|
|
||||||
std::vector<ComplexD> raw(n);
|
|
||||||
for (int j = 0; j < n; ++j) {
|
|
||||||
inner_t sum = Zero();
|
|
||||||
for (uint64_t ss = 0; ss < oSites; ++ss)
|
|
||||||
sum += all_ip_h[(uint64_t)j * oSites + ss];
|
|
||||||
raw[j] = ComplexD(Reduce(TensorRemove(sum)));
|
|
||||||
}
|
|
||||||
grid->GlobalSumVector(&raw[0], n);
|
|
||||||
for (int j = 0; j < n; ++j) c[j] = raw[j];
|
|
||||||
|
|
||||||
for (int j = 0; j < n; ++j) h_basis_v[j].ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
void batchUpdate(Field &vec,
|
|
||||||
const std::vector<Field> &basis,
|
|
||||||
const std::vector<ComplexD> &c)
|
|
||||||
{
|
|
||||||
int n = (int)basis.size();
|
|
||||||
if (n == 0) return;
|
|
||||||
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
typedef decltype(basis[0].View(AcceleratorRead)) View;
|
|
||||||
|
|
||||||
GridBase *grid = vec.Grid();
|
|
||||||
uint64_t oSites = grid->oSites();
|
|
||||||
uint64_t nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
// Split complex coefficients into real/imag double arrays on device.
|
|
||||||
// Using doubles avoids potential ComplexD-device-code compatibility issues.
|
|
||||||
hostVector<double> h_re(n), h_im(n);
|
|
||||||
deviceVector<double> d_re(n), d_im(n);
|
|
||||||
for (int k = 0; k < n; ++k) {
|
|
||||||
h_re[k] = c[k].real();
|
|
||||||
h_im[k] = c[k].imag();
|
|
||||||
}
|
|
||||||
acceleratorCopyToDevice(&h_re[0], &d_re[0], n * sizeof(double));
|
|
||||||
acceleratorCopyToDevice(&h_im[0], &d_im[0], n * sizeof(double));
|
|
||||||
double *re_p = &d_re[0];
|
|
||||||
double *im_p = &d_im[0];
|
|
||||||
|
|
||||||
// Basis views
|
|
||||||
hostVector<View> h_basis_v(n);
|
|
||||||
deviceVector<View> d_basis_v(n);
|
|
||||||
for (int k = 0; k < n; ++k) {
|
|
||||||
h_basis_v[k] = basis[k].View(AcceleratorRead);
|
|
||||||
acceleratorPut(d_basis_v[k], h_basis_v[k]);
|
|
||||||
}
|
|
||||||
View *basis_vp = &d_basis_v[0];
|
|
||||||
|
|
||||||
// Single kernel: vec[ss] -= sum_k (re[k] + i*im[k]) * basis[k][ss]
|
|
||||||
autoView(vec_v, vec, AcceleratorWrite);
|
|
||||||
accelerator_for(ss, oSites, nsimd, {
|
|
||||||
auto v = coalescedRead(vec_v[ss]);
|
|
||||||
for (int k = 0; k < n; ++k) {
|
|
||||||
auto b = coalescedRead(basis_vp[k][ss]);
|
|
||||||
v = v - re_p[k] * b - timesI(im_p[k] * b);
|
|
||||||
}
|
|
||||||
coalescedWrite(vec_v[ss], v);
|
|
||||||
});
|
|
||||||
|
|
||||||
for (int k = 0; k < n; ++k) h_basis_v[k].ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Full reorthogonalization using two-pass Classical Gram-Schmidt (CGS2).
|
|
||||||
// Each pass calls batchInnerProducts (1 GPU sync) + batchUpdate (1 sync),
|
|
||||||
// replacing the old 2n GPU syncs per pass from sequential MGS.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
void reorthogonalize(Field &vec, const std::vector<Field> &basis)
|
|
||||||
{
|
|
||||||
if (basis.empty()) return;
|
|
||||||
std::vector<ComplexD> c;
|
|
||||||
for (int pass = 0; pass < 2; ++pass) {
|
|
||||||
batchInnerProducts(vec, basis, c);
|
|
||||||
batchUpdate(vec, basis, c);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Implicit restart: given the Nm-step bidiagonalization and its SVD,
|
|
||||||
// compress to Nk steps via implicit QR shifts applied to B_k.
|
|
||||||
//
|
|
||||||
// The "shifts" are the Nm - Nk singular values we want to deflate
|
|
||||||
// (those NOT in the desired set). We apply them as implicit QR steps
|
|
||||||
// to the bidiagonal matrix, then update the lattice bases accordingly.
|
|
||||||
//
|
|
||||||
// After this call:
|
|
||||||
// V[0..Nk], U[0..Nk-1], alpha[0..Nk-1], beta[0..Nk-1] are updated.
|
|
||||||
// betaRestart ← new beta_Nk coupling for the next extension.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
void implicitRestart(int k, int p,
|
|
||||||
const Eigen::VectorXd &sigma,
|
|
||||||
const Eigen::MatrixXd &X,
|
|
||||||
const Eigen::MatrixXd &Y,
|
|
||||||
const Eigen::VectorXi &order,
|
|
||||||
RealD betaK,
|
|
||||||
RealD &betaRestart)
|
|
||||||
{
|
|
||||||
// Thick restart (Baglama & Reichel, Sec. 2.2):
|
|
||||||
//
|
|
||||||
// Given B_k = X Sigma Y^T, define the new p-step basis by:
|
|
||||||
// V^+_i = V_k * y_{order(i)} (right sing. vec. of B_k)
|
|
||||||
// U^+_i = U_k * x_{order(i)} (left sing. vec. of B_k)
|
|
||||||
//
|
|
||||||
// Then A V^+_i = A V_k y_{order(i)} = U_k B_k y_{order(i)}
|
|
||||||
// = sigma_{order(i)} U_k x_{order(i)} = sigma_{order(i)} U^+_i
|
|
||||||
//
|
|
||||||
// So B_p^+ = diag(sigma_{order(0)}, ..., sigma_{order(p-1)}) — DIAGONAL,
|
|
||||||
// all internal betas are zero.
|
|
||||||
//
|
|
||||||
// The residual coupling comes from A^dag U_k = V_k B_k^T + betaK V[k] e_{k-1}^T:
|
|
||||||
// A^dag U^+_{p-1} - sigma_{order(p-1)} V^+_{p-1}
|
|
||||||
// = V_k (B_k^T x_{order(p-1)} - sigma_{order(p-1)} y_{order(p-1)})
|
|
||||||
// + betaK * X(k-1, order(p-1)) * V[k]
|
|
||||||
// = betaK * X(k-1, order(p-1)) * V[k] (since B_k^T x_j = sigma_j y_j)
|
|
||||||
//
|
|
||||||
// Therefore: betaRestart = |betaK * X(k-1, order(p-1))|
|
|
||||||
// V[p] = sign(X(k-1, order(p-1))) * V[k]
|
|
||||||
|
|
||||||
// ---- Build new lattice vectors ----
|
|
||||||
std::vector<Field> Vnew, Unew;
|
|
||||||
Vnew.reserve(p + 1);
|
|
||||||
Unew.reserve(p);
|
|
||||||
|
|
||||||
for (int i = 0; i < p; ++i) {
|
|
||||||
int idx = order(i);
|
|
||||||
Field vi(Grid); vi = Zero();
|
|
||||||
for (int j = 0; j < k; ++j)
|
|
||||||
vi = vi + Y(j, idx) * V[j];
|
|
||||||
Vnew.push_back(vi);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < p; ++i) {
|
|
||||||
int idx = order(i);
|
|
||||||
Field ui(Grid); ui = Zero();
|
|
||||||
for (int j = 0; j < k; ++j)
|
|
||||||
ui = ui + X(j, idx) * U[j];
|
|
||||||
Unew.push_back(ui);
|
|
||||||
}
|
|
||||||
|
|
||||||
// New v_{p} (0-indexed: V[p]) = sign * V[k]
|
|
||||||
// From A^dag U_k = V_k B_k^T + betaK V[k] e_{k-1}^T:
|
|
||||||
// A^dag U^+_j - sigma_j V^+_j = betaK * X(k-1, order(j)) * V[k]
|
|
||||||
// The last Ritz pair (j=p-1) defines betaRestart and the sign of V[p].
|
|
||||||
// All p couplings (j=0..p-1) are stored in fvec so that buildFullB can
|
|
||||||
// reconstruct the exact column p of U^dag A V after the next extension.
|
|
||||||
RealD coeff = betaK * X(k - 1, order(p - 1));
|
|
||||||
betaRestart = std::abs(coeff);
|
|
||||||
RealD sgn = (coeff >= 0.0) ? 1.0 : -1.0;
|
|
||||||
|
|
||||||
fvec.resize(p);
|
|
||||||
for (int j = 0; j < p; ++j)
|
|
||||||
fvec[j] = betaK * X(k - 1, order(j)) * sgn;
|
|
||||||
// fvec[p-1] == betaRestart by construction
|
|
||||||
restart_col = p;
|
|
||||||
|
|
||||||
Field vp(Grid);
|
|
||||||
if (betaRestart > 1.0e-14) {
|
|
||||||
vp = sgn * V[k];
|
|
||||||
} else {
|
|
||||||
betaRestart = 0.0;
|
|
||||||
vp = Zero();
|
|
||||||
}
|
|
||||||
Vnew.push_back(vp); // V[p]
|
|
||||||
|
|
||||||
// ---- New alpha, beta ----
|
|
||||||
// B_p^+ is diagonal: alpha^+_i = sigma_{order(i)}, all internal beta = 0
|
|
||||||
std::vector<RealD> alpha_new(p), beta_new(p);
|
|
||||||
for (int i = 0; i < p; ++i) alpha_new[i] = sigma(order(i));
|
|
||||||
for (int i = 0; i < p - 1; ++i) beta_new[i] = 0.0;
|
|
||||||
beta_new[p - 1] = betaRestart;
|
|
||||||
|
|
||||||
// ---- Commit new state ----
|
|
||||||
V = Vnew;
|
|
||||||
U = Unew;
|
|
||||||
alpha = alpha_new;
|
|
||||||
beta = beta_new;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage
|
|
||||||
<< "IRLBA restart: compressed to " << p << " steps,"
|
|
||||||
<< " new beta_p = " << betaRestart << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
// Extract the desired singular triplets into the public output vectors.
|
|
||||||
// ------------------------------------------------------------------
|
|
||||||
void extractTriplets(int m,
|
|
||||||
const Eigen::VectorXd &sigma,
|
|
||||||
const Eigen::MatrixXd &X,
|
|
||||||
const Eigen::MatrixXd &Y,
|
|
||||||
const Eigen::VectorXi &order,
|
|
||||||
int nout)
|
|
||||||
{
|
|
||||||
singularValues.resize(nout);
|
|
||||||
leftVectors.clear(); leftVectors.reserve(nout);
|
|
||||||
rightVectors.clear(); rightVectors.reserve(nout);
|
|
||||||
|
|
||||||
for (int i = 0; i < nout; ++i) {
|
|
||||||
int idx = order(i);
|
|
||||||
singularValues[i] = sigma(idx);
|
|
||||||
|
|
||||||
// Left singular vector of A: svec_L = U_m * x_i
|
|
||||||
Field svL(Grid); svL = Zero();
|
|
||||||
for (int j = 0; j < m && j < (int)U.size(); ++j)
|
|
||||||
svL = svL + X(j, idx) * U[j];
|
|
||||||
leftVectors.push_back(svL);
|
|
||||||
|
|
||||||
// Right singular vector of A: svec_R = V_m * y_i
|
|
||||||
Field svR(Grid); svR = Zero();
|
|
||||||
for (int j = 0; j < m && j < (int)V.size(); ++j)
|
|
||||||
svR = svR + Y(j, idx) * V[j];
|
|
||||||
rightVectors.push_back(svR);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,732 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/SchurRedBlack.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_SCHUR_RED_BLACK_H
|
|
||||||
#define GRID_SCHUR_RED_BLACK_H
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Red black Schur decomposition
|
|
||||||
*
|
|
||||||
* M = (Mee Meo) = (1 0 ) (Mee 0 ) (1 Mee^{-1} Meo)
|
|
||||||
* (Moe Moo) (Moe Mee^-1 1 ) (0 Moo-Moe Mee^-1 Meo) (0 1 )
|
|
||||||
* = L D U
|
|
||||||
*
|
|
||||||
* L^-1 = (1 0 )
|
|
||||||
* (-MoeMee^{-1} 1 )
|
|
||||||
* L^{dag} = ( 1 Mee^{-dag} Moe^{dag} )
|
|
||||||
* ( 0 1 )
|
|
||||||
* L^{-d} = ( 1 -Mee^{-dag} Moe^{dag} )
|
|
||||||
* ( 0 1 )
|
|
||||||
*
|
|
||||||
* U^-1 = (1 -Mee^{-1} Meo)
|
|
||||||
* (0 1 )
|
|
||||||
* U^{dag} = ( 1 0)
|
|
||||||
* (Meo^dag Mee^{-dag} 1)
|
|
||||||
* U^{-dag} = ( 1 0)
|
|
||||||
* (-Meo^dag Mee^{-dag} 1)
|
|
||||||
***********************
|
|
||||||
* M psi = eta
|
|
||||||
***********************
|
|
||||||
*Odd
|
|
||||||
* i) D_oo psi_o = L^{-1} eta_o
|
|
||||||
* eta_o' = (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
|
|
||||||
*
|
|
||||||
* Wilson:
|
|
||||||
* (D_oo)^{\dag} D_oo psi_o = (D_oo)^dag L^{-1} eta_o
|
|
||||||
* Stag:
|
|
||||||
* D_oo psi_o = L^{-1} eta = (eta_o - Moe Mee^{-1} eta_e)
|
|
||||||
*
|
|
||||||
* L^-1 eta_o= (1 0 ) (e
|
|
||||||
* (-MoeMee^{-1} 1 )
|
|
||||||
*
|
|
||||||
*Even
|
|
||||||
* ii) Mee psi_e + Meo psi_o = src_e
|
|
||||||
*
|
|
||||||
* => sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* TODO: Other options:
|
|
||||||
*
|
|
||||||
* a) change checkerboards for Schur e<->o
|
|
||||||
*
|
|
||||||
* Left precon by Moo^-1
|
|
||||||
* b) Doo^{dag} M_oo^-dag Moo^-1 Doo psi_0 = (D_oo)^dag M_oo^-dag Moo^-1 L^{-1} eta_o
|
|
||||||
* eta_o' = (D_oo)^dag M_oo^-dag Moo^-1 (eta_o - Moe Mee^{-1} eta_e)
|
|
||||||
*
|
|
||||||
* Right precon by Moo^-1
|
|
||||||
* c) M_oo^-dag Doo^{dag} Doo Moo^-1 phi_0 = M_oo^-dag (D_oo)^dag L^{-1} eta_o
|
|
||||||
* eta_o' = M_oo^-dag (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
|
|
||||||
* psi_o = M_oo^-1 phi_o
|
|
||||||
* TODO: Deflation
|
|
||||||
*/
|
|
||||||
namespace Grid {
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Use base class to share code
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Take a matrix and form a Red Black solver calling a Herm solver
|
|
||||||
// Use of RB info prevents making SchurRedBlackSolve conform to standard interface
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class SchurRedBlackBase {
|
|
||||||
protected:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
OperatorFunction<Field> & _HermitianRBSolver;
|
|
||||||
int CBfactorise;
|
|
||||||
bool subGuess;
|
|
||||||
bool useSolnAsInitGuess; // if true user-supplied solution vector is used as initial guess for solver
|
|
||||||
public:
|
|
||||||
|
|
||||||
SchurRedBlackBase(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false) :
|
|
||||||
_HermitianRBSolver(HermitianRBSolver),
|
|
||||||
useSolnAsInitGuess(_solnAsInitGuess)
|
|
||||||
{
|
|
||||||
CBfactorise = 0;
|
|
||||||
subtractGuess(initSubGuess);
|
|
||||||
};
|
|
||||||
void subtractGuess(const bool initSubGuess)
|
|
||||||
{
|
|
||||||
subGuess = initSubGuess;
|
|
||||||
}
|
|
||||||
bool isSubtractGuess(void)
|
|
||||||
{
|
|
||||||
return subGuess;
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Shared code
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
void operator() (Matrix & _Matrix,const Field &in, Field &out){
|
|
||||||
ZeroGuesser<Field> guess;
|
|
||||||
(*this)(_Matrix,in,out,guess);
|
|
||||||
}
|
|
||||||
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out)
|
|
||||||
{
|
|
||||||
ZeroGuesser<Field> guess;
|
|
||||||
(*this)(_Matrix,in,out,guess);
|
|
||||||
}
|
|
||||||
|
|
||||||
void RedBlackSource(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &src_o)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
Field tmp(grid);
|
|
||||||
int nblock = in.size();
|
|
||||||
for(int b=0;b<nblock;b++){
|
|
||||||
RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// James can write his own deflated guesser
|
|
||||||
// with optimised code for the inner products
|
|
||||||
// RedBlackSolveSplitGrid();
|
|
||||||
// RedBlackSolve(_Matrix,src_o,sol_o);
|
|
||||||
|
|
||||||
void RedBlackSolution(Matrix &_Matrix, const std::vector<Field> &in, const std::vector<Field> &sol_o, std::vector<Field> &out)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
Field tmp(grid);
|
|
||||||
int nblock = in.size();
|
|
||||||
for(int b=0;b<nblock;b++) {
|
|
||||||
pickCheckerboard(Even,tmp,in[b]);
|
|
||||||
RedBlackSolution(_Matrix,sol_o[b],tmp,out[b]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Guesser>
|
|
||||||
void operator()(Matrix &_Matrix, const std::vector<Field> &in, std::vector<Field> &out,Guesser &guess)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
int nblock = in.size();
|
|
||||||
|
|
||||||
std::vector<Field> src_o(nblock,grid);
|
|
||||||
std::vector<Field> sol_o(nblock,grid);
|
|
||||||
|
|
||||||
std::vector<Field> guess_save;
|
|
||||||
|
|
||||||
Field resid(fgrid);
|
|
||||||
Field tmp(grid);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// Prepare RedBlack source
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
RedBlackSource(_Matrix,in,src_o);
|
|
||||||
// for(int b=0;b<nblock;b++){
|
|
||||||
// RedBlackSource(_Matrix,in[b],tmp,src_o[b]);
|
|
||||||
// }
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// Make the guesses
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
if ( subGuess ) guess_save.resize(nblock,grid);
|
|
||||||
|
|
||||||
|
|
||||||
if(useSolnAsInitGuess) {
|
|
||||||
for(int b=0;b<nblock;b++){
|
|
||||||
pickCheckerboard(Odd, sol_o[b], out[b]);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
guess(src_o, sol_o);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( subGuess ) {
|
|
||||||
for(int b=0;b<nblock;b++){
|
|
||||||
guess_save[b] = sol_o[b];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
//////////////////////////////////////////////////////////////
|
|
||||||
// Call the block solver
|
|
||||||
//////////////////////////////////////////////////////////////
|
|
||||||
std::cout<<GridLogMessage << "SchurRedBlackBase calling the solver for "<<nblock<<" RHS" <<std::endl;
|
|
||||||
RedBlackSolve(_Matrix,src_o,sol_o);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// A2A boolean behavioural control & reconstruct other checkerboard
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
for(int b=0;b<nblock;b++) {
|
|
||||||
|
|
||||||
if (subGuess) sol_o[b] = sol_o[b] - guess_save[b];
|
|
||||||
|
|
||||||
///////// Needs even source //////////////
|
|
||||||
pickCheckerboard(Even,tmp,in[b]);
|
|
||||||
RedBlackSolution(_Matrix,sol_o[b],tmp,out[b]);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Check unprec residual if possible
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
if ( ! subGuess ) {
|
|
||||||
_Matrix.M(out[b],resid);
|
|
||||||
resid = resid-in[b];
|
|
||||||
RealD ns = norm2(in[b]);
|
|
||||||
RealD nr = norm2(resid);
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage<< "SchurRedBlackBase solver true unprec resid["<<b<<"] "<<std::sqrt(nr/ns) << std::endl;
|
|
||||||
} else {
|
|
||||||
std::cout<<GridLogMessage<< "SchurRedBlackBase Guess subtracted after solve["<<b<<"] " << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
template<class Guesser>
|
|
||||||
void operator() (Matrix & _Matrix,const Field &in, Field &out,Guesser &guess){
|
|
||||||
|
|
||||||
// FIXME CGdiagonalMee not implemented virtual function
|
|
||||||
// FIXME use CBfactorise to control schur decomp
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field resid(fgrid);
|
|
||||||
Field src_o(grid);
|
|
||||||
Field src_e(grid);
|
|
||||||
Field sol_o(grid);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// RedBlack source
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
RedBlackSource(_Matrix,in,src_e,src_o);
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
// Construct the guess
|
|
||||||
////////////////////////////////
|
|
||||||
if(useSolnAsInitGuess) {
|
|
||||||
pickCheckerboard(Odd, sol_o, out);
|
|
||||||
} else {
|
|
||||||
guess(src_o,sol_o);
|
|
||||||
}
|
|
||||||
|
|
||||||
Field guess_save(grid);
|
|
||||||
guess_save = sol_o;
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////
|
|
||||||
// Call the red-black solver
|
|
||||||
//////////////////////////////////////////////////////////////
|
|
||||||
RedBlackSolve(_Matrix,src_o,sol_o);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// Fionn A2A boolean behavioural control
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
if (subGuess) sol_o= sol_o-guess_save;
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// RedBlack solution needs the even source
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
RedBlackSolution(_Matrix,sol_o,src_e,out);
|
|
||||||
|
|
||||||
// Verify the unprec residual
|
|
||||||
if ( ! subGuess ) {
|
|
||||||
_Matrix.M(out,resid);
|
|
||||||
resid = resid-in;
|
|
||||||
RealD ns = norm2(in);
|
|
||||||
RealD nr = norm2(resid);
|
|
||||||
|
|
||||||
std::cout<<GridLogMessage << "SchurRedBlackBase solver true unprec resid "<< std::sqrt(nr/ns) << std::endl;
|
|
||||||
} else {
|
|
||||||
std::cout << GridLogMessage << "SchurRedBlackBase Guess subtracted after solve." << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Override in derived.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
virtual void RedBlackSource (Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o) =0;
|
|
||||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol) =0;
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o) =0;
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)=0;
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class SchurRedBlackStaggeredSolve : public SchurRedBlackBase<Field> {
|
|
||||||
public:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
|
|
||||||
SchurRedBlackStaggeredSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false)
|
|
||||||
: SchurRedBlackBase<Field> (HermitianRBSolver,initSubGuess,_solnAsInitGuess)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Override RedBlack specialisation
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field Mtmp(grid);
|
|
||||||
|
|
||||||
pickCheckerboard(Even,src_e,src);
|
|
||||||
pickCheckerboard(Odd ,src_o,src);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// src_o = (source_o - Moe MeeInv source_e)
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
|
||||||
tmp=src_o-Mtmp; GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
|
||||||
|
|
||||||
_Matrix.Mooee(tmp,src_o); // Extra factor of "m" in source from dumb choice of matrix norm.
|
|
||||||
}
|
|
||||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e_c,Field &sol)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field sol_e(grid);
|
|
||||||
Field src_e(grid);
|
|
||||||
|
|
||||||
src_e = src_e_c; // Const correctness
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
_Matrix.Meooe(sol_o,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
src_e = src_e-tmp; GRID_ASSERT( src_e.Checkerboard() ==Even);
|
|
||||||
_Matrix.MooeeInv(src_e,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
|
|
||||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
setCheckerboard(sol,sol_o); GRID_ASSERT( sol_o.Checkerboard() ==Odd );
|
|
||||||
}
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
|
||||||
{
|
|
||||||
SchurStaggeredOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); GRID_ASSERT(sol_o.Checkerboard()==Odd);
|
|
||||||
};
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
|
||||||
{
|
|
||||||
SchurStaggeredOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
template<class Field> using SchurRedBlackStagSolve = SchurRedBlackStaggeredSolve<Field>;
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Site diagonal has Mooee on it.
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class SchurRedBlackDiagMooeeSolve : public SchurRedBlackBase<Field> {
|
|
||||||
public:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
|
|
||||||
SchurRedBlackDiagMooeeSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false)
|
|
||||||
: SchurRedBlackBase<Field> (HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
|
|
||||||
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Override RedBlack specialisation
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field Mtmp(grid);
|
|
||||||
|
|
||||||
pickCheckerboard(Even,src_e,src);
|
|
||||||
pickCheckerboard(Odd ,src_o,src);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
|
||||||
tmp=src_o-Mtmp; GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
|
||||||
|
|
||||||
// get the right MpcDag
|
|
||||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
_HermOpEO.MpcDag(tmp,src_o); GRID_ASSERT(src_o.Checkerboard() ==Odd);
|
|
||||||
|
|
||||||
}
|
|
||||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field sol_e(grid);
|
|
||||||
Field src_e_i(grid);
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
_Matrix.Meooe(sol_o,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
src_e_i = src_e-tmp; GRID_ASSERT( src_e_i.Checkerboard() ==Even);
|
|
||||||
_Matrix.MooeeInv(src_e_i,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
|
|
||||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
setCheckerboard(sol,sol_o); GRID_ASSERT( sol_o.Checkerboard() ==Odd );
|
|
||||||
}
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
|
||||||
{
|
|
||||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o); GRID_ASSERT(sol_o.Checkerboard()==Odd);
|
|
||||||
};
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
|
||||||
{
|
|
||||||
SchurDiagMooeeOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class NonHermitianSchurRedBlackDiagMooeeSolve : public SchurRedBlackBase<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
|
|
||||||
NonHermitianSchurRedBlackDiagMooeeSolve(OperatorFunction<Field>& RBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false)
|
|
||||||
: SchurRedBlackBase<Field>(RBSolver, initSubGuess, _solnAsInitGuess) {};
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Override RedBlack specialisation
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
virtual void RedBlackSource(Matrix& _Matrix, const Field& src, Field& src_e, Field& src_o)
|
|
||||||
{
|
|
||||||
GridBase* grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase* fgrid = _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field Mtmp(grid);
|
|
||||||
|
|
||||||
pickCheckerboard(Even, src_e, src);
|
|
||||||
pickCheckerboard(Odd , src_o, src);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(src_e, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
|
||||||
_Matrix.Meooe (tmp, Mtmp); GRID_ASSERT( Mtmp.Checkerboard() == Odd );
|
|
||||||
src_o -= Mtmp; GRID_ASSERT( src_o.Checkerboard() == Odd );
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void RedBlackSolution(Matrix& _Matrix, const Field& sol_o, const Field& src_e, Field& sol)
|
|
||||||
{
|
|
||||||
GridBase* grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase* fgrid = _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field sol_e(grid);
|
|
||||||
Field src_e_i(grid);
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
_Matrix.Meooe(sol_o, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
|
||||||
src_e_i = src_e - tmp; GRID_ASSERT( src_e_i.Checkerboard() == Even );
|
|
||||||
_Matrix.MooeeInv(src_e_i, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
|
||||||
|
|
||||||
setCheckerboard(sol, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
|
||||||
setCheckerboard(sol, sol_o); GRID_ASSERT( sol_o.Checkerboard() == Odd );
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void RedBlackSolve(Matrix& _Matrix, const Field& src_o, Field& sol_o)
|
|
||||||
{
|
|
||||||
NonHermitianSchurDiagMooeeOperator<Matrix,Field> _OpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o); GRID_ASSERT(sol_o.Checkerboard() == Odd);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void RedBlackSolve(Matrix& _Matrix, const std::vector<Field>& src_o, std::vector<Field>& sol_o)
|
|
||||||
{
|
|
||||||
NonHermitianSchurDiagMooeeOperator<Matrix,Field> _OpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Site diagonal is identity, left preconditioned by Mee^inv
|
|
||||||
// ( 1 - Mee^inv Meo Moo^inv Moe ) phi = Mee_inv ( Mee - Meo Moo^inv Moe Mee^inv ) phi = Mee_inv eta
|
|
||||||
//
|
|
||||||
// Solve:
|
|
||||||
// ( 1 - Mee^inv Meo Moo^inv Moe )^dag ( 1 - Mee^inv Meo Moo^inv Moe ) phi = ( 1 - Mee^inv Meo Moo^inv Moe )^dag Mee_inv eta
|
|
||||||
//
|
|
||||||
// Old notation e<->o
|
|
||||||
//
|
|
||||||
// Left precon by Moo^-1
|
|
||||||
// b) (Doo^{dag} M_oo^-dag) (Moo^-1 Doo) psi_o = [ (D_oo)^dag M_oo^-dag ] Moo^-1 L^{-1} eta_o
|
|
||||||
// eta_o' = (D_oo)^dag M_oo^-dag Moo^-1 (eta_o - Moe Mee^{-1} eta_e)
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class SchurRedBlackDiagOneSolve : public SchurRedBlackBase<Field> {
|
|
||||||
public:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations Schur trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
SchurRedBlackDiagOneSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false)
|
|
||||||
: SchurRedBlackBase<Field>(HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
|
|
||||||
|
|
||||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
SchurDiagOneOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field Mtmp(grid);
|
|
||||||
|
|
||||||
pickCheckerboard(Even,src_e,src);
|
|
||||||
pickCheckerboard(Odd ,src_o,src);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// src_o = Mpcdag *MooeeInv * (source_o - Moe MeeInv source_e)
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
|
||||||
Mtmp=src_o-Mtmp;
|
|
||||||
_Matrix.MooeeInv(Mtmp,tmp); GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
|
||||||
|
|
||||||
// get the right MpcDag
|
|
||||||
_HermOpEO.MpcDag(tmp,src_o); GRID_ASSERT(src_o.Checkerboard() ==Odd);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field sol_e(grid);
|
|
||||||
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
_Matrix.Meooe(sol_o,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
tmp = src_e-tmp; GRID_ASSERT( src_e.Checkerboard() ==Even);
|
|
||||||
_Matrix.MooeeInv(tmp,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
|
|
||||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
setCheckerboard(sol,sol_o); GRID_ASSERT( sol_o.Checkerboard() ==Odd );
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
|
||||||
{
|
|
||||||
SchurDiagOneOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
|
||||||
};
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
|
||||||
{
|
|
||||||
SchurDiagOneOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Site diagonal is identity, right preconditioned by Mee^inv
|
|
||||||
// ( 1 - Meo Moo^inv Moe Mee^inv ) phi =( 1 - Meo Moo^inv Moe Mee^inv ) Mee psi = = eta = eta
|
|
||||||
//=> psi = MeeInv phi
|
|
||||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class Field> class SchurRedBlackDiagTwoSolve : public SchurRedBlackBase<Field> {
|
|
||||||
public:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations Schur trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
SchurRedBlackDiagTwoSolve(OperatorFunction<Field> &HermitianRBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false)
|
|
||||||
: SchurRedBlackBase<Field>(HermitianRBSolver,initSubGuess,_solnAsInitGuess) {};
|
|
||||||
|
|
||||||
virtual void RedBlackSource(Matrix & _Matrix,const Field &src, Field &src_e,Field &src_o)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field Mtmp(grid);
|
|
||||||
|
|
||||||
pickCheckerboard(Even,src_e,src);
|
|
||||||
pickCheckerboard(Odd ,src_o,src);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(src_e,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
_Matrix.Meooe (tmp,Mtmp); GRID_ASSERT( Mtmp.Checkerboard() ==Odd);
|
|
||||||
tmp=src_o-Mtmp; GRID_ASSERT( tmp.Checkerboard() ==Odd);
|
|
||||||
|
|
||||||
// get the right MpcDag
|
|
||||||
_HermOpEO.MpcDag(tmp,src_o); GRID_ASSERT(src_o.Checkerboard() ==Odd);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void RedBlackSolution(Matrix & _Matrix,const Field &sol_o, const Field &src_e,Field &sol)
|
|
||||||
{
|
|
||||||
GridBase *grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase *fgrid= _Matrix.Grid();
|
|
||||||
|
|
||||||
Field sol_o_i(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
Field sol_e(grid);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// MooeeInv due to pecond
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(sol_o,tmp);
|
|
||||||
sol_o_i = tmp;
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
_Matrix.Meooe(sol_o_i,tmp); GRID_ASSERT( tmp.Checkerboard() ==Even);
|
|
||||||
tmp = src_e-tmp; GRID_ASSERT( src_e.Checkerboard() ==Even);
|
|
||||||
_Matrix.MooeeInv(tmp,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
|
|
||||||
setCheckerboard(sol,sol_e); GRID_ASSERT( sol_e.Checkerboard() ==Even);
|
|
||||||
setCheckerboard(sol,sol_o_i); GRID_ASSERT( sol_o_i.Checkerboard() ==Odd );
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const Field &src_o, Field &sol_o)
|
|
||||||
{
|
|
||||||
SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
|
||||||
};
|
|
||||||
virtual void RedBlackSolve (Matrix & _Matrix,const std::vector<Field> &src_o, std::vector<Field> &sol_o)
|
|
||||||
{
|
|
||||||
SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_HermOpEO,src_o,sol_o);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Field> class NonHermitianSchurRedBlackDiagTwoSolve : public SchurRedBlackBase<Field>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
typedef CheckerBoardedSparseMatrixBase<Field> Matrix;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Wrap the usual normal equations Schur trick
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
NonHermitianSchurRedBlackDiagTwoSolve(OperatorFunction<Field>& RBSolver, const bool initSubGuess = false,
|
|
||||||
const bool _solnAsInitGuess = false)
|
|
||||||
: SchurRedBlackBase<Field>(RBSolver, initSubGuess, _solnAsInitGuess) {};
|
|
||||||
|
|
||||||
virtual void RedBlackSource(Matrix& _Matrix, const Field& src, Field& src_e, Field& src_o)
|
|
||||||
{
|
|
||||||
GridBase* grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase* fgrid = _Matrix.Grid();
|
|
||||||
|
|
||||||
Field tmp(grid);
|
|
||||||
Field Mtmp(grid);
|
|
||||||
|
|
||||||
pickCheckerboard(Even, src_e, src);
|
|
||||||
pickCheckerboard(Odd , src_o, src);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// src_o = Mdag * (source_o - Moe MeeInv source_e)
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(src_e, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
|
||||||
_Matrix.Meooe (tmp, Mtmp); GRID_ASSERT( Mtmp.Checkerboard() == Odd );
|
|
||||||
src_o -= Mtmp; GRID_ASSERT( src_o.Checkerboard() == Odd );
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void RedBlackSolution(Matrix& _Matrix, const Field& sol_o, const Field& src_e, Field& sol)
|
|
||||||
{
|
|
||||||
GridBase* grid = _Matrix.RedBlackGrid();
|
|
||||||
GridBase* fgrid = _Matrix.Grid();
|
|
||||||
|
|
||||||
Field sol_o_i(grid);
|
|
||||||
Field tmp(grid);
|
|
||||||
Field sol_e(grid);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// MooeeInv due to pecond
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
_Matrix.MooeeInv(sol_o, tmp);
|
|
||||||
sol_o_i = tmp;
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
_Matrix.Meooe(sol_o_i, tmp); GRID_ASSERT( tmp.Checkerboard() == Even );
|
|
||||||
tmp = src_e - tmp; GRID_ASSERT( src_e.Checkerboard() == Even );
|
|
||||||
_Matrix.MooeeInv(tmp, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
|
||||||
|
|
||||||
setCheckerboard(sol, sol_e); GRID_ASSERT( sol_e.Checkerboard() == Even );
|
|
||||||
setCheckerboard(sol, sol_o_i); GRID_ASSERT( sol_o_i.Checkerboard() == Odd );
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual void RedBlackSolve(Matrix& _Matrix, const Field& src_o, Field& sol_o)
|
|
||||||
{
|
|
||||||
NonHermitianSchurDiagTwoOperator<Matrix,Field> _OpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o);
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual void RedBlackSolve(Matrix& _Matrix, const std::vector<Field>& src_o, std::vector<Field>& sol_o)
|
|
||||||
{
|
|
||||||
NonHermitianSchurDiagTwoOperator<Matrix,Field> _OpEO(_Matrix);
|
|
||||||
this->_HermitianRBSolver(_OpEO, src_o, sol_o);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,931 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Chulwoo Jung <chulwoo@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LANC_H
|
|
||||||
#define GRID_LANC_H
|
|
||||||
|
|
||||||
#include <string.h> //memset
|
|
||||||
|
|
||||||
#ifdef USE_LAPACK
|
|
||||||
#ifdef USE_MKL
|
|
||||||
#include<mkl_lapack.h>
|
|
||||||
#else
|
|
||||||
void LAPACK_dstegr (char *jobz, char *range, int *n, double *d, double *e,
|
|
||||||
double *vl, double *vu, int *il, int *iu, double *abstol,
|
|
||||||
int *m, double *w, double *z, int *ldz, int *isuppz,
|
|
||||||
double *work, int *lwork, int *iwork, int *liwork,
|
|
||||||
int *info);
|
|
||||||
//#include <lapacke/lapacke.h>
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
//#include <Grid/algorithms/densematrix/DenseMatrix.h>
|
|
||||||
|
|
||||||
// eliminate temorary vector in calc()
|
|
||||||
#define MEM_SAVE
|
|
||||||
|
|
||||||
namespace Grid
|
|
||||||
{
|
|
||||||
|
|
||||||
struct Bisection
|
|
||||||
{
|
|
||||||
|
|
||||||
#if 0
|
|
||||||
static void get_eig2 (int row_num, std::vector < RealD > &ALPHA,
|
|
||||||
std::vector < RealD > &BETA,
|
|
||||||
std::vector < RealD > &eig)
|
|
||||||
{
|
|
||||||
int i, j;
|
|
||||||
std::vector < RealD > evec1 (row_num + 3);
|
|
||||||
std::vector < RealD > evec2 (row_num + 3);
|
|
||||||
RealD eps2;
|
|
||||||
ALPHA[1] = 0.;
|
|
||||||
BETHA[1] = 0.;
|
|
||||||
for (i = 0; i < row_num - 1; i++)
|
|
||||||
{
|
|
||||||
ALPHA[i + 1] = A[i * (row_num + 1)].real ();
|
|
||||||
BETHA[i + 2] = A[i * (row_num + 1) + 1].real ();
|
|
||||||
}
|
|
||||||
ALPHA[row_num] = A[(row_num - 1) * (row_num + 1)].real ();
|
|
||||||
bisec (ALPHA, BETHA, row_num, 1, row_num, 1e-10, 1e-10, evec1, eps2);
|
|
||||||
bisec (ALPHA, BETHA, row_num, 1, row_num, 1e-16, 1e-16, evec2, eps2);
|
|
||||||
|
|
||||||
// Do we really need to sort here?
|
|
||||||
int begin = 1;
|
|
||||||
int end = row_num;
|
|
||||||
int swapped = 1;
|
|
||||||
while (swapped)
|
|
||||||
{
|
|
||||||
swapped = 0;
|
|
||||||
for (i = begin; i < end; i++)
|
|
||||||
{
|
|
||||||
if (mag (evec2[i]) > mag (evec2[i + 1]))
|
|
||||||
{
|
|
||||||
swap (evec2 + i, evec2 + i + 1);
|
|
||||||
swapped = 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
end--;
|
|
||||||
for (i = end - 1; i >= begin; i--)
|
|
||||||
{
|
|
||||||
if (mag (evec2[i]) > mag (evec2[i + 1]))
|
|
||||||
{
|
|
||||||
swap (evec2 + i, evec2 + i + 1);
|
|
||||||
swapped = 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
begin++;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (i = 0; i < row_num; i++)
|
|
||||||
{
|
|
||||||
for (j = 0; j < row_num; j++)
|
|
||||||
{
|
|
||||||
if (i == j)
|
|
||||||
H[i * row_num + j] = evec2[i + 1];
|
|
||||||
else
|
|
||||||
H[i * row_num + j] = 0.;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static void bisec (std::vector < RealD > &c,
|
|
||||||
std::vector < RealD > &b,
|
|
||||||
int n,
|
|
||||||
int m1,
|
|
||||||
int m2,
|
|
||||||
RealD eps1,
|
|
||||||
RealD relfeh, std::vector < RealD > &x, RealD & eps2)
|
|
||||||
{
|
|
||||||
std::vector < RealD > wu (n + 2);
|
|
||||||
|
|
||||||
RealD h, q, x1, xu, x0, xmin, xmax;
|
|
||||||
int i, a, k;
|
|
||||||
|
|
||||||
b[1] = 0.0;
|
|
||||||
xmin = c[n] - fabs (b[n]);
|
|
||||||
xmax = c[n] + fabs (b[n]);
|
|
||||||
for (i = 1; i < n; i++)
|
|
||||||
{
|
|
||||||
h = fabs (b[i]) + fabs (b[i + 1]);
|
|
||||||
if (c[i] + h > xmax)
|
|
||||||
xmax = c[i] + h;
|
|
||||||
if (c[i] - h < xmin)
|
|
||||||
xmin = c[i] - h;
|
|
||||||
}
|
|
||||||
xmax *= 2.;
|
|
||||||
|
|
||||||
eps2 = relfeh * ((xmin + xmax) > 0.0 ? xmax : -xmin);
|
|
||||||
if (eps1 <= 0.0)
|
|
||||||
eps1 = eps2;
|
|
||||||
eps2 = 0.5 * eps1 + 7.0 * (eps2);
|
|
||||||
x0 = xmax;
|
|
||||||
for (i = m1; i <= m2; i++)
|
|
||||||
{
|
|
||||||
x[i] = xmax;
|
|
||||||
wu[i] = xmin;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (k = m2; k >= m1; k--)
|
|
||||||
{
|
|
||||||
xu = xmin;
|
|
||||||
i = k;
|
|
||||||
do
|
|
||||||
{
|
|
||||||
if (xu < wu[i])
|
|
||||||
{
|
|
||||||
xu = wu[i];
|
|
||||||
i = m1 - 1;
|
|
||||||
}
|
|
||||||
i--;
|
|
||||||
}
|
|
||||||
while (i >= m1);
|
|
||||||
if (x0 > x[k])
|
|
||||||
x0 = x[k];
|
|
||||||
while ((x0 - xu) > 2 * relfeh * (fabs (xu) + fabs (x0)) + eps1)
|
|
||||||
{
|
|
||||||
x1 = (xu + x0) / 2;
|
|
||||||
|
|
||||||
a = 0;
|
|
||||||
q = 1.0;
|
|
||||||
for (i = 1; i <= n; i++)
|
|
||||||
{
|
|
||||||
q =
|
|
||||||
c[i] - x1 -
|
|
||||||
((q != 0.0) ? b[i] * b[i] / q : fabs (b[i]) / relfeh);
|
|
||||||
if (q < 0)
|
|
||||||
a++;
|
|
||||||
}
|
|
||||||
// printf("x1=%0.14e a=%d\n",x1,a);
|
|
||||||
if (a < k)
|
|
||||||
{
|
|
||||||
if (a < m1)
|
|
||||||
{
|
|
||||||
xu = x1;
|
|
||||||
wu[m1] = x1;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
xu = x1;
|
|
||||||
wu[a + 1] = x1;
|
|
||||||
if (x[a] > x1)
|
|
||||||
x[a] = x1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
x0 = x1;
|
|
||||||
}
|
|
||||||
printf ("x0=%0.14e xu=%0.14e k=%d\n", x0, xu, k);
|
|
||||||
x[k] = (x0 + xu) / 2;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Implicitly restarted lanczos
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
|
|
||||||
template < class Field > class SimpleLanczos
|
|
||||||
{
|
|
||||||
|
|
||||||
const RealD small = 1.0e-16;
|
|
||||||
public:
|
|
||||||
int lock;
|
|
||||||
int get;
|
|
||||||
int Niter;
|
|
||||||
int converged;
|
|
||||||
|
|
||||||
int Nstop; // Number of evecs checked for convergence
|
|
||||||
int Nk; // Number of converged sought
|
|
||||||
int Np; // Np -- Number of spare vecs in kryloc space
|
|
||||||
int Nm; // Nm -- total number of vectors
|
|
||||||
|
|
||||||
|
|
||||||
RealD OrthoTime;
|
|
||||||
|
|
||||||
RealD eresid;
|
|
||||||
|
|
||||||
// SortEigen < Field > _sort;
|
|
||||||
|
|
||||||
LinearFunction < Field > &_Linop;
|
|
||||||
|
|
||||||
// OperatorFunction < Field > &_poly;
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// Constructor
|
|
||||||
/////////////////////////
|
|
||||||
void init (void)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
// void Abort (int ff, std::vector < RealD > &evals, DenseVector < Denstd::vector < RealD > >&evecs);
|
|
||||||
|
|
||||||
SimpleLanczos (LinearFunction < Field > &Linop, // op
|
|
||||||
// OperatorFunction < Field > &poly, // polynmial
|
|
||||||
int _Nstop, // sought vecs
|
|
||||||
int _Nk, // sought vecs
|
|
||||||
int _Nm, // spare vecs
|
|
||||||
RealD _eresid, // resid in lmdue deficit
|
|
||||||
int _Niter): // Max iterations
|
|
||||||
|
|
||||||
_Linop (Linop),
|
|
||||||
// _poly (poly),
|
|
||||||
Nstop (_Nstop), Nk (_Nk), Nm (_Nm), eresid (_eresid), Niter (_Niter)
|
|
||||||
{
|
|
||||||
Np = Nm - Nk;
|
|
||||||
assert (Np > 0);
|
|
||||||
};
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// Sanity checked this routine (step) against Saad.
|
|
||||||
/////////////////////////
|
|
||||||
void RitzMatrix (std::vector < Field > &evec, int k)
|
|
||||||
{
|
|
||||||
|
|
||||||
if (1)
|
|
||||||
return;
|
|
||||||
|
|
||||||
GridBase *grid = evec[0].Grid();
|
|
||||||
Field w (grid);
|
|
||||||
std::cout << GridLogMessage << "RitzMatrix " << std::endl;
|
|
||||||
for (int i = 0; i < k; i++)
|
|
||||||
{
|
|
||||||
_Linop(evec[i], w);
|
|
||||||
// _poly(_Linop,evec[i],w);
|
|
||||||
std::cout << GridLogMessage << "[" << i << "] ";
|
|
||||||
for (int j = 0; j < k; j++)
|
|
||||||
{
|
|
||||||
ComplexD in = innerProduct (evec[j], w);
|
|
||||||
if (fabs ((double) i - j) > 1)
|
|
||||||
{
|
|
||||||
if (abs (in) > 1.0e-9)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage << "oops" << std::endl;
|
|
||||||
abort ();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
std::cout << GridLogMessage << " 0 ";
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage << " " << in << " ";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void step (std::vector < RealD > &lmd,
|
|
||||||
std::vector < RealD > &lme,
|
|
||||||
Field & last, Field & current, Field & next, uint64_t k)
|
|
||||||
{
|
|
||||||
if (lmd.size () <= k)
|
|
||||||
lmd.resize (k + Nm);
|
|
||||||
if (lme.size () <= k)
|
|
||||||
lme.resize (k + Nm);
|
|
||||||
|
|
||||||
|
|
||||||
// _poly(_Linop,current,next ); // 3. wk:=Avk−βkv_{k−1}
|
|
||||||
_Linop(current, next); // 3. wk:=Avk−βkv_{k−1}
|
|
||||||
if (k > 0)
|
|
||||||
{
|
|
||||||
next -= lme[k - 1] * last;
|
|
||||||
}
|
|
||||||
// std::cout<<GridLogMessage << "<last|next>" << innerProduct(last,next) <<std::endl;
|
|
||||||
|
|
||||||
ComplexD zalph = innerProduct (current, next); // 4. αk:=(wk,vk)
|
|
||||||
RealD alph = real (zalph);
|
|
||||||
|
|
||||||
next = next - alph * current; // 5. wk:=wk−αkvk
|
|
||||||
// std::cout<<GridLogMessage << "<current|next>" << innerProduct(current,next) <<std::endl;
|
|
||||||
|
|
||||||
RealD beta = normalise (next); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
|
|
||||||
// 7. vk+1 := wk/βk+1
|
|
||||||
// norm=beta;
|
|
||||||
|
|
||||||
int interval = Nm / 100 + 1;
|
|
||||||
if ((k % interval) == 0)
|
|
||||||
std::
|
|
||||||
cout << GridLogMessage << k << " : alpha = " << zalph << " beta " <<
|
|
||||||
beta << std::endl;
|
|
||||||
const RealD tiny = 1.0e-20;
|
|
||||||
if (beta < tiny)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage << " beta is tiny " << beta << std::
|
|
||||||
endl;
|
|
||||||
}
|
|
||||||
lmd[k] = alph;
|
|
||||||
lme[k] = beta;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
void qr_decomp (std::vector < RealD > &lmd,
|
|
||||||
std::vector < RealD > &lme,
|
|
||||||
int Nk,
|
|
||||||
int Nm,
|
|
||||||
std::vector < RealD > &Qt, RealD Dsh, int kmin, int kmax)
|
|
||||||
{
|
|
||||||
int k = kmin - 1;
|
|
||||||
RealD x;
|
|
||||||
|
|
||||||
RealD Fden = 1.0 / hypot (lmd[k] - Dsh, lme[k]);
|
|
||||||
RealD c = (lmd[k] - Dsh) * Fden;
|
|
||||||
RealD s = -lme[k] * Fden;
|
|
||||||
|
|
||||||
RealD tmpa1 = lmd[k];
|
|
||||||
RealD tmpa2 = lmd[k + 1];
|
|
||||||
RealD tmpb = lme[k];
|
|
||||||
|
|
||||||
lmd[k] = c * c * tmpa1 + s * s * tmpa2 - 2.0 * c * s * tmpb;
|
|
||||||
lmd[k + 1] = s * s * tmpa1 + c * c * tmpa2 + 2.0 * c * s * tmpb;
|
|
||||||
lme[k] = c * s * (tmpa1 - tmpa2) + (c * c - s * s) * tmpb;
|
|
||||||
x = -s * lme[k + 1];
|
|
||||||
lme[k + 1] = c * lme[k + 1];
|
|
||||||
|
|
||||||
for (int i = 0; i < Nk; ++i)
|
|
||||||
{
|
|
||||||
RealD Qtmp1 = Qt[i + Nm * k];
|
|
||||||
RealD Qtmp2 = Qt[i + Nm * (k + 1)];
|
|
||||||
Qt[i + Nm * k] = c * Qtmp1 - s * Qtmp2;
|
|
||||||
Qt[i + Nm * (k + 1)] = s * Qtmp1 + c * Qtmp2;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Givens transformations
|
|
||||||
for (int k = kmin; k < kmax - 1; ++k)
|
|
||||||
{
|
|
||||||
|
|
||||||
RealD Fden = 1.0 / hypot (x, lme[k - 1]);
|
|
||||||
RealD c = lme[k - 1] * Fden;
|
|
||||||
RealD s = -x * Fden;
|
|
||||||
|
|
||||||
RealD tmpa1 = lmd[k];
|
|
||||||
RealD tmpa2 = lmd[k + 1];
|
|
||||||
RealD tmpb = lme[k];
|
|
||||||
|
|
||||||
lmd[k] = c * c * tmpa1 + s * s * tmpa2 - 2.0 * c * s * tmpb;
|
|
||||||
lmd[k + 1] = s * s * tmpa1 + c * c * tmpa2 + 2.0 * c * s * tmpb;
|
|
||||||
lme[k] = c * s * (tmpa1 - tmpa2) + (c * c - s * s) * tmpb;
|
|
||||||
lme[k - 1] = c * lme[k - 1] - s * x;
|
|
||||||
|
|
||||||
if (k != kmax - 2)
|
|
||||||
{
|
|
||||||
x = -s * lme[k + 1];
|
|
||||||
lme[k + 1] = c * lme[k + 1];
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < Nk; ++i)
|
|
||||||
{
|
|
||||||
RealD Qtmp1 = Qt[i + Nm * k];
|
|
||||||
RealD Qtmp2 = Qt[i + Nm * (k + 1)];
|
|
||||||
Qt[i + Nm * k] = c * Qtmp1 - s * Qtmp2;
|
|
||||||
Qt[i + Nm * (k + 1)] = s * Qtmp1 + c * Qtmp2;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#if 0
|
|
||||||
#ifdef USE_LAPACK
|
|
||||||
#ifdef USE_MKL
|
|
||||||
#define LAPACK_INT MKL_INT
|
|
||||||
#else
|
|
||||||
#define LAPACK_INT long long
|
|
||||||
#endif
|
|
||||||
void diagonalize_lapack (std::vector < RealD > &lmd, std::vector < RealD > &lme, int N1, // all
|
|
||||||
int N2, // get
|
|
||||||
GridBase * grid)
|
|
||||||
{
|
|
||||||
const int size = Nm;
|
|
||||||
LAPACK_INT NN = N1;
|
|
||||||
double evals_tmp[NN];
|
|
||||||
double DD[NN];
|
|
||||||
double EE[NN];
|
|
||||||
for (int i = 0; i < NN; i++)
|
|
||||||
for (int j = i - 1; j <= i + 1; j++)
|
|
||||||
if (j < NN && j >= 0)
|
|
||||||
{
|
|
||||||
if (i == j)
|
|
||||||
DD[i] = lmd[i];
|
|
||||||
if (i == j)
|
|
||||||
evals_tmp[i] = lmd[i];
|
|
||||||
if (j == (i - 1))
|
|
||||||
EE[j] = lme[j];
|
|
||||||
}
|
|
||||||
LAPACK_INT evals_found;
|
|
||||||
LAPACK_INT lwork =
|
|
||||||
((18 * NN) >
|
|
||||||
(1 + 4 * NN + NN * NN) ? (18 * NN) : (1 + 4 * NN + NN * NN));
|
|
||||||
LAPACK_INT liwork = 3 + NN * 10;
|
|
||||||
LAPACK_INT iwork[liwork];
|
|
||||||
double work[lwork];
|
|
||||||
LAPACK_INT isuppz[2 * NN];
|
|
||||||
char jobz = 'N'; // calculate evals only
|
|
||||||
char range = 'I'; // calculate il-th to iu-th evals
|
|
||||||
// char range = 'A'; // calculate all evals
|
|
||||||
char uplo = 'U'; // refer to upper half of original matrix
|
|
||||||
char compz = 'I'; // Compute eigenvectors of tridiagonal matrix
|
|
||||||
int ifail[NN];
|
|
||||||
LAPACK_INT info;
|
|
||||||
// int total = QMP_get_number_of_nodes();
|
|
||||||
// int node = QMP_get_node_number();
|
|
||||||
// GridBase *grid = evec[0]._grid;
|
|
||||||
int total = grid->_Nprocessors;
|
|
||||||
int node = grid->_processor;
|
|
||||||
int interval = (NN / total) + 1;
|
|
||||||
double vl = 0.0, vu = 0.0;
|
|
||||||
LAPACK_INT il = interval * node + 1, iu = interval * (node + 1);
|
|
||||||
if (iu > NN)
|
|
||||||
iu = NN;
|
|
||||||
double tol = 0.0;
|
|
||||||
if (1)
|
|
||||||
{
|
|
||||||
memset (evals_tmp, 0, sizeof (double) * NN);
|
|
||||||
if (il <= NN)
|
|
||||||
{
|
|
||||||
printf ("total=%d node=%d il=%d iu=%d\n", total, node, il, iu);
|
|
||||||
#ifdef USE_MKL
|
|
||||||
dstegr (&jobz, &range, &NN,
|
|
||||||
#else
|
|
||||||
LAPACK_dstegr (&jobz, &range, &NN,
|
|
||||||
#endif
|
|
||||||
(double *) DD, (double *) EE, &vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A'
|
|
||||||
&tol, // tolerance
|
|
||||||
&evals_found, evals_tmp, (double *) NULL, &NN,
|
|
||||||
isuppz, work, &lwork, iwork, &liwork, &info);
|
|
||||||
for (int i = iu - 1; i >= il - 1; i--)
|
|
||||||
{
|
|
||||||
printf ("node=%d evals_found=%d evals_tmp[%d] = %g\n", node,
|
|
||||||
evals_found, i - (il - 1), evals_tmp[i - (il - 1)]);
|
|
||||||
evals_tmp[i] = evals_tmp[i - (il - 1)];
|
|
||||||
if (il > 1)
|
|
||||||
evals_tmp[i - (il - 1)] = 0.;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
{
|
|
||||||
grid->GlobalSumVector (evals_tmp, NN);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// cheating a bit. It is better to sort instead of just reversing it, but the document of the routine says evals are sorted in increasing order. qr gives evals in decreasing order.
|
|
||||||
}
|
|
||||||
#undef LAPACK_INT
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
void diagonalize (std::vector < RealD > &lmd,
|
|
||||||
std::vector < RealD > &lme,
|
|
||||||
int N2, int N1, GridBase * grid)
|
|
||||||
{
|
|
||||||
|
|
||||||
#ifdef USE_LAPACK
|
|
||||||
const int check_lapack = 0; // just use lapack if 0, check against lapack if 1
|
|
||||||
|
|
||||||
if (!check_lapack)
|
|
||||||
return diagonalize_lapack (lmd, lme, N2, N1, grid);
|
|
||||||
|
|
||||||
// diagonalize_lapack(lmd2,lme2,Nm2,Nm,Qt,grid);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static RealD normalise (Field & v)
|
|
||||||
{
|
|
||||||
RealD nn = norm2 (v);
|
|
||||||
nn = sqrt (nn);
|
|
||||||
v = v * (1.0 / nn);
|
|
||||||
return nn;
|
|
||||||
}
|
|
||||||
|
|
||||||
void orthogonalize (Field & w, std::vector < Field > &evec, int k)
|
|
||||||
{
|
|
||||||
double t0 = -usecond () / 1e6;
|
|
||||||
typedef typename Field::scalar_type MyComplex;
|
|
||||||
MyComplex ip;
|
|
||||||
|
|
||||||
if (0)
|
|
||||||
{
|
|
||||||
for (int j = 0; j < k; ++j)
|
|
||||||
{
|
|
||||||
normalise (evec[j]);
|
|
||||||
for (int i = 0; i < j; i++)
|
|
||||||
{
|
|
||||||
ip = innerProduct (evec[i], evec[j]); // are the evecs normalised? ; this assumes so.
|
|
||||||
evec[j] = evec[j] - ip * evec[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int j = 0; j < k; ++j)
|
|
||||||
{
|
|
||||||
ip = innerProduct (evec[j], w); // are the evecs normalised? ; this assumes so.
|
|
||||||
w = w - ip * evec[j];
|
|
||||||
}
|
|
||||||
normalise (w);
|
|
||||||
t0 += usecond () / 1e6;
|
|
||||||
OrthoTime += t0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void setUnit_Qt (int Nm, std::vector < RealD > &Qt)
|
|
||||||
{
|
|
||||||
for (int i = 0; i < Qt.size (); ++i)
|
|
||||||
Qt[i] = 0.0;
|
|
||||||
for (int k = 0; k < Nm; ++k)
|
|
||||||
Qt[k + k * Nm] = 1.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void calc (std::vector < RealD > &eval, const Field & src, int &Nconv)
|
|
||||||
{
|
|
||||||
|
|
||||||
GridBase *grid = src.Grid();
|
|
||||||
// assert(grid == src._grid);
|
|
||||||
|
|
||||||
std::
|
|
||||||
cout << GridLogMessage << " -- Nk = " << Nk << " Np = " << Np << std::
|
|
||||||
endl;
|
|
||||||
std::cout << GridLogMessage << " -- Nm = " << Nm << std::endl;
|
|
||||||
std::cout << GridLogMessage << " -- size of eval = " << eval.
|
|
||||||
size () << std::endl;
|
|
||||||
|
|
||||||
// assert(c.size() && Nm == eval.size());
|
|
||||||
|
|
||||||
std::vector < RealD > lme (Nm);
|
|
||||||
std::vector < RealD > lmd (Nm);
|
|
||||||
|
|
||||||
|
|
||||||
Field current (grid);
|
|
||||||
Field last (grid);
|
|
||||||
Field next (grid);
|
|
||||||
|
|
||||||
Nconv = 0;
|
|
||||||
|
|
||||||
RealD beta_k;
|
|
||||||
|
|
||||||
// Set initial vector
|
|
||||||
// (uniform vector) Why not src??
|
|
||||||
// evec[0] = 1.0;
|
|
||||||
current = src;
|
|
||||||
std::cout << GridLogMessage << "norm2(src)= " << norm2 (src) << std::
|
|
||||||
endl;
|
|
||||||
normalise (current);
|
|
||||||
std::
|
|
||||||
cout << GridLogMessage << "norm2(evec[0])= " << norm2 (current) <<
|
|
||||||
std::endl;
|
|
||||||
|
|
||||||
// Initial Nk steps
|
|
||||||
OrthoTime = 0.;
|
|
||||||
double t0 = usecond () / 1e6;
|
|
||||||
RealD norm; // sqrt norm of last vector
|
|
||||||
|
|
||||||
uint64_t iter = 0;
|
|
||||||
|
|
||||||
bool initted = false;
|
|
||||||
std::vector < RealD > low (Nstop * 10);
|
|
||||||
std::vector < RealD > high (Nstop * 10);
|
|
||||||
RealD cont = 0.;
|
|
||||||
while (1) {
|
|
||||||
cont = 0.;
|
|
||||||
std::vector < RealD > lme2 (Nm);
|
|
||||||
std::vector < RealD > lmd2 (Nm);
|
|
||||||
for (uint64_t k = 0; k < Nm; ++k, iter++) {
|
|
||||||
step (lmd, lme, last, current, next, iter);
|
|
||||||
last = current;
|
|
||||||
current = next;
|
|
||||||
}
|
|
||||||
double t1 = usecond () / 1e6;
|
|
||||||
std::cout << GridLogMessage << "IRL::Initial steps: " << t1 -
|
|
||||||
t0 << "seconds" << std::endl;
|
|
||||||
t0 = t1;
|
|
||||||
std::
|
|
||||||
cout << GridLogMessage << "IRL::Initial steps:OrthoTime " <<
|
|
||||||
OrthoTime << "seconds" << std::endl;
|
|
||||||
|
|
||||||
// getting eigenvalues
|
|
||||||
lmd2.resize (iter + 2);
|
|
||||||
lme2.resize (iter + 2);
|
|
||||||
for (uint64_t k = 0; k < iter; ++k) {
|
|
||||||
lmd2[k + 1] = lmd[k];
|
|
||||||
lme2[k + 2] = lme[k];
|
|
||||||
}
|
|
||||||
t1 = usecond () / 1e6;
|
|
||||||
std::cout << GridLogMessage << "IRL:: copy: " << t1 -
|
|
||||||
t0 << "seconds" << std::endl;
|
|
||||||
t0 = t1;
|
|
||||||
{
|
|
||||||
int total = grid->_Nprocessors;
|
|
||||||
int node = grid->_processor;
|
|
||||||
int interval = (Nstop / total) + 1;
|
|
||||||
int iu = (iter + 1) - (interval * node + 1);
|
|
||||||
int il = (iter + 1) - (interval * (node + 1));
|
|
||||||
std::vector < RealD > eval2 (iter + 3);
|
|
||||||
RealD eps2;
|
|
||||||
Bisection::bisec (lmd2, lme2, iter, il, iu, 1e-16, 1e-10, eval2,
|
|
||||||
eps2);
|
|
||||||
// diagonalize(eval2,lme2,iter,Nk,grid);
|
|
||||||
RealD diff = 0.;
|
|
||||||
for (int i = il; i <= iu; i++) {
|
|
||||||
if (initted)
|
|
||||||
diff =
|
|
||||||
fabs (eval2[i] - high[iu-i]) / (fabs (eval2[i]) +
|
|
||||||
fabs (high[iu-i]));
|
|
||||||
if (initted && (diff > eresid))
|
|
||||||
cont = 1.;
|
|
||||||
if (initted)
|
|
||||||
printf ("eval[%d]=%0.14e %0.14e, %0.14e\n", i, eval2[i],
|
|
||||||
high[iu-i], diff);
|
|
||||||
high[iu-i] = eval2[i];
|
|
||||||
}
|
|
||||||
il = (interval * node + 1);
|
|
||||||
iu = (interval * (node + 1));
|
|
||||||
Bisection::bisec (lmd2, lme2, iter, il, iu, 1e-16, 1e-10, eval2,
|
|
||||||
eps2);
|
|
||||||
for (int i = il; i <= iu; i++) {
|
|
||||||
if (initted)
|
|
||||||
diff =
|
|
||||||
fabs (eval2[i] - low[i]) / (fabs (eval2[i]) +
|
|
||||||
fabs (low[i]));
|
|
||||||
if (initted && (diff > eresid))
|
|
||||||
cont = 1.;
|
|
||||||
if (initted)
|
|
||||||
printf ("eval[%d]=%0.14e %0.14e, %0.14e\n", i, eval2[i],
|
|
||||||
low[i], diff);
|
|
||||||
low[i] = eval2[i];
|
|
||||||
}
|
|
||||||
t1 = usecond () / 1e6;
|
|
||||||
std::cout << GridLogMessage << "IRL:: diagonalize: " << t1 -
|
|
||||||
t0 << "seconds" << std::endl;
|
|
||||||
t0 = t1;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (uint64_t k = 0; k < Nk; ++k) {
|
|
||||||
// eval[k] = eval2[k];
|
|
||||||
}
|
|
||||||
if (initted)
|
|
||||||
{
|
|
||||||
grid->GlobalSumVector (&cont, 1);
|
|
||||||
if (cont < 1.) return;
|
|
||||||
}
|
|
||||||
initted = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#if 0
|
|
||||||
|
|
||||||
/**
|
|
||||||
There is some matrix Q such that for any vector y
|
|
||||||
Q.e_1 = y and Q is unitary.
|
|
||||||
**/
|
|
||||||
template < class T >
|
|
||||||
static T orthQ (DenseMatrix < T > &Q, std::vector < T > y)
|
|
||||||
{
|
|
||||||
int N = y.size (); //Matrix Size
|
|
||||||
Fill (Q, 0.0);
|
|
||||||
T tau;
|
|
||||||
for (int i = 0; i < N; i++)
|
|
||||||
{
|
|
||||||
Q[i][0] = y[i];
|
|
||||||
}
|
|
||||||
T sig = conj (y[0]) * y[0];
|
|
||||||
T tau0 = fabs (sqrt (sig));
|
|
||||||
|
|
||||||
for (int j = 1; j < N; j++)
|
|
||||||
{
|
|
||||||
sig += conj (y[j]) * y[j];
|
|
||||||
tau = abs (sqrt (sig));
|
|
||||||
|
|
||||||
if (abs (tau0) > 0.0)
|
|
||||||
{
|
|
||||||
|
|
||||||
T gam = conj ((y[j] / tau) / tau0);
|
|
||||||
for (int k = 0; k <= j - 1; k++)
|
|
||||||
{
|
|
||||||
Q[k][j] = -gam * y[k];
|
|
||||||
}
|
|
||||||
Q[j][j] = tau0 / tau;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Q[j - 1][j] = 1.0;
|
|
||||||
}
|
|
||||||
tau0 = tau;
|
|
||||||
}
|
|
||||||
return tau;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
There is some matrix Q such that for any vector y
|
|
||||||
Q.e_k = y and Q is unitary.
|
|
||||||
**/
|
|
||||||
template < class T >
|
|
||||||
static T orthU (DenseMatrix < T > &Q, std::vector < T > y)
|
|
||||||
{
|
|
||||||
T tau = orthQ (Q, y);
|
|
||||||
SL (Q);
|
|
||||||
return tau;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
Wind up with a matrix with the first con rows untouched
|
|
||||||
|
|
||||||
say con = 2
|
|
||||||
Q is such that Qdag H Q has {x, x, val, 0, 0, 0, 0, ...} as 1st colum
|
|
||||||
and the matrix is upper hessenberg
|
|
||||||
and with f and Q appropriately modidied with Q is the arnoldi factorization
|
|
||||||
|
|
||||||
**/
|
|
||||||
|
|
||||||
template < class T > static void Lock (DenseMatrix < T > &H, ///Hess mtx
|
|
||||||
DenseMatrix < T > &Q, ///Lock Transform
|
|
||||||
T val, ///value to be locked
|
|
||||||
int con, ///number already locked
|
|
||||||
RealD small, int dfg, bool herm)
|
|
||||||
{
|
|
||||||
//ForceTridiagonal(H);
|
|
||||||
|
|
||||||
int M = H.dim;
|
|
||||||
DenseVector < T > vec;
|
|
||||||
Resize (vec, M - con);
|
|
||||||
|
|
||||||
DenseMatrix < T > AH;
|
|
||||||
Resize (AH, M - con, M - con);
|
|
||||||
AH = GetSubMtx (H, con, M, con, M);
|
|
||||||
|
|
||||||
DenseMatrix < T > QQ;
|
|
||||||
Resize (QQ, M - con, M - con);
|
|
||||||
|
|
||||||
Unity (Q);
|
|
||||||
Unity (QQ);
|
|
||||||
|
|
||||||
DenseVector < T > evals;
|
|
||||||
Resize (evals, M - con);
|
|
||||||
DenseMatrix < T > evecs;
|
|
||||||
Resize (evecs, M - con, M - con);
|
|
||||||
|
|
||||||
Wilkinson < T > (AH, evals, evecs, small);
|
|
||||||
|
|
||||||
int k = 0;
|
|
||||||
RealD cold = abs (val - evals[k]);
|
|
||||||
for (int i = 1; i < M - con; i++)
|
|
||||||
{
|
|
||||||
RealD cnew = abs (val - evals[i]);
|
|
||||||
if (cnew < cold)
|
|
||||||
{
|
|
||||||
k = i;
|
|
||||||
cold = cnew;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
vec = evecs[k];
|
|
||||||
|
|
||||||
ComplexD tau;
|
|
||||||
orthQ (QQ, vec);
|
|
||||||
//orthQM(QQ,AH,vec);
|
|
||||||
|
|
||||||
AH = Hermitian (QQ) * AH;
|
|
||||||
AH = AH * QQ;
|
|
||||||
|
|
||||||
for (int i = con; i < M; i++)
|
|
||||||
{
|
|
||||||
for (int j = con; j < M; j++)
|
|
||||||
{
|
|
||||||
Q[i][j] = QQ[i - con][j - con];
|
|
||||||
H[i][j] = AH[i - con][j - con];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int j = M - 1; j > con + 2; j--)
|
|
||||||
{
|
|
||||||
|
|
||||||
DenseMatrix < T > U;
|
|
||||||
Resize (U, j - 1 - con, j - 1 - con);
|
|
||||||
DenseVector < T > z;
|
|
||||||
Resize (z, j - 1 - con);
|
|
||||||
T nm = norm (z);
|
|
||||||
for (int k = con + 0; k < j - 1; k++)
|
|
||||||
{
|
|
||||||
z[k - con] = conj (H (j, k + 1));
|
|
||||||
}
|
|
||||||
normalise (z);
|
|
||||||
|
|
||||||
RealD tmp = 0;
|
|
||||||
for (int i = 0; i < z.size () - 1; i++)
|
|
||||||
{
|
|
||||||
tmp = tmp + abs (z[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tmp < small / ((RealD) z.size () - 1.0))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
tau = orthU (U, z);
|
|
||||||
|
|
||||||
DenseMatrix < T > Hb;
|
|
||||||
Resize (Hb, j - 1 - con, M);
|
|
||||||
|
|
||||||
for (int a = 0; a < M; a++)
|
|
||||||
{
|
|
||||||
for (int b = 0; b < j - 1 - con; b++)
|
|
||||||
{
|
|
||||||
T sum = 0;
|
|
||||||
for (int c = 0; c < j - 1 - con; c++)
|
|
||||||
{
|
|
||||||
sum += H[a][con + 1 + c] * U[c][b];
|
|
||||||
} //sum += H(a,con+1+c)*U(c,b);}
|
|
||||||
Hb[b][a] = sum;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int k = con + 1; k < j; k++)
|
|
||||||
{
|
|
||||||
for (int l = 0; l < M; l++)
|
|
||||||
{
|
|
||||||
H[l][k] = Hb[k - 1 - con][l];
|
|
||||||
}
|
|
||||||
} //H(Hb[k-1-con][l] , l,k);}}
|
|
||||||
|
|
||||||
DenseMatrix < T > Qb;
|
|
||||||
Resize (Qb, M, M);
|
|
||||||
|
|
||||||
for (int a = 0; a < M; a++)
|
|
||||||
{
|
|
||||||
for (int b = 0; b < j - 1 - con; b++)
|
|
||||||
{
|
|
||||||
T sum = 0;
|
|
||||||
for (int c = 0; c < j - 1 - con; c++)
|
|
||||||
{
|
|
||||||
sum += Q[a][con + 1 + c] * U[c][b];
|
|
||||||
} //sum += Q(a,con+1+c)*U(c,b);}
|
|
||||||
Qb[b][a] = sum;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int k = con + 1; k < j; k++)
|
|
||||||
{
|
|
||||||
for (int l = 0; l < M; l++)
|
|
||||||
{
|
|
||||||
Q[l][k] = Qb[k - 1 - con][l];
|
|
||||||
}
|
|
||||||
} //Q(Qb[k-1-con][l] , l,k);}}
|
|
||||||
|
|
||||||
DenseMatrix < T > Hc;
|
|
||||||
Resize (Hc, M, M);
|
|
||||||
|
|
||||||
for (int a = 0; a < j - 1 - con; a++)
|
|
||||||
{
|
|
||||||
for (int b = 0; b < M; b++)
|
|
||||||
{
|
|
||||||
T sum = 0;
|
|
||||||
for (int c = 0; c < j - 1 - con; c++)
|
|
||||||
{
|
|
||||||
sum += conj (U[c][a]) * H[con + 1 + c][b];
|
|
||||||
} //sum += conj( U(c,a) )*H(con+1+c,b);}
|
|
||||||
Hc[b][a] = sum;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int k = 0; k < M; k++)
|
|
||||||
{
|
|
||||||
for (int l = con + 1; l < j; l++)
|
|
||||||
{
|
|
||||||
H[l][k] = Hc[k][l - 1 - con];
|
|
||||||
}
|
|
||||||
} //H(Hc[k][l-1-con] , l,k);}}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
@@ -1,611 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/Aggregates.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <Grid/algorithms/iterative/PrecGeneralisedConjugateResidualNonHermitian.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
inline RealD AggregatePowerLaw(RealD x)
|
|
||||||
{
|
|
||||||
// return std::pow(x,-4);
|
|
||||||
// return std::pow(x,-3);
|
|
||||||
return std::pow(x,-5);
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Fobj,class CComplex,int nbasis>
|
|
||||||
class Aggregation {
|
|
||||||
public:
|
|
||||||
constexpr int Nbasis(void) { return nbasis; };
|
|
||||||
|
|
||||||
typedef iVector<CComplex,nbasis > siteVector;
|
|
||||||
typedef Lattice<siteVector> CoarseVector;
|
|
||||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
|
||||||
|
|
||||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
|
||||||
typedef Lattice<Fobj > FineField;
|
|
||||||
|
|
||||||
GridBase *CoarseGrid;
|
|
||||||
GridBase *FineGrid;
|
|
||||||
std::vector<Lattice<Fobj> > subspace;
|
|
||||||
int checkerboard;
|
|
||||||
int Checkerboard(void){return checkerboard;}
|
|
||||||
Aggregation(GridBase *_CoarseGrid,GridBase *_FineGrid,int _checkerboard) :
|
|
||||||
CoarseGrid(_CoarseGrid),
|
|
||||||
FineGrid(_FineGrid),
|
|
||||||
subspace(nbasis,_FineGrid),
|
|
||||||
checkerboard(_checkerboard)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
void Orthogonalise(void){
|
|
||||||
CoarseScalar InnerProd(CoarseGrid);
|
|
||||||
// std::cout << GridLogMessage <<" Block Gramm-Schmidt pass 1"<<std::endl;
|
|
||||||
blockOrthogonalise(InnerProd,subspace);
|
|
||||||
}
|
|
||||||
void ProjectToSubspace(CoarseVector &CoarseVec,const FineField &FineVec){
|
|
||||||
blockProject(CoarseVec,FineVec,subspace);
|
|
||||||
}
|
|
||||||
void PromoteFromSubspace(const CoarseVector &CoarseVec,FineField &FineVec){
|
|
||||||
FineVec.Checkerboard() = subspace[0].Checkerboard();
|
|
||||||
blockPromote(CoarseVec,FineVec,subspace);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void CreateSubspaceRandom(GridParallelRNG &RNG) {
|
|
||||||
int nn=nbasis;
|
|
||||||
RealD scale;
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
for(int b=0;b<nn;b++){
|
|
||||||
subspace[b] = Zero();
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
subspace[b] = noise;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual void CreateSubspace(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,int nn=nbasis)
|
|
||||||
{
|
|
||||||
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
ConjugateGradient<FineField> CG(1.0e-4,2000,false);
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
|
|
||||||
for(int b=0;b<nn;b++){
|
|
||||||
|
|
||||||
subspace[b] = Zero();
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise ["<<b<<"] <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
|
|
||||||
for(int i=0;i<4;i++){
|
|
||||||
|
|
||||||
CG(hermop,noise,subspace[b]);
|
|
||||||
|
|
||||||
noise = subspace[b];
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "filtered["<<b<<"] <f|MdagM|f> "<<norm2(Mn)<<std::endl;
|
|
||||||
subspace[b] = noise;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void CreateSubspaceGCR(GridParallelRNG &RNG,LinearOperatorBase<FineField> &DiracOp,int nn=nbasis)
|
|
||||||
{
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
TrivialPrecon<FineField> simple_fine;
|
|
||||||
// PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,10,DiracOp,simple_fine,30,30);
|
|
||||||
// PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,10,DiracOp,simple_fine,12,12);
|
|
||||||
// PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,30,DiracOp,simple_fine,12,12);
|
|
||||||
PrecGeneralisedConjugateResidualNonHermitian<FineField> GCR(0.001,30,DiracOp,simple_fine,10,10);
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField src(FineGrid);
|
|
||||||
FineField guess(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
|
|
||||||
for(int b=0;b<nn;b++){
|
|
||||||
|
|
||||||
subspace[b] = Zero();
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
DiracOp.Op(noise,Mn); std::cout<<GridLogMessage << "noise ["<<b<<"] <n|Op|n> "<<innerProduct(noise,Mn)<<std::endl;
|
|
||||||
|
|
||||||
for(int i=0;i<3;i++){
|
|
||||||
// void operator() (const Field &src, Field &psi){
|
|
||||||
#if 1
|
|
||||||
if (i==0)std::cout << GridLogMessage << " inverting on noise "<<std::endl;
|
|
||||||
src = noise;
|
|
||||||
guess=Zero();
|
|
||||||
GCR(src,guess);
|
|
||||||
subspace[b] = guess;
|
|
||||||
#else
|
|
||||||
if (i==0)std::cout << GridLogMessage << " inverting on zero "<<std::endl;
|
|
||||||
src=Zero();
|
|
||||||
guess = noise;
|
|
||||||
GCR(src,guess);
|
|
||||||
subspace[b] = guess;
|
|
||||||
#endif
|
|
||||||
noise = subspace[b];
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
DiracOp.Op(noise,Mn); std::cout<<GridLogMessage << "filtered["<<b<<"] <f|Op|f> "<<innerProduct(noise,Mn)<<" <f|OpDagOp|f>"<<norm2(Mn)<<std::endl;
|
|
||||||
subspace[b] = noise;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// World of possibilities here. But have tried quite a lot of experiments (250+ jobs run on Summit)
|
|
||||||
// and this is the best I found
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
|
||||||
int nn,
|
|
||||||
double hi,
|
|
||||||
double lo,
|
|
||||||
int orderfilter,
|
|
||||||
int ordermin,
|
|
||||||
int orderstep,
|
|
||||||
double filterlo
|
|
||||||
) {
|
|
||||||
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
|
|
||||||
// New normalised noise
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<" Chebyshev subspace pass-1 : ord "<<orderfilter<<" ["<<lo<<","<<hi<<"]"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<" Chebyshev subspace pass-2 : nbasis"<<nn<<" min "
|
|
||||||
<<ordermin<<" step "<<orderstep
|
|
||||||
<<" lo"<<filterlo<<std::endl;
|
|
||||||
|
|
||||||
// Initial matrix element
|
|
||||||
hermop.Op(noise,Mn); std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
|
|
||||||
int b =0;
|
|
||||||
{
|
|
||||||
ComplexD ip;
|
|
||||||
// Filter
|
|
||||||
Chebyshev<FineField> Cheb(lo,hi,orderfilter);
|
|
||||||
Cheb(hermop,noise,Mn);
|
|
||||||
// normalise
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
subspace[b] = Mn;
|
|
||||||
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
ip= innerProduct(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|Op|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
|
||||||
|
|
||||||
hermop.AdjOp(Mn,tmp);
|
|
||||||
ip = innerProduct(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|AdjOp|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
|
||||||
b++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate a full sequence of Chebyshevs
|
|
||||||
{
|
|
||||||
lo=filterlo;
|
|
||||||
noise=Mn;
|
|
||||||
|
|
||||||
FineField T0(FineGrid); T0 = noise;
|
|
||||||
FineField T1(FineGrid);
|
|
||||||
FineField T2(FineGrid);
|
|
||||||
FineField y(FineGrid);
|
|
||||||
|
|
||||||
FineField *Tnm = &T0;
|
|
||||||
FineField *Tn = &T1;
|
|
||||||
FineField *Tnp = &T2;
|
|
||||||
|
|
||||||
// Tn=T1 = (xscale M + mscale)in
|
|
||||||
RealD xscale = 2.0/(hi-lo);
|
|
||||||
RealD mscale = -(hi+lo)/(hi-lo);
|
|
||||||
hermop.HermOp(T0,y);
|
|
||||||
T1=y*xscale+noise*mscale;
|
|
||||||
|
|
||||||
for(int n=2;n<=ordermin+orderstep*(nn-2);n++){
|
|
||||||
|
|
||||||
hermop.HermOp(*Tn,y);
|
|
||||||
|
|
||||||
autoView( y_v , y, AcceleratorWrite);
|
|
||||||
autoView( Tn_v , (*Tn), AcceleratorWrite);
|
|
||||||
autoView( Tnp_v , (*Tnp), AcceleratorWrite);
|
|
||||||
autoView( Tnm_v , (*Tnm), AcceleratorWrite);
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
accelerator_for(ss, FineGrid->oSites(), Nsimd, {
|
|
||||||
coalescedWrite(y_v[ss],xscale*y_v(ss)+mscale*Tn_v(ss));
|
|
||||||
coalescedWrite(Tnp_v[ss],2.0*y_v(ss)-Tnm_v(ss));
|
|
||||||
});
|
|
||||||
|
|
||||||
// Possible more fine grained control is needed than a linear sweep,
|
|
||||||
// but huge productivity gain if this is simple algorithm and not a tunable
|
|
||||||
int m =1;
|
|
||||||
if ( n>=ordermin ) m=n-ordermin;
|
|
||||||
if ( (m%orderstep)==0 ) {
|
|
||||||
Mn=*Tnp;
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
subspace[b] = Mn;
|
|
||||||
|
|
||||||
|
|
||||||
ComplexD ip;
|
|
||||||
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
ip= innerProduct(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|Op|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
|
||||||
|
|
||||||
hermop.AdjOp(Mn,tmp);
|
|
||||||
ip = innerProduct(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|AdjOp|n> "<<norm2(tmp)<<" "<<ip<<std::endl;
|
|
||||||
|
|
||||||
b++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cycle pointers to avoid copies
|
|
||||||
FineField *swizzle = Tnm;
|
|
||||||
Tnm =Tn;
|
|
||||||
Tn =Tnp;
|
|
||||||
Tnp =swizzle;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
GRID_ASSERT(b==nn);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
virtual void CreateSubspacePolyCheby(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
|
||||||
int nn,
|
|
||||||
double hi,
|
|
||||||
double lo1,
|
|
||||||
int orderfilter,
|
|
||||||
double lo2,
|
|
||||||
int orderstep)
|
|
||||||
{
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
|
|
||||||
// New normalised noise
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<<" CreateSubspacePolyCheby "<<std::endl;
|
|
||||||
// Initial matrix element
|
|
||||||
hermop.Op(noise,Mn);
|
|
||||||
std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
|
|
||||||
int b =0;
|
|
||||||
{
|
|
||||||
// Filter
|
|
||||||
std::cout << GridLogMessage << "Cheby "<<lo1<<","<<hi<<" "<<orderstep<<std::endl;
|
|
||||||
Chebyshev<FineField> Cheb(lo1,hi,orderfilter);
|
|
||||||
Cheb(hermop,noise,Mn);
|
|
||||||
// normalise
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
subspace[b] = Mn;
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate a full sequence of Chebyshevs
|
|
||||||
for(int n=1;n<nn;n++){
|
|
||||||
std::cout << GridLogMessage << "Cheby "<<lo2<<","<<hi<<" "<<orderstep<<std::endl;
|
|
||||||
Chebyshev<FineField> Cheb(lo2,hi,orderstep);
|
|
||||||
Cheb(hermop,subspace[n-1],Mn);
|
|
||||||
|
|
||||||
for(int m=0;m<n;m++){
|
|
||||||
ComplexD c = innerProduct(subspace[m],Mn);
|
|
||||||
Mn = Mn - c*subspace[m];
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalise
|
|
||||||
scale = std::pow(norm2(Mn),-0.5);
|
|
||||||
Mn=Mn*scale;
|
|
||||||
|
|
||||||
subspace[n]=Mn;
|
|
||||||
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<n<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<n<<"] <n|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void CreateSubspaceChebyshev(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
|
||||||
int nn,
|
|
||||||
double hi,
|
|
||||||
double lo,
|
|
||||||
int orderfilter
|
|
||||||
) {
|
|
||||||
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
|
|
||||||
// New normalised noise
|
|
||||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : ord "<<orderfilter<<" ["<<lo<<","<<hi<<"]"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : nbasis "<<nn<<std::endl;
|
|
||||||
|
|
||||||
|
|
||||||
for(int b =0;b<nbasis;b++)
|
|
||||||
{
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
// Initial matrix element
|
|
||||||
hermop.Op(noise,Mn);
|
|
||||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
|
|
||||||
// Filter
|
|
||||||
Chebyshev<FineField> Cheb(lo,hi,orderfilter);
|
|
||||||
Cheb(hermop,noise,Mn);
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
|
|
||||||
// Refine
|
|
||||||
Chebyshev<FineField> PowerLaw(lo,hi,1000,AggregatePowerLaw);
|
|
||||||
noise = Mn;
|
|
||||||
PowerLaw(hermop,noise,Mn);
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
|
|
||||||
// normalise
|
|
||||||
subspace[b] = Mn;
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void CreateSubspaceChebyshevPowerLaw(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
|
||||||
int nn,
|
|
||||||
double hi,
|
|
||||||
int orderfilter
|
|
||||||
) {
|
|
||||||
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
|
|
||||||
// New normalised noise
|
|
||||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : ord "<<orderfilter<<" [0,"<<hi<<"]"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<" Chebyshev subspace pure noise : nbasis "<<nn<<std::endl;
|
|
||||||
|
|
||||||
for(int b =0;b<nbasis;b++)
|
|
||||||
{
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
// Initial matrix element
|
|
||||||
hermop.Op(noise,Mn);
|
|
||||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
// Filter
|
|
||||||
Chebyshev<FineField> Cheb(0.0,hi,orderfilter,AggregatePowerLaw);
|
|
||||||
Cheb(hermop,noise,Mn);
|
|
||||||
// normalise
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
subspace[b] = Mn;
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
virtual void CreateSubspaceChebyshevNew(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
|
||||||
double hi
|
|
||||||
) {
|
|
||||||
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
|
|
||||||
// New normalised noise
|
|
||||||
for(int b =0;b<nbasis;b++)
|
|
||||||
{
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
// Initial matrix element
|
|
||||||
hermop.Op(noise,Mn);
|
|
||||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
// Filter
|
|
||||||
//#opt2(x) = acheb(x,3,90,300)* acheb(x,1,90,50) * acheb(x,0.5,90,200) * acheb(x,0.05,90,400) * acheb(x,0.01,90,1500)
|
|
||||||
/*266
|
|
||||||
Chebyshev<FineField> Cheb1(3.0,hi,300);
|
|
||||||
Chebyshev<FineField> Cheb2(1.0,hi,50);
|
|
||||||
Chebyshev<FineField> Cheb3(0.5,hi,300);
|
|
||||||
Chebyshev<FineField> Cheb4(0.05,hi,500);
|
|
||||||
Chebyshev<FineField> Cheb5(0.01,hi,2000);
|
|
||||||
*/
|
|
||||||
/* 242 */
|
|
||||||
/*
|
|
||||||
Chebyshev<FineField> Cheb3(0.1,hi,300);
|
|
||||||
Chebyshev<FineField> Cheb2(0.02,hi,1000);
|
|
||||||
Chebyshev<FineField> Cheb1(0.003,hi,2000);
|
|
||||||
8?
|
|
||||||
*/
|
|
||||||
/* How many??
|
|
||||||
*/
|
|
||||||
Chebyshev<FineField> Cheb2(0.001,hi,2500); // 169 iters on HDCG after refine
|
|
||||||
Chebyshev<FineField> Cheb1(0.02,hi,600);
|
|
||||||
|
|
||||||
// Chebyshev<FineField> Cheb2(0.001,hi,1500);
|
|
||||||
// Chebyshev<FineField> Cheb1(0.02,hi,600);
|
|
||||||
Cheb1(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
|
||||||
hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb1 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
Cheb2(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
|
||||||
hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb2 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
// Cheb3(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
|
||||||
// hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb3 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
// Cheb4(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
|
||||||
// hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb4 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
// Cheb5(hermop,noise,Mn); scale = std::pow(norm2(Mn),-0.5); noise=Mn*scale;
|
|
||||||
// hermop.Op(noise,tmp); std::cout<<GridLogMessage << "Cheb5 <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
subspace[b] = noise;
|
|
||||||
hermop.Op(subspace[b],tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<< " norm " << norm2(noise)<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual void CreateSubspaceMultishift(GridParallelRNG &RNG,LinearOperatorBase<FineField> &hermop,
|
|
||||||
double Lo,double tol,int maxit)
|
|
||||||
{
|
|
||||||
|
|
||||||
RealD scale;
|
|
||||||
|
|
||||||
FineField noise(FineGrid);
|
|
||||||
FineField Mn(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
|
|
||||||
// New normalised noise
|
|
||||||
std::cout << GridLogMessage<<" Multishift subspace : Lo "<<Lo<<std::endl;
|
|
||||||
|
|
||||||
// Filter
|
|
||||||
// [ 1/6(x+Lo) - 1/2(x+2Lo) + 1/2(x+3Lo) -1/6(x+4Lo) = Lo^3 /[ (x+1Lo)(x+2Lo)(x+3Lo)(x+4Lo) ]
|
|
||||||
//
|
|
||||||
// 1/(x+Lo) - 1/(x+2 Lo)
|
|
||||||
double epsilon = Lo/3;
|
|
||||||
std::vector<RealD> alpha({1.0/6.0,-1.0/2.0,1.0/2.0,-1.0/6.0});
|
|
||||||
std::vector<RealD> shifts({Lo,Lo+epsilon,Lo+2*epsilon,Lo+3*epsilon});
|
|
||||||
std::vector<RealD> tols({tol,tol,tol,tol});
|
|
||||||
std::cout << "sizes "<<alpha.size()<<" "<<shifts.size()<<" "<<tols.size()<<std::endl;
|
|
||||||
|
|
||||||
MultiShiftFunction msf(4,0.0,95.0);
|
|
||||||
std::cout << "msf constructed "<<std::endl;
|
|
||||||
msf.poles=shifts;
|
|
||||||
msf.residues=alpha;
|
|
||||||
msf.tolerances=tols;
|
|
||||||
msf.norm=0.0;
|
|
||||||
msf.order=alpha.size();
|
|
||||||
ConjugateGradientMultiShift<FineField> MSCG(maxit,msf);
|
|
||||||
|
|
||||||
for(int b =0;b<nbasis;b++)
|
|
||||||
{
|
|
||||||
gaussian(RNG,noise);
|
|
||||||
scale = std::pow(norm2(noise),-0.5);
|
|
||||||
noise=noise*scale;
|
|
||||||
|
|
||||||
// Initial matrix element
|
|
||||||
hermop.Op(noise,Mn);
|
|
||||||
if(b==0) std::cout<<GridLogMessage << "noise <n|MdagM|n> "<<norm2(Mn)<<std::endl;
|
|
||||||
|
|
||||||
MSCG(hermop,noise,Mn);
|
|
||||||
scale = std::pow(norm2(Mn),-0.5); Mn=Mn*scale;
|
|
||||||
subspace[b] = Mn;
|
|
||||||
hermop.Op(Mn,tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
virtual void RefineSubspace(LinearOperatorBase<FineField> &hermop,
|
|
||||||
double Lo,double tol,int maxit)
|
|
||||||
{
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
for(int b =0;b<nbasis;b++)
|
|
||||||
{
|
|
||||||
ConjugateGradient<FineField> CGsloppy(tol,maxit,false);
|
|
||||||
ShiftedHermOpLinearOperator<FineField> ShiftedFineHermOp(hermop,Lo);
|
|
||||||
tmp=Zero();
|
|
||||||
CGsloppy(hermop,subspace[b],tmp);
|
|
||||||
RealD scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
|
||||||
subspace[b]=tmp;
|
|
||||||
hermop.Op(subspace[b],tmp);
|
|
||||||
std::cout<<GridLogMessage << "filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual void RefineSubspaceHDCG(LinearOperatorBase<FineField> &hermop,
|
|
||||||
TwoLevelADEF2mrhs<FineField,CoarseVector> & theHDCG,
|
|
||||||
int nrhs)
|
|
||||||
{
|
|
||||||
std::vector<FineField> src_mrhs(nrhs,FineGrid);
|
|
||||||
std::vector<FineField> res_mrhs(nrhs,FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
for(int b =0;b<nbasis;b+=nrhs)
|
|
||||||
{
|
|
||||||
tmp = subspace[b];
|
|
||||||
RealD scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
|
||||||
subspace[b] =tmp;
|
|
||||||
hermop.Op(subspace[b],tmp);
|
|
||||||
std::cout<<GridLogMessage << "before filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
|
|
||||||
for(int r=0;r<MIN(nbasis-b,nrhs);r++){
|
|
||||||
src_mrhs[r] = subspace[b+r];
|
|
||||||
}
|
|
||||||
for(int r=0;r<nrhs;r++){
|
|
||||||
res_mrhs[r] = Zero();
|
|
||||||
}
|
|
||||||
theHDCG(src_mrhs,res_mrhs);
|
|
||||||
|
|
||||||
for(int r=0;r<MIN(nbasis-b,nrhs);r++){
|
|
||||||
tmp = res_mrhs[r];
|
|
||||||
RealD scale = std::pow(norm2(tmp),-0.5); tmp=tmp*scale;
|
|
||||||
subspace[b+r]=tmp;
|
|
||||||
}
|
|
||||||
hermop.Op(subspace[b],tmp);
|
|
||||||
std::cout<<GridLogMessage << "after filt ["<<b<<"] <n|MdagM|n> "<<norm2(tmp)<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,837 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/CoarsenedMatrix.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_ALGORITHM_COARSENED_MATRIX_H
|
|
||||||
#define GRID_ALGORITHM_COARSENED_MATRIX_H
|
|
||||||
|
|
||||||
#include <Grid/qcd/QCD.h> // needed for Dagger(Yes|No), Inverse(Yes|No)
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class vobj,class CComplex>
|
|
||||||
inline void blockMaskedInnerProduct(Lattice<CComplex> &CoarseInner,
|
|
||||||
const Lattice<decltype(innerProduct(vobj(),vobj()))> &FineMask,
|
|
||||||
const Lattice<vobj> &fineX,
|
|
||||||
const Lattice<vobj> &fineY)
|
|
||||||
{
|
|
||||||
typedef decltype(innerProduct(vobj(),vobj())) dotp;
|
|
||||||
|
|
||||||
GridBase *coarse(CoarseInner.Grid());
|
|
||||||
GridBase *fine (fineX.Grid());
|
|
||||||
|
|
||||||
Lattice<dotp> fine_inner(fine); fine_inner.Checkerboard() = fineX.Checkerboard();
|
|
||||||
Lattice<dotp> fine_inner_msk(fine);
|
|
||||||
|
|
||||||
// Multiply could be fused with innerProduct
|
|
||||||
// Single block sum kernel could do both masks.
|
|
||||||
fine_inner = localInnerProduct(fineX,fineY);
|
|
||||||
mult(fine_inner_msk, fine_inner,FineMask);
|
|
||||||
blockSum(CoarseInner,fine_inner_msk);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fine Object == (per site) type of fine field
|
|
||||||
// nbasis == number of deflation vectors
|
|
||||||
template<class Fobj,class CComplex,int nbasis>
|
|
||||||
class CoarsenedMatrix : public CheckerBoardedSparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
|
||||||
public:
|
|
||||||
|
|
||||||
typedef iVector<CComplex,nbasis > siteVector;
|
|
||||||
typedef Lattice<CComplex > CoarseComplexField;
|
|
||||||
typedef Lattice<siteVector> CoarseVector;
|
|
||||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
|
||||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
|
||||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
|
||||||
typedef Lattice<Fobj > FineField;
|
|
||||||
typedef CoarseVector FermionField;
|
|
||||||
|
|
||||||
// enrich interface, use default implementation as in FermionOperator ///////
|
|
||||||
void Dminus(CoarseVector const& in, CoarseVector& out) { out = in; }
|
|
||||||
void DminusDag(CoarseVector const& in, CoarseVector& out) { out = in; }
|
|
||||||
void ImportPhysicalFermionSource(CoarseVector const& input, CoarseVector& imported) { imported = input; }
|
|
||||||
void ImportUnphysicalFermion(CoarseVector const& input, CoarseVector& imported) { imported = input; }
|
|
||||||
void ExportPhysicalFermionSolution(CoarseVector const& solution, CoarseVector& exported) { exported = solution; };
|
|
||||||
void ExportPhysicalFermionSource(CoarseVector const& solution, CoarseVector& exported) { exported = solution; };
|
|
||||||
|
|
||||||
////////////////////
|
|
||||||
// Data members
|
|
||||||
////////////////////
|
|
||||||
Geometry geom;
|
|
||||||
GridBase * _grid;
|
|
||||||
GridBase* _cbgrid;
|
|
||||||
int hermitian;
|
|
||||||
|
|
||||||
CartesianStencil<siteVector,siteVector,DefaultImplParams> Stencil;
|
|
||||||
CartesianStencil<siteVector,siteVector,DefaultImplParams> StencilEven;
|
|
||||||
CartesianStencil<siteVector,siteVector,DefaultImplParams> StencilOdd;
|
|
||||||
|
|
||||||
std::vector<CoarseMatrix> A;
|
|
||||||
std::vector<CoarseMatrix> Aeven;
|
|
||||||
std::vector<CoarseMatrix> Aodd;
|
|
||||||
|
|
||||||
CoarseMatrix AselfInv;
|
|
||||||
CoarseMatrix AselfInvEven;
|
|
||||||
CoarseMatrix AselfInvOdd;
|
|
||||||
|
|
||||||
deviceVector<RealD> dag_factor;
|
|
||||||
|
|
||||||
///////////////////////
|
|
||||||
// Interface
|
|
||||||
///////////////////////
|
|
||||||
GridBase * Grid(void) { return _grid; }; // this is all the linalg routines need to know
|
|
||||||
GridBase * RedBlackGrid() { return _cbgrid; };
|
|
||||||
|
|
||||||
int ConstEE() { return 0; }
|
|
||||||
|
|
||||||
void M (const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
conformable(_grid,in.Grid());
|
|
||||||
conformable(in.Grid(),out.Grid());
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
|
|
||||||
SimpleCompressor<siteVector> compressor;
|
|
||||||
|
|
||||||
Stencil.HaloExchange(in,compressor);
|
|
||||||
autoView( in_v , in, AcceleratorRead);
|
|
||||||
autoView( out_v , out, AcceleratorWrite);
|
|
||||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
|
||||||
int npoint = geom.npoint;
|
|
||||||
typedef LatticeView<Cobj> Aview;
|
|
||||||
|
|
||||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
|
||||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++) {
|
|
||||||
hAcceleratorViewContainer[p] = A[p].View(AcceleratorRead);
|
|
||||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
|
||||||
}
|
|
||||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
|
||||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
|
||||||
|
|
||||||
int osites=Grid()->oSites();
|
|
||||||
|
|
||||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
for(int point=0;point<npoint;point++){
|
|
||||||
|
|
||||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(Stencil_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
|
||||||
};
|
|
||||||
|
|
||||||
void Mdag (const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
if(hermitian) {
|
|
||||||
// corresponds to Petrov-Galerkin coarsening
|
|
||||||
return M(in,out);
|
|
||||||
} else {
|
|
||||||
// corresponds to Galerkin coarsening
|
|
||||||
return MdagNonHermitian(in, out);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
void MdagNonHermitian(const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
conformable(_grid,in.Grid());
|
|
||||||
conformable(in.Grid(),out.Grid());
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
|
|
||||||
SimpleCompressor<siteVector> compressor;
|
|
||||||
|
|
||||||
Stencil.HaloExchange(in,compressor);
|
|
||||||
autoView( in_v , in, AcceleratorRead);
|
|
||||||
autoView( out_v , out, AcceleratorWrite);
|
|
||||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
|
||||||
int npoint = geom.npoint;
|
|
||||||
typedef LatticeView<Cobj> Aview;
|
|
||||||
|
|
||||||
|
|
||||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
|
||||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++) {
|
|
||||||
hAcceleratorViewContainer[p] = A[p].View(AcceleratorRead);
|
|
||||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
|
||||||
}
|
|
||||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
|
||||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
|
||||||
|
|
||||||
int osites=Grid()->oSites();
|
|
||||||
|
|
||||||
deviceVector<int> points(geom.npoint);
|
|
||||||
for(int p=0; p<geom.npoint; p++) {
|
|
||||||
acceleratorPut(points[p],geom.points_dagger[p]);
|
|
||||||
}
|
|
||||||
auto points_p = &points[0];
|
|
||||||
|
|
||||||
RealD* dag_factor_p = &dag_factor[0];
|
|
||||||
|
|
||||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
for(int p=0;p<npoint;p++){
|
|
||||||
int point = points_p[p];
|
|
||||||
|
|
||||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(Stencil_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + dag_factor_p[b*nbasis+bb]*coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
void MdirComms(const CoarseVector &in)
|
|
||||||
{
|
|
||||||
SimpleCompressor<siteVector> compressor;
|
|
||||||
Stencil.HaloExchange(in,compressor);
|
|
||||||
}
|
|
||||||
void MdirCalc(const CoarseVector &in, CoarseVector &out, int point)
|
|
||||||
{
|
|
||||||
conformable(_grid,in.Grid());
|
|
||||||
conformable(_grid,out.Grid());
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
|
|
||||||
typedef LatticeView<Cobj> Aview;
|
|
||||||
|
|
||||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
|
||||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++) {
|
|
||||||
hAcceleratorViewContainer[p] = A[p].View(AcceleratorRead);
|
|
||||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
|
||||||
}
|
|
||||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
|
||||||
|
|
||||||
autoView( out_v , out, AcceleratorWrite);
|
|
||||||
autoView( in_v , in, AcceleratorRead);
|
|
||||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
|
||||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
|
||||||
|
|
||||||
accelerator_for(sss, Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
SE=Stencil_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(Stencil_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
for(int p=0;p<geom.npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
|
||||||
}
|
|
||||||
void MdirAll(const CoarseVector &in,std::vector<CoarseVector> &out)
|
|
||||||
{
|
|
||||||
this->MdirComms(in);
|
|
||||||
int ndir=geom.npoint-1;
|
|
||||||
if ((out.size()!=ndir)&&(out.size()!=ndir+1)) {
|
|
||||||
std::cout <<"MdirAll out size "<< out.size()<<std::endl;
|
|
||||||
std::cout <<"MdirAll ndir "<< ndir<<std::endl;
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
for(int p=0;p<ndir;p++){
|
|
||||||
MdirCalc(in,out[p],p);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
void Mdir(const CoarseVector &in, CoarseVector &out, int dir, int disp){
|
|
||||||
|
|
||||||
this->MdirComms(in);
|
|
||||||
|
|
||||||
MdirCalc(in,out,geom.point(dir,disp));
|
|
||||||
};
|
|
||||||
|
|
||||||
void Mdiag(const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
int point=geom.npoint-1;
|
|
||||||
MdirCalc(in, out, point); // No comms
|
|
||||||
};
|
|
||||||
|
|
||||||
void Mooee(const CoarseVector &in, CoarseVector &out) {
|
|
||||||
MooeeInternal(in, out, DaggerNo, InverseNo);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MooeeInv(const CoarseVector &in, CoarseVector &out) {
|
|
||||||
MooeeInternal(in, out, DaggerNo, InverseYes);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MooeeDag(const CoarseVector &in, CoarseVector &out) {
|
|
||||||
MooeeInternal(in, out, DaggerYes, InverseNo);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MooeeInvDag(const CoarseVector &in, CoarseVector &out) {
|
|
||||||
MooeeInternal(in, out, DaggerYes, InverseYes);
|
|
||||||
}
|
|
||||||
|
|
||||||
void Meooe(const CoarseVector &in, CoarseVector &out) {
|
|
||||||
if(in.Checkerboard() == Odd) {
|
|
||||||
DhopEO(in, out, DaggerNo);
|
|
||||||
} else {
|
|
||||||
DhopOE(in, out, DaggerNo);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void MeooeDag(const CoarseVector &in, CoarseVector &out) {
|
|
||||||
if(in.Checkerboard() == Odd) {
|
|
||||||
DhopEO(in, out, DaggerYes);
|
|
||||||
} else {
|
|
||||||
DhopOE(in, out, DaggerYes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void Dhop(const CoarseVector &in, CoarseVector &out, int dag) {
|
|
||||||
conformable(in.Grid(), _grid); // verifies full grid
|
|
||||||
conformable(in.Grid(), out.Grid());
|
|
||||||
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
|
|
||||||
DhopInternal(Stencil, A, in, out, dag);
|
|
||||||
}
|
|
||||||
|
|
||||||
void DhopOE(const CoarseVector &in, CoarseVector &out, int dag) {
|
|
||||||
conformable(in.Grid(), _cbgrid); // verifies half grid
|
|
||||||
conformable(in.Grid(), out.Grid()); // drops the cb check
|
|
||||||
|
|
||||||
GRID_ASSERT(in.Checkerboard() == Even);
|
|
||||||
out.Checkerboard() = Odd;
|
|
||||||
|
|
||||||
DhopInternal(StencilEven, Aodd, in, out, dag);
|
|
||||||
}
|
|
||||||
|
|
||||||
void DhopEO(const CoarseVector &in, CoarseVector &out, int dag) {
|
|
||||||
conformable(in.Grid(), _cbgrid); // verifies half grid
|
|
||||||
conformable(in.Grid(), out.Grid()); // drops the cb check
|
|
||||||
|
|
||||||
GRID_ASSERT(in.Checkerboard() == Odd);
|
|
||||||
out.Checkerboard() = Even;
|
|
||||||
|
|
||||||
DhopInternal(StencilOdd, Aeven, in, out, dag);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MooeeInternal(const CoarseVector &in, CoarseVector &out, int dag, int inv) {
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
GRID_ASSERT(in.Checkerboard() == Odd || in.Checkerboard() == Even);
|
|
||||||
|
|
||||||
CoarseMatrix *Aself = nullptr;
|
|
||||||
if(in.Grid()->_isCheckerBoarded) {
|
|
||||||
if(in.Checkerboard() == Odd) {
|
|
||||||
Aself = (inv) ? &AselfInvOdd : &Aodd[geom.npoint-1];
|
|
||||||
DselfInternal(StencilOdd, *Aself, in, out, dag);
|
|
||||||
} else {
|
|
||||||
Aself = (inv) ? &AselfInvEven : &Aeven[geom.npoint-1];
|
|
||||||
DselfInternal(StencilEven, *Aself, in, out, dag);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Aself = (inv) ? &AselfInv : &A[geom.npoint-1];
|
|
||||||
DselfInternal(Stencil, *Aself, in, out, dag);
|
|
||||||
}
|
|
||||||
GRID_ASSERT(Aself != nullptr);
|
|
||||||
}
|
|
||||||
|
|
||||||
void DselfInternal(CartesianStencil<siteVector,siteVector,DefaultImplParams> &st, CoarseMatrix &a,
|
|
||||||
const CoarseVector &in, CoarseVector &out, int dag) {
|
|
||||||
int point = geom.npoint-1;
|
|
||||||
autoView( out_v, out, AcceleratorWrite);
|
|
||||||
autoView( in_v, in, AcceleratorRead);
|
|
||||||
autoView( st_v, st, AcceleratorRead);
|
|
||||||
autoView( a_v, a, AcceleratorRead);
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
|
||||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
|
||||||
|
|
||||||
RealD* dag_factor_p = &dag_factor[0];
|
|
||||||
|
|
||||||
if(dag) {
|
|
||||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
SE=st_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + dag_factor_p[b*nbasis+bb]*coalescedRead(a_v[ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
SE=st_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + coalescedRead(a_v[ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void DhopInternal(CartesianStencil<siteVector,siteVector,DefaultImplParams> &st, std::vector<CoarseMatrix> &a,
|
|
||||||
const CoarseVector &in, CoarseVector &out, int dag) {
|
|
||||||
SimpleCompressor<siteVector> compressor;
|
|
||||||
|
|
||||||
st.HaloExchange(in,compressor);
|
|
||||||
autoView( in_v, in, AcceleratorRead);
|
|
||||||
autoView( out_v, out, AcceleratorWrite);
|
|
||||||
autoView( st_v , st, AcceleratorRead);
|
|
||||||
typedef LatticeView<Cobj> Aview;
|
|
||||||
|
|
||||||
// determine in what order we need the points
|
|
||||||
int npoint = geom.npoint-1;
|
|
||||||
deviceVector<int> points(npoint);
|
|
||||||
for(int p=0; p<npoint; p++) {
|
|
||||||
int val = (dag && !hermitian) ? geom.points_dagger[p] : p;
|
|
||||||
acceleratorPut(points[p], val);
|
|
||||||
}
|
|
||||||
auto points_p = &points[0];
|
|
||||||
|
|
||||||
deviceVector<Aview> AcceleratorViewContainer(geom.npoint);
|
|
||||||
hostVector<Aview> hAcceleratorViewContainer(geom.npoint);
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++) {
|
|
||||||
hAcceleratorViewContainer[p] = a[p].View(AcceleratorRead);
|
|
||||||
acceleratorPut(AcceleratorViewContainer[p],hAcceleratorViewContainer[p]);
|
|
||||||
}
|
|
||||||
Aview *Aview_p = & AcceleratorViewContainer[0];
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
typedef decltype(coalescedRead(in_v[0])) calcVector;
|
|
||||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
|
||||||
|
|
||||||
RealD* dag_factor_p = &dag_factor[0];
|
|
||||||
|
|
||||||
if(dag) {
|
|
||||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
for(int p=0;p<npoint;p++){
|
|
||||||
int point = points_p[p];
|
|
||||||
SE=st_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + dag_factor_p[b*nbasis+bb]*coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
accelerator_for(sss, in.Grid()->oSites()*nbasis, Nsimd, {
|
|
||||||
int ss = sss/nbasis;
|
|
||||||
int b = sss%nbasis;
|
|
||||||
calcComplex res = Zero();
|
|
||||||
calcVector nbr;
|
|
||||||
int ptype;
|
|
||||||
StencilEntry *SE;
|
|
||||||
|
|
||||||
for(int p=0;p<npoint;p++){
|
|
||||||
int point = points_p[p];
|
|
||||||
SE=st_v.GetEntry(ptype,point,ss);
|
|
||||||
|
|
||||||
if(SE->_is_local) {
|
|
||||||
nbr = coalescedReadPermute(in_v[SE->_offset],ptype,SE->_permute);
|
|
||||||
} else {
|
|
||||||
nbr = coalescedRead(st_v.CommBuf()[SE->_offset]);
|
|
||||||
}
|
|
||||||
acceleratorSynchronise();
|
|
||||||
|
|
||||||
for(int bb=0;bb<nbasis;bb++) {
|
|
||||||
res = res + coalescedRead(Aview_p[point][ss](b,bb))*nbr(bb);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int p=0;p<npoint;p++) hAcceleratorViewContainer[p].ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
CoarsenedMatrix(GridCartesian &CoarseGrid, int hermitian_=0) :
|
|
||||||
_grid(&CoarseGrid),
|
|
||||||
_cbgrid(new GridRedBlackCartesian(&CoarseGrid)),
|
|
||||||
geom(CoarseGrid._ndimension),
|
|
||||||
hermitian(hermitian_),
|
|
||||||
Stencil(&CoarseGrid,geom.npoint,Even,geom.directions,geom.displacements),
|
|
||||||
StencilEven(_cbgrid,geom.npoint,Even,geom.directions,geom.displacements),
|
|
||||||
StencilOdd(_cbgrid,geom.npoint,Odd,geom.directions,geom.displacements),
|
|
||||||
A(geom.npoint,&CoarseGrid),
|
|
||||||
Aeven(geom.npoint,_cbgrid),
|
|
||||||
Aodd(geom.npoint,_cbgrid),
|
|
||||||
AselfInv(&CoarseGrid),
|
|
||||||
AselfInvEven(_cbgrid),
|
|
||||||
AselfInvOdd(_cbgrid),
|
|
||||||
dag_factor(nbasis*nbasis)
|
|
||||||
{
|
|
||||||
fillFactor();
|
|
||||||
};
|
|
||||||
|
|
||||||
CoarsenedMatrix(GridCartesian &CoarseGrid, GridRedBlackCartesian &CoarseRBGrid, int hermitian_=0) :
|
|
||||||
|
|
||||||
_grid(&CoarseGrid),
|
|
||||||
_cbgrid(&CoarseRBGrid),
|
|
||||||
geom(CoarseGrid._ndimension),
|
|
||||||
hermitian(hermitian_),
|
|
||||||
Stencil(&CoarseGrid,geom.npoint,Even,geom.directions,geom.displacements),
|
|
||||||
StencilEven(&CoarseRBGrid,geom.npoint,Even,geom.directions,geom.displacements),
|
|
||||||
StencilOdd(&CoarseRBGrid,geom.npoint,Odd,geom.directions,geom.displacements),
|
|
||||||
A(geom.npoint,&CoarseGrid),
|
|
||||||
Aeven(geom.npoint,&CoarseRBGrid),
|
|
||||||
Aodd(geom.npoint,&CoarseRBGrid),
|
|
||||||
AselfInv(&CoarseGrid),
|
|
||||||
AselfInvEven(&CoarseRBGrid),
|
|
||||||
AselfInvOdd(&CoarseRBGrid),
|
|
||||||
dag_factor(nbasis*nbasis)
|
|
||||||
{
|
|
||||||
fillFactor();
|
|
||||||
};
|
|
||||||
|
|
||||||
void fillFactor() {
|
|
||||||
Eigen::MatrixXd dag_factor_eigen = Eigen::MatrixXd::Ones(nbasis, nbasis);
|
|
||||||
if(!hermitian) {
|
|
||||||
const int nb = nbasis/2;
|
|
||||||
dag_factor_eigen.block(0,nb,nb,nb) *= -1.0;
|
|
||||||
dag_factor_eigen.block(nb,0,nb,nb) *= -1.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// GPU readable prefactor
|
|
||||||
std::vector<RealD> h_dag_factor(nbasis*nbasis);
|
|
||||||
thread_for(i, nbasis*nbasis, {
|
|
||||||
int j = i/nbasis;
|
|
||||||
int k = i%nbasis;
|
|
||||||
h_dag_factor[i] = dag_factor_eigen(j, k);
|
|
||||||
});
|
|
||||||
acceleratorCopyToDevice(&h_dag_factor[0],&dag_factor[0],dag_factor.size()*sizeof(RealD));
|
|
||||||
}
|
|
||||||
|
|
||||||
void CoarsenOperator(GridBase *FineGrid,LinearOperatorBase<Lattice<Fobj> > &linop,
|
|
||||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
|
||||||
{
|
|
||||||
typedef Lattice<typename Fobj::tensor_reduced> FineComplexField;
|
|
||||||
typedef typename Fobj::scalar_type scalar_type;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrix "<< std::endl;
|
|
||||||
|
|
||||||
FineComplexField one(FineGrid); one=scalar_type(1.0,0.0);
|
|
||||||
FineComplexField zero(FineGrid); zero=scalar_type(0.0,0.0);
|
|
||||||
|
|
||||||
std::vector<FineComplexField> masks(geom.npoint,FineGrid);
|
|
||||||
FineComplexField imask(FineGrid); // contributions from within this block
|
|
||||||
FineComplexField omask(FineGrid); // contributions from outwith this block
|
|
||||||
|
|
||||||
FineComplexField evenmask(FineGrid);
|
|
||||||
FineComplexField oddmask(FineGrid);
|
|
||||||
|
|
||||||
FineField phi(FineGrid);
|
|
||||||
FineField tmp(FineGrid);
|
|
||||||
FineField zz(FineGrid); zz=Zero();
|
|
||||||
FineField Mphi(FineGrid);
|
|
||||||
FineField Mphie(FineGrid);
|
|
||||||
FineField Mphio(FineGrid);
|
|
||||||
std::vector<FineField> Mphi_p(geom.npoint,FineGrid);
|
|
||||||
|
|
||||||
Lattice<iScalar<vInteger> > coor (FineGrid);
|
|
||||||
Lattice<iScalar<vInteger> > bcoor(FineGrid);
|
|
||||||
Lattice<iScalar<vInteger> > bcb (FineGrid); bcb = Zero();
|
|
||||||
|
|
||||||
CoarseVector iProj(Grid());
|
|
||||||
CoarseVector oProj(Grid());
|
|
||||||
CoarseVector SelfProj(Grid());
|
|
||||||
CoarseComplexField iZProj(Grid());
|
|
||||||
CoarseComplexField oZProj(Grid());
|
|
||||||
|
|
||||||
CoarseScalar InnerProd(Grid());
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrix Orthog "<< std::endl;
|
|
||||||
// Orthogonalise the subblocks over the basis
|
|
||||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
|
||||||
|
|
||||||
// Compute the matrix elements of linop between this orthonormal
|
|
||||||
// set of vectors.
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrix masks "<< std::endl;
|
|
||||||
int self_stencil=-1;
|
|
||||||
for(int p=0;p<geom.npoint;p++)
|
|
||||||
{
|
|
||||||
int dir = geom.directions[p];
|
|
||||||
int disp = geom.displacements[p];
|
|
||||||
A[p]=Zero();
|
|
||||||
if( geom.displacements[p]==0){
|
|
||||||
self_stencil=p;
|
|
||||||
}
|
|
||||||
|
|
||||||
Integer block=(FineGrid->_rdimensions[dir])/(Grid()->_rdimensions[dir]);
|
|
||||||
|
|
||||||
LatticeCoordinate(coor,dir);
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////
|
|
||||||
// Work out even and odd block checkerboarding for fast diagonal term
|
|
||||||
///////////////////////////////////////////////////////
|
|
||||||
if ( disp==1 ) {
|
|
||||||
bcb = bcb + div(coor,block);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( disp==0 ) {
|
|
||||||
masks[p]= Zero();
|
|
||||||
} else if ( disp==1 ) {
|
|
||||||
masks[p] = where(mod(coor,block)==(block-1),one,zero);
|
|
||||||
} else if ( disp==-1 ) {
|
|
||||||
masks[p] = where(mod(coor,block)==(Integer)0,one,zero);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
evenmask = where(mod(bcb,2)==(Integer)0,one,zero);
|
|
||||||
oddmask = one-evenmask;
|
|
||||||
|
|
||||||
GRID_ASSERT(self_stencil!=-1);
|
|
||||||
|
|
||||||
for(int i=0;i<nbasis;i++){
|
|
||||||
|
|
||||||
phi=Subspace.subspace[i];
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrix vector "<<i << std::endl;
|
|
||||||
linop.OpDirAll(phi,Mphi_p);
|
|
||||||
linop.OpDiag (phi,Mphi_p[geom.npoint-1]);
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
|
|
||||||
Mphi = Mphi_p[p];
|
|
||||||
|
|
||||||
int dir = geom.directions[p];
|
|
||||||
int disp = geom.displacements[p];
|
|
||||||
|
|
||||||
if ( (disp==-1) || (!hermitian ) ) {
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
// Pick out contributions coming from this cell and neighbour cell
|
|
||||||
////////////////////////////////////////////////////////////////////////
|
|
||||||
omask = masks[p];
|
|
||||||
imask = one-omask;
|
|
||||||
|
|
||||||
for(int j=0;j<nbasis;j++){
|
|
||||||
|
|
||||||
blockMaskedInnerProduct(oZProj,omask,Subspace.subspace[j],Mphi);
|
|
||||||
|
|
||||||
autoView( iZProj_v , iZProj, AcceleratorRead) ;
|
|
||||||
autoView( oZProj_v , oZProj, AcceleratorRead) ;
|
|
||||||
autoView( A_p , A[p], AcceleratorWrite);
|
|
||||||
autoView( A_self , A[self_stencil], AcceleratorWrite);
|
|
||||||
|
|
||||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_p[ss](j,i),oZProj_v(ss)); });
|
|
||||||
if ( hermitian && (disp==-1) ) {
|
|
||||||
for(int pp=0;pp<geom.npoint;pp++){// Find the opposite link and set <j|A|i> = <i|A|j>*
|
|
||||||
int dirp = geom.directions[pp];
|
|
||||||
int dispp = geom.displacements[pp];
|
|
||||||
if ( (dirp==dir) && (dispp==1) ){
|
|
||||||
auto sft = conjugate(Cshift(oZProj,dir,1));
|
|
||||||
autoView( sft_v , sft , AcceleratorWrite);
|
|
||||||
autoView( A_pp , A[pp], AcceleratorWrite);
|
|
||||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{ coalescedWrite(A_pp[ss](i,j),sft_v(ss)); });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// Faster alternate self coupling.. use hermiticity to save 2x
|
|
||||||
///////////////////////////////////////////
|
|
||||||
{
|
|
||||||
mult(tmp,phi,evenmask); linop.Op(tmp,Mphie);
|
|
||||||
mult(tmp,phi,oddmask ); linop.Op(tmp,Mphio);
|
|
||||||
|
|
||||||
{
|
|
||||||
autoView( tmp_ , tmp, AcceleratorWrite);
|
|
||||||
autoView( evenmask_ , evenmask, AcceleratorRead);
|
|
||||||
autoView( oddmask_ , oddmask, AcceleratorRead);
|
|
||||||
autoView( Mphie_ , Mphie, AcceleratorRead);
|
|
||||||
autoView( Mphio_ , Mphio, AcceleratorRead);
|
|
||||||
accelerator_for(ss, FineGrid->oSites(), Fobj::Nsimd(),{
|
|
||||||
coalescedWrite(tmp_[ss],evenmask_(ss)*Mphie_(ss) + oddmask_(ss)*Mphio_(ss));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
blockProject(SelfProj,tmp,Subspace.subspace);
|
|
||||||
|
|
||||||
autoView( SelfProj_ , SelfProj, AcceleratorRead);
|
|
||||||
autoView( A_self , A[self_stencil], AcceleratorWrite);
|
|
||||||
|
|
||||||
accelerator_for(ss, Grid()->oSites(), Fobj::Nsimd(),{
|
|
||||||
for(int j=0;j<nbasis;j++){
|
|
||||||
coalescedWrite(A_self[ss](j,i), SelfProj_(ss)(j));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(hermitian) {
|
|
||||||
std::cout << GridLogMessage << " ForceHermitian, new code "<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
InvertSelfStencilLink(); std::cout << GridLogMessage << "Coarse self link inverted" << std::endl;
|
|
||||||
FillHalfCbs(); std::cout << GridLogMessage << "Coarse half checkerboards filled" << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
void InvertSelfStencilLink() {
|
|
||||||
std::cout << GridLogDebug << "CoarsenedMatrix::InvertSelfStencilLink" << std::endl;
|
|
||||||
int localVolume = Grid()->lSites();
|
|
||||||
|
|
||||||
typedef typename Cobj::scalar_object scalar_object;
|
|
||||||
|
|
||||||
autoView(Aself_v, A[geom.npoint-1], CpuRead);
|
|
||||||
autoView(AselfInv_v, AselfInv, CpuWrite);
|
|
||||||
thread_for(site, localVolume, { // NOTE: Not able to bring this to GPU because of Eigen + peek/poke
|
|
||||||
Eigen::MatrixXcd selfLinkEigen = Eigen::MatrixXcd::Zero(nbasis, nbasis);
|
|
||||||
Eigen::MatrixXcd selfLinkInvEigen = Eigen::MatrixXcd::Zero(nbasis, nbasis);
|
|
||||||
|
|
||||||
scalar_object selfLink = Zero();
|
|
||||||
scalar_object selfLinkInv = Zero();
|
|
||||||
|
|
||||||
Coordinate lcoor;
|
|
||||||
|
|
||||||
Grid()->LocalIndexToLocalCoor(site, lcoor);
|
|
||||||
peekLocalSite(selfLink, Aself_v, lcoor);
|
|
||||||
|
|
||||||
for (int i = 0; i < nbasis; ++i)
|
|
||||||
for (int j = 0; j < nbasis; ++j)
|
|
||||||
selfLinkEigen(i, j) = static_cast<ComplexD>(TensorRemove(selfLink(i, j)));
|
|
||||||
|
|
||||||
selfLinkInvEigen = selfLinkEigen.inverse();
|
|
||||||
|
|
||||||
for(int i = 0; i < nbasis; ++i)
|
|
||||||
for(int j = 0; j < nbasis; ++j)
|
|
||||||
selfLinkInv(i, j) = selfLinkInvEigen(i, j);
|
|
||||||
|
|
||||||
pokeLocalSite(selfLinkInv, AselfInv_v, lcoor);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
void FillHalfCbs() {
|
|
||||||
std::cout << GridLogDebug << "CoarsenedMatrix::FillHalfCbs" << std::endl;
|
|
||||||
for(int p = 0; p < geom.npoint; ++p) {
|
|
||||||
pickCheckerboard(Even, Aeven[p], A[p]);
|
|
||||||
pickCheckerboard(Odd, Aodd[p], A[p]);
|
|
||||||
}
|
|
||||||
pickCheckerboard(Even, AselfInvEven, AselfInv);
|
|
||||||
pickCheckerboard(Odd, AselfInvOdd, AselfInv);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,629 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/GeneralCoarsenedMatrix.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <Grid/qcd/QCD.h> // needed for Dagger(Yes|No), Inverse(Yes|No)
|
|
||||||
|
|
||||||
#include <Grid/lattice/PaddedCell.h>
|
|
||||||
#include <Grid/stencil/GeneralLocalStencil.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
// Fine Object == (per site) type of fine field
|
|
||||||
// nbasis == number of deflation vectors
|
|
||||||
template<class Fobj,class CComplex,int nbasis>
|
|
||||||
class GeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
|
||||||
public:
|
|
||||||
|
|
||||||
typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
|
|
||||||
typedef iVector<CComplex,nbasis > siteVector;
|
|
||||||
typedef iMatrix<CComplex,nbasis > siteMatrix;
|
|
||||||
typedef Lattice<iScalar<CComplex> > CoarseComplexField;
|
|
||||||
typedef Lattice<siteVector> CoarseVector;
|
|
||||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
|
||||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
|
||||||
typedef iVector<CComplex,nbasis > Cvec;
|
|
||||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
|
||||||
typedef Lattice<Fobj > FineField;
|
|
||||||
typedef Lattice<CComplex > FineComplexField;
|
|
||||||
typedef CoarseVector Field;
|
|
||||||
////////////////////
|
|
||||||
// Data members
|
|
||||||
////////////////////
|
|
||||||
int hermitian;
|
|
||||||
GridBase * _FineGrid;
|
|
||||||
GridCartesian * _CoarseGrid;
|
|
||||||
NonLocalStencilGeometry &geom;
|
|
||||||
PaddedCell Cell;
|
|
||||||
GeneralLocalStencil Stencil;
|
|
||||||
|
|
||||||
std::vector<CoarseMatrix> _A;
|
|
||||||
std::vector<CoarseMatrix> _Adag;
|
|
||||||
std::vector<CoarseVector> MultTemporaries;
|
|
||||||
|
|
||||||
///////////////////////
|
|
||||||
// Interface
|
|
||||||
///////////////////////
|
|
||||||
GridBase * Grid(void) { return _CoarseGrid; }; // this is all the linalg routines need to know
|
|
||||||
GridBase * FineGrid(void) { return _FineGrid; }; // this is all the linalg routines need to know
|
|
||||||
GridCartesian * CoarseGrid(void) { return _CoarseGrid; }; // this is all the linalg routines need to know
|
|
||||||
|
|
||||||
/* void ShiftMatrix(RealD shift)
|
|
||||||
{
|
|
||||||
int Nd=_FineGrid->Nd();
|
|
||||||
Coordinate zero_shift(Nd,0);
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
if ( zero_shift==geom.shifts[p] ) {
|
|
||||||
_A[p] = _A[p]+shift;
|
|
||||||
// _Adag[p] = _Adag[p]+shift;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void ProjectNearestNeighbour(RealD shift, GeneralCoarseOp &CopyMe)
|
|
||||||
{
|
|
||||||
int nfound=0;
|
|
||||||
std::cout << GridLogMessage <<"GeneralCoarsenedMatrix::ProjectNearestNeighbour "<< CopyMe._A[0].Grid()<<std::endl;
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
for(int pp=0;pp<CopyMe.geom.npoint;pp++){
|
|
||||||
// Search for the same relative shift
|
|
||||||
// Avoids brutal handling of Grid pointers
|
|
||||||
if ( CopyMe.geom.shifts[pp]==geom.shifts[p] ) {
|
|
||||||
_A[p] = CopyMe.Cell.Extract(CopyMe._A[pp]);
|
|
||||||
// _Adag[p] = CopyMe.Cell.Extract(CopyMe._Adag[pp]);
|
|
||||||
nfound++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
GRID_ASSERT(nfound==geom.npoint);
|
|
||||||
ExchangeCoarseLinks();
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
GeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridBase *FineGrid, GridCartesian * CoarseGrid)
|
|
||||||
: geom(_geom),
|
|
||||||
_FineGrid(FineGrid),
|
|
||||||
_CoarseGrid(CoarseGrid),
|
|
||||||
hermitian(1),
|
|
||||||
Cell(_geom.Depth(),_CoarseGrid),
|
|
||||||
Stencil(Cell.grids.back(),geom.shifts)
|
|
||||||
{
|
|
||||||
{
|
|
||||||
int npoint = _geom.npoint;
|
|
||||||
}
|
|
||||||
_A.resize(geom.npoint,CoarseGrid);
|
|
||||||
// _Adag.resize(geom.npoint,CoarseGrid);
|
|
||||||
}
|
|
||||||
void M (const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
Mult(_A,in,out);
|
|
||||||
}
|
|
||||||
void Mdag (const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(hermitian);
|
|
||||||
Mult(_A,in,out);
|
|
||||||
// if ( hermitian ) M(in,out);
|
|
||||||
// else Mult(_Adag,in,out);
|
|
||||||
}
|
|
||||||
void Mult (std::vector<CoarseMatrix> &A,const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
RealD tviews=0; RealD ttot=0; RealD tmult=0; RealD texch=0; RealD text=0; RealD ttemps=0; RealD tcopy=0;
|
|
||||||
RealD tmult2=0;
|
|
||||||
|
|
||||||
ttot=-usecond();
|
|
||||||
conformable(CoarseGrid(),in.Grid());
|
|
||||||
conformable(in.Grid(),out.Grid());
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
CoarseVector tin=in;
|
|
||||||
|
|
||||||
texch-=usecond();
|
|
||||||
CoarseVector pin = Cell.ExchangePeriodic(tin);
|
|
||||||
texch+=usecond();
|
|
||||||
|
|
||||||
CoarseVector pout(pin.Grid());
|
|
||||||
|
|
||||||
int npoint = geom.npoint;
|
|
||||||
typedef LatticeView<Cobj> Aview;
|
|
||||||
typedef LatticeView<Cvec> Vview;
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
|
|
||||||
int64_t osites=pin.Grid()->oSites();
|
|
||||||
|
|
||||||
RealD flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
|
|
||||||
RealD bytes = 1.0*osites*sizeof(siteMatrix)*npoint
|
|
||||||
+ 2.0*osites*sizeof(siteVector)*npoint;
|
|
||||||
|
|
||||||
{
|
|
||||||
tviews-=usecond();
|
|
||||||
autoView( in_v , pin, AcceleratorRead);
|
|
||||||
autoView( out_v , pout, AcceleratorWriteDiscard);
|
|
||||||
autoView( Stencil_v , Stencil, AcceleratorRead);
|
|
||||||
tviews+=usecond();
|
|
||||||
|
|
||||||
// Static and prereserve to keep UVM region live and not resized across multiple calls
|
|
||||||
ttemps-=usecond();
|
|
||||||
MultTemporaries.resize(npoint,pin.Grid());
|
|
||||||
ttemps+=usecond();
|
|
||||||
std::vector<Aview> AcceleratorViewContainer_h;
|
|
||||||
std::vector<Vview> AcceleratorVecViewContainer_h;
|
|
||||||
|
|
||||||
tviews-=usecond();
|
|
||||||
for(int p=0;p<npoint;p++) {
|
|
||||||
AcceleratorViewContainer_h.push_back( A[p].View(AcceleratorRead));
|
|
||||||
AcceleratorVecViewContainer_h.push_back(MultTemporaries[p].View(AcceleratorWrite));
|
|
||||||
}
|
|
||||||
tviews+=usecond();
|
|
||||||
|
|
||||||
static deviceVector<Aview> AcceleratorViewContainer; AcceleratorViewContainer.resize(npoint);
|
|
||||||
static deviceVector<Vview> AcceleratorVecViewContainer; AcceleratorVecViewContainer.resize(npoint);
|
|
||||||
|
|
||||||
auto Aview_p = &AcceleratorViewContainer[0];
|
|
||||||
auto Vview_p = &AcceleratorVecViewContainer[0];
|
|
||||||
tcopy-=usecond();
|
|
||||||
acceleratorCopyToDevice(&AcceleratorViewContainer_h[0],&AcceleratorViewContainer[0],npoint *sizeof(Aview));
|
|
||||||
acceleratorCopyToDevice(&AcceleratorVecViewContainer_h[0],&AcceleratorVecViewContainer[0],npoint *sizeof(Vview));
|
|
||||||
tcopy+=usecond();
|
|
||||||
|
|
||||||
tmult-=usecond();
|
|
||||||
accelerator_for(spb, osites*nbasis*npoint, Nsimd, {
|
|
||||||
typedef decltype(coalescedRead(in_v[0](0))) calcComplex;
|
|
||||||
int32_t ss = spb/(nbasis*npoint);
|
|
||||||
int32_t bp = spb%(nbasis*npoint);
|
|
||||||
int32_t point= bp/nbasis;
|
|
||||||
int32_t b = bp%nbasis;
|
|
||||||
auto SE = Stencil_v.GetEntry(point,ss);
|
|
||||||
auto nbr = coalescedReadGeneralPermute(in_v[SE->_offset],SE->_permute,Nd);
|
|
||||||
auto res = coalescedRead(Aview_p[point][ss](0,b))*nbr(0);
|
|
||||||
for(int bb=1;bb<nbasis;bb++) {
|
|
||||||
res = res + coalescedRead(Aview_p[point][ss](bb,b))*nbr(bb);
|
|
||||||
}
|
|
||||||
coalescedWrite(Vview_p[point][ss](b),res);
|
|
||||||
});
|
|
||||||
tmult2-=usecond();
|
|
||||||
accelerator_for(sb, osites*nbasis, Nsimd, {
|
|
||||||
int ss = sb/nbasis;
|
|
||||||
int b = sb%nbasis;
|
|
||||||
auto res = coalescedRead(Vview_p[0][ss](b));
|
|
||||||
for(int point=1;point<npoint;point++){
|
|
||||||
res = res + coalescedRead(Vview_p[point][ss](b));
|
|
||||||
}
|
|
||||||
coalescedWrite(out_v[ss](b),res);
|
|
||||||
});
|
|
||||||
tmult2+=usecond();
|
|
||||||
tmult+=usecond();
|
|
||||||
for(int p=0;p<npoint;p++) {
|
|
||||||
AcceleratorViewContainer_h[p].ViewClose();
|
|
||||||
AcceleratorVecViewContainer_h[p].ViewClose();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
text-=usecond();
|
|
||||||
out = Cell.Extract(pout);
|
|
||||||
text+=usecond();
|
|
||||||
ttot+=usecond();
|
|
||||||
|
|
||||||
std::cout << GridLogPerformance<<"Coarse 1rhs Mult Aviews "<<tviews<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Mult exch "<<texch<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Mult mult "<<tmult<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<" of which mult2 "<<tmult2<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Mult ext "<<text<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Mult temps "<<ttemps<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Mult copy "<<tcopy<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Mult tot "<<ttot<<" us"<<std::endl;
|
|
||||||
// std::cout << GridLogPerformance<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Kernel flops "<< flops<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Kernel flop/s "<< flops/tmult<<" mflop/s"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse Kernel bytes/s "<< bytes/tmult<<" MB/s"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse overall flops/s "<< flops/ttot<<" mflop/s"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance<<"Coarse total bytes "<< bytes/1e6<<" MB"<<std::endl;
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
void PopulateAdag(void)
|
|
||||||
{
|
|
||||||
for(int64_t bidx=0;bidx<CoarseGrid()->gSites() ;bidx++){
|
|
||||||
Coordinate bcoor;
|
|
||||||
CoarseGrid()->GlobalIndexToGlobalCoor(bidx,bcoor);
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
Coordinate scoor = bcoor;
|
|
||||||
for(int mu=0;mu<bcoor.size();mu++){
|
|
||||||
int L = CoarseGrid()->GlobalDimensions()[mu];
|
|
||||||
scoor[mu] = (bcoor[mu] - geom.shifts[p][mu] + L) % L; // Modulo arithmetic
|
|
||||||
}
|
|
||||||
// Flip to poke/peekLocalSite and not too bad
|
|
||||||
auto link = peekSite(_A[p],scoor);
|
|
||||||
int pp = geom.Reverse(p);
|
|
||||||
pokeSite(adj(link),_Adag[pp],bcoor);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
//
|
|
||||||
// A) Only reduced flops option is to use a padded cell of depth 4
|
|
||||||
// and apply MpcDagMpc in the padded cell.
|
|
||||||
//
|
|
||||||
// Makes for ONE application of MpcDagMpc per vector instead of 30 or 80.
|
|
||||||
// With the effective cell size around (B+8)^4 perhaps 12^4/4^4 ratio
|
|
||||||
// Cost is 81x more, same as stencil size.
|
|
||||||
//
|
|
||||||
// But: can eliminate comms and do as local dirichlet.
|
|
||||||
//
|
|
||||||
// Local exchange gauge field once.
|
|
||||||
// Apply to all vectors, local only computation.
|
|
||||||
// Must exchange ghost subcells in reverse process of PaddedCell to take inner products
|
|
||||||
//
|
|
||||||
// B) Can reduce cost: pad by 1, apply Deo (4^4+6^4+8^4+8^4 )/ (4x 4^4)
|
|
||||||
// pad by 2, apply Doe
|
|
||||||
// pad by 3, apply Deo
|
|
||||||
// then break out 8x directions; cost is ~10x MpcDagMpc per vector
|
|
||||||
//
|
|
||||||
// => almost factor of 10 in setup cost, excluding data rearrangement
|
|
||||||
//
|
|
||||||
// Intermediates -- ignore the corner terms, leave approximate and force Hermitian
|
|
||||||
// Intermediates -- pad by 2 and apply 1+8+24 = 33 times.
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
// BFM HDCG style approach: Solve a system of equations to get Aij
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
/*
|
|
||||||
* Here, k,l index which possible shift within the 3^Nd "ball" connected by MdagM.
|
|
||||||
*
|
|
||||||
* conj(phases[block]) proj[k][ block*Nvec+j ] = \sum_ball e^{i q_k . delta} < phi_{block,j} | MdagM | phi_{(block+delta),i} >
|
|
||||||
* = \sum_ball e^{iqk.delta} A_ji
|
|
||||||
*
|
|
||||||
* Must invert matrix M_k,l = e^[i q_k . delta_l]
|
|
||||||
*
|
|
||||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
|
||||||
*/
|
|
||||||
#if 0
|
|
||||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
|
||||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrix "<< std::endl;
|
|
||||||
GridBase *grid = FineGrid();
|
|
||||||
|
|
||||||
RealD tproj=0.0;
|
|
||||||
RealD teigen=0.0;
|
|
||||||
RealD tmat=0.0;
|
|
||||||
RealD tphase=0.0;
|
|
||||||
RealD tinv=0.0;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Orthogonalise the subblocks over the basis
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
CoarseScalar InnerProd(CoarseGrid());
|
|
||||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
|
||||||
|
|
||||||
const int npoint = geom.npoint;
|
|
||||||
|
|
||||||
Coordinate clatt = CoarseGrid()->GlobalDimensions();
|
|
||||||
int Nd = CoarseGrid()->Nd();
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
|
||||||
* Matrix index i is mapped to this shift via
|
|
||||||
* geom.shifts[i]
|
|
||||||
*
|
|
||||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
|
||||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
|
||||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
|
||||||
* = M_{kl} A_ji^{b.b+l}
|
|
||||||
*
|
|
||||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
|
||||||
*
|
|
||||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
|
||||||
*
|
|
||||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
|
||||||
*/
|
|
||||||
teigen-=usecond();
|
|
||||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
ComplexD ci(0.0,1.0);
|
|
||||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
|
||||||
|
|
||||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
|
||||||
ComplexD phase(0.0,0.0);
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
phase=phase+TwoPiL*geom.shifts[k][mu]*geom.shifts[l][mu];
|
|
||||||
}
|
|
||||||
phase=exp(phase*ci);
|
|
||||||
Mkl(k,l) = phase;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
invMkl = Mkl.inverse();
|
|
||||||
teigen+=usecond();
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
// Now compute the matrix elements of linop between the orthonormal
|
|
||||||
// set of vectors.
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
FineField phaV(grid); // Phased block basis vector
|
|
||||||
FineField MphaV(grid);// Matrix applied
|
|
||||||
CoarseVector coarseInner(CoarseGrid());
|
|
||||||
|
|
||||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid());
|
|
||||||
std::vector<CoarseVector> FT(npoint,CoarseGrid());
|
|
||||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
|
||||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Stick a phase on every block
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
tphase-=usecond();
|
|
||||||
CoarseComplexField coor(CoarseGrid());
|
|
||||||
CoarseComplexField pha(CoarseGrid()); pha=Zero();
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
LatticeCoordinate(coor,mu);
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
pha = pha + (TwoPiL * geom.shifts[p][mu]) * coor;
|
|
||||||
}
|
|
||||||
pha =exp(pha*ci);
|
|
||||||
phaV=Zero();
|
|
||||||
blockZAXPY(phaV,pha,Subspace.subspace[i],phaV);
|
|
||||||
tphase+=usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
// Multiple phased subspace vector by matrix and project to subspace
|
|
||||||
// Remove local bulk phase to leave relative phases
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
tmat-=usecond();
|
|
||||||
linop.Op(phaV,MphaV);
|
|
||||||
tmat+=usecond();
|
|
||||||
|
|
||||||
tproj-=usecond();
|
|
||||||
blockProject(coarseInner,MphaV,Subspace.subspace);
|
|
||||||
coarseInner = conjugate(pha) * coarseInner;
|
|
||||||
|
|
||||||
ComputeProj[p] = coarseInner;
|
|
||||||
tproj+=usecond();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
tinv-=usecond();
|
|
||||||
for(int k=0;k<npoint;k++){
|
|
||||||
FT[k] = Zero();
|
|
||||||
for(int l=0;l<npoint;l++){
|
|
||||||
FT[k]= FT[k]+ invMkl(l,k)*ComputeProj[l];
|
|
||||||
}
|
|
||||||
|
|
||||||
int osites=CoarseGrid()->oSites();
|
|
||||||
autoView( A_v , _A[k], AcceleratorWrite);
|
|
||||||
autoView( FT_v , FT[k], AcceleratorRead);
|
|
||||||
accelerator_for(sss, osites, 1, {
|
|
||||||
for(int j=0;j<nbasis;j++){
|
|
||||||
A_v[sss](i,j) = FT_v[sss](j);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
tinv+=usecond();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only needed if nonhermitian
|
|
||||||
if ( ! hermitian ) {
|
|
||||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
|
||||||
// PopulateAdag();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Need to write something to populate Adag from A
|
|
||||||
ExchangeCoarseLinks();
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator eigen "<<teigen<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator phase "<<tphase<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator mat "<<tmat <<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator proj "<<tproj<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator inv "<<tinv<<" us"<<std::endl;
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Galerkin projection of matrix
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
|
||||||
Aggregation<Fobj,CComplex,nbasis> & Subspace)
|
|
||||||
{
|
|
||||||
CoarsenOperator(linop,Subspace,Subspace);
|
|
||||||
}
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Petrov - Galerkin projection of matrix
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
|
||||||
Aggregation<Fobj,CComplex,nbasis> & U,
|
|
||||||
Aggregation<Fobj,CComplex,nbasis> & V)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrix "<< std::endl;
|
|
||||||
GridBase *grid = FineGrid();
|
|
||||||
|
|
||||||
RealD tproj=0.0;
|
|
||||||
RealD teigen=0.0;
|
|
||||||
RealD tmat=0.0;
|
|
||||||
RealD tphase=0.0;
|
|
||||||
RealD tphaseBZ=0.0;
|
|
||||||
RealD tinv=0.0;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Orthogonalise the subblocks over the basis
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
CoarseScalar InnerProd(CoarseGrid());
|
|
||||||
blockOrthogonalise(InnerProd,V.subspace);
|
|
||||||
blockOrthogonalise(InnerProd,U.subspace);
|
|
||||||
|
|
||||||
const int npoint = geom.npoint;
|
|
||||||
|
|
||||||
Coordinate clatt = CoarseGrid()->GlobalDimensions();
|
|
||||||
int Nd = CoarseGrid()->Nd();
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
|
||||||
* Matrix index i is mapped to this shift via
|
|
||||||
* geom.shifts[i]
|
|
||||||
*
|
|
||||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
|
||||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
|
||||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
|
||||||
* = M_{kl} A_ji^{b.b+l}
|
|
||||||
*
|
|
||||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
|
||||||
*
|
|
||||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
|
||||||
*
|
|
||||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
|
||||||
*/
|
|
||||||
teigen-=usecond();
|
|
||||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
ComplexD ci(0.0,1.0);
|
|
||||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
|
||||||
|
|
||||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
|
||||||
ComplexD phase(0.0,0.0);
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
phase=phase+TwoPiL*geom.shifts[k][mu]*geom.shifts[l][mu];
|
|
||||||
}
|
|
||||||
phase=exp(phase*ci);
|
|
||||||
Mkl(k,l) = phase;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
invMkl = Mkl.inverse();
|
|
||||||
teigen+=usecond();
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
// Now compute the matrix elements of linop between the orthonormal
|
|
||||||
// set of vectors.
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
FineField phaV(grid); // Phased block basis vector
|
|
||||||
FineField MphaV(grid);// Matrix applied
|
|
||||||
std::vector<FineComplexField> phaF(npoint,grid);
|
|
||||||
std::vector<CoarseComplexField> pha(npoint,CoarseGrid());
|
|
||||||
|
|
||||||
CoarseVector coarseInner(CoarseGrid());
|
|
||||||
|
|
||||||
typedef typename CComplex::scalar_type SComplex;
|
|
||||||
FineComplexField one(grid); one=SComplex(1.0);
|
|
||||||
FineComplexField zz(grid); zz = Zero();
|
|
||||||
tphase=-usecond();
|
|
||||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Stick a phase on every block
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
CoarseComplexField coor(CoarseGrid());
|
|
||||||
pha[p]=Zero();
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
LatticeCoordinate(coor,mu);
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
pha[p] = pha[p] + (TwoPiL * geom.shifts[p][mu]) * coor;
|
|
||||||
}
|
|
||||||
pha[p] =exp(pha[p]*ci);
|
|
||||||
|
|
||||||
blockZAXPY(phaF[p],pha[p],one,zz);
|
|
||||||
|
|
||||||
}
|
|
||||||
tphase+=usecond();
|
|
||||||
|
|
||||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid());
|
|
||||||
std::vector<CoarseVector> FT(npoint,CoarseGrid());
|
|
||||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
|
||||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
|
||||||
tphaseBZ-=usecond();
|
|
||||||
phaV = phaF[p]*V.subspace[i];
|
|
||||||
tphaseBZ+=usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
// Multiple phased subspace vector by matrix and project to subspace
|
|
||||||
// Remove local bulk phase to leave relative phases
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
tmat-=usecond();
|
|
||||||
linop.Op(phaV,MphaV);
|
|
||||||
tmat+=usecond();
|
|
||||||
// std::cout << i << " " <<p << " MphaV "<<norm2(MphaV)<<" "<<norm2(phaV)<<std::endl;
|
|
||||||
|
|
||||||
tproj-=usecond();
|
|
||||||
blockProject(coarseInner,MphaV,U.subspace);
|
|
||||||
coarseInner = conjugate(pha[p]) * coarseInner;
|
|
||||||
|
|
||||||
ComputeProj[p] = coarseInner;
|
|
||||||
tproj+=usecond();
|
|
||||||
// std::cout << i << " " <<p << " ComputeProj "<<norm2(ComputeProj[p])<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
tinv-=usecond();
|
|
||||||
for(int k=0;k<npoint;k++){
|
|
||||||
FT[k] = Zero();
|
|
||||||
for(int l=0;l<npoint;l++){
|
|
||||||
FT[k]= FT[k]+ invMkl(l,k)*ComputeProj[l];
|
|
||||||
}
|
|
||||||
|
|
||||||
int osites=CoarseGrid()->oSites();
|
|
||||||
autoView( A_v , _A[k], AcceleratorWrite);
|
|
||||||
autoView( FT_v , FT[k], AcceleratorRead);
|
|
||||||
accelerator_for(sss, osites, 1, {
|
|
||||||
for(int j=0;j<nbasis;j++){
|
|
||||||
A_v[sss](i,j) = FT_v[sss](j);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
tinv+=usecond();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only needed if nonhermitian
|
|
||||||
if ( ! hermitian ) {
|
|
||||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
|
||||||
// PopulateAdag();
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
std::cout << " _A["<<p<<"] "<<norm2(_A[p])<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Need to write something to populate Adag from A
|
|
||||||
ExchangeCoarseLinks();
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator eigen "<<teigen<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator phase "<<tphase<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator mat "<<tmat <<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator proj "<<tproj<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator inv "<<tinv<<" us"<<std::endl;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
void ExchangeCoarseLinks(void){
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
_A[p] = Cell.ExchangePeriodic(_A[p]);
|
|
||||||
// _Adag[p]= Cell.ExchangePeriodic(_Adag[p]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual void Mdiag (const Field &in, Field &out){ GRID_ASSERT(0);};
|
|
||||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);};
|
|
||||||
virtual void MdirAll (const Field &in, std::vector<Field> &out){assert(0);};
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,729 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/GeneralCoarsenedMatrixMultiRHS.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
// Fine Object == (per site) type of fine field
|
|
||||||
// nbasis == number of deflation vectors
|
|
||||||
template<class Fobj,class CComplex,int nbasis>
|
|
||||||
class MultiGeneralCoarsenedMatrix : public SparseMatrixBase<Lattice<iVector<CComplex,nbasis > > > {
|
|
||||||
public:
|
|
||||||
typedef typename CComplex::scalar_object SComplex;
|
|
||||||
typedef GeneralCoarsenedMatrix<Fobj,CComplex,nbasis> GeneralCoarseOp;
|
|
||||||
typedef MultiGeneralCoarsenedMatrix<Fobj,CComplex,nbasis> MultiGeneralCoarseOp;
|
|
||||||
|
|
||||||
typedef iVector<CComplex,nbasis > siteVector;
|
|
||||||
typedef iMatrix<CComplex,nbasis > siteMatrix;
|
|
||||||
typedef iVector<SComplex,nbasis > calcVector;
|
|
||||||
typedef iMatrix<SComplex,nbasis > calcMatrix;
|
|
||||||
typedef Lattice<iScalar<CComplex> > CoarseComplexField;
|
|
||||||
typedef Lattice<siteVector> CoarseVector;
|
|
||||||
typedef Lattice<iMatrix<CComplex,nbasis > > CoarseMatrix;
|
|
||||||
typedef iMatrix<CComplex,nbasis > Cobj;
|
|
||||||
typedef iVector<CComplex,nbasis > Cvec;
|
|
||||||
typedef Lattice< CComplex > CoarseScalar; // used for inner products on fine field
|
|
||||||
typedef Lattice<Fobj > FineField;
|
|
||||||
typedef Lattice<CComplex > FineComplexField;
|
|
||||||
typedef CoarseVector Field;
|
|
||||||
|
|
||||||
////////////////////
|
|
||||||
// Data members
|
|
||||||
////////////////////
|
|
||||||
GridCartesian * _CoarseGridMulti;
|
|
||||||
NonLocalStencilGeometry geom;
|
|
||||||
NonLocalStencilGeometry geom_srhs;
|
|
||||||
PaddedCell Cell;
|
|
||||||
GeneralLocalStencil Stencil;
|
|
||||||
|
|
||||||
deviceVector<calcVector> BLAS_B;
|
|
||||||
deviceVector<calcVector> BLAS_C;
|
|
||||||
std::vector<deviceVector<calcMatrix> > BLAS_A;
|
|
||||||
|
|
||||||
std::vector<deviceVector<ComplexD *> > BLAS_AP;
|
|
||||||
std::vector<deviceVector<ComplexD *> > BLAS_BP;
|
|
||||||
deviceVector<ComplexD *> BLAS_CP;
|
|
||||||
|
|
||||||
///////////////////////
|
|
||||||
// Interface
|
|
||||||
///////////////////////
|
|
||||||
GridBase * Grid(void) { return _CoarseGridMulti; }; // this is all the linalg routines need to know
|
|
||||||
GridCartesian * CoarseGrid(void) { return _CoarseGridMulti; }; // this is all the linalg routines need to know
|
|
||||||
|
|
||||||
// Can be used to do I/O on the operator matrices externally
|
|
||||||
void SetMatrix (int p,CoarseMatrix & A)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(A.size()==geom_srhs.npoint);
|
|
||||||
GridtoBLAS(A[p],BLAS_A[p]);
|
|
||||||
}
|
|
||||||
void GetMatrix (int p,CoarseMatrix & A)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(A.size()==geom_srhs.npoint);
|
|
||||||
BLAStoGrid(A[p],BLAS_A[p]);
|
|
||||||
}
|
|
||||||
void CopyMatrix (GeneralCoarseOp &_Op)
|
|
||||||
{
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
auto Aup = _Op.Cell.Extract(_Op._A[p]);
|
|
||||||
//Unpadded
|
|
||||||
GridtoBLAS(Aup,BLAS_A[p]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
void CheckMatrix (GeneralCoarseOp &_Op)
|
|
||||||
{
|
|
||||||
std::cout <<"************* Checking the little direc operator mRHS"<<std::endl;
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
//Unpadded
|
|
||||||
auto Aup = _Op.Cell.Extract(_Op._A[p]);
|
|
||||||
auto Ack = Aup;
|
|
||||||
BLAStoGrid(Ack,BLAS_A[p]);
|
|
||||||
std::cout << p<<" Ack "<<norm2(Ack)<<std::endl;
|
|
||||||
std::cout << p<<" Aup "<<norm2(Aup)<<std::endl;
|
|
||||||
}
|
|
||||||
std::cout <<"************* "<<std::endl;
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
MultiGeneralCoarsenedMatrix(NonLocalStencilGeometry &_geom,GridCartesian *CoarseGridMulti) :
|
|
||||||
_CoarseGridMulti(CoarseGridMulti),
|
|
||||||
geom_srhs(_geom),
|
|
||||||
geom(_CoarseGridMulti,_geom.hops,_geom.skip+1),
|
|
||||||
Cell(geom.Depth(),_CoarseGridMulti),
|
|
||||||
Stencil(Cell.grids.back(),geom.shifts) // padded cell stencil
|
|
||||||
{
|
|
||||||
int32_t padded_sites = Cell.grids.back()->lSites();
|
|
||||||
int32_t unpadded_sites = CoarseGridMulti->lSites();
|
|
||||||
|
|
||||||
int32_t nrhs = CoarseGridMulti->FullDimensions()[0]; // # RHS
|
|
||||||
int32_t orhs = nrhs/CComplex::Nsimd();
|
|
||||||
|
|
||||||
padded_sites = padded_sites/nrhs;
|
|
||||||
unpadded_sites = unpadded_sites/nrhs;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Device data vector storage
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
BLAS_A.resize(geom.npoint);
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
BLAS_A[p].resize (unpadded_sites); // no ghost zone, npoint elements
|
|
||||||
}
|
|
||||||
|
|
||||||
BLAS_B.resize(nrhs *padded_sites); // includes ghost zone
|
|
||||||
BLAS_C.resize(nrhs *unpadded_sites); // no ghost zone
|
|
||||||
BLAS_AP.resize(geom.npoint);
|
|
||||||
BLAS_BP.resize(geom.npoint);
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
BLAS_AP[p].resize(unpadded_sites);
|
|
||||||
BLAS_BP[p].resize(unpadded_sites);
|
|
||||||
}
|
|
||||||
BLAS_CP.resize(unpadded_sites);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Pointers to data
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// Site identity mapping for A
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
for(int ss=0;ss<unpadded_sites;ss++){
|
|
||||||
ComplexD *ptr = (ComplexD *)&BLAS_A[p][ss];
|
|
||||||
acceleratorPut(BLAS_AP[p][ss],ptr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Site identity mapping for C
|
|
||||||
for(int ss=0;ss<unpadded_sites;ss++){
|
|
||||||
ComplexD *ptr = (ComplexD *)&BLAS_C[ss*nrhs];
|
|
||||||
acceleratorPut(BLAS_CP[ss],ptr);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Neighbour table is more complicated
|
|
||||||
int32_t j=0; // Interior point counter (unpadded)
|
|
||||||
for(int32_t s=0;s<padded_sites;s++){ // 4 volume, padded
|
|
||||||
int ghost_zone=0;
|
|
||||||
for(int32_t point = 0 ; point < geom.npoint; point++){
|
|
||||||
int i=s*orhs*geom.npoint+point;
|
|
||||||
if( Stencil._entries[i]._wrap ) { // stencil is indexed by the oSite of the CoarseGridMulti, hence orhs factor
|
|
||||||
ghost_zone=1; // If general stencil wrapped in any direction, wrap=1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if( ghost_zone==0) {
|
|
||||||
for(int32_t point = 0 ; point < geom.npoint; point++){
|
|
||||||
int i=s*orhs*geom.npoint+point;
|
|
||||||
int32_t nbr = Stencil._entries[i]._offset*CComplex::Nsimd(); // oSite -> lSite
|
|
||||||
GRID_ASSERT(nbr<BLAS_B.size());
|
|
||||||
ComplexD * ptr = (ComplexD *)&BLAS_B[nbr];
|
|
||||||
acceleratorPut(BLAS_BP[point][j],ptr); // neighbour indexing in ghost zone volume
|
|
||||||
}
|
|
||||||
j++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
GRID_ASSERT(j==unpadded_sites);
|
|
||||||
}
|
|
||||||
template<class vobj> void GridtoBLAS(const Lattice<vobj> &from,deviceVector<typename vobj::scalar_object> &to)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
GridBase *Fg = from.Grid();
|
|
||||||
GRID_ASSERT(!Fg->_isCheckerBoarded);
|
|
||||||
int nd = Fg->_ndimension;
|
|
||||||
|
|
||||||
to.resize(Fg->lSites());
|
|
||||||
|
|
||||||
Coordinate LocalLatt = Fg->LocalDimensions();
|
|
||||||
size_t nsite = 1;
|
|
||||||
for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// do the index calc on the GPU
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
Coordinate f_ostride = Fg->_ostride;
|
|
||||||
Coordinate f_istride = Fg->_istride;
|
|
||||||
Coordinate f_rdimensions = Fg->_rdimensions;
|
|
||||||
|
|
||||||
autoView(from_v,from,AcceleratorRead);
|
|
||||||
auto to_v = &to[0];
|
|
||||||
|
|
||||||
const int words=sizeof(vobj)/sizeof(vector_type);
|
|
||||||
accelerator_for(idx,nsite,1,{
|
|
||||||
|
|
||||||
Coordinate from_coor, base;
|
|
||||||
Lexicographic::CoorFromIndex(base,idx,LocalLatt);
|
|
||||||
for(int i=0;i<nd;i++){
|
|
||||||
from_coor[i] = base[i];
|
|
||||||
}
|
|
||||||
int from_oidx = 0; for(int d=0;d<nd;d++) from_oidx+=f_ostride[d]*(from_coor[d]%f_rdimensions[d]);
|
|
||||||
int from_lane = 0; for(int d=0;d<nd;d++) from_lane+=f_istride[d]*(from_coor[d]/f_rdimensions[d]);
|
|
||||||
|
|
||||||
const vector_type* from = (const vector_type *)&from_v[from_oidx];
|
|
||||||
scalar_type* to = (scalar_type *)&to_v[idx];
|
|
||||||
|
|
||||||
scalar_type stmp;
|
|
||||||
for(int w=0;w<words;w++){
|
|
||||||
stmp = getlane(from[w], from_lane);
|
|
||||||
to[w] = stmp;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
template<class vobj> void BLAStoGrid(Lattice<vobj> &grid,deviceVector<typename vobj::scalar_object> &in)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
GridBase *Tg = grid.Grid();
|
|
||||||
GRID_ASSERT(!Tg->_isCheckerBoarded);
|
|
||||||
int nd = Tg->_ndimension;
|
|
||||||
|
|
||||||
GRID_ASSERT(in.size()==Tg->lSites());
|
|
||||||
|
|
||||||
Coordinate LocalLatt = Tg->LocalDimensions();
|
|
||||||
size_t nsite = 1;
|
|
||||||
for(int i=0;i<nd;i++) nsite *= LocalLatt[i];
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// do the index calc on the GPU
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
Coordinate t_ostride = Tg->_ostride;
|
|
||||||
Coordinate t_istride = Tg->_istride;
|
|
||||||
Coordinate t_rdimensions = Tg->_rdimensions;
|
|
||||||
|
|
||||||
autoView(to_v,grid,AcceleratorWrite);
|
|
||||||
auto from_v = &in[0];
|
|
||||||
|
|
||||||
const int words=sizeof(vobj)/sizeof(vector_type);
|
|
||||||
accelerator_for(idx,nsite,1,{
|
|
||||||
|
|
||||||
Coordinate to_coor, base;
|
|
||||||
Lexicographic::CoorFromIndex(base,idx,LocalLatt);
|
|
||||||
for(int i=0;i<nd;i++){
|
|
||||||
to_coor[i] = base[i];
|
|
||||||
}
|
|
||||||
int to_oidx = 0; for(int d=0;d<nd;d++) to_oidx+=t_ostride[d]*(to_coor[d]%t_rdimensions[d]);
|
|
||||||
int to_lane = 0; for(int d=0;d<nd;d++) to_lane+=t_istride[d]*(to_coor[d]/t_rdimensions[d]);
|
|
||||||
|
|
||||||
vector_type* to = (vector_type *)&to_v[to_oidx];
|
|
||||||
scalar_type* from = (scalar_type *)&from_v[idx];
|
|
||||||
|
|
||||||
scalar_type stmp;
|
|
||||||
for(int w=0;w<words;w++){
|
|
||||||
stmp=from[w];
|
|
||||||
putlane(to[w], stmp, to_lane);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
void CoarsenOperator(LinearOperatorBase<Lattice<Fobj> > &linop,
|
|
||||||
Aggregation<Fobj,CComplex,nbasis> & Subspace,
|
|
||||||
GridBase *CoarseGrid)
|
|
||||||
{
|
|
||||||
#if 0
|
|
||||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
|
|
||||||
|
|
||||||
GridBase *grid = Subspace.FineGrid;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Orthogonalise the subblocks over the basis
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
CoarseScalar InnerProd(CoarseGrid);
|
|
||||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
|
||||||
|
|
||||||
const int npoint = geom_srhs.npoint;
|
|
||||||
|
|
||||||
Coordinate clatt = CoarseGrid->GlobalDimensions();
|
|
||||||
int Nd = CoarseGrid->Nd();
|
|
||||||
/*
|
|
||||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
|
||||||
* Matrix index i is mapped to this shift via
|
|
||||||
* geom.shifts[i]
|
|
||||||
*
|
|
||||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
|
||||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
|
||||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
|
||||||
* = M_{kl} A_ji^{b.b+l}
|
|
||||||
*
|
|
||||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
|
||||||
*
|
|
||||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
|
||||||
*
|
|
||||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
|
||||||
*/
|
|
||||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
ComplexD ci(0.0,1.0);
|
|
||||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
|
||||||
|
|
||||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
|
||||||
ComplexD phase(0.0,0.0);
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
|
|
||||||
}
|
|
||||||
phase=exp(phase*ci);
|
|
||||||
Mkl(k,l) = phase;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
invMkl = Mkl.inverse();
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
// Now compute the matrix elements of linop between the orthonormal
|
|
||||||
// set of vectors.
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
FineField phaV(grid); // Phased block basis vector
|
|
||||||
FineField MphaV(grid);// Matrix applied
|
|
||||||
std::vector<FineComplexField> phaF(npoint,grid);
|
|
||||||
std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
|
|
||||||
|
|
||||||
CoarseVector coarseInner(CoarseGrid);
|
|
||||||
|
|
||||||
typedef typename CComplex::scalar_type SComplex;
|
|
||||||
FineComplexField one(grid); one=SComplex(1.0);
|
|
||||||
FineComplexField zz(grid); zz = Zero();
|
|
||||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Stick a phase on every block
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
CoarseComplexField coor(CoarseGrid);
|
|
||||||
pha[p]=Zero();
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
LatticeCoordinate(coor,mu);
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
|
|
||||||
}
|
|
||||||
pha[p] =exp(pha[p]*ci);
|
|
||||||
|
|
||||||
blockZAXPY(phaF[p],pha[p],one,zz);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Could save on temporary storage here
|
|
||||||
std::vector<CoarseMatrix> _A;
|
|
||||||
_A.resize(geom_srhs.npoint,CoarseGrid);
|
|
||||||
|
|
||||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
|
|
||||||
CoarseVector FT(CoarseGrid);
|
|
||||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
|
||||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
|
||||||
|
|
||||||
phaV = phaF[p]*Subspace.subspace[i];
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
// Multiple phased subspace vector by matrix and project to subspace
|
|
||||||
// Remove local bulk phase to leave relative phases
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
linop.Op(phaV,MphaV);
|
|
||||||
|
|
||||||
// Fixme, could use batched block projector here
|
|
||||||
blockProject(coarseInner,MphaV,Subspace.subspace);
|
|
||||||
|
|
||||||
coarseInner = conjugate(pha[p]) * coarseInner;
|
|
||||||
|
|
||||||
ComputeProj[p] = coarseInner;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
|
|
||||||
for(int k=0;k<npoint;k++){
|
|
||||||
|
|
||||||
FT = Zero();
|
|
||||||
for(int l=0;l<npoint;l++){
|
|
||||||
FT= FT+ invMkl(l,k)*ComputeProj[l];
|
|
||||||
}
|
|
||||||
|
|
||||||
int osites=CoarseGrid->oSites();
|
|
||||||
autoView( A_v , _A[k], AcceleratorWrite);
|
|
||||||
autoView( FT_v , FT, AcceleratorRead);
|
|
||||||
accelerator_for(sss, osites, 1, {
|
|
||||||
for(int j=0;j<nbasis;j++){
|
|
||||||
A_v[sss](i,j) = FT_v[sss](j);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only needed if nonhermitian
|
|
||||||
// if ( ! hermitian ) {
|
|
||||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
|
||||||
// PopulateAdag();
|
|
||||||
// }
|
|
||||||
// Need to write something to populate Adag from A
|
|
||||||
|
|
||||||
for(int p=0;p<geom_srhs.npoint;p++){
|
|
||||||
GridtoBLAS(_A[p],BLAS_A[p]);
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
Grid : Message : 11698.730546 s : CoarsenOperator eigen 1334 us
|
|
||||||
Grid : Message : 11698.730563 s : CoarsenOperator phase 34729 us
|
|
||||||
Grid : Message : 11698.730565 s : CoarsenOperator phaseBZ 2423814 us
|
|
||||||
Grid : Message : 11698.730566 s : CoarsenOperator mat 127890998 us
|
|
||||||
Grid : Message : 11698.730567 s : CoarsenOperator proj 515840840 us
|
|
||||||
Grid : Message : 11698.730568 s : CoarsenOperator inv 103948313 us
|
|
||||||
Takes 600s to compute matrix elements, DOMINATED by the block project.
|
|
||||||
Easy to speed up with the batched block project.
|
|
||||||
Store npoint vectors, get npoint x Nbasis block projection, and 81 fold faster.
|
|
||||||
|
|
||||||
// Block project below taks to 240s
|
|
||||||
Grid : Message : 328.193418 s : CoarsenOperator phase 38338 us
|
|
||||||
Grid : Message : 328.193434 s : CoarsenOperator phaseBZ 1711226 us
|
|
||||||
Grid : Message : 328.193436 s : CoarsenOperator mat 122213270 us
|
|
||||||
//Grid : Message : 328.193438 s : CoarsenOperator proj 1181154 us <-- this is mistimed
|
|
||||||
//Grid : Message : 11698.730568 s : CoarsenOperator inv 103948313 us <-- Cut this ~10x if lucky by loop fusion
|
|
||||||
*/
|
|
||||||
#else
|
|
||||||
RealD tproj=0.0;
|
|
||||||
RealD tmat=0.0;
|
|
||||||
RealD tphase=0.0;
|
|
||||||
RealD tphaseBZ=0.0;
|
|
||||||
RealD tinv=0.0;
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<< "GeneralCoarsenMatrixMrhs "<< std::endl;
|
|
||||||
|
|
||||||
GridBase *grid = Subspace.FineGrid;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Orthogonalise the subblocks over the basis
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
CoarseScalar InnerProd(CoarseGrid);
|
|
||||||
blockOrthogonalise(InnerProd,Subspace.subspace);
|
|
||||||
|
|
||||||
|
|
||||||
MultiRHSBlockProject<Lattice<Fobj> > Projector;
|
|
||||||
Projector.Allocate(nbasis,grid,CoarseGrid);
|
|
||||||
Projector.ImportBasis(Subspace.subspace);
|
|
||||||
|
|
||||||
const int npoint = geom_srhs.npoint;
|
|
||||||
|
|
||||||
Coordinate clatt = CoarseGrid->GlobalDimensions();
|
|
||||||
int Nd = CoarseGrid->Nd();
|
|
||||||
/*
|
|
||||||
* Here, k,l index which possible momentum/shift within the N-points connected by MdagM.
|
|
||||||
* Matrix index i is mapped to this shift via
|
|
||||||
* geom.shifts[i]
|
|
||||||
*
|
|
||||||
* conj(pha[block]) proj[k (which mom)][j (basis vec cpt)][block]
|
|
||||||
* = \sum_{l in ball} e^{i q_k . delta_l} < phi_{block,j} | MdagM | phi_{(block+delta_l),i} >
|
|
||||||
* = \sum_{l in ball} e^{iqk.delta_l} A_ji^{b.b+l}
|
|
||||||
* = M_{kl} A_ji^{b.b+l}
|
|
||||||
*
|
|
||||||
* Must assemble and invert matrix M_k,l = e^[i q_k . delta_l]
|
|
||||||
*
|
|
||||||
* Where q_k = delta_k . (2*M_PI/global_nb[mu])
|
|
||||||
*
|
|
||||||
* Then A{ji}^{b,b+l} = M^{-1}_{lm} ComputeProj_{m,b,i,j}
|
|
||||||
*/
|
|
||||||
Eigen::MatrixXcd Mkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
Eigen::MatrixXcd invMkl = Eigen::MatrixXcd::Zero(npoint,npoint);
|
|
||||||
ComplexD ci(0.0,1.0);
|
|
||||||
for(int k=0;k<npoint;k++){ // Loop over momenta
|
|
||||||
|
|
||||||
for(int l=0;l<npoint;l++){ // Loop over nbr relative
|
|
||||||
ComplexD phase(0.0,0.0);
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
phase=phase+TwoPiL*geom_srhs.shifts[k][mu]*geom_srhs.shifts[l][mu];
|
|
||||||
}
|
|
||||||
phase=exp(phase*ci);
|
|
||||||
Mkl(k,l) = phase;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
invMkl = Mkl.inverse();
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
// Now compute the matrix elements of linop between the orthonormal
|
|
||||||
// set of vectors.
|
|
||||||
///////////////////////////////////////////////////////////////////////
|
|
||||||
FineField phaV(grid); // Phased block basis vector
|
|
||||||
FineField MphaV(grid);// Matrix applied
|
|
||||||
std::vector<FineComplexField> phaF(npoint,grid);
|
|
||||||
std::vector<CoarseComplexField> pha(npoint,CoarseGrid);
|
|
||||||
|
|
||||||
CoarseVector coarseInner(CoarseGrid);
|
|
||||||
|
|
||||||
tphase=-usecond();
|
|
||||||
typedef typename CComplex::scalar_type SComplex;
|
|
||||||
FineComplexField one(grid); one=SComplex(1.0);
|
|
||||||
FineComplexField zz(grid); zz = Zero();
|
|
||||||
for(int p=0;p<npoint;p++){ // Loop over momenta in npoint
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Stick a phase on every block
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
CoarseComplexField coor(CoarseGrid);
|
|
||||||
pha[p]=Zero();
|
|
||||||
for(int mu=0;mu<Nd;mu++){
|
|
||||||
LatticeCoordinate(coor,mu);
|
|
||||||
RealD TwoPiL = M_PI * 2.0/ clatt[mu];
|
|
||||||
pha[p] = pha[p] + (TwoPiL * geom_srhs.shifts[p][mu]) * coor;
|
|
||||||
}
|
|
||||||
pha[p] =exp(pha[p]*ci);
|
|
||||||
|
|
||||||
blockZAXPY(phaF[p],pha[p],one,zz);
|
|
||||||
}
|
|
||||||
tphase+=usecond();
|
|
||||||
|
|
||||||
// Could save on temporary storage here
|
|
||||||
std::vector<CoarseMatrix> _A;
|
|
||||||
_A.resize(geom_srhs.npoint,CoarseGrid);
|
|
||||||
|
|
||||||
// Count use small chunks than npoint == 81 and save memory
|
|
||||||
int batch = 9;
|
|
||||||
std::vector<FineField> _MphaV(batch,grid);
|
|
||||||
std::vector<CoarseVector> TmpProj(batch,CoarseGrid);
|
|
||||||
|
|
||||||
std::vector<CoarseVector> ComputeProj(npoint,CoarseGrid);
|
|
||||||
CoarseVector FT(CoarseGrid);
|
|
||||||
for(int i=0;i<nbasis;i++){// Loop over basis vectors
|
|
||||||
std::cout << GridLogMessage<< "CoarsenMatrixColoured vec "<<i<<"/"<<nbasis<< std::endl;
|
|
||||||
|
|
||||||
// std::cout << GridLogMessage << " phasing the fine vector "<<std::endl;
|
|
||||||
// Fixme : do this in batches
|
|
||||||
for(int p=0;p<npoint;p+=batch){ // Loop over momenta in npoint
|
|
||||||
|
|
||||||
for(int b=0;b<MIN(batch,npoint-p);b++){
|
|
||||||
tphaseBZ-=usecond();
|
|
||||||
phaV = phaF[p+b]*Subspace.subspace[i];
|
|
||||||
tphaseBZ+=usecond();
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
// Multiple phased subspace vector by matrix and project to subspace
|
|
||||||
// Remove local bulk phase to leave relative phases
|
|
||||||
/////////////////////////////////////////////////////////////////////
|
|
||||||
// Memory footprint was an issue
|
|
||||||
tmat-=usecond();
|
|
||||||
linop.Op(phaV,MphaV);
|
|
||||||
_MphaV[b] = MphaV;
|
|
||||||
tmat+=usecond();
|
|
||||||
}
|
|
||||||
|
|
||||||
// std::cout << GridLogMessage << " Calling block project "<<std::endl;
|
|
||||||
tproj-=usecond();
|
|
||||||
Projector.blockProject(_MphaV,TmpProj);
|
|
||||||
tproj+=usecond();
|
|
||||||
|
|
||||||
// std::cout << GridLogMessage << " conj phasing the coarse vectors "<<std::endl;
|
|
||||||
for(int b=0;b<MIN(batch,npoint-p);b++){
|
|
||||||
ComputeProj[p+b] = conjugate(pha[p+b])*TmpProj[b];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Could do this with a block promote or similar BLAS call via the MultiRHSBlockProjector with a const matrix.
|
|
||||||
|
|
||||||
// std::cout << GridLogMessage << " Starting FT inv "<<std::endl;
|
|
||||||
tinv-=usecond();
|
|
||||||
for(int k=0;k<npoint;k++){
|
|
||||||
FT = Zero();
|
|
||||||
// 81 kernel calls as many ComputeProj vectors
|
|
||||||
// Could fuse with a vector of views, but ugly
|
|
||||||
// Could unroll the expression and run fewer kernels -- much more attractive
|
|
||||||
// Could also do non blocking.
|
|
||||||
#if 0
|
|
||||||
for(int l=0;l<npoint;l++){
|
|
||||||
FT= FT+ invMkl(l,k)*ComputeProj[l];
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
const int radix = 9;
|
|
||||||
int ll;
|
|
||||||
for(ll=0;ll+radix-1<npoint;ll+=radix){
|
|
||||||
// When ll = npoint-radix, ll+radix-1 = npoint-1, and we do it all.
|
|
||||||
FT = FT
|
|
||||||
+ invMkl(ll+0,k)*ComputeProj[ll+0]
|
|
||||||
+ invMkl(ll+1,k)*ComputeProj[ll+1]
|
|
||||||
+ invMkl(ll+2,k)*ComputeProj[ll+2]
|
|
||||||
+ invMkl(ll+3,k)*ComputeProj[ll+3]
|
|
||||||
+ invMkl(ll+4,k)*ComputeProj[ll+4]
|
|
||||||
+ invMkl(ll+5,k)*ComputeProj[ll+5]
|
|
||||||
+ invMkl(ll+6,k)*ComputeProj[ll+6]
|
|
||||||
+ invMkl(ll+7,k)*ComputeProj[ll+7]
|
|
||||||
+ invMkl(ll+8,k)*ComputeProj[ll+8];
|
|
||||||
}
|
|
||||||
for(int l=ll;l<npoint;l++){
|
|
||||||
FT= FT+ invMkl(l,k)*ComputeProj[l];
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// 1 kernel call -- must be cheaper
|
|
||||||
int osites=CoarseGrid->oSites();
|
|
||||||
autoView( A_v , _A[k], AcceleratorWrite);
|
|
||||||
autoView( FT_v , FT, AcceleratorRead);
|
|
||||||
accelerator_for(sss, osites, 1, {
|
|
||||||
for(int j=0;j<nbasis;j++){
|
|
||||||
A_v[sss](i,j) = FT_v[sss](j);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
tinv+=usecond();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only needed if nonhermitian
|
|
||||||
// if ( ! hermitian ) {
|
|
||||||
// std::cout << GridLogMessage<<"PopulateAdag "<<std::endl;
|
|
||||||
// PopulateAdag();
|
|
||||||
// }
|
|
||||||
// Need to write something to populate Adag from A
|
|
||||||
// std::cout << GridLogMessage << " Calling GridtoBLAS "<<std::endl;
|
|
||||||
for(int p=0;p<geom_srhs.npoint;p++){
|
|
||||||
GridtoBLAS(_A[p],BLAS_A[p]);
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator phase "<<tphase<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator phaseBZ "<<tphaseBZ<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator mat "<<tmat <<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator proj "<<tproj<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"CoarsenOperator inv "<<tinv<<" us"<<std::endl;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
void Mdag(const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
this->M(in,out);
|
|
||||||
}
|
|
||||||
void M (const CoarseVector &in, CoarseVector &out)
|
|
||||||
{
|
|
||||||
// std::cout << GridLogMessage << "New Mrhs coarse"<<std::endl;
|
|
||||||
conformable(CoarseGrid(),in.Grid());
|
|
||||||
conformable(in.Grid(),out.Grid());
|
|
||||||
out.Checkerboard() = in.Checkerboard();
|
|
||||||
|
|
||||||
RealD t_tot;
|
|
||||||
RealD t_exch;
|
|
||||||
RealD t_GtoB;
|
|
||||||
RealD t_BtoG;
|
|
||||||
RealD t_mult;
|
|
||||||
|
|
||||||
t_tot=-usecond();
|
|
||||||
CoarseVector tin=in;
|
|
||||||
t_exch=-usecond();
|
|
||||||
CoarseVector pin = Cell.ExchangePeriodic(tin); //padded input
|
|
||||||
t_exch+=usecond();
|
|
||||||
|
|
||||||
CoarseVector pout(pin.Grid());
|
|
||||||
|
|
||||||
int npoint = geom.npoint;
|
|
||||||
typedef calcMatrix* Aview;
|
|
||||||
typedef LatticeView<Cvec> Vview;
|
|
||||||
|
|
||||||
const int Nsimd = CComplex::Nsimd();
|
|
||||||
|
|
||||||
int64_t nrhs =pin.Grid()->GlobalDimensions()[0];
|
|
||||||
GRID_ASSERT(nrhs>=1);
|
|
||||||
|
|
||||||
RealD flops,bytes;
|
|
||||||
int64_t osites=in.Grid()->oSites(); // unpadded
|
|
||||||
int64_t unpadded_vol = CoarseGrid()->lSites()/nrhs;
|
|
||||||
|
|
||||||
flops = 1.0* npoint * nbasis * nbasis * 8.0 * osites * CComplex::Nsimd();
|
|
||||||
bytes = 1.0*osites*sizeof(siteMatrix)*npoint/pin.Grid()->GlobalDimensions()[0]
|
|
||||||
+ 2.0*osites*sizeof(siteVector)*npoint;
|
|
||||||
|
|
||||||
|
|
||||||
t_GtoB=-usecond();
|
|
||||||
GridtoBLAS(pin,BLAS_B);
|
|
||||||
t_GtoB+=usecond();
|
|
||||||
|
|
||||||
GridBLAS BLAS;
|
|
||||||
|
|
||||||
t_mult=-usecond();
|
|
||||||
for(int p=0;p<geom.npoint;p++){
|
|
||||||
RealD c = 1.0;
|
|
||||||
if (p==0) c = 0.0;
|
|
||||||
ComplexD beta(c);
|
|
||||||
|
|
||||||
BLAS.gemmBatched(nbasis,nrhs,nbasis,
|
|
||||||
ComplexD(1.0),
|
|
||||||
BLAS_AP[p],
|
|
||||||
BLAS_BP[p],
|
|
||||||
ComplexD(c),
|
|
||||||
BLAS_CP);
|
|
||||||
}
|
|
||||||
BLAS.synchronise();
|
|
||||||
t_mult+=usecond();
|
|
||||||
|
|
||||||
t_BtoG=-usecond();
|
|
||||||
BLAStoGrid(out,BLAS_C);
|
|
||||||
t_BtoG+=usecond();
|
|
||||||
t_tot+=usecond();
|
|
||||||
/*
|
|
||||||
std::cout << GridLogMessage << "New Mrhs coarse DONE "<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"Coarse Mult exch "<<t_exch<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"Coarse Mult mult "<<t_mult<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"Coarse Mult GtoB "<<t_GtoB<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"Coarse Mult BtoG "<<t_BtoG<<" us"<<std::endl;
|
|
||||||
std::cout << GridLogMessage<<"Coarse Mult tot "<<t_tot<<" us"<<std::endl;
|
|
||||||
*/
|
|
||||||
// std::cout << GridLogMessage<<std::endl;
|
|
||||||
// std::cout << GridLogMessage<<"Coarse Kernel flops "<< flops<<std::endl;
|
|
||||||
// std::cout << GridLogMessage<<"Coarse Kernel flop/s "<< flops/t_mult<<" mflop/s"<<std::endl;
|
|
||||||
// std::cout << GridLogMessage<<"Coarse Kernel bytes/s "<< bytes/t_mult/1000<<" GB/s"<<std::endl;
|
|
||||||
// std::cout << GridLogMessage<<"Coarse overall flops/s "<< flops/t_tot<<" mflop/s"<<std::endl;
|
|
||||||
// std::cout << GridLogMessage<<"Coarse total bytes "<< bytes/1e6<<" MB"<<std::endl;
|
|
||||||
};
|
|
||||||
virtual void Mdiag (const Field &in, Field &out){ GRID_ASSERT(0);};
|
|
||||||
virtual void Mdir (const Field &in, Field &out,int dir, int disp){assert(0);};
|
|
||||||
virtual void MdirAll (const Field &in, std::vector<Field> &out){assert(0);};
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,238 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/algorithms/GeneralCoarsenedMatrix.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <pboyle@bnl.gov>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
|
||||||
// Geometry class in cartesian case
|
|
||||||
/////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
class Geometry {
|
|
||||||
public:
|
|
||||||
int npoint;
|
|
||||||
int base;
|
|
||||||
std::vector<int> directions ;
|
|
||||||
std::vector<int> displacements;
|
|
||||||
std::vector<int> points_dagger;
|
|
||||||
|
|
||||||
Geometry(int _d) {
|
|
||||||
|
|
||||||
base = (_d==5) ? 1:0;
|
|
||||||
|
|
||||||
// make coarse grid stencil for 4d , not 5d
|
|
||||||
if ( _d==5 ) _d=4;
|
|
||||||
|
|
||||||
npoint = 2*_d+1;
|
|
||||||
directions.resize(npoint);
|
|
||||||
displacements.resize(npoint);
|
|
||||||
points_dagger.resize(npoint);
|
|
||||||
for(int d=0;d<_d;d++){
|
|
||||||
directions[d ] = d+base;
|
|
||||||
directions[d+_d] = d+base;
|
|
||||||
displacements[d ] = +1;
|
|
||||||
displacements[d+_d]= -1;
|
|
||||||
points_dagger[d ] = d+_d;
|
|
||||||
points_dagger[d+_d] = d;
|
|
||||||
}
|
|
||||||
directions [2*_d]=0;
|
|
||||||
displacements[2*_d]=0;
|
|
||||||
points_dagger[2*_d]=2*_d;
|
|
||||||
}
|
|
||||||
|
|
||||||
int point(int dir, int disp) {
|
|
||||||
GRID_ASSERT(disp == -1 || disp == 0 || disp == 1);
|
|
||||||
GRID_ASSERT(base+0 <= dir && dir < base+4);
|
|
||||||
|
|
||||||
// directions faster index = new indexing
|
|
||||||
// 4d (base = 0):
|
|
||||||
// point 0 1 2 3 4 5 6 7 8
|
|
||||||
// dir 0 1 2 3 0 1 2 3 0
|
|
||||||
// disp +1 +1 +1 +1 -1 -1 -1 -1 0
|
|
||||||
// 5d (base = 1):
|
|
||||||
// point 0 1 2 3 4 5 6 7 8
|
|
||||||
// dir 1 2 3 4 1 2 3 4 0
|
|
||||||
// disp +1 +1 +1 +1 -1 -1 -1 -1 0
|
|
||||||
|
|
||||||
// displacements faster index = old indexing
|
|
||||||
// 4d (base = 0):
|
|
||||||
// point 0 1 2 3 4 5 6 7 8
|
|
||||||
// dir 0 0 1 1 2 2 3 3 0
|
|
||||||
// disp +1 -1 +1 -1 +1 -1 +1 -1 0
|
|
||||||
// 5d (base = 1):
|
|
||||||
// point 0 1 2 3 4 5 6 7 8
|
|
||||||
// dir 1 1 2 2 3 3 4 4 0
|
|
||||||
// disp +1 -1 +1 -1 +1 -1 +1 -1 0
|
|
||||||
|
|
||||||
if(dir == 0 and disp == 0)
|
|
||||||
return 8;
|
|
||||||
else // New indexing
|
|
||||||
return (1 - disp) / 2 * 4 + dir - base;
|
|
||||||
// else // Old indexing
|
|
||||||
// return (4 * (dir - base) + 1 - disp) / 2;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////
|
|
||||||
// Less local equivalent of Geometry class in cartesian case
|
|
||||||
/////////////////////////////////////////////////////////////////
|
|
||||||
class NonLocalStencilGeometry {
|
|
||||||
public:
|
|
||||||
// int depth;
|
|
||||||
int skip;
|
|
||||||
int hops;
|
|
||||||
int npoint;
|
|
||||||
std::vector<Coordinate> shifts;
|
|
||||||
Coordinate stencil_size;
|
|
||||||
Coordinate stencil_lo;
|
|
||||||
Coordinate stencil_hi;
|
|
||||||
GridCartesian *grid;
|
|
||||||
GridCartesian *Grid() {return grid;};
|
|
||||||
int Depth(void){return 1;}; // Ghost zone depth
|
|
||||||
int Hops(void){return hops;}; // # of hops=> level of corner fill in in stencil
|
|
||||||
int DimSkip(void){return skip;};
|
|
||||||
|
|
||||||
virtual ~NonLocalStencilGeometry() {};
|
|
||||||
|
|
||||||
int Reverse(int point)
|
|
||||||
{
|
|
||||||
int Nd = Grid()->Nd();
|
|
||||||
Coordinate shft = shifts[point];
|
|
||||||
Coordinate rev(Nd);
|
|
||||||
for(int mu=0;mu<Nd;mu++) rev[mu]= -shft[mu];
|
|
||||||
for(int p=0;p<npoint;p++){
|
|
||||||
if(rev==shifts[p]){
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
void BuildShifts(void)
|
|
||||||
{
|
|
||||||
this->shifts.resize(0);
|
|
||||||
int Nd = this->grid->Nd();
|
|
||||||
|
|
||||||
int dd = this->DimSkip();
|
|
||||||
for(int s0=this->stencil_lo[dd+0];s0<=this->stencil_hi[dd+0];s0++){
|
|
||||||
for(int s1=this->stencil_lo[dd+1];s1<=this->stencil_hi[dd+1];s1++){
|
|
||||||
for(int s2=this->stencil_lo[dd+2];s2<=this->stencil_hi[dd+2];s2++){
|
|
||||||
for(int s3=this->stencil_lo[dd+3];s3<=this->stencil_hi[dd+3];s3++){
|
|
||||||
Coordinate sft(Nd,0);
|
|
||||||
sft[dd+0] = s0;
|
|
||||||
sft[dd+1] = s1;
|
|
||||||
sft[dd+2] = s2;
|
|
||||||
sft[dd+3] = s3;
|
|
||||||
int nhops = abs(s0)+abs(s1)+abs(s2)+abs(s3);
|
|
||||||
if(nhops<=this->hops) this->shifts.push_back(sft);
|
|
||||||
}}}}
|
|
||||||
this->npoint = this->shifts.size();
|
|
||||||
std::cout << GridLogMessage << "NonLocalStencilGeometry has "<< this->npoint << " terms in stencil "<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
NonLocalStencilGeometry(GridCartesian *_coarse_grid,int _hops,int _skip) : grid(_coarse_grid), hops(_hops), skip(_skip)
|
|
||||||
{
|
|
||||||
Coordinate latt = grid->GlobalDimensions();
|
|
||||||
stencil_size.resize(grid->Nd());
|
|
||||||
stencil_lo.resize(grid->Nd());
|
|
||||||
stencil_hi.resize(grid->Nd());
|
|
||||||
for(int d=0;d<grid->Nd();d++){
|
|
||||||
if ( latt[d] == 1 ) {
|
|
||||||
stencil_lo[d] = 0;
|
|
||||||
stencil_hi[d] = 0;
|
|
||||||
stencil_size[d]= 1;
|
|
||||||
} else if ( latt[d] == 2 ) {
|
|
||||||
stencil_lo[d] = -1;
|
|
||||||
stencil_hi[d] = 0;
|
|
||||||
stencil_size[d]= 2;
|
|
||||||
} else if ( latt[d] > 2 ) {
|
|
||||||
stencil_lo[d] = -1;
|
|
||||||
stencil_hi[d] = 1;
|
|
||||||
stencil_size[d]= 3;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this->BuildShifts();
|
|
||||||
};
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
// Need to worry about red-black now
|
|
||||||
class NonLocalStencilGeometry4D : public NonLocalStencilGeometry {
|
|
||||||
public:
|
|
||||||
virtual int DerivedDimSkip(void) { return 0;};
|
|
||||||
NonLocalStencilGeometry4D(GridCartesian *Coarse,int _hops) : NonLocalStencilGeometry(Coarse,_hops,0) { };
|
|
||||||
virtual ~NonLocalStencilGeometry4D() {};
|
|
||||||
};
|
|
||||||
class NonLocalStencilGeometry5D : public NonLocalStencilGeometry {
|
|
||||||
public:
|
|
||||||
virtual int DerivedDimSkip(void) { return 1; };
|
|
||||||
NonLocalStencilGeometry5D(GridCartesian *Coarse,int _hops) : NonLocalStencilGeometry(Coarse,_hops,1) { };
|
|
||||||
virtual ~NonLocalStencilGeometry5D() {};
|
|
||||||
};
|
|
||||||
/*
|
|
||||||
* Bunch of different options classes
|
|
||||||
*/
|
|
||||||
class NextToNextToNextToNearestStencilGeometry4D : public NonLocalStencilGeometry4D {
|
|
||||||
public:
|
|
||||||
NextToNextToNextToNearestStencilGeometry4D(GridCartesian *Coarse) : NonLocalStencilGeometry4D(Coarse,4)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
|
||||||
class NextToNextToNextToNearestStencilGeometry5D : public NonLocalStencilGeometry5D {
|
|
||||||
public:
|
|
||||||
NextToNextToNextToNearestStencilGeometry5D(GridCartesian *Coarse) : NonLocalStencilGeometry5D(Coarse,4)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
|
||||||
class NextToNearestStencilGeometry4D : public NonLocalStencilGeometry4D {
|
|
||||||
public:
|
|
||||||
NextToNearestStencilGeometry4D(GridCartesian *Coarse) : NonLocalStencilGeometry4D(Coarse,2)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
|
||||||
class NextToNearestStencilGeometry5D : public NonLocalStencilGeometry5D {
|
|
||||||
public:
|
|
||||||
NextToNearestStencilGeometry5D(GridCartesian *Coarse) : NonLocalStencilGeometry5D(Coarse,2)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
|
||||||
class NearestStencilGeometry4D : public NonLocalStencilGeometry4D {
|
|
||||||
public:
|
|
||||||
NearestStencilGeometry4D(GridCartesian *Coarse) : NonLocalStencilGeometry4D(Coarse,1)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
|
||||||
class NearestStencilGeometry5D : public NonLocalStencilGeometry5D {
|
|
||||||
public:
|
|
||||||
NearestStencilGeometry5D(GridCartesian *Coarse) : NonLocalStencilGeometry5D(Coarse,1)
|
|
||||||
{
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/AlignedAllocator.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<typename _Tp>
|
|
||||||
class alignedAllocator {
|
|
||||||
public:
|
|
||||||
typedef std::size_t size_type;
|
|
||||||
typedef std::ptrdiff_t difference_type;
|
|
||||||
typedef _Tp* pointer;
|
|
||||||
typedef const _Tp* const_pointer;
|
|
||||||
typedef _Tp& reference;
|
|
||||||
typedef const _Tp& const_reference;
|
|
||||||
typedef _Tp value_type;
|
|
||||||
|
|
||||||
template<typename _Tp1> struct rebind { typedef alignedAllocator<_Tp1> other; };
|
|
||||||
alignedAllocator() throw() { }
|
|
||||||
alignedAllocator(const alignedAllocator&) throw() { }
|
|
||||||
template<typename _Tp1> alignedAllocator(const alignedAllocator<_Tp1>&) throw() { }
|
|
||||||
~alignedAllocator() throw() { }
|
|
||||||
pointer address(reference __x) const { return &__x; }
|
|
||||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
|
||||||
|
|
||||||
pointer allocate(size_type __n, const void* _p= 0)
|
|
||||||
{
|
|
||||||
size_type bytes = __n*sizeof(_Tp);
|
|
||||||
profilerAllocate(bytes);
|
|
||||||
_Tp *ptr = (_Tp*) MemoryManager::CpuAllocate(bytes);
|
|
||||||
if ( (_Tp*)ptr == (_Tp *) NULL ) {
|
|
||||||
printf("Grid CPU Allocator got NULL for %lu bytes\n",(unsigned long) bytes );
|
|
||||||
}
|
|
||||||
GRID_ASSERT( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
void deallocate(pointer __p, size_type __n)
|
|
||||||
{
|
|
||||||
size_type bytes = __n * sizeof(_Tp);
|
|
||||||
profilerFree(bytes);
|
|
||||||
MemoryManager::CpuFree((void *)__p,bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME: hack for the copy constructor: it must be avoided to avoid single thread loop
|
|
||||||
void construct(pointer __p, const _Tp& __val) { };
|
|
||||||
void construct(pointer __p) { };
|
|
||||||
void destroy(pointer __p) { };
|
|
||||||
};
|
|
||||||
template<typename _Tp> inline bool operator==(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return true; }
|
|
||||||
template<typename _Tp> inline bool operator!=(const alignedAllocator<_Tp>&, const alignedAllocator<_Tp>&){ return false; }
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Unified virtual memory
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<typename _Tp>
|
|
||||||
class uvmAllocator {
|
|
||||||
public:
|
|
||||||
typedef std::size_t size_type;
|
|
||||||
typedef std::ptrdiff_t difference_type;
|
|
||||||
typedef _Tp* pointer;
|
|
||||||
typedef const _Tp* const_pointer;
|
|
||||||
typedef _Tp& reference;
|
|
||||||
typedef const _Tp& const_reference;
|
|
||||||
typedef _Tp value_type;
|
|
||||||
|
|
||||||
template<typename _Tp1> struct rebind { typedef uvmAllocator<_Tp1> other; };
|
|
||||||
uvmAllocator() throw() { }
|
|
||||||
uvmAllocator(const uvmAllocator&) throw() { }
|
|
||||||
template<typename _Tp1> uvmAllocator(const uvmAllocator<_Tp1>&) throw() { }
|
|
||||||
~uvmAllocator() throw() { }
|
|
||||||
pointer address(reference __x) const { return &__x; }
|
|
||||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
|
||||||
|
|
||||||
pointer allocate(size_type __n, const void* _p= 0)
|
|
||||||
{
|
|
||||||
size_type bytes = __n*sizeof(_Tp);
|
|
||||||
profilerAllocate(bytes);
|
|
||||||
_Tp *ptr = (_Tp*) MemoryManager::SharedAllocate(bytes);
|
|
||||||
if ( (_Tp*)ptr == (_Tp *) NULL ) {
|
|
||||||
printf("Grid Shared Allocator got NULL for %lu bytes\n",(unsigned long) bytes );
|
|
||||||
}
|
|
||||||
GRID_ASSERT( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
void deallocate(pointer __p, size_type __n)
|
|
||||||
{
|
|
||||||
size_type bytes = __n * sizeof(_Tp);
|
|
||||||
profilerFree(bytes);
|
|
||||||
MemoryManager::SharedFree((void *)__p,bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
void construct(pointer __p, const _Tp& __val) { new((void *)__p) _Tp(__val); };
|
|
||||||
void construct(pointer __p) { };
|
|
||||||
void destroy(pointer __p) { };
|
|
||||||
};
|
|
||||||
template<typename _Tp> inline bool operator==(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return true; }
|
|
||||||
template<typename _Tp> inline bool operator!=(const uvmAllocator<_Tp>&, const uvmAllocator<_Tp>&){ return false; }
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Device memory
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<typename _Tp>
|
|
||||||
class devAllocator {
|
|
||||||
public:
|
|
||||||
typedef std::size_t size_type;
|
|
||||||
typedef std::ptrdiff_t difference_type;
|
|
||||||
typedef _Tp* pointer;
|
|
||||||
typedef const _Tp* const_pointer;
|
|
||||||
typedef _Tp& reference;
|
|
||||||
typedef const _Tp& const_reference;
|
|
||||||
typedef _Tp value_type;
|
|
||||||
|
|
||||||
template<typename _Tp1> struct rebind { typedef devAllocator<_Tp1> other; };
|
|
||||||
devAllocator() throw() { }
|
|
||||||
devAllocator(const devAllocator&) throw() { }
|
|
||||||
template<typename _Tp1> devAllocator(const devAllocator<_Tp1>&) throw() { }
|
|
||||||
~devAllocator() throw() { }
|
|
||||||
pointer address(reference __x) const { return &__x; }
|
|
||||||
size_type max_size() const throw() { return size_t(-1) / sizeof(_Tp); }
|
|
||||||
|
|
||||||
pointer allocate(size_type __n, const void* _p= 0)
|
|
||||||
{
|
|
||||||
size_type bytes = __n*sizeof(_Tp);
|
|
||||||
profilerAllocate(bytes);
|
|
||||||
_Tp *ptr = (_Tp*) MemoryManager::AcceleratorAllocate(bytes);
|
|
||||||
if ( (_Tp*)ptr == (_Tp *) NULL ) {
|
|
||||||
printf("Grid Device Allocator got NULL for %lu bytes\n",(unsigned long) bytes );
|
|
||||||
}
|
|
||||||
GRID_ASSERT( ( (_Tp*)ptr != (_Tp *)NULL ) );
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
void deallocate(pointer __p, size_type __n)
|
|
||||||
{
|
|
||||||
size_type bytes = __n * sizeof(_Tp);
|
|
||||||
profilerFree(bytes);
|
|
||||||
MemoryManager::AcceleratorFree((void *)__p,bytes);
|
|
||||||
}
|
|
||||||
void construct(pointer __p, const _Tp& __val) { };
|
|
||||||
void construct(pointer __p) { };
|
|
||||||
void destroy(pointer __p) { };
|
|
||||||
};
|
|
||||||
template<typename _Tp> inline bool operator==(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return true; }
|
|
||||||
template<typename _Tp> inline bool operator!=(const devAllocator<_Tp>&, const devAllocator<_Tp>&){ return false; }
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Template typedefs
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class T> using hostVector = std::vector<T,alignedAllocator<T> >; // Needs autoview
|
|
||||||
template<class T> using Vector = std::vector<T,uvmAllocator<T> >; // Really want to deprecate
|
|
||||||
template<class T> using uvmVector = std::vector<T,uvmAllocator<T> >; // auto migrating page
|
|
||||||
template<class T> using deviceVector = std::vector<T,devAllocator<T> >; // device vector
|
|
||||||
|
|
||||||
/*
|
|
||||||
template<class T> class vecView
|
|
||||||
{
|
|
||||||
protected:
|
|
||||||
T * data;
|
|
||||||
uint64_t size;
|
|
||||||
ViewMode mode;
|
|
||||||
void * cpu_ptr;
|
|
||||||
public:
|
|
||||||
// Rvalue accessor
|
|
||||||
accelerator_inline T & operator[](size_t i) const { return this->data[i]; };
|
|
||||||
vecView(Vector<T> &refer_to_me,ViewMode _mode)
|
|
||||||
{
|
|
||||||
cpu_ptr = &refer_to_me[0];
|
|
||||||
size = refer_to_me.size();
|
|
||||||
mode = _mode;
|
|
||||||
data =(T *) MemoryManager::ViewOpen(cpu_ptr,
|
|
||||||
size*sizeof(T),
|
|
||||||
mode,
|
|
||||||
AdviseDefault);
|
|
||||||
}
|
|
||||||
void ViewClose(void)
|
|
||||||
{ // Inform the manager
|
|
||||||
MemoryManager::ViewClose(this->cpu_ptr,this->mode);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class T> vecView<T> VectorView(Vector<T> &vec,ViewMode _mode)
|
|
||||||
{
|
|
||||||
vecView<T> ret(vec,_mode); // does the open
|
|
||||||
return ret; // must be closed
|
|
||||||
}
|
|
||||||
|
|
||||||
#define autoVecView(v_v,v,mode) \
|
|
||||||
auto v_v = VectorView(v,mode); \
|
|
||||||
ViewCloser<decltype(v_v)> _autoView##v_v(v_v);
|
|
||||||
*/
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
#pragma once
|
|
||||||
#include <Grid/allocator/MemoryStats.h>
|
|
||||||
#include <Grid/allocator/MemoryManager.h>
|
|
||||||
#include <Grid/allocator/AlignedAllocator.h>
|
|
||||||
@@ -1,362 +0,0 @@
|
|||||||
#include <Grid/GridCore.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/*Allocation types, saying which pointer cache should be used*/
|
|
||||||
#define Cpu (0)
|
|
||||||
#define CpuHuge (1)
|
|
||||||
#define CpuSmall (2)
|
|
||||||
#define Acc (3)
|
|
||||||
#define AccHuge (4)
|
|
||||||
#define AccSmall (5)
|
|
||||||
#define Shared (6)
|
|
||||||
#define SharedHuge (7)
|
|
||||||
#define SharedSmall (8)
|
|
||||||
#undef GRID_MM_VERBOSE
|
|
||||||
uint64_t total_shared;
|
|
||||||
uint64_t total_device;
|
|
||||||
uint64_t total_host;;
|
|
||||||
|
|
||||||
#if defined(__has_feature)
|
|
||||||
#if __has_feature(leak_sanitizer)
|
|
||||||
#define ASAN_LEAK_CHECK
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef ASAN_LEAK_CHECK
|
|
||||||
#include <sanitizer/asan_interface.h>
|
|
||||||
#include <sanitizer/common_interface_defs.h>
|
|
||||||
#include <sanitizer/lsan_interface.h>
|
|
||||||
#define LEAK_CHECK(A) { __lsan_do_recoverable_leak_check(); }
|
|
||||||
#else
|
|
||||||
#define LEAK_CHECK(A) { }
|
|
||||||
#endif
|
|
||||||
|
|
||||||
void MemoryManager::DisplayMallinfo(void)
|
|
||||||
{
|
|
||||||
#ifdef __linux__
|
|
||||||
struct mallinfo mi; // really want mallinfo2, but glibc version isn't uniform
|
|
||||||
|
|
||||||
mi = mallinfo();
|
|
||||||
|
|
||||||
std::cout << "MemoryManager: Total non-mmapped bytes (arena): "<< (size_t)mi.arena<<std::endl;
|
|
||||||
std::cout << "MemoryManager: # of free chunks (ordblks): "<< (size_t)mi.ordblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: # of free fastbin blocks (smblks): "<< (size_t)mi.smblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: # of mapped regions (hblks): "<< (size_t)mi.hblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: Bytes in mapped regions (hblkhd): "<< (size_t)mi.hblkhd<<std::endl;
|
|
||||||
std::cout << "MemoryManager: Max. total allocated space (usmblks): "<< (size_t)mi.usmblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: Free bytes held in fastbins (fsmblks): "<< (size_t)mi.fsmblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: Total allocated space (uordblks): "<< (size_t)mi.uordblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: Total free space (fordblks): "<< (size_t)mi.fordblks<<std::endl;
|
|
||||||
std::cout << "MemoryManager: Topmost releasable block (keepcost): "<< (size_t)mi.keepcost<<std::endl;
|
|
||||||
#endif
|
|
||||||
LEAK_CHECK();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::PrintBytes(void)
|
|
||||||
{
|
|
||||||
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
|
||||||
std::cout << " MemoryManager : PrintBytes "<<std::endl;
|
|
||||||
std::cout << " MemoryManager : ------------------------------------ "<<std::endl;
|
|
||||||
std::cout << " MemoryManager : "<<(total_shared>>20)<<" shared Mbytes "<<std::endl;
|
|
||||||
std::cout << " MemoryManager : "<<(total_device>>20)<<" accelerator Mbytes "<<std::endl;
|
|
||||||
std::cout << " MemoryManager : "<<(total_host>>20) <<" cpu Mbytes "<<std::endl;
|
|
||||||
uint64_t cacheBytes;
|
|
||||||
cacheBytes = CacheBytes[Cpu];
|
|
||||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" cpu cache Mbytes "<<std::endl;
|
|
||||||
cacheBytes = CacheBytes[Acc];
|
|
||||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" acc cache Mbytes "<<std::endl;
|
|
||||||
cacheBytes = CacheBytes[Shared];
|
|
||||||
std::cout << " MemoryManager : "<<(cacheBytes>>20) <<" shared cache Mbytes "<<std::endl;
|
|
||||||
|
|
||||||
#ifdef GRID_CUDA
|
|
||||||
cuda_mem();
|
|
||||||
#endif
|
|
||||||
DisplayMallinfo();
|
|
||||||
}
|
|
||||||
|
|
||||||
uint64_t MemoryManager::DeviceCacheBytes() { return CacheBytes[Acc] + CacheBytes[AccHuge] + CacheBytes[AccSmall]; }
|
|
||||||
uint64_t MemoryManager::HostCacheBytes() { return CacheBytes[Cpu] + CacheBytes[CpuHuge] + CacheBytes[CpuSmall]; }
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Data tables for recently freed pooiniter caches
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
MemoryManager::AllocationCacheEntry MemoryManager::Entries[MemoryManager::NallocType][MemoryManager::NallocCacheMax];
|
|
||||||
int MemoryManager::Victim[MemoryManager::NallocType];
|
|
||||||
int MemoryManager::Ncache[MemoryManager::NallocType] = { 2, 0, 8, 8, 0, 16, 8, 0, 16 };
|
|
||||||
uint64_t MemoryManager::CacheBytes[MemoryManager::NallocType];
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Actual allocation and deallocation utils
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
void *MemoryManager::AcceleratorAllocate(size_t bytes)
|
|
||||||
{
|
|
||||||
total_device+=bytes;
|
|
||||||
void *ptr = (void *) Lookup(bytes,Acc);
|
|
||||||
if ( ptr == (void *) NULL ) {
|
|
||||||
ptr = (void *) acceleratorAllocDevice(bytes);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"AcceleratorAllocate "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void MemoryManager::AcceleratorFree (void *ptr,size_t bytes)
|
|
||||||
{
|
|
||||||
total_device-=bytes;
|
|
||||||
void *__freeme = Insert(ptr,bytes,Acc);
|
|
||||||
if ( __freeme ) {
|
|
||||||
acceleratorFreeDevice(__freeme);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"AcceleratorFree "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
void *MemoryManager::SharedAllocate(size_t bytes)
|
|
||||||
{
|
|
||||||
total_shared+=bytes;
|
|
||||||
void *ptr = (void *) Lookup(bytes,Shared);
|
|
||||||
if ( ptr == (void *) NULL ) {
|
|
||||||
ptr = (void *) acceleratorAllocShared(bytes);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"SharedAllocate "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void MemoryManager::SharedFree (void *ptr,size_t bytes)
|
|
||||||
{
|
|
||||||
total_shared-=bytes;
|
|
||||||
void *__freeme = Insert(ptr,bytes,Shared);
|
|
||||||
if ( __freeme ) {
|
|
||||||
acceleratorFreeShared(__freeme);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"SharedFree "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#ifdef GRID_UVM
|
|
||||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
|
||||||
{
|
|
||||||
total_host+=bytes;
|
|
||||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
|
||||||
if ( ptr == (void *) NULL ) {
|
|
||||||
ptr = (void *) acceleratorAllocShared(bytes);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"CpuAllocate "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
|
||||||
{
|
|
||||||
total_host-=bytes;
|
|
||||||
NotifyDeletion(_ptr);
|
|
||||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
|
||||||
if ( __freeme ) {
|
|
||||||
acceleratorFreeShared(__freeme);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"CpuFree "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
void *MemoryManager::CpuAllocate(size_t bytes)
|
|
||||||
{
|
|
||||||
total_host+=bytes;
|
|
||||||
void *ptr = (void *) Lookup(bytes,Cpu);
|
|
||||||
if ( ptr == (void *) NULL ) {
|
|
||||||
ptr = (void *) acceleratorAllocCpu(bytes);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"CpuAllocate "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void MemoryManager::CpuFree (void *_ptr,size_t bytes)
|
|
||||||
{
|
|
||||||
total_host-=bytes;
|
|
||||||
NotifyDeletion(_ptr);
|
|
||||||
void *__freeme = Insert(_ptr,bytes,Cpu);
|
|
||||||
if ( __freeme ) {
|
|
||||||
acceleratorFreeCpu(__freeme);
|
|
||||||
}
|
|
||||||
#ifdef GRID_MM_VERBOSE
|
|
||||||
std::cout <<"CpuFree "<<std::endl;
|
|
||||||
PrintBytes();
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
//////////////////////////////////////////
|
|
||||||
// call only once
|
|
||||||
//////////////////////////////////////////
|
|
||||||
void MemoryManager::Init(void)
|
|
||||||
{
|
|
||||||
|
|
||||||
char * str;
|
|
||||||
int Nc;
|
|
||||||
|
|
||||||
str= getenv("GRID_ALLOC_NCACHE_LARGE");
|
|
||||||
if ( str ) {
|
|
||||||
Nc = atoi(str);
|
|
||||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
|
||||||
Ncache[Cpu]=Nc;
|
|
||||||
Ncache[Acc]=Nc;
|
|
||||||
Ncache[Shared]=Nc;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
str= getenv("GRID_ALLOC_NCACHE_HUGE");
|
|
||||||
if ( str ) {
|
|
||||||
Nc = atoi(str);
|
|
||||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
|
||||||
Ncache[CpuHuge]=Nc;
|
|
||||||
Ncache[AccHuge]=Nc;
|
|
||||||
Ncache[SharedHuge]=Nc;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
str= getenv("GRID_ALLOC_NCACHE_SMALL");
|
|
||||||
if ( str ) {
|
|
||||||
Nc = atoi(str);
|
|
||||||
if ( (Nc>=0) && (Nc < NallocCacheMax)) {
|
|
||||||
Ncache[CpuSmall]=Nc;
|
|
||||||
Ncache[AccSmall]=Nc;
|
|
||||||
Ncache[SharedSmall]=Nc;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::InitMessage(void) {
|
|
||||||
|
|
||||||
#ifndef GRID_UVM
|
|
||||||
std::cout << GridLogMessage << "MemoryManager Cache "<< MemoryManager::DeviceMaxBytes <<" bytes "<<std::endl;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() setting up"<<std::endl;
|
|
||||||
#ifdef ALLOCATION_CACHE
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent host allocations: SMALL "<<Ncache[CpuSmall]<<" LARGE "<<Ncache[Cpu]<<" HUGE "<<Ncache[CpuHuge]<<std::endl;
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent device allocations: SMALL "<<Ncache[AccSmall]<<" LARGE "<<Ncache[Acc]<<" Huge "<<Ncache[AccHuge]<<std::endl;
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() cache pool for recent shared allocations: SMALL "<<Ncache[SharedSmall]<<" LARGE "<<Ncache[Shared]<<" Huge "<<Ncache[SharedHuge]<<std::endl;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef GRID_UVM
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Unified memory space"<<std::endl;
|
|
||||||
#ifdef GRID_CUDA
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMallocManaged"<<std::endl;
|
|
||||||
#endif
|
|
||||||
#ifdef GRID_HIP
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMallocManaged"<<std::endl;
|
|
||||||
#endif
|
|
||||||
#ifdef GRID_SYCL
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_shared"<<std::endl;
|
|
||||||
#endif
|
|
||||||
#else
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Non unified: Caching accelerator data in dedicated memory"<<std::endl;
|
|
||||||
#ifdef GRID_CUDA
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using cudaMalloc"<<std::endl;
|
|
||||||
#endif
|
|
||||||
#ifdef GRID_HIP
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using hipMalloc"<<std::endl;
|
|
||||||
#endif
|
|
||||||
#ifdef GRID_SYCL
|
|
||||||
std::cout << GridLogMessage<< "MemoryManager::Init() Using SYCL malloc_device"<<std::endl;
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
void *MemoryManager::Insert(void *ptr,size_t bytes,int type)
|
|
||||||
{
|
|
||||||
#ifdef ALLOCATION_CACHE
|
|
||||||
int cache;
|
|
||||||
if (bytes < GRID_ALLOC_SMALL_LIMIT) cache = type + 2;
|
|
||||||
else if (bytes >= GRID_ALLOC_HUGE_LIMIT) cache = type + 1;
|
|
||||||
else cache = type;
|
|
||||||
|
|
||||||
return Insert(ptr,bytes,Entries[cache],Ncache[cache],Victim[cache],CacheBytes[cache]);
|
|
||||||
#else
|
|
||||||
return ptr;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void *MemoryManager::Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim, uint64_t &cacheBytes)
|
|
||||||
{
|
|
||||||
#ifdef GRID_OMP
|
|
||||||
GRID_ASSERT(omp_in_parallel()==0);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
if (ncache == 0) return ptr;
|
|
||||||
|
|
||||||
void * ret = NULL;
|
|
||||||
int v = -1;
|
|
||||||
|
|
||||||
for(int e=0;e<ncache;e++) {
|
|
||||||
if ( entries[e].valid==0 ) {
|
|
||||||
v=e;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( v==-1 ) {
|
|
||||||
v=victim;
|
|
||||||
victim = (victim+1)%ncache;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( entries[v].valid ) {
|
|
||||||
ret = entries[v].address;
|
|
||||||
cacheBytes -= entries[v].bytes;
|
|
||||||
entries[v].valid = 0;
|
|
||||||
entries[v].address = NULL;
|
|
||||||
entries[v].bytes = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
entries[v].address=ptr;
|
|
||||||
entries[v].bytes =bytes;
|
|
||||||
entries[v].valid =1;
|
|
||||||
cacheBytes += bytes;
|
|
||||||
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
void *MemoryManager::Lookup(size_t bytes,int type)
|
|
||||||
{
|
|
||||||
#ifdef ALLOCATION_CACHE
|
|
||||||
int cache;
|
|
||||||
if (bytes < GRID_ALLOC_SMALL_LIMIT) cache = type + 2;
|
|
||||||
else if (bytes >= GRID_ALLOC_HUGE_LIMIT) cache = type + 1;
|
|
||||||
else cache = type;
|
|
||||||
|
|
||||||
return Lookup(bytes,Entries[cache],Ncache[cache],CacheBytes[cache]);
|
|
||||||
#else
|
|
||||||
return NULL;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void *MemoryManager::Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t & cacheBytes)
|
|
||||||
{
|
|
||||||
#ifdef GRID_OMP
|
|
||||||
GRID_ASSERT(omp_in_parallel()==0);
|
|
||||||
#endif
|
|
||||||
for(int e=0;e<ncache;e++){
|
|
||||||
if ( entries[e].valid && ( entries[e].bytes == bytes ) ) {
|
|
||||||
entries[e].valid = 0;
|
|
||||||
cacheBytes -= entries[e].bytes;
|
|
||||||
return entries[e].address;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,227 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/MemoryManager.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
#include <list>
|
|
||||||
#include <unordered_map>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
// Move control to configure.ac and Config.h?
|
|
||||||
|
|
||||||
#define GRID_ALLOC_SMALL_LIMIT (4096)
|
|
||||||
#define GRID_ALLOC_HUGE_LIMIT (2147483648)
|
|
||||||
|
|
||||||
#define STRINGIFY(x) #x
|
|
||||||
#define TOSTRING(x) STRINGIFY(x)
|
|
||||||
#define FILE_LINE __FILE__ ":" TOSTRING(__LINE__)
|
|
||||||
#define AUDIT(a) MemoryManager::Audit(FILE_LINE)
|
|
||||||
|
|
||||||
/*Pinning pages is costly*/
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Advise the LatticeAccelerator class
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
enum ViewAdvise {
|
|
||||||
AdviseDefault = 0x0, // Regular data
|
|
||||||
AdviseInfrequentUse = 0x1 // Advise that the data is used infrequently. This can
|
|
||||||
// significantly influence performance of bulk storage.
|
|
||||||
|
|
||||||
// AdviseTransient = 0x2, // Data will mostly be read. On some architectures
|
|
||||||
// enables read-only copies of memory to be kept on
|
|
||||||
// host and device.
|
|
||||||
|
|
||||||
// AdviseAcceleratorWriteDiscard = 0x4 // Field will be written in entirety on device
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
// View Access Mode
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
enum ViewMode {
|
|
||||||
AcceleratorRead = 0x01,
|
|
||||||
AcceleratorWrite = 0x02,
|
|
||||||
AcceleratorWriteDiscard = 0x04,
|
|
||||||
CpuRead = 0x08,
|
|
||||||
CpuWrite = 0x10,
|
|
||||||
CpuWriteDiscard = 0x10 // same for now
|
|
||||||
};
|
|
||||||
|
|
||||||
struct MemoryStatus {
|
|
||||||
uint64_t DeviceBytes;
|
|
||||||
uint64_t DeviceLRUBytes;
|
|
||||||
uint64_t DeviceMaxBytes;
|
|
||||||
uint64_t HostToDeviceBytes;
|
|
||||||
uint64_t DeviceToHostBytes;
|
|
||||||
uint64_t HostToDeviceXfer;
|
|
||||||
uint64_t DeviceToHostXfer;
|
|
||||||
uint64_t DeviceEvictions;
|
|
||||||
uint64_t DeviceDestroy;
|
|
||||||
uint64_t DeviceAllocCacheBytes;
|
|
||||||
uint64_t HostAllocCacheBytes;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
class MemoryManager {
|
|
||||||
private:
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
// For caching recently freed allocations
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
typedef struct {
|
|
||||||
void *address;
|
|
||||||
size_t bytes;
|
|
||||||
int valid;
|
|
||||||
} AllocationCacheEntry;
|
|
||||||
|
|
||||||
static const int NallocCacheMax=128;
|
|
||||||
static const int NallocType=9;
|
|
||||||
static AllocationCacheEntry Entries[NallocType][NallocCacheMax];
|
|
||||||
static int Victim[NallocType];
|
|
||||||
static int Ncache[NallocType];
|
|
||||||
static uint64_t CacheBytes[NallocType];
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Free pool
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
static void *Insert(void *ptr,size_t bytes,int type) ;
|
|
||||||
static void *Lookup(size_t bytes,int type) ;
|
|
||||||
static void *Insert(void *ptr,size_t bytes,AllocationCacheEntry *entries,int ncache,int &victim,uint64_t &cbytes) ;
|
|
||||||
static void *Lookup(size_t bytes,AllocationCacheEntry *entries,int ncache,uint64_t &cbytes) ;
|
|
||||||
|
|
||||||
public:
|
|
||||||
static void PrintBytes(void);
|
|
||||||
static void Audit(std::string s);
|
|
||||||
static void Init(void);
|
|
||||||
static void InitMessage(void);
|
|
||||||
static void *AcceleratorAllocate(size_t bytes);
|
|
||||||
static void AcceleratorFree (void *ptr,size_t bytes);
|
|
||||||
static void *SharedAllocate(size_t bytes);
|
|
||||||
static void SharedFree (void *ptr,size_t bytes);
|
|
||||||
static void *CpuAllocate(size_t bytes);
|
|
||||||
static void CpuFree (void *ptr,size_t bytes);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////
|
|
||||||
// Footprint tracking
|
|
||||||
////////////////////////////////////////////////////////
|
|
||||||
static uint64_t DeviceBytes;
|
|
||||||
static uint64_t DeviceLRUBytes;
|
|
||||||
static uint64_t DeviceMaxBytes;
|
|
||||||
static uint64_t HostToDeviceBytes;
|
|
||||||
static uint64_t DeviceToHostBytes;
|
|
||||||
static uint64_t HostToDeviceXfer;
|
|
||||||
static uint64_t DeviceToHostXfer;
|
|
||||||
static uint64_t DeviceEvictions;
|
|
||||||
static uint64_t DeviceDestroy;
|
|
||||||
|
|
||||||
static uint64_t DeviceCacheBytes();
|
|
||||||
static uint64_t HostCacheBytes();
|
|
||||||
|
|
||||||
static MemoryStatus GetFootprint(void) {
|
|
||||||
MemoryStatus stat;
|
|
||||||
stat.DeviceBytes = DeviceBytes;
|
|
||||||
stat.DeviceLRUBytes = DeviceLRUBytes;
|
|
||||||
stat.DeviceMaxBytes = DeviceMaxBytes;
|
|
||||||
stat.HostToDeviceBytes = HostToDeviceBytes;
|
|
||||||
stat.DeviceToHostBytes = DeviceToHostBytes;
|
|
||||||
stat.HostToDeviceXfer = HostToDeviceXfer;
|
|
||||||
stat.DeviceToHostXfer = DeviceToHostXfer;
|
|
||||||
stat.DeviceEvictions = DeviceEvictions;
|
|
||||||
stat.DeviceDestroy = DeviceDestroy;
|
|
||||||
stat.DeviceAllocCacheBytes = DeviceCacheBytes();
|
|
||||||
stat.HostAllocCacheBytes = HostCacheBytes();
|
|
||||||
return stat;
|
|
||||||
};
|
|
||||||
|
|
||||||
private:
|
|
||||||
#ifndef GRID_UVM
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Data tables for ViewCache
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
typedef std::list<uint64_t> LRU_t;
|
|
||||||
typedef typename LRU_t::iterator LRUiterator;
|
|
||||||
typedef struct {
|
|
||||||
int LRU_valid;
|
|
||||||
LRUiterator LRU_entry;
|
|
||||||
uint64_t CpuPtr;
|
|
||||||
uint64_t AccPtr;
|
|
||||||
size_t bytes;
|
|
||||||
uint32_t transient;
|
|
||||||
uint32_t state;
|
|
||||||
uint32_t accLock;
|
|
||||||
uint32_t cpuLock;
|
|
||||||
} AcceleratorViewEntry;
|
|
||||||
|
|
||||||
typedef std::unordered_map<uint64_t,AcceleratorViewEntry> AccViewTable_t;
|
|
||||||
typedef typename AccViewTable_t::iterator AccViewTableIterator ;
|
|
||||||
|
|
||||||
static AccViewTable_t AccViewTable;
|
|
||||||
static LRU_t LRU;
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Device motion
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
static void Create(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
|
||||||
static void EvictVictims(uint64_t bytes); // Frees up <bytes>
|
|
||||||
static void Evict(AcceleratorViewEntry &AccCache);
|
|
||||||
static void Flush(AcceleratorViewEntry &AccCache);
|
|
||||||
static void Clone(AcceleratorViewEntry &AccCache);
|
|
||||||
static void AccDiscard(AcceleratorViewEntry &AccCache);
|
|
||||||
static void CpuDiscard(AcceleratorViewEntry &AccCache);
|
|
||||||
|
|
||||||
// static void LRUupdate(AcceleratorViewEntry &AccCache);
|
|
||||||
static void LRUinsert(AcceleratorViewEntry &AccCache);
|
|
||||||
static void LRUremove(AcceleratorViewEntry &AccCache);
|
|
||||||
|
|
||||||
// manage entries in the table
|
|
||||||
static int EntryPresent(uint64_t CpuPtr);
|
|
||||||
static void EntryCreate(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
|
||||||
static void EntryErase (uint64_t CpuPtr);
|
|
||||||
static AccViewTableIterator EntryLookup(uint64_t CpuPtr);
|
|
||||||
static void EntrySet (uint64_t CpuPtr,AcceleratorViewEntry &entry);
|
|
||||||
|
|
||||||
static void AcceleratorViewClose(uint64_t AccPtr);
|
|
||||||
static uint64_t AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
|
||||||
static void CpuViewClose(uint64_t Ptr);
|
|
||||||
static uint64_t CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
public:
|
|
||||||
static void DisplayMallinfo(void);
|
|
||||||
static void NotifyDeletion(void * CpuPtr);
|
|
||||||
static void Print(void);
|
|
||||||
static void PrintAll(void);
|
|
||||||
static void PrintState( void* CpuPtr);
|
|
||||||
static int isOpen (void* CpuPtr);
|
|
||||||
static void ViewClose(void* CpuPtr,ViewMode mode);
|
|
||||||
static void *ViewOpen (void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint);
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,606 +0,0 @@
|
|||||||
#include <Grid/GridCore.h>
|
|
||||||
#ifndef GRID_UVM
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#define MAXLINE 512
|
|
||||||
static char print_buffer [ MAXLINE ];
|
|
||||||
|
|
||||||
#define mprintf(...) snprintf (print_buffer,MAXLINE, __VA_ARGS__ ); std::cout << GridLogMemory << print_buffer << std::endl;
|
|
||||||
#define dprintf(...) snprintf (print_buffer,MAXLINE, __VA_ARGS__ ); std::cout << GridLogDebug << print_buffer << std::endl;
|
|
||||||
//#define dprintf(...)
|
|
||||||
//#define mprintf(...)
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
// For caching copies of data on device
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
MemoryManager::AccViewTable_t MemoryManager::AccViewTable;
|
|
||||||
MemoryManager::LRU_t MemoryManager::LRU;
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////
|
|
||||||
// Footprint tracking
|
|
||||||
////////////////////////////////////////////////////////
|
|
||||||
uint64_t MemoryManager::DeviceBytes;
|
|
||||||
uint64_t MemoryManager::DeviceLRUBytes;
|
|
||||||
uint64_t MemoryManager::DeviceMaxBytes = 1024*1024*128;
|
|
||||||
uint64_t MemoryManager::HostToDeviceBytes;
|
|
||||||
uint64_t MemoryManager::DeviceToHostBytes;
|
|
||||||
uint64_t MemoryManager::HostToDeviceXfer;
|
|
||||||
uint64_t MemoryManager::DeviceToHostXfer;
|
|
||||||
uint64_t MemoryManager::DeviceEvictions;
|
|
||||||
uint64_t MemoryManager::DeviceDestroy;
|
|
||||||
|
|
||||||
////////////////////////////////////
|
|
||||||
// Priority ordering for unlocked entries
|
|
||||||
// Empty
|
|
||||||
// CpuDirty
|
|
||||||
// Consistent
|
|
||||||
// AccDirty
|
|
||||||
////////////////////////////////////
|
|
||||||
#define Empty (0x0) /*Entry unoccupied */
|
|
||||||
#define CpuDirty (0x1) /*CPU copy is golden, Acc buffer MAY not be allocated*/
|
|
||||||
#define Consistent (0x2) /*ACC copy AND CPU copy are valid */
|
|
||||||
#define AccDirty (0x4) /*ACC copy is golden */
|
|
||||||
#define EvictNext (0x8) /*Priority for eviction*/
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Mechanics of data table maintenance
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
int MemoryManager::EntryPresent(uint64_t CpuPtr)
|
|
||||||
{
|
|
||||||
if(AccViewTable.empty()) return 0;
|
|
||||||
|
|
||||||
auto count = AccViewTable.count(CpuPtr); GRID_ASSERT((count==0)||(count==1));
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
void MemoryManager::EntryCreate(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(!EntryPresent(CpuPtr));
|
|
||||||
AcceleratorViewEntry AccCache;
|
|
||||||
AccCache.CpuPtr = CpuPtr;
|
|
||||||
AccCache.AccPtr = (uint64_t)NULL;
|
|
||||||
AccCache.bytes = bytes;
|
|
||||||
AccCache.state = CpuDirty;
|
|
||||||
AccCache.LRU_valid=0;
|
|
||||||
AccCache.transient=0;
|
|
||||||
AccCache.accLock=0;
|
|
||||||
AccCache.cpuLock=0;
|
|
||||||
AccViewTable[CpuPtr] = AccCache;
|
|
||||||
}
|
|
||||||
MemoryManager::AccViewTableIterator MemoryManager::EntryLookup(uint64_t CpuPtr)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(EntryPresent(CpuPtr));
|
|
||||||
auto AccCacheIterator = AccViewTable.find(CpuPtr);
|
|
||||||
GRID_ASSERT(AccCacheIterator!=AccViewTable.end());
|
|
||||||
return AccCacheIterator;
|
|
||||||
}
|
|
||||||
void MemoryManager::EntryErase(uint64_t CpuPtr)
|
|
||||||
{
|
|
||||||
auto AccCache = EntryLookup(CpuPtr);
|
|
||||||
AccViewTable.erase(CpuPtr);
|
|
||||||
}
|
|
||||||
void MemoryManager::LRUinsert(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(AccCache.LRU_valid==0);
|
|
||||||
if (AccCache.transient) {
|
|
||||||
LRU.push_back(AccCache.CpuPtr);
|
|
||||||
AccCache.LRU_entry = --LRU.end();
|
|
||||||
} else {
|
|
||||||
LRU.push_front(AccCache.CpuPtr);
|
|
||||||
AccCache.LRU_entry = LRU.begin();
|
|
||||||
}
|
|
||||||
AccCache.LRU_valid = 1;
|
|
||||||
DeviceLRUBytes+=AccCache.bytes;
|
|
||||||
}
|
|
||||||
void MemoryManager::LRUremove(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(AccCache.LRU_valid==1);
|
|
||||||
LRU.erase(AccCache.LRU_entry);
|
|
||||||
AccCache.LRU_valid = 0;
|
|
||||||
DeviceLRUBytes-=AccCache.bytes;
|
|
||||||
}
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
// Accelerator cache motion & consistency logic
|
|
||||||
/////////////////////////////////////////////////
|
|
||||||
void MemoryManager::AccDiscard(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
///////////////////////////////////////////////////////////
|
|
||||||
// Remove from Accelerator, remove entry, without flush
|
|
||||||
// Cannot be locked. If allocated Must be in LRU pool.
|
|
||||||
///////////////////////////////////////////////////////////
|
|
||||||
GRID_ASSERT(AccCache.state!=Empty);
|
|
||||||
|
|
||||||
dprintf("MemoryManager: Discard(%lx) %lx",(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr);
|
|
||||||
GRID_ASSERT(AccCache.accLock==0);
|
|
||||||
GRID_ASSERT(AccCache.cpuLock==0);
|
|
||||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
|
||||||
if(AccCache.AccPtr) {
|
|
||||||
AcceleratorFree((void *)AccCache.AccPtr,AccCache.bytes);
|
|
||||||
DeviceDestroy++;
|
|
||||||
DeviceBytes -=AccCache.bytes;
|
|
||||||
LRUremove(AccCache);
|
|
||||||
AccCache.AccPtr=(uint64_t) NULL;
|
|
||||||
dprintf("MemoryManager: Free(%lx) LRU %ld Total %ld",(uint64_t)AccCache.AccPtr,DeviceLRUBytes,DeviceBytes);
|
|
||||||
}
|
|
||||||
uint64_t CpuPtr = AccCache.CpuPtr;
|
|
||||||
EntryErase(CpuPtr);
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::Evict(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
///////////////////////////////////////////////////////////////////////////
|
|
||||||
// Make CPU consistent, remove from Accelerator, remove from LRU, LEAVE CPU only entry
|
|
||||||
// Cannot be acclocked. If allocated must be in LRU pool.
|
|
||||||
//
|
|
||||||
// Nov 2022... Felix issue: Allocating two CpuPtrs, can have an entry in LRU-q with CPUlock.
|
|
||||||
// and require to evict the AccPtr copy. Eviction was a mistake in CpuViewOpen
|
|
||||||
// but there is a weakness where CpuLock entries are attempted for erase
|
|
||||||
// Take these OUT LRU queue when CPU locked?
|
|
||||||
// Cannot take out the table as cpuLock data is important.
|
|
||||||
///////////////////////////////////////////////////////////////////////////
|
|
||||||
GRID_ASSERT(AccCache.state!=Empty);
|
|
||||||
|
|
||||||
mprintf("MemoryManager: Evict CpuPtr %lx AccPtr %lx cpuLock %ld accLock %ld",
|
|
||||||
(uint64_t)AccCache.CpuPtr,(uint64_t)AccCache.AccPtr,
|
|
||||||
(uint64_t)AccCache.cpuLock,(uint64_t)AccCache.accLock);
|
|
||||||
if (AccCache.accLock!=0) return;
|
|
||||||
if (AccCache.cpuLock!=0) return;
|
|
||||||
if(AccCache.state==AccDirty) {
|
|
||||||
Flush(AccCache);
|
|
||||||
}
|
|
||||||
if(AccCache.AccPtr) {
|
|
||||||
AcceleratorFree((void *)AccCache.AccPtr,AccCache.bytes);
|
|
||||||
LRUremove(AccCache);
|
|
||||||
AccCache.AccPtr=(uint64_t)NULL;
|
|
||||||
AccCache.state=CpuDirty; // CPU primary now
|
|
||||||
DeviceBytes -=AccCache.bytes;
|
|
||||||
dprintf("MemoryManager: Free(AccPtr %lx) footprint now %ld ",(uint64_t)AccCache.AccPtr,DeviceBytes);
|
|
||||||
}
|
|
||||||
// uint64_t CpuPtr = AccCache.CpuPtr;
|
|
||||||
DeviceEvictions++;
|
|
||||||
// EntryErase(CpuPtr);
|
|
||||||
}
|
|
||||||
void MemoryManager::Flush(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(AccCache.state==AccDirty);
|
|
||||||
GRID_ASSERT(AccCache.cpuLock==0);
|
|
||||||
GRID_ASSERT(AccCache.accLock==0);
|
|
||||||
GRID_ASSERT(AccCache.AccPtr!=(uint64_t)NULL);
|
|
||||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
|
||||||
acceleratorCopyFromDevice((void *)AccCache.AccPtr,(void *)AccCache.CpuPtr,AccCache.bytes);
|
|
||||||
mprintf("MemoryManager: acceleratorCopyFromDevice Flush size %ld AccPtr %lx -> CpuPtr %lx",(uint64_t)AccCache.bytes,(uint64_t)AccCache.AccPtr,(uint64_t)AccCache.CpuPtr); fflush(stdout);
|
|
||||||
DeviceToHostBytes+=AccCache.bytes;
|
|
||||||
DeviceToHostXfer++;
|
|
||||||
AccCache.state=Consistent;
|
|
||||||
}
|
|
||||||
void MemoryManager::Clone(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(AccCache.state==CpuDirty);
|
|
||||||
GRID_ASSERT(AccCache.cpuLock==0);
|
|
||||||
GRID_ASSERT(AccCache.accLock==0);
|
|
||||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
|
||||||
if(AccCache.AccPtr==(uint64_t)NULL){
|
|
||||||
AccCache.AccPtr=(uint64_t)AcceleratorAllocate(AccCache.bytes);
|
|
||||||
DeviceBytes+=AccCache.bytes;
|
|
||||||
}
|
|
||||||
mprintf("MemoryManager: acceleratorCopyToDevice Clone size %ld AccPtr %lx <- CpuPtr %lx",
|
|
||||||
(uint64_t)AccCache.bytes,
|
|
||||||
(uint64_t)AccCache.AccPtr,(uint64_t)AccCache.CpuPtr); fflush(stdout);
|
|
||||||
acceleratorCopyToDevice((void *)AccCache.CpuPtr,(void *)AccCache.AccPtr,AccCache.bytes);
|
|
||||||
HostToDeviceBytes+=AccCache.bytes;
|
|
||||||
HostToDeviceXfer++;
|
|
||||||
AccCache.state=Consistent;
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::CpuDiscard(AcceleratorViewEntry &AccCache)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(AccCache.state!=Empty);
|
|
||||||
GRID_ASSERT(AccCache.cpuLock==0);
|
|
||||||
GRID_ASSERT(AccCache.accLock==0);
|
|
||||||
GRID_ASSERT(AccCache.CpuPtr!=(uint64_t)NULL);
|
|
||||||
if(AccCache.AccPtr==(uint64_t)NULL){
|
|
||||||
AccCache.AccPtr=(uint64_t)AcceleratorAllocate(AccCache.bytes);
|
|
||||||
DeviceBytes+=AccCache.bytes;
|
|
||||||
}
|
|
||||||
AccCache.state=AccDirty;
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// View management
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
void MemoryManager::ViewClose(void* Ptr,ViewMode mode)
|
|
||||||
{
|
|
||||||
if( (mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard) ){
|
|
||||||
dprintf("AcceleratorViewClose %lx",(uint64_t)Ptr);
|
|
||||||
AcceleratorViewClose((uint64_t)Ptr);
|
|
||||||
} else if( (mode==CpuRead)||(mode==CpuWrite)){
|
|
||||||
CpuViewClose((uint64_t)Ptr);
|
|
||||||
} else {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void *MemoryManager::ViewOpen(void* _CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
|
||||||
{
|
|
||||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
|
||||||
if( (mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard) ){
|
|
||||||
dprintf("AcceleratorViewOpen %lx",(uint64_t)CpuPtr);
|
|
||||||
return (void *) AcceleratorViewOpen(CpuPtr,bytes,mode,hint);
|
|
||||||
} else if( (mode==CpuRead)||(mode==CpuWrite)){
|
|
||||||
return (void *)CpuViewOpen(CpuPtr,bytes,mode,hint);
|
|
||||||
} else {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void MemoryManager::EvictVictims(uint64_t bytes)
|
|
||||||
{
|
|
||||||
if(bytes>=DeviceMaxBytes) {
|
|
||||||
printf("EvictVictims bytes %ld DeviceMaxBytes %ld\n",bytes,DeviceMaxBytes);
|
|
||||||
}
|
|
||||||
GRID_ASSERT(bytes<DeviceMaxBytes);
|
|
||||||
while(bytes+DeviceLRUBytes > DeviceMaxBytes){
|
|
||||||
if ( DeviceLRUBytes > 0){
|
|
||||||
GRID_ASSERT(LRU.size()>0);
|
|
||||||
uint64_t victim = LRU.back(); // From the LRU
|
|
||||||
auto AccCacheIterator = EntryLookup(victim);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
Evict(AccCache);
|
|
||||||
} else {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
uint64_t MemoryManager::AcceleratorViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint)
|
|
||||||
{
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Find if present, otherwise get or force an empty
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
if ( EntryPresent(CpuPtr)==0 ){
|
|
||||||
EntryCreate(CpuPtr,bytes,mode,hint);
|
|
||||||
}
|
|
||||||
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
if (!AccCache.AccPtr) {
|
|
||||||
EvictVictims(bytes);
|
|
||||||
}
|
|
||||||
GRID_ASSERT((mode==AcceleratorRead)||(mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard));
|
|
||||||
|
|
||||||
GRID_ASSERT(AccCache.cpuLock==0); // Programming error
|
|
||||||
|
|
||||||
if(AccCache.state!=Empty) {
|
|
||||||
dprintf("ViewOpen found entry %lx %lx : sizes %ld %ld accLock %ld",
|
|
||||||
(uint64_t)AccCache.CpuPtr,
|
|
||||||
(uint64_t)CpuPtr,
|
|
||||||
(uint64_t)AccCache.bytes,
|
|
||||||
(uint64_t)bytes,
|
|
||||||
(uint64_t)AccCache.accLock);
|
|
||||||
GRID_ASSERT(AccCache.CpuPtr == CpuPtr);
|
|
||||||
GRID_ASSERT(AccCache.bytes ==bytes);
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
* State transitions and actions
|
|
||||||
*
|
|
||||||
* Action State StateNext Flush Clone
|
|
||||||
*
|
|
||||||
* AccRead Empty Consistent - Y
|
|
||||||
* AccWrite Empty AccDirty - Y
|
|
||||||
* AccRead CpuDirty Consistent - Y
|
|
||||||
* AccWrite CpuDirty AccDirty - Y
|
|
||||||
* AccRead Consistent Consistent - -
|
|
||||||
* AccWrite Consistent AccDirty - -
|
|
||||||
* AccRead AccDirty AccDirty - -
|
|
||||||
* AccWrite AccDirty AccDirty - -
|
|
||||||
*/
|
|
||||||
if(AccCache.state==Empty) {
|
|
||||||
GRID_ASSERT(AccCache.LRU_valid==0);
|
|
||||||
AccCache.CpuPtr = CpuPtr;
|
|
||||||
AccCache.AccPtr = (uint64_t)NULL;
|
|
||||||
AccCache.bytes = bytes;
|
|
||||||
AccCache.state = CpuDirty; // Cpu starts primary
|
|
||||||
if(mode==AcceleratorWriteDiscard){
|
|
||||||
CpuDiscard(AccCache);
|
|
||||||
AccCache.state = AccDirty; // Empty + AcceleratorWrite=> AccDirty
|
|
||||||
} else if(mode==AcceleratorWrite){
|
|
||||||
Clone(AccCache);
|
|
||||||
AccCache.state = AccDirty; // Empty + AcceleratorWrite=> AccDirty
|
|
||||||
} else {
|
|
||||||
Clone(AccCache);
|
|
||||||
AccCache.state = Consistent; // Empty + AccRead => Consistent
|
|
||||||
}
|
|
||||||
AccCache.accLock= 1;
|
|
||||||
dprintf("Copied Empty entry into device accLock= %d",AccCache.accLock);
|
|
||||||
} else if(AccCache.state==CpuDirty ){
|
|
||||||
if(mode==AcceleratorWriteDiscard) {
|
|
||||||
CpuDiscard(AccCache);
|
|
||||||
AccCache.state = AccDirty; // CpuDirty + AcceleratorWrite=> AccDirty
|
|
||||||
} else if(mode==AcceleratorWrite) {
|
|
||||||
Clone(AccCache);
|
|
||||||
AccCache.state = AccDirty; // CpuDirty + AcceleratorWrite=> AccDirty
|
|
||||||
} else {
|
|
||||||
Clone(AccCache);
|
|
||||||
AccCache.state = Consistent; // CpuDirty + AccRead => Consistent
|
|
||||||
}
|
|
||||||
AccCache.accLock++;
|
|
||||||
dprintf("CpuDirty entry into device ++accLock= %d",AccCache.accLock);
|
|
||||||
} else if(AccCache.state==Consistent) {
|
|
||||||
if((mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard))
|
|
||||||
AccCache.state = AccDirty; // Consistent + AcceleratorWrite=> AccDirty
|
|
||||||
else
|
|
||||||
AccCache.state = Consistent; // Consistent + AccRead => Consistent
|
|
||||||
AccCache.accLock++;
|
|
||||||
dprintf("Consistent entry into device ++accLock= %d",AccCache.accLock);
|
|
||||||
} else if(AccCache.state==AccDirty) {
|
|
||||||
if((mode==AcceleratorWrite)||(mode==AcceleratorWriteDiscard))
|
|
||||||
AccCache.state = AccDirty; // AccDirty + AcceleratorWrite=> AccDirty
|
|
||||||
else
|
|
||||||
AccCache.state = AccDirty; // AccDirty + AccRead => AccDirty
|
|
||||||
AccCache.accLock++;
|
|
||||||
dprintf("AccDirty entry ++accLock= %d",AccCache.accLock);
|
|
||||||
} else {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT(AccCache.accLock>0);
|
|
||||||
// If view is opened on device must remove from LRU
|
|
||||||
if(AccCache.LRU_valid==1){
|
|
||||||
// must possibly remove from LRU as now locked on GPU
|
|
||||||
dprintf("AccCache entry removed from LRU ");
|
|
||||||
LRUremove(AccCache);
|
|
||||||
}
|
|
||||||
|
|
||||||
int transient =hint;
|
|
||||||
AccCache.transient= transient? EvictNext : 0;
|
|
||||||
|
|
||||||
return AccCache.AccPtr;
|
|
||||||
}
|
|
||||||
////////////////////////////////////
|
|
||||||
// look up & decrement lock count
|
|
||||||
////////////////////////////////////
|
|
||||||
void MemoryManager::AcceleratorViewClose(uint64_t CpuPtr)
|
|
||||||
{
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
|
|
||||||
GRID_ASSERT(AccCache.cpuLock==0);
|
|
||||||
GRID_ASSERT(AccCache.accLock>0);
|
|
||||||
|
|
||||||
AccCache.accLock--;
|
|
||||||
// Move to LRU queue if not locked and close on device
|
|
||||||
if(AccCache.accLock==0) {
|
|
||||||
dprintf("AccleratorViewClose %lx AccLock decremented to %ld move to LRU queue",(uint64_t)CpuPtr,(uint64_t)AccCache.accLock);
|
|
||||||
LRUinsert(AccCache);
|
|
||||||
} else {
|
|
||||||
dprintf("AccleratorViewClose %lx AccLock decremented to %ld",(uint64_t)CpuPtr,(uint64_t)AccCache.accLock);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void MemoryManager::CpuViewClose(uint64_t CpuPtr)
|
|
||||||
{
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
|
|
||||||
GRID_ASSERT(AccCache.cpuLock>0);
|
|
||||||
GRID_ASSERT(AccCache.accLock==0);
|
|
||||||
|
|
||||||
AccCache.cpuLock--;
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
* Action State StateNext Flush Clone
|
|
||||||
*
|
|
||||||
* CpuRead Empty CpuDirty - -
|
|
||||||
* CpuWrite Empty CpuDirty - -
|
|
||||||
* CpuRead CpuDirty CpuDirty - -
|
|
||||||
* CpuWrite CpuDirty CpuDirty - -
|
|
||||||
* CpuRead Consistent Consistent - -
|
|
||||||
* CpuWrite Consistent CpuDirty - -
|
|
||||||
* CpuRead AccDirty Consistent Y -
|
|
||||||
* CpuWrite AccDirty CpuDirty Y -
|
|
||||||
*/
|
|
||||||
uint64_t MemoryManager::CpuViewOpen(uint64_t CpuPtr,size_t bytes,ViewMode mode,ViewAdvise transient)
|
|
||||||
{
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Find if present, otherwise get or force an empty
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
if ( EntryPresent(CpuPtr)==0 ){
|
|
||||||
EntryCreate(CpuPtr,bytes,mode,transient);
|
|
||||||
}
|
|
||||||
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
|
|
||||||
// CPU doesn't need to free space
|
|
||||||
// if (!AccCache.AccPtr) {
|
|
||||||
// EvictVictims(bytes);
|
|
||||||
// }
|
|
||||||
|
|
||||||
GRID_ASSERT((mode==CpuRead)||(mode==CpuWrite));
|
|
||||||
GRID_ASSERT(AccCache.accLock==0); // Programming error
|
|
||||||
|
|
||||||
if(AccCache.state!=Empty) {
|
|
||||||
GRID_ASSERT(AccCache.CpuPtr == CpuPtr);
|
|
||||||
GRID_ASSERT(AccCache.bytes==bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
if(AccCache.state==Empty) {
|
|
||||||
AccCache.CpuPtr = CpuPtr;
|
|
||||||
AccCache.AccPtr = (uint64_t)NULL;
|
|
||||||
AccCache.bytes = bytes;
|
|
||||||
AccCache.state = CpuDirty; // Empty + CpuRead/CpuWrite => CpuDirty
|
|
||||||
AccCache.accLock= 0;
|
|
||||||
AccCache.cpuLock= 1;
|
|
||||||
} else if(AccCache.state==CpuDirty ){
|
|
||||||
// AccPtr dont care, deferred allocate
|
|
||||||
AccCache.state = CpuDirty; // CpuDirty +CpuRead/CpuWrite => CpuDirty
|
|
||||||
AccCache.cpuLock++;
|
|
||||||
} else if(AccCache.state==Consistent) {
|
|
||||||
GRID_ASSERT(AccCache.AccPtr != (uint64_t)NULL);
|
|
||||||
if(mode==CpuWrite)
|
|
||||||
AccCache.state = CpuDirty; // Consistent +CpuWrite => CpuDirty
|
|
||||||
else
|
|
||||||
AccCache.state = Consistent; // Consistent +CpuRead => Consistent
|
|
||||||
AccCache.cpuLock++;
|
|
||||||
} else if(AccCache.state==AccDirty) {
|
|
||||||
GRID_ASSERT(AccCache.AccPtr != (uint64_t)NULL);
|
|
||||||
Flush(AccCache);
|
|
||||||
if(mode==CpuWrite) AccCache.state = CpuDirty; // AccDirty +CpuWrite => CpuDirty, Flush
|
|
||||||
else AccCache.state = Consistent; // AccDirty +CpuRead => Consistent, Flush
|
|
||||||
AccCache.cpuLock++;
|
|
||||||
} else {
|
|
||||||
GRID_ASSERT(0); // should be unreachable
|
|
||||||
}
|
|
||||||
|
|
||||||
AccCache.transient= transient? EvictNext : 0;
|
|
||||||
|
|
||||||
return AccCache.CpuPtr;
|
|
||||||
}
|
|
||||||
void MemoryManager::NotifyDeletion(void *_ptr)
|
|
||||||
{
|
|
||||||
// Look up in ViewCache
|
|
||||||
uint64_t ptr = (uint64_t)_ptr;
|
|
||||||
if(EntryPresent(ptr)) {
|
|
||||||
auto e = EntryLookup(ptr);
|
|
||||||
AccDiscard(e->second);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void MemoryManager::Print(void)
|
|
||||||
{
|
|
||||||
PrintBytes();
|
|
||||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
|
||||||
std::cout << GridLogMessage << "Memory Manager " << std::endl;
|
|
||||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceBytes << " bytes allocated on device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceLRUBytes<< " bytes evictable on device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceMaxBytes<< " bytes max on device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << HostToDeviceXfer << " transfers to device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceToHostXfer << " transfers from device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << HostToDeviceBytes<< " bytes transfered to device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceToHostBytes<< " bytes transfered from device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceEvictions << " Evictions from device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << DeviceDestroy << " Destroyed vectors on device " << std::endl;
|
|
||||||
std::cout << GridLogMessage << AccViewTable.size()<< " vectors " << LRU.size()<<" evictable"<< std::endl;
|
|
||||||
acceleratorMem();
|
|
||||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
|
||||||
}
|
|
||||||
void MemoryManager::PrintAll(void)
|
|
||||||
{
|
|
||||||
Print();
|
|
||||||
std::cout << GridLogMessage << std::endl;
|
|
||||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
|
||||||
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
|
||||||
for(auto it=AccViewTable.begin();it!=AccViewTable.end();it++){
|
|
||||||
auto &AccCache = it->second;
|
|
||||||
|
|
||||||
std::string str;
|
|
||||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
|
||||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
|
||||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
|
||||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
|
||||||
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
|
||||||
<< "\t" << AccCache.cpuLock
|
|
||||||
<< "\t" << AccCache.accLock
|
|
||||||
<< "\t" << AccCache.LRU_valid<<std::endl;
|
|
||||||
}
|
|
||||||
std::cout << GridLogMessage << "--------------------------------------------" << std::endl;
|
|
||||||
|
|
||||||
};
|
|
||||||
int MemoryManager::isOpen (void* _CpuPtr)
|
|
||||||
{
|
|
||||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
|
||||||
if ( EntryPresent(CpuPtr) ){
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
return AccCache.cpuLock+AccCache.accLock;
|
|
||||||
} else {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void MemoryManager::Audit(std::string s)
|
|
||||||
{
|
|
||||||
uint64_t CpuBytes=0;
|
|
||||||
uint64_t AccBytes=0;
|
|
||||||
uint64_t LruBytes1=0;
|
|
||||||
uint64_t LruBytes2=0;
|
|
||||||
uint64_t LruCnt=0;
|
|
||||||
|
|
||||||
std::cout << " Memory Manager::Audit() from "<<s<<std::endl;
|
|
||||||
for(auto it=LRU.begin();it!=LRU.end();it++){
|
|
||||||
uint64_t cpuPtr = *it;
|
|
||||||
GRID_ASSERT(EntryPresent(cpuPtr));
|
|
||||||
auto AccCacheIterator = EntryLookup(cpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
LruBytes2+=AccCache.bytes;
|
|
||||||
GRID_ASSERT(AccCache.LRU_valid==1);
|
|
||||||
GRID_ASSERT(AccCache.LRU_entry==it);
|
|
||||||
}
|
|
||||||
std::cout << " Memory Manager::Audit() LRU queue matches table entries "<<std::endl;
|
|
||||||
|
|
||||||
for(auto it=AccViewTable.begin();it!=AccViewTable.end();it++){
|
|
||||||
auto &AccCache = it->second;
|
|
||||||
|
|
||||||
std::string str;
|
|
||||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
|
||||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
|
||||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
|
||||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
|
||||||
|
|
||||||
CpuBytes+=AccCache.bytes;
|
|
||||||
if( AccCache.AccPtr ) AccBytes+=AccCache.bytes;
|
|
||||||
if( AccCache.LRU_valid ) LruBytes1+=AccCache.bytes;
|
|
||||||
if( AccCache.LRU_valid ) LruCnt++;
|
|
||||||
|
|
||||||
if ( AccCache.cpuLock || AccCache.accLock ) {
|
|
||||||
GRID_ASSERT(AccCache.LRU_valid==0);
|
|
||||||
|
|
||||||
std::cout << GridLogError << s<< "\n\t 0x"<<std::hex<<AccCache.CpuPtr<<std::dec
|
|
||||||
<< "\t0x"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
|
||||||
<< "\t cpuLock " << AccCache.cpuLock
|
|
||||||
<< "\t accLock " << AccCache.accLock
|
|
||||||
<< "\t LRUvalid " << AccCache.LRU_valid<<std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
GRID_ASSERT( AccCache.cpuLock== 0 ) ;
|
|
||||||
GRID_ASSERT( AccCache.accLock== 0 ) ;
|
|
||||||
}
|
|
||||||
std::cout << " Memory Manager::Audit() no locked table entries "<<std::endl;
|
|
||||||
GRID_ASSERT(LruBytes1==LruBytes2);
|
|
||||||
GRID_ASSERT(LruBytes1==DeviceLRUBytes);
|
|
||||||
std::cout << " Memory Manager::Audit() evictable bytes matches sum over table "<<std::endl;
|
|
||||||
GRID_ASSERT(AccBytes==DeviceBytes);
|
|
||||||
std::cout << " Memory Manager::Audit() device bytes matches sum over table "<<std::endl;
|
|
||||||
GRID_ASSERT(LruCnt == LRU.size());
|
|
||||||
std::cout << " Memory Manager::Audit() LRU entry count matches "<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
void MemoryManager::PrintState(void* _CpuPtr)
|
|
||||||
{
|
|
||||||
uint64_t CpuPtr = (uint64_t)_CpuPtr;
|
|
||||||
|
|
||||||
if ( EntryPresent(CpuPtr) ){
|
|
||||||
auto AccCacheIterator = EntryLookup(CpuPtr);
|
|
||||||
auto & AccCache = AccCacheIterator->second;
|
|
||||||
std::string str;
|
|
||||||
if ( AccCache.state==Empty ) str = std::string("Empty");
|
|
||||||
if ( AccCache.state==CpuDirty ) str = std::string("CpuDirty");
|
|
||||||
if ( AccCache.state==AccDirty ) str = std::string("AccDirty");
|
|
||||||
if ( AccCache.state==Consistent)str = std::string("Consistent");
|
|
||||||
if ( AccCache.state==EvictNext) str = std::string("EvictNext");
|
|
||||||
|
|
||||||
std::cout << GridLogMessage << "CpuAddr\t\tAccAddr\t\tState\t\tcpuLock\taccLock\tLRU_valid "<<std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tx"<<std::hex<<AccCache.CpuPtr<<std::dec
|
|
||||||
<< "\tx"<<std::hex<<AccCache.AccPtr<<std::dec<<"\t" <<str
|
|
||||||
<< "\t" << AccCache.cpuLock
|
|
||||||
<< "\t" << AccCache.accLock
|
|
||||||
<< "\t" << AccCache.LRU_valid<<std::endl;
|
|
||||||
|
|
||||||
} else {
|
|
||||||
std::cout << GridLogMessage << "No Entry in AccCache table." << std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
#include <Grid/GridCore.h>
|
|
||||||
#ifdef GRID_UVM
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// View management is 1:1 address space mapping
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
uint64_t MemoryManager::DeviceBytes;
|
|
||||||
uint64_t MemoryManager::DeviceLRUBytes;
|
|
||||||
uint64_t MemoryManager::DeviceMaxBytes = 1024*1024*128;
|
|
||||||
uint64_t MemoryManager::HostToDeviceBytes;
|
|
||||||
uint64_t MemoryManager::DeviceToHostBytes;
|
|
||||||
uint64_t MemoryManager::HostToDeviceXfer;
|
|
||||||
uint64_t MemoryManager::DeviceToHostXfer;
|
|
||||||
uint64_t MemoryManager::DeviceEvictions;
|
|
||||||
uint64_t MemoryManager::DeviceDestroy;
|
|
||||||
|
|
||||||
void MemoryManager::Audit(std::string s){};
|
|
||||||
void MemoryManager::ViewClose(void* AccPtr,ViewMode mode){};
|
|
||||||
void *MemoryManager::ViewOpen(void* CpuPtr,size_t bytes,ViewMode mode,ViewAdvise hint){ return CpuPtr; };
|
|
||||||
int MemoryManager::isOpen (void* CpuPtr) { return 0;}
|
|
||||||
void MemoryManager::PrintState(void* CpuPtr)
|
|
||||||
{
|
|
||||||
std::cout << GridLogMessage << "Host<->Device memory movement not currently managed by Grid." << std::endl;
|
|
||||||
};
|
|
||||||
void MemoryManager::Print(void){};
|
|
||||||
void MemoryManager::PrintAll(void){};
|
|
||||||
void MemoryManager::NotifyDeletion(void *ptr){};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/MemoryStats.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
std::string sizeString(size_t bytes);
|
|
||||||
|
|
||||||
struct MemoryStats
|
|
||||||
{
|
|
||||||
size_t totalAllocated{0}, maxAllocated{0},
|
|
||||||
currentlyAllocated{0}, totalFreed{0};
|
|
||||||
};
|
|
||||||
|
|
||||||
class MemoryProfiler
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
static MemoryStats *stats;
|
|
||||||
static bool debug;
|
|
||||||
};
|
|
||||||
|
|
||||||
#define memString(bytes) std::to_string(bytes) + " (" + sizeString(bytes) + ")"
|
|
||||||
#define profilerDebugPrint \
|
|
||||||
if (MemoryProfiler::stats) \
|
|
||||||
{ \
|
|
||||||
auto s = MemoryProfiler::stats; \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] Stats " << MemoryProfiler::stats << std::endl; \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] total : " << memString(s->totalAllocated) \
|
|
||||||
<< std::endl; \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] max : " << memString(s->maxAllocated) \
|
|
||||||
<< std::endl; \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] current: " << memString(s->currentlyAllocated) \
|
|
||||||
<< std::endl; \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] freed : " << memString(s->totalFreed) \
|
|
||||||
<< std::endl; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define profilerAllocate(bytes) \
|
|
||||||
if (MemoryProfiler::stats) \
|
|
||||||
{ \
|
|
||||||
auto s = MemoryProfiler::stats; \
|
|
||||||
s->totalAllocated += (bytes); \
|
|
||||||
s->currentlyAllocated += (bytes); \
|
|
||||||
s->maxAllocated = std::max(s->maxAllocated, s->currentlyAllocated); \
|
|
||||||
} \
|
|
||||||
if (MemoryProfiler::debug) \
|
|
||||||
{ \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] allocating " << memString(bytes) << std::endl; \
|
|
||||||
profilerDebugPrint; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define profilerFree(bytes) \
|
|
||||||
if (MemoryProfiler::stats) \
|
|
||||||
{ \
|
|
||||||
auto s = MemoryProfiler::stats; \
|
|
||||||
s->totalFreed += (bytes); \
|
|
||||||
s->currentlyAllocated -= (bytes); \
|
|
||||||
} \
|
|
||||||
if (MemoryProfiler::debug) \
|
|
||||||
{ \
|
|
||||||
std::cout << GridLogDebug << "[Memory debug] freeing " << memString(bytes) << std::endl; \
|
|
||||||
profilerDebugPrint; \
|
|
||||||
}
|
|
||||||
|
|
||||||
void check_huge_pages(void *Buf,uint64_t BYTES);
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/cartesian/Cartesian_base.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Guido Cossu <guido.cossu@ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CARTESIAN_BASE_H
|
|
||||||
#define GRID_CARTESIAN_BASE_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// Commicator provides information on the processor grid
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
// unsigned long _ndimension;
|
|
||||||
// Coordinate _processors; // processor grid
|
|
||||||
// int _processor; // linear processor rank
|
|
||||||
// Coordinate _processor_coor; // linear processor rank
|
|
||||||
//////////////////////////////////////////////////////////////////////
|
|
||||||
class GridBase : public CartesianCommunicator , public GridThread {
|
|
||||||
|
|
||||||
public:
|
|
||||||
int dummy;
|
|
||||||
// Give Lattice access
|
|
||||||
template<class object> friend class Lattice;
|
|
||||||
|
|
||||||
GridBase(const Coordinate & processor_grid) : CartesianCommunicator(processor_grid) { LocallyPeriodic=0;};
|
|
||||||
|
|
||||||
GridBase(const Coordinate & processor_grid,
|
|
||||||
const CartesianCommunicator &parent,
|
|
||||||
int &split_rank)
|
|
||||||
: CartesianCommunicator(processor_grid,parent,split_rank) {LocallyPeriodic=0;};
|
|
||||||
|
|
||||||
GridBase(const Coordinate & processor_grid,
|
|
||||||
const CartesianCommunicator &parent)
|
|
||||||
: CartesianCommunicator(processor_grid,parent,dummy) {LocallyPeriodic=0;};
|
|
||||||
|
|
||||||
virtual ~GridBase() = default;
|
|
||||||
|
|
||||||
// Physics Grid information.
|
|
||||||
Coordinate _simd_layout;// Which dimensions get relayed out over simd lanes.
|
|
||||||
Coordinate _fdimensions;// (full) Global dimensions of array prior to cb removal
|
|
||||||
Coordinate _gdimensions;// Global dimensions of array after cb removal
|
|
||||||
Coordinate _ldimensions;// local dimensions of array with processor images removed
|
|
||||||
Coordinate _rdimensions;// Reduced local dimensions with simd lane images and processor images removed
|
|
||||||
Coordinate _ostride; // Outer stride for each dimension
|
|
||||||
Coordinate _istride; // Inner stride i.e. within simd lane
|
|
||||||
int _osites; // _isites*_osites = product(dimensions).
|
|
||||||
int _isites;
|
|
||||||
int64_t _fsites; // _isites*_osites = product(dimensions).
|
|
||||||
int64_t _gsites;
|
|
||||||
Coordinate _slice_block;// subslice information
|
|
||||||
Coordinate _slice_stride;
|
|
||||||
Coordinate _slice_nblock;
|
|
||||||
|
|
||||||
Coordinate _lstart; // local start of array in gcoors _processor_coor[d]*_ldimensions[d]
|
|
||||||
Coordinate _lend ; // local end of array in gcoors _processor_coor[d]*_ldimensions[d]+_ldimensions_[d]-1
|
|
||||||
|
|
||||||
bool _isCheckerBoarded;
|
|
||||||
int LocallyPeriodic;
|
|
||||||
Coordinate _checker_dim_mask;
|
|
||||||
int _checker_dim;
|
|
||||||
|
|
||||||
public:
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Checkerboarding interface is virtual and overridden by
|
|
||||||
// GridCartesian / GridRedBlackCartesian
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
virtual int CheckerBoarded(int dim) =0;
|
|
||||||
virtual int CheckerBoard(const Coordinate &site)=0;
|
|
||||||
virtual int CheckerBoardDestination(int source_cb,int shift,int dim)=0;
|
|
||||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite)=0;
|
|
||||||
virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int cb)=0;
|
|
||||||
virtual int CheckerBoardFromOindex (int Oindex)=0;
|
|
||||||
virtual int CheckerBoardFromOindexTable (int Oindex)=0;
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Local layout calculations
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// These routines are key. Subdivide the linearised cartesian index into
|
|
||||||
// "inner" index identifying which simd lane of object<vFcomplex> is associated with coord
|
|
||||||
// "outer" index identifying which element of _odata in class "Lattice" is associated with coord.
|
|
||||||
//
|
|
||||||
// Compared to, say, Blitz++ we simply need to store BOTH an inner stride and an outer
|
|
||||||
// stride per dimension. The cost of evaluating the indexing information is doubled for an n-dimensional
|
|
||||||
// coordinate. Note, however, for data parallel operations the "inner" indexing cost is not paid and all
|
|
||||||
// lanes are operated upon simultaneously.
|
|
||||||
|
|
||||||
virtual int oIndex(Coordinate &coor)
|
|
||||||
{
|
|
||||||
int idx=0;
|
|
||||||
// Works with either global or local coordinates
|
|
||||||
for(int d=0;d<_ndimension;d++) idx+=_ostride[d]*(coor[d]%_rdimensions[d]);
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
virtual int iIndex(Coordinate &lcoor)
|
|
||||||
{
|
|
||||||
int idx=0;
|
|
||||||
for(int d=0;d<_ndimension;d++) idx+=_istride[d]*(lcoor[d]/_rdimensions[d]);
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
inline int oIndexReduced(Coordinate &ocoor)
|
|
||||||
{
|
|
||||||
int idx=0;
|
|
||||||
// ocoor is already reduced so can eliminate the modulo operation
|
|
||||||
// for fast indexing and inline the routine
|
|
||||||
for(int d=0;d<_ndimension;d++) idx+=_ostride[d]*ocoor[d];
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
inline void oCoorFromOindex (Coordinate& coor,int Oindex){
|
|
||||||
Lexicographic::CoorFromIndex(coor,Oindex,_rdimensions);
|
|
||||||
}
|
|
||||||
|
|
||||||
inline void InOutCoorToLocalCoor (Coordinate &ocoor, Coordinate &icoor, Coordinate &lcoor) {
|
|
||||||
lcoor.resize(_ndimension);
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
lcoor[d] = ocoor[d] + _rdimensions[d] * icoor[d];
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
// SIMD lane addressing
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
inline void iCoorFromIindex(Coordinate &coor,int lane)
|
|
||||||
{
|
|
||||||
Lexicographic::CoorFromIndex(coor,lane,_simd_layout);
|
|
||||||
}
|
|
||||||
|
|
||||||
inline int PermuteDim(int dimension){
|
|
||||||
return _simd_layout[dimension]>1;
|
|
||||||
}
|
|
||||||
inline int PermuteType(int dimension){
|
|
||||||
int permute_type=0;
|
|
||||||
//
|
|
||||||
// Best way to encode this would be to present a mask
|
|
||||||
// for which simd dimensions are rotated, and the rotation
|
|
||||||
// size. If there is only one simd dimension rotated, this is just
|
|
||||||
// a permute.
|
|
||||||
//
|
|
||||||
// Cases: PermuteType == 1,2,4,8
|
|
||||||
// Distance should be either 0,1,2..
|
|
||||||
//
|
|
||||||
if ( _simd_layout[dimension] > 2 ) {
|
|
||||||
for(int d=0;d<_ndimension;d++){
|
|
||||||
if ( d != dimension ) GRID_ASSERT ( (_simd_layout[d]==1) );
|
|
||||||
}
|
|
||||||
permute_type = RotateBit; // How to specify distance; this is not just direction.
|
|
||||||
return permute_type;
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int d=_ndimension-1;d>dimension;d--){
|
|
||||||
if (_simd_layout[d]>1 ) permute_type++;
|
|
||||||
}
|
|
||||||
return permute_type;
|
|
||||||
}
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Array sizing queries
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
inline int iSites(void) const { return _isites; };
|
|
||||||
inline int Nsimd(void) const { return _isites; };// Synonymous with iSites
|
|
||||||
inline int oSites(void) const { return _osites; };
|
|
||||||
inline int lSites(void) const { return _isites*_osites; };
|
|
||||||
inline int64_t gSites(void) const { return (int64_t)_isites*(int64_t)_osites*(int64_t)_Nprocessors; };
|
|
||||||
inline int Nd (void) const { return _ndimension;};
|
|
||||||
|
|
||||||
inline const Coordinate &LocalStarts(void) { return _lstart; };
|
|
||||||
inline const Coordinate &FullDimensions(void) { return _fdimensions;};
|
|
||||||
inline const Coordinate &GlobalDimensions(void) { return _gdimensions;};
|
|
||||||
inline const Coordinate &LocalDimensions(void) { return _ldimensions;};
|
|
||||||
inline const Coordinate &VirtualLocalDimensions(void) { return _ldimensions;};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Utility to print the full decomposition details
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
void show_decomposition(){
|
|
||||||
std::cout << GridLogMessage << "\tFull Dimensions : " << _fdimensions << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tSIMD layout : " << _simd_layout << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tGlobal Dimensions : " << _gdimensions << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tLocal Dimensions : " << _ldimensions << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tReduced Dimensions : " << _rdimensions << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tOuter strides : " << _ostride << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tInner strides : " << _istride << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tiSites : " << _isites << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\toSites : " << _osites << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tlSites : " << lSites() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tgSites : " << gSites() << std::endl;
|
|
||||||
std::cout << GridLogMessage << "\tNd : " << _ndimension << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Global addressing
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
void GlobalIndexToGlobalCoor(int64_t gidx,Coordinate &gcoor){
|
|
||||||
GRID_ASSERT(gidx< gSites());
|
|
||||||
Lexicographic::CoorFromIndex(gcoor,gidx,_gdimensions);
|
|
||||||
}
|
|
||||||
void LocalIndexToLocalCoor(int lidx,Coordinate &lcoor){
|
|
||||||
GRID_ASSERT(lidx<lSites());
|
|
||||||
Lexicographic::CoorFromIndex(lcoor,lidx,_ldimensions);
|
|
||||||
}
|
|
||||||
void GlobalCoorToGlobalIndex(const Coordinate & gcoor,int64_t & gidx){
|
|
||||||
gidx=0;
|
|
||||||
int mult=1;
|
|
||||||
for(int mu=0;mu<_ndimension;mu++) {
|
|
||||||
gidx+=mult*gcoor[mu];
|
|
||||||
mult*=_gdimensions[mu];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void GlobalCoorToProcessorCoorLocalCoor(Coordinate &pcoor,Coordinate &lcoor,const Coordinate &gcoor)
|
|
||||||
{
|
|
||||||
pcoor.resize(_ndimension);
|
|
||||||
lcoor.resize(_ndimension);
|
|
||||||
for(int mu=0;mu<_ndimension;mu++){
|
|
||||||
int _fld = _fdimensions[mu]/_processors[mu];
|
|
||||||
pcoor[mu] = gcoor[mu]/_fld;
|
|
||||||
lcoor[mu] = gcoor[mu]%_fld;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void GlobalCoorToRankIndex(int &rank, int &o_idx, int &i_idx ,const Coordinate &gcoor)
|
|
||||||
{
|
|
||||||
Coordinate pcoor;
|
|
||||||
Coordinate lcoor;
|
|
||||||
GlobalCoorToProcessorCoorLocalCoor(pcoor,lcoor,gcoor);
|
|
||||||
rank = RankFromProcessorCoor(pcoor);
|
|
||||||
/*
|
|
||||||
Coordinate cblcoor(lcoor);
|
|
||||||
for(int d=0;d<cblcoor.size();d++){
|
|
||||||
if( this->CheckerBoarded(d) ) {
|
|
||||||
cblcoor[d] = lcoor[d]/2;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
i_idx= iIndex(lcoor);
|
|
||||||
o_idx= oIndex(lcoor);
|
|
||||||
}
|
|
||||||
|
|
||||||
void RankIndexToGlobalCoor(int rank, int o_idx, int i_idx , Coordinate &gcoor)
|
|
||||||
{
|
|
||||||
gcoor.resize(_ndimension);
|
|
||||||
Coordinate coor(_ndimension);
|
|
||||||
|
|
||||||
ProcessorCoorFromRank(rank,coor);
|
|
||||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] = _ldimensions[mu]*coor[mu];
|
|
||||||
|
|
||||||
iCoorFromIindex(coor,i_idx);
|
|
||||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] += _rdimensions[mu]*coor[mu];
|
|
||||||
|
|
||||||
oCoorFromOindex (coor,o_idx);
|
|
||||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] += coor[mu];
|
|
||||||
|
|
||||||
}
|
|
||||||
void RankIndexCbToFullGlobalCoor(int rank, int o_idx, int i_idx, int cb,Coordinate &fcoor)
|
|
||||||
{
|
|
||||||
RankIndexToGlobalCoor(rank,o_idx,i_idx ,fcoor);
|
|
||||||
if(CheckerBoarded(0)){
|
|
||||||
fcoor[0] = fcoor[0]*2+cb;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void ProcessorCoorLocalCoorToGlobalCoor(Coordinate &Pcoor,Coordinate &Lcoor,Coordinate &gcoor)
|
|
||||||
{
|
|
||||||
gcoor.resize(_ndimension);
|
|
||||||
for(int mu=0;mu<_ndimension;mu++) gcoor[mu] = Pcoor[mu]*_ldimensions[mu]+Lcoor[mu];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,179 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/cartesian/Cartesian_full.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CARTESIAN_FULL_H
|
|
||||||
#define GRID_CARTESIAN_FULL_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Grid Support.
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
class GridCartesian: public GridBase {
|
|
||||||
|
|
||||||
public:
|
|
||||||
int dummy;
|
|
||||||
// Coordinate _checker_dim_mask;
|
|
||||||
virtual int CheckerBoardFromOindexTable (int Oindex) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardFromOindex (int Oindex)
|
|
||||||
{
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoarded(int dim) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoard(const Coordinate &site){
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardDestination(int cb,int shift,int dim){
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift, int ocb){
|
|
||||||
return shift;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift, int osite){
|
|
||||||
return shift;
|
|
||||||
}
|
|
||||||
/////////////////////////////////////////////////////////////////////////
|
|
||||||
// Constructor takes a parent grid and possibly subdivides communicator.
|
|
||||||
/////////////////////////////////////////////////////////////////////////
|
|
||||||
GridCartesian(const Coordinate &dimensions,
|
|
||||||
const Coordinate &simd_layout,
|
|
||||||
const Coordinate &processor_grid,
|
|
||||||
const GridCartesian &parent) : GridBase(processor_grid,parent,dummy)
|
|
||||||
{
|
|
||||||
Init(dimensions,simd_layout,processor_grid);
|
|
||||||
}
|
|
||||||
GridCartesian(const Coordinate &dimensions,
|
|
||||||
const Coordinate &simd_layout,
|
|
||||||
const Coordinate &processor_grid,
|
|
||||||
const GridCartesian &parent,int &split_rank) : GridBase(processor_grid,parent,split_rank)
|
|
||||||
{
|
|
||||||
Init(dimensions,simd_layout,processor_grid);
|
|
||||||
}
|
|
||||||
/////////////////////////////////////////////////////////////////////////
|
|
||||||
// Construct from comm world
|
|
||||||
/////////////////////////////////////////////////////////////////////////
|
|
||||||
GridCartesian(const Coordinate &dimensions,
|
|
||||||
const Coordinate &simd_layout,
|
|
||||||
const Coordinate &processor_grid) : GridBase(processor_grid)
|
|
||||||
{
|
|
||||||
Init(dimensions,simd_layout,processor_grid);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual ~GridCartesian() = default;
|
|
||||||
|
|
||||||
void Init(const Coordinate &dimensions,
|
|
||||||
const Coordinate &simd_layout,
|
|
||||||
const Coordinate &processor_grid)
|
|
||||||
{
|
|
||||||
///////////////////////
|
|
||||||
// Grid information
|
|
||||||
///////////////////////
|
|
||||||
_isCheckerBoarded = false;
|
|
||||||
_ndimension = dimensions.size();
|
|
||||||
|
|
||||||
_fdimensions.resize(_ndimension);
|
|
||||||
_gdimensions.resize(_ndimension);
|
|
||||||
_ldimensions.resize(_ndimension);
|
|
||||||
_rdimensions.resize(_ndimension);
|
|
||||||
_simd_layout.resize(_ndimension);
|
|
||||||
_checker_dim_mask.resize(_ndimension);;
|
|
||||||
_checker_dim = -1;
|
|
||||||
_lstart.resize(_ndimension);
|
|
||||||
_lend.resize(_ndimension);
|
|
||||||
|
|
||||||
_ostride.resize(_ndimension);
|
|
||||||
_istride.resize(_ndimension);
|
|
||||||
|
|
||||||
_fsites = _gsites = _osites = _isites = 1;
|
|
||||||
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
_checker_dim_mask[d]=0;
|
|
||||||
|
|
||||||
_fdimensions[d] = dimensions[d]; // Global dimensions
|
|
||||||
_gdimensions[d] = _fdimensions[d]; // Global dimensions
|
|
||||||
_simd_layout[d] = simd_layout[d];
|
|
||||||
_fsites = _fsites * _fdimensions[d];
|
|
||||||
_gsites = _gsites * _gdimensions[d];
|
|
||||||
|
|
||||||
// Use a reduced simd grid
|
|
||||||
_ldimensions[d] = _gdimensions[d] / _processors[d]; //local dimensions
|
|
||||||
//std::cout << _ldimensions[d] << " " << _gdimensions[d] << " " << _processors[d] << std::endl;
|
|
||||||
GRID_ASSERT(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
|
||||||
|
|
||||||
_rdimensions[d] = _ldimensions[d] / _simd_layout[d]; //overdecomposition
|
|
||||||
GRID_ASSERT(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
|
||||||
|
|
||||||
_lstart[d] = _processor_coor[d] * _ldimensions[d];
|
|
||||||
_lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
|
|
||||||
_osites *= _rdimensions[d];
|
|
||||||
_isites *= _simd_layout[d];
|
|
||||||
|
|
||||||
// Addressing support
|
|
||||||
if (d == 0)
|
|
||||||
{
|
|
||||||
_ostride[d] = 1;
|
|
||||||
_istride[d] = 1;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_ostride[d] = _ostride[d - 1] * _rdimensions[d - 1];
|
|
||||||
_istride[d] = _istride[d - 1] * _simd_layout[d - 1];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////
|
|
||||||
// subplane information
|
|
||||||
///////////////////////
|
|
||||||
_slice_block.resize(_ndimension);
|
|
||||||
_slice_stride.resize(_ndimension);
|
|
||||||
_slice_nblock.resize(_ndimension);
|
|
||||||
|
|
||||||
int block = 1;
|
|
||||||
int nblock = 1;
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
nblock *= _rdimensions[d];
|
|
||||||
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
nblock /= _rdimensions[d];
|
|
||||||
_slice_block[d] = block;
|
|
||||||
_slice_stride[d] = _ostride[d] * _rdimensions[d];
|
|
||||||
_slice_nblock[d] = nblock;
|
|
||||||
block = block * _rdimensions[d];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,306 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/cartesian/Cartesian_red_black.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_CARTESIAN_RED_BLACK_H
|
|
||||||
#define GRID_CARTESIAN_RED_BLACK_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
static const int CbRed =0;
|
|
||||||
static const int CbBlack=1;
|
|
||||||
static const int Even =CbRed;
|
|
||||||
static const int Odd =CbBlack;
|
|
||||||
|
|
||||||
accelerator_inline int RedBlackCheckerBoardFromOindex (int oindex,const Coordinate &rdim,const Coordinate &chk_dim_msk)
|
|
||||||
{
|
|
||||||
int nd=rdim.size();
|
|
||||||
Coordinate coor(nd);
|
|
||||||
|
|
||||||
Lexicographic::CoorFromIndex(coor,oindex,rdim);
|
|
||||||
|
|
||||||
int linear=0;
|
|
||||||
for(int d=0;d<nd;d++){
|
|
||||||
if(chk_dim_msk[d])
|
|
||||||
linear=linear+coor[d];
|
|
||||||
}
|
|
||||||
return (linear&0x1);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Specialise this for red black grids storing half the data like a chess board.
|
|
||||||
class GridRedBlackCartesian : public GridBase
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
// Coordinate _checker_dim_mask;
|
|
||||||
// int _checker_dim;
|
|
||||||
std::vector<int> _checker_board;
|
|
||||||
|
|
||||||
virtual int isCheckerBoarded(void) const { return 1; };
|
|
||||||
virtual int CheckerBoarded(int dim){
|
|
||||||
if( dim==_checker_dim) return 1;
|
|
||||||
else return 0;
|
|
||||||
}
|
|
||||||
virtual int CheckerBoard(const Coordinate &site){
|
|
||||||
int linear=0;
|
|
||||||
GRID_ASSERT(site.size()==_ndimension);
|
|
||||||
for(int d=0;d<_ndimension;d++){
|
|
||||||
if(_checker_dim_mask[d])
|
|
||||||
linear=linear+site[d];
|
|
||||||
}
|
|
||||||
return (linear&0x1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Depending on the cb of site, we toggle source cb.
|
|
||||||
// for block #b, element #e = (b, e)
|
|
||||||
// we need
|
|
||||||
virtual int CheckerBoardShiftForCB(int source_cb,int dim,int shift,int ocb){
|
|
||||||
if(dim != _checker_dim) return shift;
|
|
||||||
|
|
||||||
int fulldim =_fdimensions[dim];
|
|
||||||
shift = (shift+fulldim)%fulldim;
|
|
||||||
|
|
||||||
// Probably faster with table lookup;
|
|
||||||
// or by looping over x,y,z and multiply rather than computing checkerboard.
|
|
||||||
|
|
||||||
if ( (source_cb+ocb)&1 ) {
|
|
||||||
return (shift)/2;
|
|
||||||
} else {
|
|
||||||
return (shift+1)/2;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardFromOindexTable (int Oindex) {
|
|
||||||
return _checker_board[Oindex];
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardFromOindex (int Oindex)
|
|
||||||
{
|
|
||||||
Coordinate ocoor;
|
|
||||||
oCoorFromOindex(ocoor,Oindex);
|
|
||||||
return CheckerBoard(ocoor);
|
|
||||||
}
|
|
||||||
virtual int CheckerBoardShift(int source_cb,int dim,int shift,int osite){
|
|
||||||
|
|
||||||
if(dim != _checker_dim) return shift;
|
|
||||||
|
|
||||||
int ocb=CheckerBoardFromOindex(osite);
|
|
||||||
|
|
||||||
return CheckerBoardShiftForCB(source_cb,dim,shift,ocb);
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual int CheckerBoardDestination(int source_cb,int shift,int dim){
|
|
||||||
if ( _checker_dim_mask[dim] ) {
|
|
||||||
// If _fdimensions[checker_dim] is odd, then shifting by 1 in other dims
|
|
||||||
// does NOT cause a parity hop.
|
|
||||||
int add=(dim==_checker_dim) ? 0 : _fdimensions[_checker_dim];
|
|
||||||
if ( (shift+add) &0x1) {
|
|
||||||
return 1-source_cb;
|
|
||||||
} else {
|
|
||||||
return source_cb;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return source_cb;
|
|
||||||
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
// Create Redblack from original grid; require full grid pointer ?
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
GridRedBlackCartesian(const GridBase *base) : GridBase(base->_processors,*base)
|
|
||||||
{
|
|
||||||
int dims = base->_ndimension;
|
|
||||||
Coordinate checker_dim_mask(dims,1);
|
|
||||||
int checker_dim = 0;
|
|
||||||
Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim);
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
// Create redblack from original grid, with non-trivial checker dim mask
|
|
||||||
////////////////////////////////////////////////////////////
|
|
||||||
GridRedBlackCartesian(const GridBase *base,
|
|
||||||
const Coordinate &checker_dim_mask,
|
|
||||||
int checker_dim
|
|
||||||
) : GridBase(base->_processors,*base)
|
|
||||||
{
|
|
||||||
Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim) ;
|
|
||||||
}
|
|
||||||
|
|
||||||
virtual ~GridRedBlackCartesian() = default;
|
|
||||||
|
|
||||||
void Init(const Coordinate &dimensions,
|
|
||||||
const Coordinate &simd_layout,
|
|
||||||
const Coordinate &processor_grid,
|
|
||||||
const Coordinate &checker_dim_mask,
|
|
||||||
int checker_dim)
|
|
||||||
{
|
|
||||||
|
|
||||||
_isCheckerBoarded = true;
|
|
||||||
_checker_dim = checker_dim;
|
|
||||||
GRID_ASSERT(checker_dim_mask[checker_dim] == 1);
|
|
||||||
_ndimension = dimensions.size();
|
|
||||||
GRID_ASSERT(checker_dim_mask.size() == _ndimension);
|
|
||||||
GRID_ASSERT(processor_grid.size() == _ndimension);
|
|
||||||
GRID_ASSERT(simd_layout.size() == _ndimension);
|
|
||||||
|
|
||||||
_fdimensions.resize(_ndimension);
|
|
||||||
_gdimensions.resize(_ndimension);
|
|
||||||
_ldimensions.resize(_ndimension);
|
|
||||||
_rdimensions.resize(_ndimension);
|
|
||||||
_simd_layout.resize(_ndimension);
|
|
||||||
_lstart.resize(_ndimension);
|
|
||||||
_lend.resize(_ndimension);
|
|
||||||
|
|
||||||
_ostride.resize(_ndimension);
|
|
||||||
_istride.resize(_ndimension);
|
|
||||||
|
|
||||||
_fsites = _gsites = _osites = _isites = 1;
|
|
||||||
|
|
||||||
_checker_dim_mask = checker_dim_mask;
|
|
||||||
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
_fdimensions[d] = dimensions[d];
|
|
||||||
_gdimensions[d] = _fdimensions[d];
|
|
||||||
_fsites = _fsites * _fdimensions[d];
|
|
||||||
_gsites = _gsites * _gdimensions[d];
|
|
||||||
|
|
||||||
if (d == _checker_dim)
|
|
||||||
{
|
|
||||||
GRID_ASSERT((_gdimensions[d] & 0x1) == 0);
|
|
||||||
_gdimensions[d] = _gdimensions[d] / 2; // Remove a checkerboard
|
|
||||||
_gsites /= 2;
|
|
||||||
}
|
|
||||||
_ldimensions[d] = _gdimensions[d] / _processors[d];
|
|
||||||
GRID_ASSERT(_ldimensions[d] * _processors[d] == _gdimensions[d]);
|
|
||||||
_lstart[d] = _processor_coor[d] * _ldimensions[d];
|
|
||||||
_lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
|
|
||||||
|
|
||||||
// Use a reduced simd grid
|
|
||||||
_simd_layout[d] = simd_layout[d];
|
|
||||||
_rdimensions[d] = _ldimensions[d] / _simd_layout[d]; // this is not checking if this is integer
|
|
||||||
GRID_ASSERT(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
|
|
||||||
GRID_ASSERT(_rdimensions[d] > 0);
|
|
||||||
|
|
||||||
// all elements of a simd vector must have same checkerboard.
|
|
||||||
// If Ls vectorised, this must still be the case; e.g. dwf rb5d
|
|
||||||
if (_simd_layout[d] > 1)
|
|
||||||
{
|
|
||||||
if (checker_dim_mask[d])
|
|
||||||
{
|
|
||||||
assert((_rdimensions[d] & 0x1) == 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_osites *= _rdimensions[d];
|
|
||||||
_isites *= _simd_layout[d];
|
|
||||||
|
|
||||||
// Addressing support
|
|
||||||
if (d == 0)
|
|
||||||
{
|
|
||||||
_ostride[d] = 1;
|
|
||||||
_istride[d] = 1;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_ostride[d] = _ostride[d - 1] * _rdimensions[d - 1];
|
|
||||||
_istride[d] = _istride[d - 1] * _simd_layout[d - 1];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// subplane information
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
_slice_block.resize(_ndimension);
|
|
||||||
_slice_stride.resize(_ndimension);
|
|
||||||
_slice_nblock.resize(_ndimension);
|
|
||||||
|
|
||||||
int block = 1;
|
|
||||||
int nblock = 1;
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
nblock *= _rdimensions[d];
|
|
||||||
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
nblock /= _rdimensions[d];
|
|
||||||
_slice_block[d] = block;
|
|
||||||
_slice_stride[d] = _ostride[d] * _rdimensions[d];
|
|
||||||
_slice_nblock[d] = nblock;
|
|
||||||
block = block * _rdimensions[d];
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
// Create a checkerboard lookup table
|
|
||||||
////////////////////////////////////////////////
|
|
||||||
int rvol = 1;
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
rvol = rvol * _rdimensions[d];
|
|
||||||
}
|
|
||||||
_checker_board.resize(rvol);
|
|
||||||
for (int osite = 0; osite < _osites; osite++)
|
|
||||||
{
|
|
||||||
_checker_board[osite] = CheckerBoardFromOindex(osite);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
protected:
|
|
||||||
virtual int oIndex(Coordinate &coor)
|
|
||||||
{
|
|
||||||
int idx = 0;
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
if (d == _checker_dim)
|
|
||||||
{
|
|
||||||
idx += _ostride[d] * ((coor[d] / 2) % _rdimensions[d]);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
idx += _ostride[d] * (coor[d] % _rdimensions[d]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return idx;
|
|
||||||
};
|
|
||||||
|
|
||||||
virtual int iIndex(Coordinate &lcoor)
|
|
||||||
{
|
|
||||||
int idx = 0;
|
|
||||||
for (int d = 0; d < _ndimension; d++)
|
|
||||||
{
|
|
||||||
if (d == _checker_dim)
|
|
||||||
{
|
|
||||||
idx += _istride[d] * (lcoor[d] / (2 * _rdimensions[d]));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
idx += _istride[d] * (lcoor[d] / _rdimensions[d]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,949 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/communicator/Communicator_mpi.cc
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#include <Grid/GridCore.h>
|
|
||||||
#include <Grid/communicator/SharedMemory.h>
|
|
||||||
|
|
||||||
void GridAbort(void) { MPI_Abort(MPI_COMM_WORLD,SIGABRT); }
|
|
||||||
extern void * Grid_backtrace_buffer[_NBACKTRACE];
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
Grid_MPI_Comm CartesianCommunicator::communicator_world;
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
uint64_t checksum_index = 1;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
// First initialise of comms system
|
|
||||||
////////////////////////////////////////////
|
|
||||||
void CartesianCommunicator::Init(int *argc, char ***argv)
|
|
||||||
{
|
|
||||||
|
|
||||||
int flag;
|
|
||||||
int provided;
|
|
||||||
|
|
||||||
MPI_Initialized(&flag); // needed to coexist with other libs apparently
|
|
||||||
if ( !flag ) {
|
|
||||||
|
|
||||||
#ifndef GRID_COMMS_THREADS
|
|
||||||
nCommThreads=1;
|
|
||||||
// wrong results here too
|
|
||||||
// For now: comms-overlap leads to wrong results in Benchmark_wilson even on single node MPI runs
|
|
||||||
// other comms schemes are ok
|
|
||||||
MPI_Init_thread(argc,argv,MPI_THREAD_SERIALIZED,&provided);
|
|
||||||
#else
|
|
||||||
MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided);
|
|
||||||
#endif
|
|
||||||
//If only 1 comms thread we require any threading mode other than SINGLE, but for multiple comms threads we need MULTIPLE
|
|
||||||
if( (nCommThreads == 1) && (provided == MPI_THREAD_SINGLE) ) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
if( (nCommThreads > 1) && (provided != MPI_THREAD_MULTIPLE) ) {
|
|
||||||
GRID_ASSERT(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Never clean up as done once.
|
|
||||||
MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
|
|
||||||
|
|
||||||
Grid_quiesce_nodes();
|
|
||||||
GlobalSharedMemory::Init(communicator_world);
|
|
||||||
GlobalSharedMemory::SharedMemoryAllocate(
|
|
||||||
GlobalSharedMemory::MAX_MPI_SHM_BYTES,
|
|
||||||
GlobalSharedMemory::Hugepages);
|
|
||||||
Grid_unquiesce_nodes();
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////////////
|
|
||||||
// Use cartesian communicators now even in MPI3
|
|
||||||
///////////////////////////////////////////////////////////////////////////
|
|
||||||
void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
|
|
||||||
{
|
|
||||||
int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
int CartesianCommunicator::RankFromProcessorCoor(Coordinate &coor)
|
|
||||||
{
|
|
||||||
int rank;
|
|
||||||
int ierr=MPI_Cart_rank (communicator, &coor[0], &rank);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
return rank;
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::ProcessorCoorFromRank(int rank, Coordinate &coor)
|
|
||||||
{
|
|
||||||
coor.resize(_ndimension);
|
|
||||||
int ierr=MPI_Cart_coords (communicator, rank, _ndimension,&coor[0]);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Initialises from communicator_world
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors)
|
|
||||||
{
|
|
||||||
MPI_Comm optimal_comm;
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
// Remap using the shared memory optimising routine
|
|
||||||
// The remap creates a comm which must be freed
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
GlobalSharedMemory::OptimalCommunicator (processors,optimal_comm,_shm_processors);
|
|
||||||
InitFromMPICommunicator(processors,optimal_comm);
|
|
||||||
SetCommunicator(optimal_comm);
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// Free the temp communicator
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
MPI_Comm_free(&optimal_comm);
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////
|
|
||||||
// Try to subdivide communicator
|
|
||||||
//////////////////////////////////
|
|
||||||
CartesianCommunicator::CartesianCommunicator(const Coordinate &processors,const CartesianCommunicator &parent,int &srank)
|
|
||||||
{
|
|
||||||
_ndimension = processors.size(); GRID_ASSERT(_ndimension>=1);
|
|
||||||
int parent_ndimension = parent._ndimension; GRID_ASSERT(_ndimension >= parent._ndimension);
|
|
||||||
Coordinate parent_processor_coor(_ndimension,0);
|
|
||||||
Coordinate parent_processors (_ndimension,1);
|
|
||||||
Coordinate shm_processors (_ndimension,1);
|
|
||||||
// Can make 5d grid from 4d etc...
|
|
||||||
int pad = _ndimension-parent_ndimension;
|
|
||||||
for(int d=0;d<parent_ndimension;d++){
|
|
||||||
parent_processor_coor[pad+d]=parent._processor_coor[d];
|
|
||||||
parent_processors [pad+d]=parent._processors[d];
|
|
||||||
shm_processors [pad+d]=parent._shm_processors[d];
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// split the communicator
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// int Nparent = parent._processors ;
|
|
||||||
int Nparent;
|
|
||||||
MPI_Comm_size(parent.communicator,&Nparent);
|
|
||||||
|
|
||||||
int childsize=1;
|
|
||||||
for(int d=0;d<processors.size();d++) {
|
|
||||||
childsize *= processors[d];
|
|
||||||
}
|
|
||||||
int Nchild = Nparent/childsize;
|
|
||||||
GRID_ASSERT (childsize * Nchild == Nparent);
|
|
||||||
|
|
||||||
Coordinate ccoor(_ndimension); // coor within subcommunicator
|
|
||||||
Coordinate scoor(_ndimension); // coor of split within parent
|
|
||||||
Coordinate ssize(_ndimension); // coor of split within parent
|
|
||||||
|
|
||||||
for(int d=0;d<_ndimension;d++){
|
|
||||||
ccoor[d] = parent_processor_coor[d] % processors[d];
|
|
||||||
scoor[d] = parent_processor_coor[d] / processors[d];
|
|
||||||
ssize[d] = parent_processors[d] / processors[d];
|
|
||||||
if ( processors[d] < shm_processors[d] ) shm_processors[d] = processors[d]; // subnode splitting.
|
|
||||||
}
|
|
||||||
|
|
||||||
// rank within subcomm ; srank is rank of subcomm within blocks of subcomms
|
|
||||||
int crank;
|
|
||||||
// Mpi uses the reverse Lexico convention to us; so reversed routines called
|
|
||||||
Lexicographic::IndexFromCoorReversed(ccoor,crank,processors); // processors is the split grid dimensions
|
|
||||||
Lexicographic::IndexFromCoorReversed(scoor,srank,ssize); // ssize is the number of split grids
|
|
||||||
|
|
||||||
MPI_Comm comm_split;
|
|
||||||
if ( Nchild > 1 ) {
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Split the communicator
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
int ierr= MPI_Comm_split(parent.communicator,srank,crank,&comm_split);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
|
|
||||||
} else {
|
|
||||||
srank = 0;
|
|
||||||
int ierr = MPI_Comm_dup (parent.communicator,&comm_split);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Set up from the new split communicator
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
InitFromMPICommunicator(processors,comm_split);
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Take the right SHM buffers
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
SetCommunicator(comm_split);
|
|
||||||
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
// Free the temp communicator
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
MPI_Comm_free(&comm_split);
|
|
||||||
|
|
||||||
if(0){
|
|
||||||
std::cout << " ndim " <<_ndimension<<" " << parent._ndimension << std::endl;
|
|
||||||
for(int d=0;d<processors.size();d++){
|
|
||||||
std::cout << d<< " " << _processor_coor[d] <<" " << ccoor[d]<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for(int d=0;d<processors.size();d++){
|
|
||||||
GRID_ASSERT(_processor_coor[d] == ccoor[d] );
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void CartesianCommunicator::InitFromMPICommunicator(const Coordinate &processors, MPI_Comm communicator_base)
|
|
||||||
{
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
// Creates communicator, and the communicator_halo
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
_ndimension = processors.size();
|
|
||||||
_processor_coor.resize(_ndimension);
|
|
||||||
|
|
||||||
/////////////////////////////////
|
|
||||||
// Count the requested nodes
|
|
||||||
/////////////////////////////////
|
|
||||||
_Nprocessors=1;
|
|
||||||
_processors = processors;
|
|
||||||
for(int i=0;i<_ndimension;i++){
|
|
||||||
_Nprocessors*=_processors[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
Coordinate periodic(_ndimension,1);
|
|
||||||
MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],0,&communicator);
|
|
||||||
MPI_Comm_rank(communicator,&_processor);
|
|
||||||
MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
|
|
||||||
|
|
||||||
if ( 0 && (communicator_base != communicator_world) ) {
|
|
||||||
std::cout << "InitFromMPICommunicator Cartesian communicator created with a non-world communicator"<<std::endl;
|
|
||||||
std::cout << " new communicator rank "<<_processor<< " coor ["<<_ndimension<<"] ";
|
|
||||||
for(int d=0;d<_processors.size();d++){
|
|
||||||
std::cout << _processor_coor[d]<<" ";
|
|
||||||
}
|
|
||||||
std::cout << std::endl;
|
|
||||||
}
|
|
||||||
|
|
||||||
int Size;
|
|
||||||
MPI_Comm_size(communicator,&Size);
|
|
||||||
|
|
||||||
communicator_halo.resize (2*_ndimension);
|
|
||||||
for(int i=0;i<_ndimension*2;i++){
|
|
||||||
MPI_Comm_dup(communicator,&communicator_halo[i]);
|
|
||||||
}
|
|
||||||
GRID_ASSERT(Size==_Nprocessors);
|
|
||||||
}
|
|
||||||
|
|
||||||
CartesianCommunicator::~CartesianCommunicator()
|
|
||||||
{
|
|
||||||
int MPI_is_finalised;
|
|
||||||
MPI_Finalized(&MPI_is_finalised);
|
|
||||||
if (communicator && !MPI_is_finalised) {
|
|
||||||
MPI_Comm_free(&communicator);
|
|
||||||
for(int i=0;i<communicator_halo.size();i++){
|
|
||||||
MPI_Comm_free(&communicator_halo[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#ifdef USE_GRID_REDUCTION
|
|
||||||
void CartesianCommunicator::GlobalSum(float &f){
|
|
||||||
FlightRecorder::StepLog("GlobalSumP2P");
|
|
||||||
CartesianCommunicator::GlobalSumP2P(f);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalSum(double &d)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("GlobalSumP2P");
|
|
||||||
CartesianCommunicator::GlobalSumP2P(d);
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
void CartesianCommunicator::GlobalSum(float &f){
|
|
||||||
FlightRecorder::StepLog("AllReduce float");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalSum(double &d)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("AllReduce double");
|
|
||||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
void CartesianCommunicator::GlobalSum(uint32_t &u){
|
|
||||||
FlightRecorder::StepLog("AllReduce uint32_t");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalSum(uint64_t &u){
|
|
||||||
FlightRecorder::StepLog("AllReduce uint64_t");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalSumVector(uint64_t* u,int N){
|
|
||||||
FlightRecorder::StepLog("AllReduceVector");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,u,N,MPI_UINT64_T,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalXOR(uint32_t &u){
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalXOR(uint64_t &u){
|
|
||||||
FlightRecorder::StepLog("GlobalXOR");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalMax(float &f)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("GlobalMax");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_MAX,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalMax(double &d)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("GlobalMax");
|
|
||||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_MAX,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalSumVector(float *f,int N)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("GlobalSumVector(float *)");
|
|
||||||
int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::GlobalSumVector(double *d,int N)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("GlobalSumVector(double *)");
|
|
||||||
int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
|
|
||||||
void CartesianCommunicator::SendToRecvFromBegin(std::vector<MpiCommsRequest_t> &list,
|
|
||||||
void *xmit,
|
|
||||||
int dest,
|
|
||||||
void *recv,
|
|
||||||
int from,
|
|
||||||
uint64_t bytes,int dir)
|
|
||||||
{
|
|
||||||
MPI_Request xrq;
|
|
||||||
MPI_Request rrq;
|
|
||||||
|
|
||||||
GRID_ASSERT(dest != _processor);
|
|
||||||
GRID_ASSERT(from != _processor);
|
|
||||||
int tag;
|
|
||||||
|
|
||||||
tag= dir+from*32;
|
|
||||||
int ierr=MPI_Irecv(recv,(int)( bytes/sizeof(int32_t)), MPI_INT32_T,from,tag,communicator,&rrq);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
list.push_back(rrq);
|
|
||||||
|
|
||||||
tag= dir+_processor*32;
|
|
||||||
ierr =MPI_Isend(xmit,(int)(bytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator,&xrq);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
list.push_back(xrq);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::CommsComplete(std::vector<MpiCommsRequest_t> &list)
|
|
||||||
{
|
|
||||||
int nreq=list.size();
|
|
||||||
|
|
||||||
if (nreq==0) return;
|
|
||||||
|
|
||||||
std::vector<MPI_Status> status(nreq);
|
|
||||||
int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
list.resize(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Basic Halo comms primitive
|
|
||||||
void CartesianCommunicator::SendToRecvFrom(void *xmit,
|
|
||||||
int dest,
|
|
||||||
void *recv,
|
|
||||||
int from,
|
|
||||||
uint64_t bytes)
|
|
||||||
{
|
|
||||||
std::vector<MpiCommsRequest_t> reqs(0);
|
|
||||||
|
|
||||||
int myrank = _processor;
|
|
||||||
int ierr;
|
|
||||||
|
|
||||||
// Enforce no UVM in comms, device or host OK
|
|
||||||
GRID_ASSERT(acceleratorIsCommunicable(xmit));
|
|
||||||
GRID_ASSERT(acceleratorIsCommunicable(recv));
|
|
||||||
|
|
||||||
// Give the CPU to MPI immediately; can use threads to overlap optionally
|
|
||||||
// printf("proc %d SendToRecvFrom %d bytes Sendrecv \n",_processor,bytes);
|
|
||||||
ierr=MPI_Sendrecv(xmit,(int)(bytes/sizeof(int32_t)),MPI_INT32_T,dest,myrank,
|
|
||||||
recv,(int)(bytes/sizeof(int32_t)),MPI_INT32_T,from, from,
|
|
||||||
communicator,MPI_STATUS_IGNORE);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
|
|
||||||
}
|
|
||||||
// Basic Halo comms primitive
|
|
||||||
double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
|
|
||||||
int dest, int dox,
|
|
||||||
void *recv,
|
|
||||||
int from, int dor,
|
|
||||||
uint64_t bytes,int dir)
|
|
||||||
{
|
|
||||||
std::vector<CommsRequest_t> list;
|
|
||||||
double offbytes = StencilSendToRecvFromPrepare(list,xmit,dest,dox,recv,from,dor,bytes,bytes,dir);
|
|
||||||
offbytes += StencilSendToRecvFromBegin(list,xmit,xmit,dest,dox,recv,recv,from,dor,bytes,bytes,dir);
|
|
||||||
StencilSendToRecvFromComplete(list,dir);
|
|
||||||
return offbytes;
|
|
||||||
}
|
|
||||||
int CartesianCommunicator::IsOffNode(int rank)
|
|
||||||
{
|
|
||||||
int grank = ShmRanks[rank];
|
|
||||||
if ( grank == MPI_UNDEFINED ) return true;
|
|
||||||
else return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifdef ACCELERATOR_AWARE_MPI
|
|
||||||
void CartesianCommunicator::StencilSendToRecvFromPollIRecv(std::vector<CommsRequest_t> &list) {};
|
|
||||||
void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector<CommsRequest_t> &list) {};
|
|
||||||
double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector<CommsRequest_t> &list,
|
|
||||||
void *xmit,
|
|
||||||
int dest,int dox,
|
|
||||||
void *recv,
|
|
||||||
int from,int dor,
|
|
||||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
|
||||||
{
|
|
||||||
return 0.0; // Do nothing -- no preparation required
|
|
||||||
}
|
|
||||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
|
||||||
void *xmit,void *xmit_comp,
|
|
||||||
int dest,int dox,
|
|
||||||
void *recv,void *recv_comp,
|
|
||||||
int from,int dor,
|
|
||||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
|
||||||
{
|
|
||||||
int ncomm =communicator_halo.size();
|
|
||||||
int commdir=dir%ncomm;
|
|
||||||
|
|
||||||
MPI_Request xrq;
|
|
||||||
MPI_Request rrq;
|
|
||||||
|
|
||||||
int ierr;
|
|
||||||
int gdest = ShmRanks[dest];
|
|
||||||
int gfrom = ShmRanks[from];
|
|
||||||
int gme = ShmRanks[_processor];
|
|
||||||
|
|
||||||
GRID_ASSERT(dest != _processor);
|
|
||||||
GRID_ASSERT(from != _processor);
|
|
||||||
GRID_ASSERT(gme == ShmRank);
|
|
||||||
double off_node_bytes=0.0;
|
|
||||||
int tag;
|
|
||||||
|
|
||||||
if ( dor ) {
|
|
||||||
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
|
||||||
tag= dir+from*32;
|
|
||||||
// std::cout << " StencilSendToRecvFrom "<<dir<<" MPI_Irecv "<<std::hex<<recv<<std::dec<<std::endl;
|
|
||||||
ierr=MPI_Irecv(recv_comp,(int)(rbytes/sizeof(int32_t)), MPI_INT32_T,from,tag,communicator_halo[commdir],&rrq);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
list.push_back(rrq);
|
|
||||||
off_node_bytes+=rbytes;
|
|
||||||
}
|
|
||||||
#ifdef NVLINK_GET
|
|
||||||
else {
|
|
||||||
void *shm = (void *) this->ShmBufferTranslate(from,xmit);
|
|
||||||
GRID_ASSERT(shm!=NULL);
|
|
||||||
// std::cout << " StencilSendToRecvFrom "<<dir<<" CopyDeviceToDevice recv "<<std::hex<<recv<<" remote "<<shm <<std::dec<<std::endl;
|
|
||||||
acceleratorCopyDeviceToDeviceAsynch(shm,recv,rbytes);
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
// This is a NVLINK PUT
|
|
||||||
if (dox) {
|
|
||||||
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
|
||||||
tag= dir+_processor*32;
|
|
||||||
ierr =MPI_Isend(xmit_comp,(int)(xbytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator_halo[commdir],&xrq);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
list.push_back(xrq);
|
|
||||||
off_node_bytes+=xbytes;
|
|
||||||
} else {
|
|
||||||
#ifndef NVLINK_GET
|
|
||||||
void *shm = (void *) this->ShmBufferTranslate(dest,recv);
|
|
||||||
GRID_ASSERT(shm!=NULL);
|
|
||||||
acceleratorCopyDeviceToDeviceAsynch(xmit,shm,xbytes);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return off_node_bytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
|
||||||
{
|
|
||||||
int nreq=list.size();
|
|
||||||
/*finishes Get/Put*/
|
|
||||||
acceleratorCopySynchronise();
|
|
||||||
|
|
||||||
if (nreq==0) return;
|
|
||||||
std::vector<MPI_Status> status(nreq);
|
|
||||||
int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
list.resize(0);
|
|
||||||
this->StencilBarrier();
|
|
||||||
}
|
|
||||||
|
|
||||||
#else /* NOT ... ACCELERATOR_AWARE_MPI */
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// Pipeline mode through host memory
|
|
||||||
///////////////////////////////////////////
|
|
||||||
/*
|
|
||||||
* In prepare (phase 1):
|
|
||||||
* PHASE 1: (prepare)
|
|
||||||
* - post MPI receive buffers asynch
|
|
||||||
* - post device - host send buffer transfer asynch
|
|
||||||
* PHASE 2: (Begin)
|
|
||||||
* - complete all copies
|
|
||||||
* - post MPI send asynch
|
|
||||||
* - post device - device transfers
|
|
||||||
* PHASE 3: (Complete)
|
|
||||||
* - MPI_waitall
|
|
||||||
* - host-device transfers
|
|
||||||
*
|
|
||||||
*********************************
|
|
||||||
* NB could split this further:
|
|
||||||
*--------------------------------
|
|
||||||
* PHASE 1: (Prepare)
|
|
||||||
* - post MPI receive buffers asynch
|
|
||||||
* - post device - host send buffer transfer asynch
|
|
||||||
* PHASE 2: (BeginInterNode)
|
|
||||||
* - complete all copies
|
|
||||||
* - post MPI send asynch
|
|
||||||
* PHASE 3: (BeginIntraNode)
|
|
||||||
* - post device - device transfers
|
|
||||||
* PHASE 4: (Complete)
|
|
||||||
* - MPI_waitall
|
|
||||||
* - host-device transfers asynch
|
|
||||||
* - (complete all copies)
|
|
||||||
*/
|
|
||||||
double CartesianCommunicator::StencilSendToRecvFromPrepare(std::vector<CommsRequest_t> &list,
|
|
||||||
void *xmit,
|
|
||||||
int dest,int dox,
|
|
||||||
void *recv,
|
|
||||||
int from,int dor,
|
|
||||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
|
||||||
{
|
|
||||||
/*
|
|
||||||
* Bring sequence from Stencil.h down to lower level.
|
|
||||||
* Assume using XeLink is ok
|
|
||||||
*/
|
|
||||||
int ncomm =communicator_halo.size();
|
|
||||||
int commdir=dir%ncomm;
|
|
||||||
|
|
||||||
MPI_Request xrq;
|
|
||||||
MPI_Request rrq;
|
|
||||||
|
|
||||||
int ierr;
|
|
||||||
int gdest = ShmRanks[dest];
|
|
||||||
int gfrom = ShmRanks[from];
|
|
||||||
int gme = ShmRanks[_processor];
|
|
||||||
|
|
||||||
GRID_ASSERT(dest != _processor);
|
|
||||||
GRID_ASSERT(from != _processor);
|
|
||||||
GRID_ASSERT(gme == ShmRank);
|
|
||||||
double off_node_bytes=0.0;
|
|
||||||
int tag;
|
|
||||||
|
|
||||||
void * host_recv = NULL;
|
|
||||||
void * host_xmit = NULL;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* PHASE 1: (Prepare)
|
|
||||||
* - post MPI receive buffers asynch
|
|
||||||
* - post device - host send buffer transfer asynch
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
rbytes += 8;
|
|
||||||
xbytes += 8;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
if ( dor ) {
|
|
||||||
if ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) {
|
|
||||||
tag= dir+from*32;
|
|
||||||
host_recv = this->HostBufferMalloc(rbytes);
|
|
||||||
ierr=MPI_Irecv(host_recv,(int)(rbytes/sizeof(int32_t)), MPI_INT32_T,from,tag,communicator_halo[commdir],&rrq);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
CommsRequest_t srq;
|
|
||||||
srq.PacketType = InterNodeRecv;
|
|
||||||
srq.bytes = rbytes;
|
|
||||||
srq.req = rrq;
|
|
||||||
srq.host_buf = host_recv;
|
|
||||||
srq.device_buf = recv;
|
|
||||||
srq.tag = tag;
|
|
||||||
list.push_back(srq);
|
|
||||||
off_node_bytes+=rbytes;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dox) {
|
|
||||||
if ( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) {
|
|
||||||
|
|
||||||
tag= dir+_processor*32;
|
|
||||||
|
|
||||||
host_xmit = this->HostBufferMalloc(xbytes);
|
|
||||||
CommsRequest_t srq;
|
|
||||||
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
uint64_t xbytes_data = xbytes - 8;
|
|
||||||
srq.ev = acceleratorCopyFromDeviceAsynch(xmit, host_xmit,xbytes_data); // Make this Asynch
|
|
||||||
GRID_ASSERT(xbytes % 8 == 0);
|
|
||||||
// flip one bit so that a zero buffer is not consistent
|
|
||||||
uint64_t xsum = checksum_gpu((uint64_t*)xmit, xbytes_data / 8) ^ (checksum_index + 1 + 1000 * tag);
|
|
||||||
*(uint64_t*)(((char*)host_xmit) + xbytes_data) = xsum;
|
|
||||||
#else
|
|
||||||
srq.ev = acceleratorCopyFromDeviceAsynch(xmit, host_xmit,xbytes); // Make this Asynch
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// ierr =MPI_Isend(host_xmit, xbytes, MPI_CHAR,dest,tag,communicator_halo[commdir],&xrq);
|
|
||||||
// GRID_ASSERT(ierr==0);
|
|
||||||
// off_node_bytes+=xbytes;
|
|
||||||
|
|
||||||
srq.PacketType = InterNodeXmit;
|
|
||||||
srq.bytes = xbytes;
|
|
||||||
// srq.req = xrq;
|
|
||||||
srq.host_buf = host_xmit;
|
|
||||||
srq.device_buf = xmit;
|
|
||||||
srq.tag = tag;
|
|
||||||
srq.dest = dest;
|
|
||||||
srq.commdir = commdir;
|
|
||||||
list.push_back(srq);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return off_node_bytes;
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
* In the interest of better pipelining, poll for completion on each DtoH and
|
|
||||||
* start MPI_ISend in the meantime
|
|
||||||
*/
|
|
||||||
void CartesianCommunicator::StencilSendToRecvFromPollIRecv(std::vector<CommsRequest_t> &list)
|
|
||||||
{
|
|
||||||
int pending = 0;
|
|
||||||
do {
|
|
||||||
|
|
||||||
pending = 0;
|
|
||||||
|
|
||||||
for(int idx = 0; idx<list.size();idx++){
|
|
||||||
|
|
||||||
if ( list[idx].PacketType==InterNodeRecv ) {
|
|
||||||
|
|
||||||
int flag = 0;
|
|
||||||
MPI_Status status;
|
|
||||||
int ierr = MPI_Test(&list[idx].req,&flag,&status);
|
|
||||||
assert(ierr==0);
|
|
||||||
|
|
||||||
if ( flag ) {
|
|
||||||
// std::cout << " PollIrecv "<<idx<<" flag "<<flag<<std::endl;
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
acceleratorCopyToDeviceAsynch(list[idx].host_buf,list[idx].device_buf,list[idx].bytes - 8);
|
|
||||||
#else
|
|
||||||
acceleratorCopyToDeviceAsynch(list[idx].host_buf,list[idx].device_buf,list[idx].bytes);
|
|
||||||
#endif
|
|
||||||
list[idx].PacketType=InterNodeReceiveHtoD;
|
|
||||||
} else {
|
|
||||||
pending ++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// std::cout << " PollIrecv "<<pending<<" pending requests"<<std::endl;
|
|
||||||
} while ( pending );
|
|
||||||
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::StencilSendToRecvFromPollDtoH(std::vector<CommsRequest_t> &list)
|
|
||||||
{
|
|
||||||
int pending = 0;
|
|
||||||
do {
|
|
||||||
|
|
||||||
pending = 0;
|
|
||||||
|
|
||||||
for(int idx = 0; idx<list.size();idx++){
|
|
||||||
|
|
||||||
if ( list[idx].PacketType==InterNodeXmit ) {
|
|
||||||
|
|
||||||
if ( acceleratorEventIsComplete(list[idx].ev) ) {
|
|
||||||
|
|
||||||
void *host_xmit = list[idx].host_buf;
|
|
||||||
uint64_t xbytes = list[idx].bytes;
|
|
||||||
int dest = list[idx].dest;
|
|
||||||
int tag = list[idx].tag;
|
|
||||||
int commdir = list[idx].commdir;
|
|
||||||
///////////////////
|
|
||||||
// Send packet
|
|
||||||
///////////////////
|
|
||||||
|
|
||||||
// std::cout << " DtoH is complete for index "<<idx<<" calling MPI_Isend "<<std::endl;
|
|
||||||
|
|
||||||
MPI_Request xrq;
|
|
||||||
int ierr =MPI_Isend(host_xmit, (int)(xbytes/sizeof(int32_t)), MPI_INT32_T,dest,tag,communicator_halo[commdir],&xrq);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
|
|
||||||
list[idx].req = xrq; // Update the MPI request in the list
|
|
||||||
|
|
||||||
list[idx].PacketType=InterNodeXmitISend;
|
|
||||||
|
|
||||||
} else {
|
|
||||||
// not done, so return to polling loop
|
|
||||||
pending++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} while (pending);
|
|
||||||
}
|
|
||||||
|
|
||||||
double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
|
|
||||||
void *xmit,void *xmit_comp,
|
|
||||||
int dest,int dox,
|
|
||||||
void *recv,void *recv_comp,
|
|
||||||
int from,int dor,
|
|
||||||
uint64_t xbytes,uint64_t rbytes,int dir)
|
|
||||||
{
|
|
||||||
int ncomm =communicator_halo.size();
|
|
||||||
int commdir=dir%ncomm;
|
|
||||||
|
|
||||||
MPI_Request xrq;
|
|
||||||
MPI_Request rrq;
|
|
||||||
|
|
||||||
int ierr;
|
|
||||||
int gdest = ShmRanks[dest];
|
|
||||||
int gfrom = ShmRanks[from];
|
|
||||||
int gme = ShmRanks[_processor];
|
|
||||||
|
|
||||||
GRID_ASSERT(dest != _processor);
|
|
||||||
GRID_ASSERT(from != _processor);
|
|
||||||
GRID_ASSERT(gme == ShmRank);
|
|
||||||
double off_node_bytes=0.0;
|
|
||||||
int tag;
|
|
||||||
|
|
||||||
void * host_xmit = NULL;
|
|
||||||
|
|
||||||
////////////////////////////////
|
|
||||||
// Receives already posted
|
|
||||||
// Copies already started
|
|
||||||
////////////////////////////////
|
|
||||||
/*
|
|
||||||
* PHASE 2: (Begin)
|
|
||||||
* - complete all copies
|
|
||||||
* - post MPI send asynch
|
|
||||||
*/
|
|
||||||
#ifdef NVLINK_GET
|
|
||||||
if ( dor ) {
|
|
||||||
|
|
||||||
if ( ! ( (gfrom ==MPI_UNDEFINED) || Stencil_force_mpi ) ) {
|
|
||||||
// Intranode
|
|
||||||
void *shm = (void *) this->ShmBufferTranslate(from,xmit);
|
|
||||||
GRID_ASSERT(shm!=NULL);
|
|
||||||
|
|
||||||
CommsRequest_t srq;
|
|
||||||
|
|
||||||
srq.ev = acceleratorCopyDeviceToDeviceAsynch(shm,recv,rbytes);
|
|
||||||
|
|
||||||
srq.PacketType = IntraNodeRecv;
|
|
||||||
srq.bytes = xbytes;
|
|
||||||
// srq.req = xrq;
|
|
||||||
srq.host_buf = NULL;
|
|
||||||
srq.device_buf = xmit;
|
|
||||||
srq.tag = -1;
|
|
||||||
srq.dest = dest;
|
|
||||||
srq.commdir = dir;
|
|
||||||
list.push_back(srq);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
if (dox) {
|
|
||||||
|
|
||||||
if ( !( (gdest == MPI_UNDEFINED) || Stencil_force_mpi ) ) {
|
|
||||||
// Intranode
|
|
||||||
void *shm = (void *) this->ShmBufferTranslate(dest,recv);
|
|
||||||
GRID_ASSERT(shm!=NULL);
|
|
||||||
|
|
||||||
CommsRequest_t srq;
|
|
||||||
|
|
||||||
srq.ev = acceleratorCopyDeviceToDeviceAsynch(xmit,shm,xbytes);
|
|
||||||
|
|
||||||
srq.PacketType = IntraNodeXmit;
|
|
||||||
srq.bytes = xbytes;
|
|
||||||
// srq.req = xrq;
|
|
||||||
srq.host_buf = NULL;
|
|
||||||
srq.device_buf = xmit;
|
|
||||||
srq.tag = -1;
|
|
||||||
srq.dest = dest;
|
|
||||||
srq.commdir = dir;
|
|
||||||
list.push_back(srq);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
return off_node_bytes;
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &list,int dir)
|
|
||||||
{
|
|
||||||
acceleratorCopySynchronise(); // Complete all pending copy transfers D2D
|
|
||||||
|
|
||||||
std::vector<MPI_Status> status;
|
|
||||||
std::vector<MPI_Request> MpiRequests;
|
|
||||||
|
|
||||||
for(int r=0;r<list.size();r++){
|
|
||||||
// Must check each Send buf is clear to reuse
|
|
||||||
if ( list[r].PacketType == InterNodeXmitISend ) MpiRequests.push_back(list[r].req);
|
|
||||||
// if ( list[r].PacketType == InterNodeRecv ) MpiRequests.push_back(list[r].req); // Already "Test" passed
|
|
||||||
}
|
|
||||||
|
|
||||||
int nreq=MpiRequests.size();
|
|
||||||
|
|
||||||
if (nreq>0) {
|
|
||||||
status.resize(MpiRequests.size());
|
|
||||||
int ierr = MPI_Waitall(MpiRequests.size(),&MpiRequests[0],&status[0]); // Sends are guaranteed in order. No harm in not completing.
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// for(int r=0;r<nreq;r++){
|
|
||||||
// if ( list[r].PacketType==InterNodeRecv ) {
|
|
||||||
// acceleratorCopyToDeviceAsynch(list[r].host_buf,list[r].device_buf,list[r].bytes);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
for(int r=0;r<list.size();r++){
|
|
||||||
if ( list[r].PacketType == InterNodeReceiveHtoD ) {
|
|
||||||
uint64_t rbytes_data = list[r].bytes - 8;
|
|
||||||
uint64_t expected_cs = *(uint64_t*)(((char*)list[r].host_buf) + rbytes_data);
|
|
||||||
uint64_t computed_cs = checksum_gpu((uint64_t*)list[r].device_buf, rbytes_data / 8) ^ (checksum_index + 1 + 1000 * list[r].tag); //
|
|
||||||
if (expected_cs != computed_cs) {
|
|
||||||
// TODO: error message, backtrace, quit
|
|
||||||
|
|
||||||
fprintf(stderr, "GRID_CHECKSUM_COMMS error:\n");
|
|
||||||
fprintf(stderr, " processor = %d\n", (int)_processor);
|
|
||||||
for(int d=0;d<_processors.size();d++)
|
|
||||||
fprintf(stderr, " processor_coord[%d] = %d\n", d, _processor_coor[d]);
|
|
||||||
fprintf(stderr, " hostname: %s\n", GridHostname());
|
|
||||||
fprintf(stderr, " expected_cs: %ld\n", expected_cs);
|
|
||||||
fprintf(stderr, " computed_cs: %ld\n", computed_cs);
|
|
||||||
fprintf(stderr, " dest: %d\n", list[r].dest);
|
|
||||||
fprintf(stderr, " tag: %d\n", list[r].tag);
|
|
||||||
fprintf(stderr, " commdir: %d\n", list[r].commdir);
|
|
||||||
fprintf(stderr, " bytes: %ld\n", (uint64_t)list[r].bytes);
|
|
||||||
|
|
||||||
fflush(stderr);
|
|
||||||
|
|
||||||
// backtrace
|
|
||||||
int symbols = backtrace(Grid_backtrace_buffer,_NBACKTRACE);
|
|
||||||
backtrace_symbols_fd(Grid_backtrace_buffer,symbols, 2);
|
|
||||||
|
|
||||||
exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
checksum_index += 1;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
list.resize(0); // Delete the list
|
|
||||||
this->HostBufferFreeAll(); // Clean up the buffer allocs
|
|
||||||
#ifndef NVLINK_GET
|
|
||||||
this->StencilBarrier(); // if PUT must check our nbrs have filled our receive buffers.
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
////////////////////////////////////////////
|
|
||||||
// END PIPELINE MODE / NO CUDA AWARE MPI
|
|
||||||
////////////////////////////////////////////
|
|
||||||
|
|
||||||
void CartesianCommunicator::StencilBarrier(void)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("NodeBarrier");
|
|
||||||
MPI_Barrier (ShmComm);
|
|
||||||
}
|
|
||||||
//void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
|
|
||||||
//{
|
|
||||||
//}
|
|
||||||
void CartesianCommunicator::Barrier(void)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("GridBarrier");
|
|
||||||
int ierr = MPI_Barrier(communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::Broadcast(int root,void* data,uint64_t bytes)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("Broadcast");
|
|
||||||
int ierr=MPI_Bcast(data,
|
|
||||||
(int)bytes,
|
|
||||||
MPI_BYTE,
|
|
||||||
root,
|
|
||||||
communicator);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
int CartesianCommunicator::RankWorld(void){
|
|
||||||
int r;
|
|
||||||
MPI_Comm_rank(communicator_world,&r);
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::BarrierWorld(void){
|
|
||||||
FlightRecorder::StepLog("BarrierWorld");
|
|
||||||
int ierr = MPI_Barrier(communicator_world);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::BroadcastWorld(int root,void* data, uint64_t bytes)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("BroadcastWorld");
|
|
||||||
int ierr= MPI_Bcast(data,
|
|
||||||
(int)bytes,
|
|
||||||
MPI_BYTE,
|
|
||||||
root,
|
|
||||||
communicator_world);
|
|
||||||
GRID_ASSERT(ierr==0);
|
|
||||||
}
|
|
||||||
|
|
||||||
void CartesianCommunicator::AllToAll(int dim,void *in,void *out,uint64_t words,uint64_t bytes)
|
|
||||||
{
|
|
||||||
Coordinate row(_ndimension,1);
|
|
||||||
GRID_ASSERT(dim>=0 && dim<_ndimension);
|
|
||||||
|
|
||||||
// Split the communicator
|
|
||||||
row[dim] = _processors[dim];
|
|
||||||
|
|
||||||
int me;
|
|
||||||
CartesianCommunicator Comm(row,*this,me);
|
|
||||||
Comm.AllToAll(in,out,words,bytes);
|
|
||||||
}
|
|
||||||
void CartesianCommunicator::AllToAll(void *in,void *out,uint64_t words,uint64_t bytes)
|
|
||||||
{
|
|
||||||
FlightRecorder::StepLog("AllToAll");
|
|
||||||
// MPI is a pain and uses "int" arguments
|
|
||||||
// 64*64*64*128*16 == 500Million elements of data.
|
|
||||||
// When 24*4 bytes multiples get 50x 10^9 >>> 2x10^9 Y2K bug.
|
|
||||||
// (Turns up on 32^3 x 64 Gparity too)
|
|
||||||
MPI_Datatype object;
|
|
||||||
int iwords;
|
|
||||||
int ibytes;
|
|
||||||
iwords = words;
|
|
||||||
ibytes = bytes;
|
|
||||||
GRID_ASSERT(words == iwords); // safe to cast to int ?
|
|
||||||
GRID_ASSERT(bytes == ibytes); // safe to cast to int ?
|
|
||||||
MPI_Type_contiguous(ibytes,MPI_BYTE,&object);
|
|
||||||
MPI_Type_commit(&object);
|
|
||||||
MPI_Alltoall(in,iwords,object,out,iwords,object,communicator);
|
|
||||||
MPI_Type_free(&object);
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,172 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/communicator/SharedMemory.cc
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#include <Grid/GridCore.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
// static data
|
|
||||||
|
|
||||||
int GlobalSharedMemory::HPEhypercube = 1;
|
|
||||||
uint64_t GlobalSharedMemory::MAX_MPI_SHM_BYTES = 1024LL*1024LL*1024LL;
|
|
||||||
int GlobalSharedMemory::Hugepages = 0;
|
|
||||||
int GlobalSharedMemory::_ShmSetup;
|
|
||||||
int GlobalSharedMemory::_ShmAlloc;
|
|
||||||
uint64_t GlobalSharedMemory::_ShmAllocBytes;
|
|
||||||
|
|
||||||
std::vector<void *> GlobalSharedMemory::WorldShmCommBufs;
|
|
||||||
#ifndef ACCELERATOR_AWARE_MPI
|
|
||||||
void * GlobalSharedMemory::HostCommBuf;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
Grid_MPI_Comm GlobalSharedMemory::WorldShmComm;
|
|
||||||
int GlobalSharedMemory::WorldShmRank;
|
|
||||||
int GlobalSharedMemory::WorldShmSize;
|
|
||||||
std::vector<int> GlobalSharedMemory::WorldShmRanks;
|
|
||||||
|
|
||||||
Grid_MPI_Comm GlobalSharedMemory::WorldComm;
|
|
||||||
int GlobalSharedMemory::WorldSize;
|
|
||||||
int GlobalSharedMemory::WorldRank;
|
|
||||||
|
|
||||||
int GlobalSharedMemory::WorldNodes;
|
|
||||||
int GlobalSharedMemory::WorldNode;
|
|
||||||
|
|
||||||
void GlobalSharedMemory::SharedMemoryFree(void)
|
|
||||||
{
|
|
||||||
GRID_ASSERT(_ShmAlloc);
|
|
||||||
GRID_ASSERT(_ShmAllocBytes>0);
|
|
||||||
for(int r=0;r<WorldShmSize;r++){
|
|
||||||
munmap(WorldShmCommBufs[r],_ShmAllocBytes);
|
|
||||||
}
|
|
||||||
_ShmAlloc = 0;
|
|
||||||
_ShmAllocBytes = 0;
|
|
||||||
}
|
|
||||||
/////////////////////////////////
|
|
||||||
// Alloc, free shmem region
|
|
||||||
/////////////////////////////////
|
|
||||||
#ifndef ACCELERATOR_AWARE_MPI
|
|
||||||
void *SharedMemory::HostBufferMalloc(size_t bytes){
|
|
||||||
void *ptr = (void *)host_heap_top;
|
|
||||||
host_heap_top += bytes;
|
|
||||||
host_heap_bytes+= bytes;
|
|
||||||
if (host_heap_bytes >= host_heap_size) {
|
|
||||||
std::cout<< " HostBufferMalloc exceeded heap size -- try increasing with --shm <MB> flag" <<std::endl;
|
|
||||||
std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
|
|
||||||
std::cout<< " Current alloc is " << (bytes/(1024*1024)) <<"MB"<<std::endl;
|
|
||||||
std::cout<< " Current bytes is " << (host_heap_bytes/(1024*1024)) <<"MB"<<std::endl;
|
|
||||||
std::cout<< " Current heap is " << (host_heap_size/(1024*1024)) <<"MB"<<std::endl;
|
|
||||||
GRID_ASSERT(host_heap_bytes<host_heap_size);
|
|
||||||
}
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void SharedMemory::HostBufferFreeAll(void) {
|
|
||||||
host_heap_top =(size_t)HostCommBuf;
|
|
||||||
host_heap_bytes=0;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
void *SharedMemory::ShmBufferMalloc(size_t bytes){
|
|
||||||
// bytes = (bytes+sizeof(vRealD))&(~(sizeof(vRealD)-1));// align up bytes
|
|
||||||
void *ptr = (void *)heap_top;
|
|
||||||
heap_top += bytes;
|
|
||||||
heap_bytes+= bytes;
|
|
||||||
if (heap_bytes >= heap_size) {
|
|
||||||
std::cout<< " ShmBufferMalloc exceeded shared heap size -- try increasing with --shm <MB> flag" <<std::endl;
|
|
||||||
std::cout<< " Parameter specified in units of MB (megabytes) " <<std::endl;
|
|
||||||
std::cout<< " Current alloc is " << (bytes/(1024*1024)) <<"MB"<<std::endl;
|
|
||||||
std::cout<< " Current bytes is " << (heap_bytes/(1024*1024)) <<"MB"<<std::endl;
|
|
||||||
std::cout<< " Current heap is " << (heap_size/(1024*1024)) <<"MB"<<std::endl;
|
|
||||||
GRID_ASSERT(heap_bytes<heap_size);
|
|
||||||
}
|
|
||||||
//std::cerr << "ShmBufferMalloc "<<std::hex<< ptr<<" - "<<((uint64_t)ptr+bytes)<<std::dec<<std::endl;
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
void SharedMemory::ShmBufferFreeAll(void) {
|
|
||||||
heap_top =(size_t)ShmBufferSelf();
|
|
||||||
heap_bytes=0;
|
|
||||||
}
|
|
||||||
void *SharedMemory::ShmBufferSelf(void)
|
|
||||||
{
|
|
||||||
//std::cerr << "ShmBufferSelf "<<ShmRank<<" "<<std::hex<< ShmCommBufs[ShmRank] <<std::dec<<std::endl;
|
|
||||||
return ShmCommBufs[ShmRank];
|
|
||||||
}
|
|
||||||
static inline int divides(int a,int b)
|
|
||||||
{
|
|
||||||
return ( b == ( (b/a)*a ) );
|
|
||||||
}
|
|
||||||
void GlobalSharedMemory::GetShmDims(const Coordinate &WorldDims,Coordinate &ShmDims)
|
|
||||||
{
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Allow user to configure through environment variable
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
char* str = getenv(("GRID_SHM_DIMS_" + std::to_string(ShmDims.size())).c_str());
|
|
||||||
if ( str ) {
|
|
||||||
std::vector<int> IntShmDims;
|
|
||||||
GridCmdOptionIntVector(std::string(str),IntShmDims);
|
|
||||||
GRID_ASSERT(IntShmDims.size() == WorldDims.size());
|
|
||||||
long ShmSize = 1;
|
|
||||||
for (int dim=0;dim<WorldDims.size();dim++) {
|
|
||||||
ShmSize *= (ShmDims[dim] = IntShmDims[dim]);
|
|
||||||
GRID_ASSERT(divides(ShmDims[dim],WorldDims[dim]));
|
|
||||||
}
|
|
||||||
GRID_ASSERT(ShmSize == WorldShmSize);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
// Powers of 2,3,5 only in prime decomposition for now
|
|
||||||
////////////////////////////////////////////////////////////////
|
|
||||||
int ndimension = WorldDims.size();
|
|
||||||
ShmDims=Coordinate(ndimension,1);
|
|
||||||
|
|
||||||
std::vector<int> primes({2,3,5});
|
|
||||||
|
|
||||||
int dim = 0;
|
|
||||||
int last_dim = ndimension - 1;
|
|
||||||
int AutoShmSize = 1;
|
|
||||||
while(AutoShmSize != WorldShmSize) {
|
|
||||||
int p;
|
|
||||||
for(p=0;p<primes.size();p++) {
|
|
||||||
int prime=primes[p];
|
|
||||||
if ( divides(prime,WorldDims[dim]/ShmDims[dim])
|
|
||||||
&& divides(prime,WorldShmSize/AutoShmSize) ) {
|
|
||||||
AutoShmSize*=prime;
|
|
||||||
ShmDims[dim]*=prime;
|
|
||||||
last_dim = dim;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (p == primes.size() && last_dim == dim) {
|
|
||||||
std::cerr << "GlobalSharedMemory::GetShmDims failed" << std::endl;
|
|
||||||
exit(EXIT_FAILURE);
|
|
||||||
}
|
|
||||||
dim=(dim+1) %ndimension;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,440 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/cshift/Cshift_common.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
extern std::vector<std::pair<int,int> > Cshift_table;
|
|
||||||
extern deviceVector<std::pair<int,int> > Cshift_table_device;
|
|
||||||
|
|
||||||
inline std::pair<int,int> *MapCshiftTable(void)
|
|
||||||
{
|
|
||||||
// GPU version
|
|
||||||
uint64_t sz=Cshift_table.size();
|
|
||||||
if (Cshift_table_device.size()!=sz ) {
|
|
||||||
Cshift_table_device.resize(sz);
|
|
||||||
}
|
|
||||||
acceleratorCopyToDevice((void *)&Cshift_table[0],
|
|
||||||
(void *)&Cshift_table_device[0],
|
|
||||||
sizeof(Cshift_table[0])*sz);
|
|
||||||
|
|
||||||
return &Cshift_table_device[0];
|
|
||||||
// CPU version use identify map
|
|
||||||
}
|
|
||||||
///////////////////////////////////////////////////////////////////
|
|
||||||
// Gather for when there is no need to SIMD split
|
|
||||||
///////////////////////////////////////////////////////////////////
|
|
||||||
template<class vobj> void
|
|
||||||
Gather_plane_simple (const Lattice<vobj> &rhs,deviceVector<vobj> &buffer,int dimension,int plane,int cbmask, int off=0)
|
|
||||||
{
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
cbmask = 0x3;
|
|
||||||
}
|
|
||||||
|
|
||||||
int so=plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
|
||||||
int e2=rhs.Grid()->_slice_block[dimension];
|
|
||||||
int ent = 0;
|
|
||||||
|
|
||||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
|
||||||
|
|
||||||
int stride=rhs.Grid()->_slice_stride[dimension];
|
|
||||||
|
|
||||||
if ( cbmask == 0x3 ) {
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o = n*stride;
|
|
||||||
int bo = n*e2;
|
|
||||||
Cshift_table[ent++] = std::pair<int,int>(off+bo+b,so+o+b);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
int bo=0;
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o = n*stride;
|
|
||||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);
|
|
||||||
if ( ocb &cbmask ) {
|
|
||||||
Cshift_table[ent++]=std::pair<int,int> (off+bo++,so+o+b);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
{
|
|
||||||
auto buffer_p = & buffer[0];
|
|
||||||
auto table = MapCshiftTable();
|
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
|
||||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
|
||||||
coalescedWrite(buffer_p[table[i].first],coalescedRead(rhs_v[table[i].second]));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////
|
|
||||||
// Gather for when there *is* need to SIMD split
|
|
||||||
///////////////////////////////////////////////////////////////////
|
|
||||||
template<class vobj> void
|
|
||||||
Gather_plane_extract(const Lattice<vobj> &rhs,
|
|
||||||
ExtractPointerArray<typename vobj::scalar_object> pointers,
|
|
||||||
int dimension,int plane,int cbmask)
|
|
||||||
{
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
cbmask = 0x3;
|
|
||||||
}
|
|
||||||
|
|
||||||
int so = plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
|
|
||||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
|
||||||
int e2=rhs.Grid()->_slice_block[dimension];
|
|
||||||
int n1=rhs.Grid()->_slice_stride[dimension];
|
|
||||||
|
|
||||||
if ( cbmask ==0x3){
|
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
|
||||||
accelerator_for(nn,e1*e2,1,{
|
|
||||||
int n = nn%e1;
|
|
||||||
int b = nn/e1;
|
|
||||||
int o = n*n1;
|
|
||||||
int offset = b+n*e2;
|
|
||||||
|
|
||||||
vobj temp =rhs_v[so+o+b];
|
|
||||||
extract<vobj>(temp,pointers,offset);
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
Coordinate rdim=rhs.Grid()->_rdimensions;
|
|
||||||
Coordinate cdm =rhs.Grid()->_checker_dim_mask;
|
|
||||||
std::cout << " Dense packed buffer WARNING " <<std::endl; // Does this get called twice once for each cb?
|
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
|
||||||
accelerator_for(nn,e1*e2,1,{
|
|
||||||
int n = nn%e1;
|
|
||||||
int b = nn/e1;
|
|
||||||
|
|
||||||
Coordinate coor;
|
|
||||||
|
|
||||||
int o=n*n1;
|
|
||||||
int oindex = o+b;
|
|
||||||
|
|
||||||
int cb = RedBlackCheckerBoardFromOindex(oindex, rdim, cdm);
|
|
||||||
|
|
||||||
int ocb=1<<cb;
|
|
||||||
int offset = b+n*e2;
|
|
||||||
|
|
||||||
if ( ocb & cbmask ) {
|
|
||||||
vobj temp =rhs_v[so+o+b];
|
|
||||||
extract<vobj>(temp,pointers,offset);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Scatter for when there is no need to SIMD split
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
template<class vobj> void Scatter_plane_simple (Lattice<vobj> &rhs,deviceVector<vobj> &buffer, int dimension,int plane,int cbmask)
|
|
||||||
{
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
cbmask=0x3;
|
|
||||||
}
|
|
||||||
|
|
||||||
int so = plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
|
|
||||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
|
||||||
int e2=rhs.Grid()->_slice_block[dimension];
|
|
||||||
int stride=rhs.Grid()->_slice_stride[dimension];
|
|
||||||
|
|
||||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
|
||||||
|
|
||||||
int ent =0;
|
|
||||||
|
|
||||||
if ( cbmask ==0x3 ) {
|
|
||||||
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o =n*rhs.Grid()->_slice_stride[dimension];
|
|
||||||
int bo =n*rhs.Grid()->_slice_block[dimension];
|
|
||||||
Cshift_table[ent++] = std::pair<int,int>(so+o+b,bo+b);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
int bo=0;
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o =n*rhs.Grid()->_slice_stride[dimension];
|
|
||||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);// Could easily be a table lookup
|
|
||||||
if ( ocb & cbmask ) {
|
|
||||||
Cshift_table[ent++]=std::pair<int,int> (so+o+b,bo++);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
auto buffer_p = & buffer[0];
|
|
||||||
auto table = MapCshiftTable();
|
|
||||||
autoView( rhs_v, rhs, AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
|
||||||
coalescedWrite(rhs_v[table[i].first],coalescedRead(buffer_p[table[i].second]));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Scatter for when there *is* need to SIMD split
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
template<class vobj> void Scatter_plane_merge(Lattice<vobj> &rhs,ExtractPointerArray<typename vobj::scalar_object> pointers,int dimension,int plane,int cbmask)
|
|
||||||
{
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
cbmask=0x3;
|
|
||||||
}
|
|
||||||
|
|
||||||
int so = plane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
|
|
||||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
|
||||||
int e2=rhs.Grid()->_slice_block[dimension];
|
|
||||||
|
|
||||||
if(cbmask ==0x3 ) {
|
|
||||||
int _slice_stride = rhs.Grid()->_slice_stride[dimension];
|
|
||||||
int _slice_block = rhs.Grid()->_slice_block[dimension];
|
|
||||||
autoView( rhs_v , rhs, AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(nn,e1*e2,1,{
|
|
||||||
int n = nn%e1;
|
|
||||||
int b = nn/e1;
|
|
||||||
int o = n*_slice_stride;
|
|
||||||
int offset = b+n*_slice_block;
|
|
||||||
merge(rhs_v[so+o+b],pointers,offset);
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
|
|
||||||
// Case of SIMD split AND checker dim cannot currently be hit, except in
|
|
||||||
// Test_cshift_red_black code.
|
|
||||||
std::cout << "Scatter_plane merge GRID_ASSERT(0); think this is buggy FIXME "<< std::endl;// think this is buggy FIXME
|
|
||||||
std::cout<<" Unthreaded warning -- buffer is not densely packed ??"<<std::endl;
|
|
||||||
GRID_ASSERT(0); // This will fail if hit on GPU
|
|
||||||
autoView( rhs_v, rhs, CpuWrite);
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o = n*rhs.Grid()->_slice_stride[dimension];
|
|
||||||
int offset = b+n*rhs.Grid()->_slice_block[dimension];
|
|
||||||
int ocb=1<<rhs.Grid()->CheckerBoardFromOindex(o+b);
|
|
||||||
if ( ocb&cbmask ) {
|
|
||||||
merge(rhs_v[so+o+b],pointers,offset);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// local to node block strided copies
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class vobj> void Copy_plane(Lattice<vobj>& lhs,const Lattice<vobj> &rhs, int dimension,int lplane,int rplane,int cbmask)
|
|
||||||
{
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
cbmask=0x3;
|
|
||||||
}
|
|
||||||
|
|
||||||
int ro = rplane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
int lo = lplane*lhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
|
|
||||||
int e1=rhs.Grid()->_slice_nblock[dimension]; // clearly loop invariant for icpc
|
|
||||||
int e2=rhs.Grid()->_slice_block[dimension];
|
|
||||||
int stride = rhs.Grid()->_slice_stride[dimension];
|
|
||||||
|
|
||||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
|
||||||
|
|
||||||
int ent=0;
|
|
||||||
|
|
||||||
if(cbmask == 0x3 ){
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o =n*stride+b;
|
|
||||||
Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o =n*stride+b;
|
|
||||||
int ocb=1<<lhs.Grid()->CheckerBoardFromOindex(o);
|
|
||||||
if ( ocb&cbmask ) {
|
|
||||||
Cshift_table[ent++] = std::pair<int,int>(lo+o,ro+o);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
auto table = MapCshiftTable();
|
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
|
||||||
autoView(lhs_v , lhs, AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(i,ent,vobj::Nsimd(),{
|
|
||||||
coalescedWrite(lhs_v[table[i].first],coalescedRead(rhs_v[table[i].second]));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> void Copy_plane_permute(Lattice<vobj>& lhs,const Lattice<vobj> &rhs, int dimension,int lplane,int rplane,int cbmask,int permute_type)
|
|
||||||
{
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
if ( !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
cbmask=0x3;
|
|
||||||
}
|
|
||||||
|
|
||||||
int ro = rplane*rhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
int lo = lplane*lhs.Grid()->_ostride[dimension]; // base offset for start of plane
|
|
||||||
|
|
||||||
int e1=rhs.Grid()->_slice_nblock[dimension];
|
|
||||||
int e2=rhs.Grid()->_slice_block [dimension];
|
|
||||||
int stride = rhs.Grid()->_slice_stride[dimension];
|
|
||||||
|
|
||||||
if(Cshift_table.size()<e1*e2) Cshift_table.resize(e1*e2); // Let it grow to biggest
|
|
||||||
|
|
||||||
int ent=0;
|
|
||||||
|
|
||||||
if ( cbmask == 0x3 ) {
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o =n*stride;
|
|
||||||
Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
|
||||||
}}
|
|
||||||
} else {
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int o =n*stride;
|
|
||||||
int ocb=1<<lhs.Grid()->CheckerBoardFromOindex(o+b);
|
|
||||||
if ( ocb&cbmask ) Cshift_table[ent++] = std::pair<int,int>(lo+o+b,ro+o+b);
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
auto table = MapCshiftTable();
|
|
||||||
autoView( rhs_v, rhs, AcceleratorRead);
|
|
||||||
autoView( lhs_v, lhs, AcceleratorWrite);
|
|
||||||
accelerator_for(i,ent,1,{
|
|
||||||
permute(lhs_v[table[i].first],rhs_v[table[i].second],permute_type);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Local to node Cshift
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
template<class vobj> void Cshift_local(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
|
||||||
{
|
|
||||||
int sshift[2];
|
|
||||||
|
|
||||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
|
||||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
|
||||||
|
|
||||||
if ( sshift[0] == sshift[1] ) {
|
|
||||||
Cshift_local(ret,rhs,dimension,shift,0x3);
|
|
||||||
} else {
|
|
||||||
Cshift_local(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
|
||||||
Cshift_local(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> void Cshift_local(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
|
||||||
{
|
|
||||||
GridBase *grid = rhs.Grid();
|
|
||||||
int fd = grid->_fdimensions[dimension];
|
|
||||||
int rd = grid->_rdimensions[dimension];
|
|
||||||
int ld = grid->_ldimensions[dimension];
|
|
||||||
int gd = grid->_gdimensions[dimension];
|
|
||||||
int ly = grid->_simd_layout[dimension];
|
|
||||||
|
|
||||||
// Map to always positive shift modulo global full dimension.
|
|
||||||
shift = (shift+fd)%fd;
|
|
||||||
|
|
||||||
// the permute type
|
|
||||||
ret.Checkerboard() = grid->CheckerBoardDestination(rhs.Checkerboard(),shift,dimension);
|
|
||||||
int permute_dim =grid->PermuteDim(dimension);
|
|
||||||
int permute_type=grid->PermuteType(dimension);
|
|
||||||
int permute_type_dist;
|
|
||||||
|
|
||||||
for(int x=0;x<rd;x++){
|
|
||||||
|
|
||||||
// int o = 0;
|
|
||||||
int bo = x * grid->_ostride[dimension];
|
|
||||||
int cb= (cbmask==0x2)? Odd : Even;
|
|
||||||
|
|
||||||
int sshift = grid->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
|
||||||
int sx = (x+sshift)%rd;
|
|
||||||
|
|
||||||
// wrap is whether sshift > rd.
|
|
||||||
// num is sshift mod rd.
|
|
||||||
//
|
|
||||||
// shift 7
|
|
||||||
//
|
|
||||||
// XoXo YcYc
|
|
||||||
// oXoX cYcY
|
|
||||||
// XoXo YcYc
|
|
||||||
// oXoX cYcY
|
|
||||||
//
|
|
||||||
// sshift --
|
|
||||||
//
|
|
||||||
// XX YY ; 3
|
|
||||||
// XX YY ; 0
|
|
||||||
// XX YY ; 3
|
|
||||||
// XX YY ; 0
|
|
||||||
//
|
|
||||||
int permute_slice=0;
|
|
||||||
if(permute_dim){
|
|
||||||
int wrap = sshift/rd; wrap=wrap % ly;
|
|
||||||
int num = sshift%rd;
|
|
||||||
|
|
||||||
if ( x< rd-num ) permute_slice=wrap;
|
|
||||||
else permute_slice = (wrap+1)%ly;
|
|
||||||
|
|
||||||
if ( (ly>2) && (permute_slice) ) {
|
|
||||||
assert(permute_type & RotateBit);
|
|
||||||
permute_type_dist = permute_type|permute_slice;
|
|
||||||
} else {
|
|
||||||
permute_type_dist = permute_type;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( permute_slice ) Copy_plane_permute(ret,rhs,dimension,x,sx,cbmask,permute_type_dist);
|
|
||||||
else Copy_plane(ret,rhs,dimension,x,sx,cbmask);
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,484 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/cshift/Cshift_mpi.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef _GRID_CSHIFT_MPI_H_
|
|
||||||
#define _GRID_CSHIFT_MPI_H_
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
extern uint64_t checksum_index;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
const int Cshift_verbose=0;
|
|
||||||
template<class vobj> Lattice<vobj> Cshift(const Lattice<vobj> &rhs,int dimension,int shift)
|
|
||||||
{
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
|
|
||||||
Lattice<vobj> ret(rhs.Grid());
|
|
||||||
|
|
||||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
|
|
||||||
// Map to always positive shift modulo global full dimension.
|
|
||||||
shift = (shift+fd)%fd;
|
|
||||||
|
|
||||||
if( shift ==0 ) {
|
|
||||||
ret = rhs;
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
//
|
|
||||||
// Potential easy fast cases:
|
|
||||||
// Shift is a multiple of the local lattice extent.
|
|
||||||
// Then need only to shift whole subvolumes
|
|
||||||
int L = rhs.Grid()->_ldimensions[dimension];
|
|
||||||
if ( (shift%L )==0 && !rhs.Grid()->CheckerBoarded(dimension) ) {
|
|
||||||
Cshift_simple(ret,rhs,dimension,shift);
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
ret.Checkerboard() = rhs.Grid()->CheckerBoardDestination(rhs.Checkerboard(),shift,dimension);
|
|
||||||
|
|
||||||
// the permute type
|
|
||||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
|
||||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
|
||||||
int splice_dim = rhs.Grid()->_simd_layout[dimension]>1 && (comm_dim);
|
|
||||||
|
|
||||||
RealD t1,t0;
|
|
||||||
t0=usecond();
|
|
||||||
if ( !comm_dim ) {
|
|
||||||
// std::cout << "CSHIFT: Cshift_local" <<std::endl;
|
|
||||||
Cshift_local(ret,rhs,dimension,shift); // Handles checkerboarding
|
|
||||||
} else if ( splice_dim ) {
|
|
||||||
// std::cout << "CSHIFT: Cshift_comms_simd call - splice_dim = " << splice_dim << " shift " << shift << " dimension = " << dimension << std::endl;
|
|
||||||
Cshift_comms_simd(ret,rhs,dimension,shift);
|
|
||||||
} else {
|
|
||||||
// std::cout << "CSHIFT: Cshift_comms" <<std::endl;
|
|
||||||
Cshift_comms(ret,rhs,dimension,shift);
|
|
||||||
}
|
|
||||||
t1=usecond();
|
|
||||||
if(Cshift_verbose) std::cout << GridLogPerformance << "Cshift took "<< (t1-t0)/1e3 << " ms"<<std::endl;
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> void Cshift_simple(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
|
||||||
{
|
|
||||||
GridBase *grid=rhs.Grid();
|
|
||||||
int comm_proc, xmit_to_rank, recv_from_rank;
|
|
||||||
|
|
||||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
int ld = rhs.Grid()->_ldimensions[dimension];
|
|
||||||
int pd = rhs.Grid()->_processors[dimension];
|
|
||||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
|
||||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
|
||||||
|
|
||||||
comm_proc = ((shift)/ld)%pd;
|
|
||||||
|
|
||||||
grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
|
|
||||||
if(comm_dim) {
|
|
||||||
|
|
||||||
int64_t bytes = sizeof(vobj) * grid->oSites();
|
|
||||||
|
|
||||||
autoView(rhs_v , rhs, AcceleratorRead);
|
|
||||||
autoView(ret_v , ret, AcceleratorWrite);
|
|
||||||
void *send_buf = (void *)&rhs_v[0];
|
|
||||||
void *recv_buf = (void *)&ret_v[0];
|
|
||||||
|
|
||||||
#ifdef ACCELERATOR_AWARE_MPI
|
|
||||||
grid->SendToRecvFrom(send_buf,
|
|
||||||
xmit_to_rank,
|
|
||||||
recv_buf,
|
|
||||||
recv_from_rank,
|
|
||||||
bytes);
|
|
||||||
#else
|
|
||||||
static hostVector<vobj> hrhs; hrhs.resize(grid->oSites());
|
|
||||||
static hostVector<vobj> hret; hret.resize(grid->oSites());
|
|
||||||
|
|
||||||
void *hsend_buf = (void *)&hrhs[0];
|
|
||||||
void *hrecv_buf = (void *)&hret[0];
|
|
||||||
|
|
||||||
acceleratorCopyFromDevice(send_buf,hsend_buf,bytes);
|
|
||||||
|
|
||||||
grid->SendToRecvFrom(hsend_buf,
|
|
||||||
xmit_to_rank,
|
|
||||||
hrecv_buf,
|
|
||||||
recv_from_rank,
|
|
||||||
bytes);
|
|
||||||
|
|
||||||
acceleratorCopyToDevice(hrecv_buf,recv_buf,bytes);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
}
|
|
||||||
template<class vobj> void Cshift_comms(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
|
||||||
{
|
|
||||||
int sshift[2];
|
|
||||||
|
|
||||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
|
||||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
|
||||||
|
|
||||||
// std::cout << "Cshift_comms dim "<<dimension<<"cb "<<rhs.Checkerboard()<<"shift "<<shift<<" sshift " << sshift[0]<<" "<<sshift[1]<<std::endl;
|
|
||||||
if ( sshift[0] == sshift[1] ) {
|
|
||||||
// std::cout << "Single pass Cshift_comms" <<std::endl;
|
|
||||||
Cshift_comms(ret,rhs,dimension,shift,0x3);
|
|
||||||
} else {
|
|
||||||
// std::cout << "Two pass Cshift_comms" <<std::endl;
|
|
||||||
Cshift_comms(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
|
||||||
Cshift_comms(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> void Cshift_comms_simd(Lattice<vobj>& ret,const Lattice<vobj> &rhs,int dimension,int shift)
|
|
||||||
{
|
|
||||||
int sshift[2];
|
|
||||||
|
|
||||||
sshift[0] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Even);
|
|
||||||
sshift[1] = rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,Odd);
|
|
||||||
|
|
||||||
// std::cout << "Cshift_comms_simd dim "<<dimension<<"cb "<<rhs.Checkerboard()<<"shift "<<shift<<" sshift " << sshift[0]<<" "<<sshift[1]<<std::endl;
|
|
||||||
if ( sshift[0] == sshift[1] ) {
|
|
||||||
// std::cout << "Single pass Cshift_comms" <<std::endl;
|
|
||||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x3);
|
|
||||||
} else {
|
|
||||||
// std::cout << "Two pass Cshift_comms" <<std::endl;
|
|
||||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x1);// if checkerboard is unfavourable take two passes
|
|
||||||
Cshift_comms_simd(ret,rhs,dimension,shift,0x2);// both with block stride loop iteration
|
|
||||||
}
|
|
||||||
}
|
|
||||||
template<class vobj> void Cshift_comms(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
|
||||||
{
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
|
|
||||||
GridBase *grid=rhs.Grid();
|
|
||||||
Lattice<vobj> temp(rhs.Grid());
|
|
||||||
|
|
||||||
int fd = rhs.Grid()->_fdimensions[dimension];
|
|
||||||
int rd = rhs.Grid()->_rdimensions[dimension];
|
|
||||||
int pd = rhs.Grid()->_processors[dimension];
|
|
||||||
int simd_layout = rhs.Grid()->_simd_layout[dimension];
|
|
||||||
int comm_dim = rhs.Grid()->_processors[dimension] >1 ;
|
|
||||||
GRID_ASSERT(simd_layout==1);
|
|
||||||
GRID_ASSERT(comm_dim==1);
|
|
||||||
GRID_ASSERT(shift>=0);
|
|
||||||
GRID_ASSERT(shift<fd);
|
|
||||||
|
|
||||||
int buffer_size = rhs.Grid()->_slice_nblock[dimension]*rhs.Grid()->_slice_block[dimension];
|
|
||||||
static deviceVector<vobj> send_buf; send_buf.resize(buffer_size);
|
|
||||||
static deviceVector<vobj> recv_buf; recv_buf.resize(buffer_size);
|
|
||||||
#ifndef ACCELERATOR_AWARE_MPI
|
|
||||||
int pad = (8 + sizeof(vobj) - 1) / sizeof(vobj);
|
|
||||||
static hostVector<vobj> hsend_buf; hsend_buf.resize(buffer_size+pad);
|
|
||||||
static hostVector<vobj> hrecv_buf; hrecv_buf.resize(buffer_size+pad);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
int cb= (cbmask==0x2)? Odd : Even;
|
|
||||||
int sshift= rhs.Grid()->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
|
||||||
RealD tcopy=0.0;
|
|
||||||
RealD tgather=0.0;
|
|
||||||
RealD tscatter=0.0;
|
|
||||||
RealD tcomms=0.0;
|
|
||||||
uint64_t xbytes=0;
|
|
||||||
for(int x=0;x<rd;x++){
|
|
||||||
|
|
||||||
int sx = (x+sshift)%rd;
|
|
||||||
int comm_proc = ((x+sshift)/rd)%pd;
|
|
||||||
|
|
||||||
if (comm_proc==0) {
|
|
||||||
FlightRecorder::StepLog("Cshift_Copy_plane");
|
|
||||||
tcopy-=usecond();
|
|
||||||
Copy_plane(ret,rhs,dimension,x,sx,cbmask);
|
|
||||||
tcopy+=usecond();
|
|
||||||
FlightRecorder::StepLog("Cshift_Copy_plane_complete");
|
|
||||||
} else {
|
|
||||||
|
|
||||||
int words = buffer_size;
|
|
||||||
if (cbmask != 0x3) words=words>>1;
|
|
||||||
|
|
||||||
int bytes = words * sizeof(vobj);
|
|
||||||
|
|
||||||
FlightRecorder::StepLog("Cshift_Gather_plane");
|
|
||||||
tgather-=usecond();
|
|
||||||
Gather_plane_simple (rhs,send_buf,dimension,sx,cbmask);
|
|
||||||
tgather+=usecond();
|
|
||||||
FlightRecorder::StepLog("Cshift_Gather_plane_complete");
|
|
||||||
|
|
||||||
// int rank = grid->_processor;
|
|
||||||
int recv_from_rank;
|
|
||||||
int xmit_to_rank;
|
|
||||||
|
|
||||||
grid->ShiftedRanks(dimension,comm_proc,xmit_to_rank,recv_from_rank);
|
|
||||||
|
|
||||||
tcomms-=usecond();
|
|
||||||
grid->Barrier();
|
|
||||||
|
|
||||||
FlightRecorder::StepLog("Cshift_SendRecv");
|
|
||||||
#ifdef ACCELERATOR_AWARE_MPI
|
|
||||||
grid->SendToRecvFrom((void *)&send_buf[0],
|
|
||||||
xmit_to_rank,
|
|
||||||
(void *)&recv_buf[0],
|
|
||||||
recv_from_rank,
|
|
||||||
bytes);
|
|
||||||
#else
|
|
||||||
// bouncy bouncy
|
|
||||||
acceleratorCopyFromDevice(&send_buf[0],&hsend_buf[0],bytes);
|
|
||||||
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
GRID_ASSERT(bytes % 8 == 0);
|
|
||||||
checksum_index++;
|
|
||||||
uint64_t xsum = checksum_gpu((uint64_t*)&send_buf[0], bytes / 8) ^ (1 + checksum_index);
|
|
||||||
*(uint64_t*)(((char*)&hsend_buf[0]) + bytes) = xsum;
|
|
||||||
bytes += 8;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
grid->SendToRecvFrom((void *)&hsend_buf[0],
|
|
||||||
xmit_to_rank,
|
|
||||||
(void *)&hrecv_buf[0],
|
|
||||||
recv_from_rank,
|
|
||||||
bytes);
|
|
||||||
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
bytes -= 8;
|
|
||||||
acceleratorCopyToDevice(&hrecv_buf[0],&recv_buf[0],bytes);
|
|
||||||
uint64_t expected_cs = *(uint64_t*)(((char*)&hrecv_buf[0]) + bytes);
|
|
||||||
uint64_t computed_cs = checksum_gpu((uint64_t*)&recv_buf[0], bytes / 8) ^ (1 + checksum_index);
|
|
||||||
std::cout << GridLogComms<< " Cshift: "
|
|
||||||
<<" dim"<<dimension
|
|
||||||
<<" shift "<<shift
|
|
||||||
<< " rank "<< grid->ThisRank()
|
|
||||||
<<" Coor "<<grid->ThisProcessorCoor()
|
|
||||||
<<" send "<<xsum<<" to "<<xmit_to_rank
|
|
||||||
<<" recv "<<computed_cs<<" from "<<recv_from_rank
|
|
||||||
<<std::endl;
|
|
||||||
GRID_ASSERT(expected_cs == computed_cs);
|
|
||||||
#else
|
|
||||||
acceleratorCopyToDevice(&hrecv_buf[0],&recv_buf[0],bytes);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif
|
|
||||||
FlightRecorder::StepLog("Cshift_SendRecv_complete");
|
|
||||||
|
|
||||||
xbytes+=bytes;
|
|
||||||
grid->Barrier();
|
|
||||||
tcomms+=usecond();
|
|
||||||
FlightRecorder::StepLog("Cshift_barrier_complete");
|
|
||||||
|
|
||||||
tscatter-=usecond();
|
|
||||||
Scatter_plane_simple (ret,recv_buf,dimension,x,cbmask);
|
|
||||||
tscatter+=usecond();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Cshift_verbose){
|
|
||||||
std::cout << GridLogPerformance << " Cshift copy "<<tcopy/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift gather "<<tgather/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift scatter "<<tscatter/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift comm "<<tcomms/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift BW "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> void Cshift_comms_simd(Lattice<vobj> &ret,const Lattice<vobj> &rhs,int dimension,int shift,int cbmask)
|
|
||||||
{
|
|
||||||
GridBase *grid=rhs.Grid();
|
|
||||||
const int Nsimd = grid->Nsimd();
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
typedef typename vobj::scalar_object scalar_object;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
|
|
||||||
int fd = grid->_fdimensions[dimension];
|
|
||||||
int rd = grid->_rdimensions[dimension];
|
|
||||||
int ld = grid->_ldimensions[dimension];
|
|
||||||
int pd = grid->_processors[dimension];
|
|
||||||
int simd_layout = grid->_simd_layout[dimension];
|
|
||||||
int comm_dim = grid->_processors[dimension] >1 ;
|
|
||||||
|
|
||||||
// std::cout << "Cshift_comms_simd dim "<< dimension << " fd "<<fd<<" rd "<<rd
|
|
||||||
// << " ld "<<ld<<" pd " << pd<<" simd_layout "<<simd_layout
|
|
||||||
// << " comm_dim " << comm_dim << " cbmask " << cbmask <<std::endl;
|
|
||||||
|
|
||||||
GRID_ASSERT(comm_dim==1);
|
|
||||||
GRID_ASSERT(simd_layout==2);
|
|
||||||
GRID_ASSERT(shift>=0);
|
|
||||||
GRID_ASSERT(shift<fd);
|
|
||||||
|
|
||||||
RealD tcopy=0.0;
|
|
||||||
RealD tgather=0.0;
|
|
||||||
RealD tscatter=0.0;
|
|
||||||
RealD tcomms=0.0;
|
|
||||||
uint64_t xbytes=0;
|
|
||||||
|
|
||||||
int permute_type=grid->PermuteType(dimension);
|
|
||||||
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
// Simd direction uses an extract/merge pair
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
int buffer_size = grid->_slice_nblock[dimension]*grid->_slice_block[dimension];
|
|
||||||
// int words = sizeof(vobj)/sizeof(vector_type);
|
|
||||||
|
|
||||||
static std::vector<deviceVector<scalar_object> > send_buf_extract; send_buf_extract.resize(Nsimd);
|
|
||||||
static std::vector<deviceVector<scalar_object> > recv_buf_extract; recv_buf_extract.resize(Nsimd);
|
|
||||||
scalar_object * recv_buf_extract_mpi;
|
|
||||||
scalar_object * send_buf_extract_mpi;
|
|
||||||
|
|
||||||
for(int s=0;s<Nsimd;s++){
|
|
||||||
send_buf_extract[s].resize(buffer_size);
|
|
||||||
recv_buf_extract[s].resize(buffer_size);
|
|
||||||
}
|
|
||||||
#ifndef ACCELERATOR_AWARE_MPI
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
buffer_size += (8 + sizeof(vobj) - 1) / sizeof(vobj);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static hostVector<vobj> hsend_buf; hsend_buf.resize(buffer_size);
|
|
||||||
static hostVector<vobj> hrecv_buf; hrecv_buf.resize(buffer_size);
|
|
||||||
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
buffer_size -= (8 + sizeof(vobj) - 1) / sizeof(vobj);
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
|
|
||||||
int bytes = buffer_size*sizeof(scalar_object);
|
|
||||||
|
|
||||||
ExtractPointerArray<scalar_object> pointers(Nsimd); //
|
|
||||||
ExtractPointerArray<scalar_object> rpointers(Nsimd); // received pointers
|
|
||||||
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// Work out what to send where
|
|
||||||
///////////////////////////////////////////
|
|
||||||
int cb = (cbmask==0x2)? Odd : Even;
|
|
||||||
int sshift= grid->CheckerBoardShiftForCB(rhs.Checkerboard(),dimension,shift,cb);
|
|
||||||
|
|
||||||
// loop over outer coord planes orthog to dim
|
|
||||||
for(int x=0;x<rd;x++){
|
|
||||||
|
|
||||||
// FIXME call local permute copy if none are offnode.
|
|
||||||
for(int i=0;i<Nsimd;i++){
|
|
||||||
pointers[i] = &send_buf_extract[i][0];
|
|
||||||
}
|
|
||||||
int sx = (x+sshift)%rd;
|
|
||||||
tgather-=usecond();
|
|
||||||
Gather_plane_extract(rhs,pointers,dimension,sx,cbmask);
|
|
||||||
tgather+=usecond();
|
|
||||||
|
|
||||||
for(int i=0;i<Nsimd;i++){
|
|
||||||
|
|
||||||
int inner_bit = (Nsimd>>(permute_type+1));
|
|
||||||
int ic= (i&inner_bit)? 1:0;
|
|
||||||
|
|
||||||
int my_coor = rd*ic + x;
|
|
||||||
int nbr_coor = my_coor+sshift;
|
|
||||||
int nbr_proc = ((nbr_coor)/ld) % pd;// relative shift in processors
|
|
||||||
|
|
||||||
int nbr_ic = (nbr_coor%ld)/rd; // inner coord of peer
|
|
||||||
int nbr_ox = (nbr_coor%rd); // outer coord of peer
|
|
||||||
int nbr_lane = (i&(~inner_bit));
|
|
||||||
|
|
||||||
int recv_from_rank;
|
|
||||||
int xmit_to_rank;
|
|
||||||
|
|
||||||
if (nbr_ic) nbr_lane|=inner_bit;
|
|
||||||
|
|
||||||
GRID_ASSERT (sx == nbr_ox);
|
|
||||||
|
|
||||||
if(nbr_proc){
|
|
||||||
grid->ShiftedRanks(dimension,nbr_proc,xmit_to_rank,recv_from_rank);
|
|
||||||
|
|
||||||
tcomms-=usecond();
|
|
||||||
grid->Barrier();
|
|
||||||
|
|
||||||
send_buf_extract_mpi = &send_buf_extract[nbr_lane][0];
|
|
||||||
recv_buf_extract_mpi = &recv_buf_extract[i][0];
|
|
||||||
#ifdef ACCELERATOR_AWARE_MPI
|
|
||||||
grid->SendToRecvFrom((void *)send_buf_extract_mpi,
|
|
||||||
xmit_to_rank,
|
|
||||||
(void *)recv_buf_extract_mpi,
|
|
||||||
recv_from_rank,
|
|
||||||
bytes);
|
|
||||||
#else
|
|
||||||
// bouncy bouncy
|
|
||||||
acceleratorCopyFromDevice((void *)send_buf_extract_mpi,(void *)&hsend_buf[0],bytes);
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
assert(bytes % 8 == 0);
|
|
||||||
checksum_index++;
|
|
||||||
uint64_t xsum = checksum_gpu((uint64_t*)send_buf_extract_mpi, bytes / 8) ^ (1 + checksum_index);
|
|
||||||
*(uint64_t*)(((char*)&hsend_buf[0]) + bytes) = xsum;
|
|
||||||
bytes += 8;
|
|
||||||
#endif
|
|
||||||
grid->SendToRecvFrom((void *)&hsend_buf[0],
|
|
||||||
xmit_to_rank,
|
|
||||||
(void *)&hrecv_buf[0],
|
|
||||||
recv_from_rank,
|
|
||||||
bytes);
|
|
||||||
#ifdef GRID_CHECKSUM_COMMS
|
|
||||||
bytes -= 8;
|
|
||||||
acceleratorCopyToDevice((void *)&hrecv_buf[0],(void *)recv_buf_extract_mpi,bytes);
|
|
||||||
uint64_t expected_cs = *(uint64_t*)(((char*)&hrecv_buf[0]) + bytes);
|
|
||||||
uint64_t computed_cs = checksum_gpu((uint64_t*)recv_buf_extract_mpi, bytes / 8) ^ (1 + checksum_index);
|
|
||||||
|
|
||||||
std::cout << GridLogComms<< " Cshift_comms_simd: "
|
|
||||||
<<" dim"<<dimension
|
|
||||||
<<" shift "<<shift
|
|
||||||
<< " rank "<< grid->ThisRank()
|
|
||||||
<<" Coor "<<grid->ThisProcessorCoor()
|
|
||||||
<<" send "<<xsum<<" to "<<xmit_to_rank
|
|
||||||
<<" recv "<<computed_cs<<" from "<<recv_from_rank
|
|
||||||
<<std::endl;
|
|
||||||
assert(expected_cs == computed_cs);
|
|
||||||
#else
|
|
||||||
acceleratorCopyToDevice((void *)&hrecv_buf[0],(void *)recv_buf_extract_mpi,bytes);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
xbytes+=bytes;
|
|
||||||
grid->Barrier();
|
|
||||||
tcomms+=usecond();
|
|
||||||
|
|
||||||
rpointers[i] = &recv_buf_extract[i][0];
|
|
||||||
} else {
|
|
||||||
rpointers[i] = &send_buf_extract[nbr_lane][0];
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
tscatter-=usecond();
|
|
||||||
Scatter_plane_merge(ret,rpointers,dimension,x,cbmask);
|
|
||||||
tscatter+=usecond();
|
|
||||||
}
|
|
||||||
if(Cshift_verbose){
|
|
||||||
std::cout << GridLogPerformance << " Cshift (s) copy "<<tcopy/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift (s) gather "<<tgather/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift (s) scatter "<<tscatter/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift (s) comm "<<tcomms/1e3<<" ms"<<std::endl;
|
|
||||||
std::cout << GridLogPerformance << " Cshift BW "<<(2.0*xbytes)/tcomms<<" MB/s "<<2*xbytes<< " Bytes "<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
#include <Grid/GridCore.h>
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
std::vector<std::pair<int,int> > Cshift_table;
|
|
||||||
deviceVector<std::pair<int,int> > Cshift_table_device;
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
-22091
File diff suppressed because it is too large
Load Diff
@@ -1,534 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_ET.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: neo <cossu@post.kek.jp>
|
|
||||||
Author: Christoph Lehner <christoph@lhnr.de
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_ET_H
|
|
||||||
#define GRID_LATTICE_ET_H
|
|
||||||
|
|
||||||
#include <iostream>
|
|
||||||
#include <tuple>
|
|
||||||
#include <typeinfo>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
// Predicated where support
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
// drop to scalar in SIMT; cleaner in fact
|
|
||||||
template <class iobj, class vobj, class robj>
|
|
||||||
accelerator_inline vobj predicatedWhere(const iobj &predicate,
|
|
||||||
const vobj &iftrue,
|
|
||||||
const robj &iffalse)
|
|
||||||
{
|
|
||||||
Integer mask = TensorRemove(predicate);
|
|
||||||
typename std::remove_const<vobj>::type ret= iffalse;
|
|
||||||
if (mask) ret=iftrue;
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
template <class iobj, class vobj, class robj>
|
|
||||||
accelerator_inline vobj predicatedWhere(const iobj &predicate,
|
|
||||||
const vobj &iftrue,
|
|
||||||
const robj &iffalse)
|
|
||||||
{
|
|
||||||
typename std::remove_const<vobj>::type ret;
|
|
||||||
|
|
||||||
typedef typename vobj::scalar_object scalar_object;
|
|
||||||
// typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
const int Nsimd = vobj::vector_type::Nsimd();
|
|
||||||
|
|
||||||
ExtractBuffer<Integer> mask(Nsimd);
|
|
||||||
ExtractBuffer<scalar_object> truevals(Nsimd);
|
|
||||||
ExtractBuffer<scalar_object> falsevals(Nsimd);
|
|
||||||
|
|
||||||
extract(iftrue, truevals);
|
|
||||||
extract(iffalse, falsevals);
|
|
||||||
extract<vInteger, Integer>(TensorRemove(predicate), mask);
|
|
||||||
|
|
||||||
for (int s = 0; s < Nsimd; s++) {
|
|
||||||
if (mask[s]) falsevals[s] = truevals[s];
|
|
||||||
}
|
|
||||||
|
|
||||||
merge(ret, falsevals);
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
//Specialization of getVectorType for lattices
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
template<typename T>
|
|
||||||
struct getVectorType<Lattice<T> >{
|
|
||||||
typedef typename Lattice<T>::vector_object type;
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
//-- recursive evaluation of expressions; --
|
|
||||||
// handle leaves of syntax tree
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
template<class sobj,
|
|
||||||
typename std::enable_if<!is_lattice<sobj>::value&&!is_lattice_expr<sobj>::value,sobj>::type * = nullptr>
|
|
||||||
accelerator_inline
|
|
||||||
sobj eval(const uint64_t ss, const sobj &arg)
|
|
||||||
{
|
|
||||||
return arg;
|
|
||||||
}
|
|
||||||
template <class lobj> accelerator_inline
|
|
||||||
auto eval(const uint64_t ss, const LatticeView<lobj> &arg) -> decltype(arg(ss))
|
|
||||||
{
|
|
||||||
return arg(ss);
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
//-- recursive evaluation of expressions; --
|
|
||||||
// whole vector return, used only for expression return type inference
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
template<class sobj> accelerator_inline
|
|
||||||
sobj vecEval(const uint64_t ss, const sobj &arg)
|
|
||||||
{
|
|
||||||
return arg;
|
|
||||||
}
|
|
||||||
template <class lobj> accelerator_inline
|
|
||||||
const lobj & vecEval(const uint64_t ss, const LatticeView<lobj> &arg)
|
|
||||||
{
|
|
||||||
return arg[ss];
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// handle nodes in syntax tree- eval one operand
|
|
||||||
// vecEval needed (but never called as all expressions offloaded) to infer the return type
|
|
||||||
// in SIMT contexts of closure.
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
template <typename Op, typename T1> accelerator_inline
|
|
||||||
auto vecEval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
-> decltype(expr.op.func( vecEval(ss, expr.arg1)))
|
|
||||||
{
|
|
||||||
return expr.op.func( vecEval(ss, expr.arg1) );
|
|
||||||
}
|
|
||||||
// vecEval two operands
|
|
||||||
template <typename Op, typename T1, typename T2> accelerator_inline
|
|
||||||
auto vecEval(const uint64_t ss, const LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
-> decltype(expr.op.func( vecEval(ss,expr.arg1),vecEval(ss,expr.arg2)))
|
|
||||||
{
|
|
||||||
return expr.op.func( vecEval(ss,expr.arg1), vecEval(ss,expr.arg2) );
|
|
||||||
}
|
|
||||||
// vecEval three operands
|
|
||||||
template <typename Op, typename T1, typename T2, typename T3> accelerator_inline
|
|
||||||
auto vecEval(const uint64_t ss, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
-> decltype(expr.op.func(vecEval(ss, expr.arg1), vecEval(ss, expr.arg2), vecEval(ss, expr.arg3)))
|
|
||||||
{
|
|
||||||
return expr.op.func(vecEval(ss, expr.arg1), vecEval(ss, expr.arg2), vecEval(ss, expr.arg3));
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// handle nodes in syntax tree- eval one operand coalesced
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
template <typename Op, typename T1> accelerator_inline
|
|
||||||
auto eval(const uint64_t ss, const LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
-> decltype(expr.op.func( eval(ss, expr.arg1)))
|
|
||||||
{
|
|
||||||
return expr.op.func( eval(ss, expr.arg1) );
|
|
||||||
}
|
|
||||||
// eval two operands
|
|
||||||
template <typename Op, typename T1, typename T2> accelerator_inline
|
|
||||||
auto eval(const uint64_t ss, const LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
-> decltype(expr.op.func( eval(ss,expr.arg1),eval(ss,expr.arg2)))
|
|
||||||
{
|
|
||||||
return expr.op.func( eval(ss,expr.arg1), eval(ss,expr.arg2) );
|
|
||||||
}
|
|
||||||
// eval three operands
|
|
||||||
template <typename Op, typename T1, typename T2, typename T3> accelerator_inline
|
|
||||||
auto eval(const uint64_t ss, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
-> decltype(expr.op.func(eval(ss, expr.arg1),
|
|
||||||
eval(ss, expr.arg2),
|
|
||||||
eval(ss, expr.arg3)))
|
|
||||||
{
|
|
||||||
#ifdef GRID_SIMT
|
|
||||||
// Handles Nsimd (vInteger) != Nsimd(ComplexD)
|
|
||||||
typedef decltype(vecEval(ss, expr.arg2)) rvobj;
|
|
||||||
typedef typename std::remove_reference<rvobj>::type vobj;
|
|
||||||
|
|
||||||
const int Nsimd = vobj::vector_type::Nsimd();
|
|
||||||
|
|
||||||
auto vpred = vecEval(ss,expr.arg1);
|
|
||||||
|
|
||||||
ExtractBuffer<Integer> mask(Nsimd);
|
|
||||||
extract<vInteger, Integer>(TensorRemove(vpred), mask);
|
|
||||||
|
|
||||||
int s = acceleratorSIMTlane(Nsimd);
|
|
||||||
return expr.op.func(mask[s],
|
|
||||||
eval(ss, expr.arg2),
|
|
||||||
eval(ss, expr.arg3));
|
|
||||||
#else
|
|
||||||
return expr.op.func(eval(ss, expr.arg1),
|
|
||||||
eval(ss, expr.arg2),
|
|
||||||
eval(ss, expr.arg3));
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// Obtain the grid from an expression, ensuring conformable. This must follow a
|
|
||||||
// tree recursion; must retain grid pointer in the LatticeView class which sucks
|
|
||||||
// Use a different method, and make it void *.
|
|
||||||
// Perhaps a conformable method.
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
accelerator_inline void GridFromExpression(GridBase *&grid, const T1 &lat) // Lattice leaf
|
|
||||||
{
|
|
||||||
lat.Conformable(grid);
|
|
||||||
}
|
|
||||||
|
|
||||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
accelerator_inline
|
|
||||||
void GridFromExpression(GridBase *&grid,const T1 ¬lat) // non-lattice leaf
|
|
||||||
{}
|
|
||||||
|
|
||||||
template <typename Op, typename T1>
|
|
||||||
accelerator_inline
|
|
||||||
void GridFromExpression(GridBase *&grid,const LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
{
|
|
||||||
GridFromExpression(grid, expr.arg1); // recurse
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename Op, typename T1, typename T2>
|
|
||||||
accelerator_inline
|
|
||||||
void GridFromExpression(GridBase *&grid, const LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
{
|
|
||||||
GridFromExpression(grid, expr.arg1); // recurse
|
|
||||||
GridFromExpression(grid, expr.arg2);
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1, typename T2, typename T3>
|
|
||||||
accelerator_inline
|
|
||||||
void GridFromExpression(GridBase *&grid, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
{
|
|
||||||
GridFromExpression(grid, expr.arg1); // recurse
|
|
||||||
GridFromExpression(grid, expr.arg2); // recurse
|
|
||||||
GridFromExpression(grid, expr.arg3); // recurse
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// Obtain the CB from an expression, ensuring conformable. This must follow a
|
|
||||||
// tree recursion
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
inline void CBFromExpression(int &cb, const T1 &lat) // Lattice leaf
|
|
||||||
{
|
|
||||||
if ((cb == Odd) || (cb == Even)) {
|
|
||||||
GRID_ASSERT(cb == lat.Checkerboard());
|
|
||||||
}
|
|
||||||
cb = lat.Checkerboard();
|
|
||||||
}
|
|
||||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
inline void CBFromExpression(int &cb, const T1 ¬lat) {} // non-lattice leaf
|
|
||||||
template <typename Op, typename T1> inline
|
|
||||||
void CBFromExpression(int &cb,const LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
{
|
|
||||||
CBFromExpression(cb, expr.arg1); // recurse AST
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1, typename T2> inline
|
|
||||||
void CBFromExpression(int &cb,const LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
{
|
|
||||||
CBFromExpression(cb, expr.arg1); // recurse AST
|
|
||||||
CBFromExpression(cb, expr.arg2); // recurse AST
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1, typename T2, typename T3>
|
|
||||||
inline void CBFromExpression(int &cb, const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
{
|
|
||||||
CBFromExpression(cb, expr.arg1); // recurse AST
|
|
||||||
CBFromExpression(cb, expr.arg2); // recurse AST
|
|
||||||
CBFromExpression(cb, expr.arg3); // recurse AST
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// ViewOpen
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
inline void ExpressionViewOpen(T1 &lat) // Lattice leaf
|
|
||||||
{
|
|
||||||
lat.ViewOpen(AcceleratorRead);
|
|
||||||
}
|
|
||||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
inline void ExpressionViewOpen(T1 ¬lat) {}
|
|
||||||
|
|
||||||
template <typename Op, typename T1> inline
|
|
||||||
void ExpressionViewOpen(LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
{
|
|
||||||
ExpressionViewOpen(expr.arg1); // recurse AST
|
|
||||||
}
|
|
||||||
|
|
||||||
template <typename Op, typename T1, typename T2> inline
|
|
||||||
void ExpressionViewOpen(LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
{
|
|
||||||
ExpressionViewOpen(expr.arg1); // recurse AST
|
|
||||||
ExpressionViewOpen(expr.arg2); // rrecurse AST
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1, typename T2, typename T3>
|
|
||||||
inline void ExpressionViewOpen(LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
{
|
|
||||||
ExpressionViewOpen(expr.arg1); // recurse AST
|
|
||||||
ExpressionViewOpen(expr.arg2); // recurse AST
|
|
||||||
ExpressionViewOpen(expr.arg3); // recurse AST
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// ViewClose
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
template <class T1,typename std::enable_if<is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
inline void ExpressionViewClose( T1 &lat) // Lattice leaf
|
|
||||||
{
|
|
||||||
lat.ViewClose();
|
|
||||||
}
|
|
||||||
template <class T1,typename std::enable_if<!is_lattice<T1>::value, T1>::type * = nullptr>
|
|
||||||
inline void ExpressionViewClose(T1 ¬lat) {}
|
|
||||||
|
|
||||||
template <typename Op, typename T1> inline
|
|
||||||
void ExpressionViewClose(LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
{
|
|
||||||
ExpressionViewClose(expr.arg1); // recurse AST
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1, typename T2> inline
|
|
||||||
void ExpressionViewClose(LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
{
|
|
||||||
ExpressionViewClose(expr.arg1); // recurse AST
|
|
||||||
ExpressionViewClose(expr.arg2); // recurse AST
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1, typename T2, typename T3>
|
|
||||||
inline void ExpressionViewClose(LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
{
|
|
||||||
ExpressionViewClose(expr.arg1); // recurse AST
|
|
||||||
ExpressionViewClose(expr.arg2); // recurse AST
|
|
||||||
ExpressionViewClose(expr.arg3); // recurse AST
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
// Unary operators and funcs
|
|
||||||
////////////////////////////////////////////
|
|
||||||
#define GridUnopClass(name, ret) \
|
|
||||||
struct name { \
|
|
||||||
template<class _arg> static auto accelerator_inline func(const _arg a) -> decltype(ret) { return ret; } \
|
|
||||||
};
|
|
||||||
|
|
||||||
GridUnopClass(UnarySub, -a);
|
|
||||||
GridUnopClass(UnaryNot, Not(a));
|
|
||||||
GridUnopClass(UnaryTrace, trace(a));
|
|
||||||
GridUnopClass(UnaryTranspose, transpose(a));
|
|
||||||
GridUnopClass(UnaryTa, Ta(a));
|
|
||||||
GridUnopClass(UnarySpTa, SpTa(a));
|
|
||||||
GridUnopClass(UnaryProjectOnGroup, ProjectOnGroup(a));
|
|
||||||
GridUnopClass(UnaryProjectOnSpGroup, ProjectOnSpGroup(a));
|
|
||||||
GridUnopClass(UnaryTimesI, timesI(a));
|
|
||||||
GridUnopClass(UnaryTimesMinusI, timesMinusI(a));
|
|
||||||
GridUnopClass(UnaryAbs, abs(a));
|
|
||||||
GridUnopClass(UnarySqrt, sqrt(a));
|
|
||||||
GridUnopClass(UnarySin, sin(a));
|
|
||||||
GridUnopClass(UnaryCos, cos(a));
|
|
||||||
GridUnopClass(UnaryAsin, asin(a));
|
|
||||||
GridUnopClass(UnaryAcos, acos(a));
|
|
||||||
GridUnopClass(UnaryLog, log(a));
|
|
||||||
GridUnopClass(UnaryExp, exp(a));
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
// Binary operators
|
|
||||||
////////////////////////////////////////////
|
|
||||||
#define GridBinOpClass(name, combination) \
|
|
||||||
struct name { \
|
|
||||||
template <class _left, class _right> \
|
|
||||||
static auto accelerator_inline \
|
|
||||||
func(const _left &lhs, const _right &rhs) \
|
|
||||||
-> decltype(combination) const \
|
|
||||||
{ \
|
|
||||||
return combination; \
|
|
||||||
} \
|
|
||||||
};
|
|
||||||
|
|
||||||
GridBinOpClass(BinaryAdd, lhs + rhs);
|
|
||||||
GridBinOpClass(BinarySub, lhs - rhs);
|
|
||||||
GridBinOpClass(BinaryMul, lhs *rhs);
|
|
||||||
GridBinOpClass(BinaryDiv, lhs /rhs);
|
|
||||||
GridBinOpClass(BinaryAnd, lhs &rhs);
|
|
||||||
GridBinOpClass(BinaryOr, lhs | rhs);
|
|
||||||
GridBinOpClass(BinaryAndAnd, lhs &&rhs);
|
|
||||||
GridBinOpClass(BinaryOrOr, lhs || rhs);
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
// Trinary conditional op
|
|
||||||
////////////////////////////////////////////////////
|
|
||||||
#define GridTrinOpClass(name, combination) \
|
|
||||||
struct name { \
|
|
||||||
template <class _predicate,class _left, class _right> \
|
|
||||||
static auto accelerator_inline \
|
|
||||||
func(const _predicate &pred, const _left &lhs, const _right &rhs) \
|
|
||||||
-> decltype(combination) const \
|
|
||||||
{ \
|
|
||||||
return combination; \
|
|
||||||
} \
|
|
||||||
};
|
|
||||||
|
|
||||||
GridTrinOpClass(TrinaryWhere,
|
|
||||||
(predicatedWhere<
|
|
||||||
typename std::remove_reference<_predicate>::type,
|
|
||||||
typename std::remove_reference<_left>::type,
|
|
||||||
typename std::remove_reference<_right>::type>(pred, lhs,rhs)));
|
|
||||||
|
|
||||||
////////////////////////////////////////////
|
|
||||||
// Operator syntactical glue
|
|
||||||
////////////////////////////////////////////
|
|
||||||
#define GRID_UNOP(name) name
|
|
||||||
#define GRID_BINOP(name) name
|
|
||||||
#define GRID_TRINOP(name) name
|
|
||||||
|
|
||||||
#define GRID_DEF_UNOP(op, name) \
|
|
||||||
template <typename T1, typename std::enable_if<is_lattice<T1>::value||is_lattice_expr<T1>::value,T1>::type * = nullptr> \
|
|
||||||
inline auto op(const T1 &arg) ->decltype(LatticeUnaryExpression<GRID_UNOP(name),T1>(GRID_UNOP(name)(), arg)) \
|
|
||||||
{ \
|
|
||||||
return LatticeUnaryExpression<GRID_UNOP(name),T1>(GRID_UNOP(name)(), arg); \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define GRID_BINOP_LEFT(op, name) \
|
|
||||||
template <typename T1, typename T2, \
|
|
||||||
typename std::enable_if<is_lattice<T1>::value||is_lattice_expr<T1>::value,T1>::type * = nullptr> \
|
|
||||||
inline auto op(const T1 &lhs, const T2 &rhs) \
|
|
||||||
->decltype(LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs,rhs)) \
|
|
||||||
{ \
|
|
||||||
return LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs,rhs);\
|
|
||||||
}
|
|
||||||
|
|
||||||
#define GRID_BINOP_RIGHT(op, name) \
|
|
||||||
template <typename T1, typename T2, \
|
|
||||||
typename std::enable_if<!is_lattice<T1>::value&&!is_lattice_expr<T1>::value,T1>::type * = nullptr, \
|
|
||||||
typename std::enable_if< is_lattice<T2>::value|| is_lattice_expr<T2>::value,T2>::type * = nullptr> \
|
|
||||||
inline auto op(const T1 &lhs, const T2 &rhs) \
|
|
||||||
->decltype(LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs, rhs)) \
|
|
||||||
{ \
|
|
||||||
return LatticeBinaryExpression<GRID_BINOP(name),T1,T2>(GRID_BINOP(name)(),lhs, rhs); \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define GRID_DEF_BINOP(op, name) \
|
|
||||||
GRID_BINOP_LEFT(op, name); \
|
|
||||||
GRID_BINOP_RIGHT(op, name);
|
|
||||||
|
|
||||||
#define GRID_DEF_TRINOP(op, name) \
|
|
||||||
template <typename T1, typename T2, typename T3> \
|
|
||||||
inline auto op(const T1 &pred, const T2 &lhs, const T3 &rhs) \
|
|
||||||
->decltype(LatticeTrinaryExpression<GRID_TRINOP(name),T1,T2,T3>(GRID_TRINOP(name)(),pred, lhs, rhs)) \
|
|
||||||
{ \
|
|
||||||
return LatticeTrinaryExpression<GRID_TRINOP(name),T1,T2,T3>(GRID_TRINOP(name)(),pred, lhs, rhs); \
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////
|
|
||||||
// Operator definitions
|
|
||||||
////////////////////////
|
|
||||||
GRID_DEF_UNOP(operator-, UnarySub);
|
|
||||||
GRID_DEF_UNOP(Not, UnaryNot);
|
|
||||||
GRID_DEF_UNOP(operator!, UnaryNot);
|
|
||||||
//GRID_DEF_UNOP(adj, UnaryAdj);
|
|
||||||
//GRID_DEF_UNOP(conjugate, UnaryConj);
|
|
||||||
GRID_DEF_UNOP(trace, UnaryTrace);
|
|
||||||
GRID_DEF_UNOP(transpose, UnaryTranspose);
|
|
||||||
GRID_DEF_UNOP(Ta, UnaryTa);
|
|
||||||
GRID_DEF_UNOP(SpTa, UnarySpTa);
|
|
||||||
GRID_DEF_UNOP(ProjectOnGroup, UnaryProjectOnGroup);
|
|
||||||
GRID_DEF_UNOP(ProjectOnSpGroup, UnaryProjectOnSpGroup);
|
|
||||||
GRID_DEF_UNOP(timesI, UnaryTimesI);
|
|
||||||
GRID_DEF_UNOP(timesMinusI, UnaryTimesMinusI);
|
|
||||||
GRID_DEF_UNOP(abs, UnaryAbs); // abs overloaded in cmath C++98; DON'T do the
|
|
||||||
// abs-fabs-dabs-labs thing
|
|
||||||
GRID_DEF_UNOP(sqrt, UnarySqrt);
|
|
||||||
GRID_DEF_UNOP(sin, UnarySin);
|
|
||||||
GRID_DEF_UNOP(cos, UnaryCos);
|
|
||||||
GRID_DEF_UNOP(asin, UnaryAsin);
|
|
||||||
GRID_DEF_UNOP(acos, UnaryAcos);
|
|
||||||
GRID_DEF_UNOP(log, UnaryLog);
|
|
||||||
GRID_DEF_UNOP(exp, UnaryExp);
|
|
||||||
|
|
||||||
GRID_DEF_BINOP(operator+, BinaryAdd);
|
|
||||||
GRID_DEF_BINOP(operator-, BinarySub);
|
|
||||||
GRID_DEF_BINOP(operator*, BinaryMul);
|
|
||||||
GRID_DEF_BINOP(operator/, BinaryDiv);
|
|
||||||
|
|
||||||
GRID_DEF_BINOP(operator&, BinaryAnd);
|
|
||||||
GRID_DEF_BINOP(operator|, BinaryOr);
|
|
||||||
GRID_DEF_BINOP(operator&&, BinaryAndAnd);
|
|
||||||
GRID_DEF_BINOP(operator||, BinaryOrOr);
|
|
||||||
|
|
||||||
GRID_DEF_TRINOP(where, TrinaryWhere);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
// Closure convenience to force expression to evaluate
|
|
||||||
/////////////////////////////////////////////////////////////
|
|
||||||
template <class Op, class T1>
|
|
||||||
auto closure(const LatticeUnaryExpression<Op, T1> &expr)
|
|
||||||
-> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1)))>::type >
|
|
||||||
{
|
|
||||||
Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1)))>::type > ret(expr);
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
template <class Op, class T1, class T2>
|
|
||||||
auto closure(const LatticeBinaryExpression<Op, T1, T2> &expr)
|
|
||||||
-> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),vecEval(0, expr.arg2)))>::type >
|
|
||||||
{
|
|
||||||
Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),vecEval(0, expr.arg2)))>::type > ret(expr);
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
template <class Op, class T1, class T2, class T3>
|
|
||||||
auto closure(const LatticeTrinaryExpression<Op, T1, T2, T3> &expr)
|
|
||||||
-> Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),
|
|
||||||
vecEval(0, expr.arg2),
|
|
||||||
vecEval(0, expr.arg3)))>::type >
|
|
||||||
{
|
|
||||||
Lattice<typename std::remove_const<decltype(expr.op.func(vecEval(0, expr.arg1),
|
|
||||||
vecEval(0, expr.arg2),
|
|
||||||
vecEval(0, expr.arg3)))>::type > ret(expr);
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
#define EXPRESSION_CLOSURE(function) \
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr> \
|
|
||||||
auto function(Expression &expr) -> decltype(function(closure(expr))) \
|
|
||||||
{ \
|
|
||||||
return function(closure(expr)); \
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#undef GRID_UNOP
|
|
||||||
#undef GRID_BINOP
|
|
||||||
#undef GRID_TRINOP
|
|
||||||
|
|
||||||
#undef GRID_DEF_UNOP
|
|
||||||
#undef GRID_DEF_BINOP
|
|
||||||
#undef GRID_DEF_TRINOP
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,324 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_arith.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christoph Lehner <christoph@lhnr.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_ARITH_H
|
|
||||||
#define GRID_LATTICE_ARITH_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// avoid copy back routines for mult, mac, sub, add
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void mult(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("mult");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
conformable(ret,rhs);
|
|
||||||
conformable(lhs,rhs);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto lhs_t = lhs_v(ss);
|
|
||||||
auto rhs_t = rhs_v(ss);
|
|
||||||
mult(&tmp,&lhs_t,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void mac(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("mac");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
conformable(lhs,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
auto lhs_t=lhs_v(ss);
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
auto tmp =ret_v(ss);
|
|
||||||
mac(&tmp,&lhs_t,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void sub(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("sub");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
conformable(lhs,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto lhs_t=lhs_v(ss);
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
sub(&tmp,&lhs_t,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void add(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("add");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
conformable(lhs,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto lhs_t=lhs_v(ss);
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
add(&tmp,&lhs_t,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// avoid copy back routines for mult, mac, sub, add
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void mult(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
|
||||||
GRID_TRACE("mult");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(lhs,ret);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
mult(&tmp,&lhs_v(ss),&rhs);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void mac(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
|
||||||
GRID_TRACE("mac");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(ret,lhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
auto tmp =ret_v(ss);
|
|
||||||
auto lhs_t=lhs_v(ss);
|
|
||||||
mac(&tmp,&lhs_t,&rhs);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void sub(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
|
||||||
GRID_TRACE("sub");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(ret,lhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto lhs_t=lhs_v(ss);
|
|
||||||
sub(&tmp,&lhs_t,&rhs);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void add(Lattice<obj1> &ret,const Lattice<obj2> &lhs,const obj3 &rhs){
|
|
||||||
GRID_TRACE("add");
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
conformable(lhs,ret);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,lhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto lhs_t=lhs_v(ss);
|
|
||||||
add(&tmp,&lhs_t,&rhs);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// avoid copy back routines for mult, mac, sub, add
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void mult(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("mult");
|
|
||||||
ret.Checkerboard() = rhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( rhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
mult(&tmp,&lhs,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void mac(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("mac");
|
|
||||||
ret.Checkerboard() = rhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( rhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
|
||||||
auto tmp =ret_v(ss);
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
mac(&tmp,&lhs,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void sub(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("sub");
|
|
||||||
ret.Checkerboard() = rhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( rhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
sub(&tmp,&lhs,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
template<class obj1,class obj2,class obj3> inline
|
|
||||||
void add(Lattice<obj1> &ret,const obj2 &lhs,const Lattice<obj3> &rhs){
|
|
||||||
GRID_TRACE("add");
|
|
||||||
ret.Checkerboard() = rhs.Checkerboard();
|
|
||||||
conformable(ret,rhs);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( rhs_v , lhs, AcceleratorRead);
|
|
||||||
accelerator_for(ss,rhs_v.size(),obj1::Nsimd(),{
|
|
||||||
decltype(coalescedRead(obj1())) tmp;
|
|
||||||
auto rhs_t=rhs_v(ss);
|
|
||||||
add(&tmp,&lhs,&rhs_t);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class sobj,class vobj> inline
|
|
||||||
void axpy(Lattice<vobj> &ret,sobj a,const Lattice<vobj> &x,const Lattice<vobj> &y){
|
|
||||||
GRID_TRACE("axpy");
|
|
||||||
ret.Checkerboard() = x.Checkerboard();
|
|
||||||
conformable(ret,x);
|
|
||||||
conformable(x,y);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( x_v , x, AcceleratorRead);
|
|
||||||
autoView( y_v , y, AcceleratorRead);
|
|
||||||
accelerator_for(ss,x_v.size(),vobj::Nsimd(),{
|
|
||||||
auto tmp = a*coalescedRead(x_v[ss])+coalescedRead(y_v[ss]);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
template<class sobj,class vobj> inline
|
|
||||||
void axpby(Lattice<vobj> &ret,sobj a,sobj b,const Lattice<vobj> &x,const Lattice<vobj> &y){
|
|
||||||
GRID_TRACE("axpby");
|
|
||||||
ret.Checkerboard() = x.Checkerboard();
|
|
||||||
conformable(ret,x);
|
|
||||||
conformable(x,y);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
autoView( x_v , x, AcceleratorRead);
|
|
||||||
autoView( y_v , y, AcceleratorRead);
|
|
||||||
accelerator_for(ss,x_v.size(),vobj::Nsimd(),{
|
|
||||||
auto tmp = a*x_v(ss)+b*y_v(ss);
|
|
||||||
coalescedWrite(ret_v[ss],tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
#define FAST_AXPY_NORM
|
|
||||||
template<class sobj,class vobj> inline
|
|
||||||
RealD axpy_norm(Lattice<vobj> &ret,sobj a,const Lattice<vobj> &x,const Lattice<vobj> &y)
|
|
||||||
{
|
|
||||||
GRID_TRACE("axpy_norm");
|
|
||||||
#ifdef FAST_AXPY_NORM
|
|
||||||
return axpy_norm_fast(ret,a,x,y);
|
|
||||||
#else
|
|
||||||
ret = a*x+y;
|
|
||||||
RealD nn=norm2(ret);
|
|
||||||
return nn;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
template<class sobj,class vobj> inline
|
|
||||||
RealD axpby_norm(Lattice<vobj> &ret,sobj a,sobj b,const Lattice<vobj> &x,const Lattice<vobj> &y)
|
|
||||||
{
|
|
||||||
GRID_TRACE("axpby_norm");
|
|
||||||
#ifdef FAST_AXPY_NORM
|
|
||||||
return axpby_norm_fast(ret,a,b,x,y);
|
|
||||||
#else
|
|
||||||
ret = a*x+b*y;
|
|
||||||
RealD nn=norm2(ret);
|
|
||||||
return nn;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Trace product
|
|
||||||
template<class obj> auto traceProduct(const Lattice<obj> &rhs_1,const Lattice<obj> &rhs_2)
|
|
||||||
-> Lattice<decltype(trace(obj()))>
|
|
||||||
{
|
|
||||||
typedef decltype(trace(obj())) robj;
|
|
||||||
Lattice<robj> ret_i(rhs_1.Grid());
|
|
||||||
autoView( rhs1 , rhs_1, AcceleratorRead);
|
|
||||||
autoView( rhs2 , rhs_2, AcceleratorRead);
|
|
||||||
autoView( ret , ret_i, AcceleratorWrite);
|
|
||||||
ret.Checkerboard() = rhs_1.Checkerboard();
|
|
||||||
accelerator_for(ss,rhs1.size(),obj::Nsimd(),{
|
|
||||||
coalescedWrite(ret[ss],traceProduct(rhs1(ss),rhs2(ss)));
|
|
||||||
});
|
|
||||||
return ret_i;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class obj1,class obj2> auto traceProduct(const Lattice<obj1> &rhs_1,const obj2 &rhs2)
|
|
||||||
-> Lattice<decltype(trace(obj1()))>
|
|
||||||
{
|
|
||||||
typedef decltype(trace(obj1())) robj;
|
|
||||||
Lattice<robj> ret_i(rhs_1.Grid());
|
|
||||||
autoView( rhs1 , rhs_1, AcceleratorRead);
|
|
||||||
autoView( ret , ret_i, AcceleratorWrite);
|
|
||||||
ret.Checkerboard() = rhs_1.Checkerboard();
|
|
||||||
accelerator_for(ss,rhs1.size(),obj1::Nsimd(),{
|
|
||||||
coalescedWrite(ret[ss],traceProduct(rhs1(ss),rhs2));
|
|
||||||
});
|
|
||||||
return ret_i;
|
|
||||||
}
|
|
||||||
template<class obj1,class obj2> auto traceProduct(const obj2 &rhs_2,const Lattice<obj1> &rhs_1)
|
|
||||||
-> Lattice<decltype(trace(obj1()))>
|
|
||||||
{
|
|
||||||
return traceProduct(rhs_1,rhs_2);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,395 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_base.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christoph Lehner <christoph@lhnr.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#define STREAMING_STORES
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
extern int GridCshiftPermuteMap[4][16];
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// The real lattice class, with normal copy and assignment semantics.
|
|
||||||
// This contains extra (host resident) grid pointer data that may be accessed by host code
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class vobj>
|
|
||||||
class Lattice : public LatticeAccelerator<vobj>
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
GridBase *Grid(void) const { return this->_grid; }
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
// Member types
|
|
||||||
///////////////////////////////////////////////////
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
typedef typename vobj::scalar_object scalar_object;
|
|
||||||
typedef vobj vector_object;
|
|
||||||
|
|
||||||
private:
|
|
||||||
void dealloc(void)
|
|
||||||
{
|
|
||||||
if( this->_odata_size ) {
|
|
||||||
alignedAllocator<vobj> alloc;
|
|
||||||
alloc.deallocate(this->_odata,this->_odata_size);
|
|
||||||
this->_odata=nullptr;
|
|
||||||
this->_odata_size=0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
void resize(uint64_t size)
|
|
||||||
{
|
|
||||||
if ( this->_odata_size != size ) {
|
|
||||||
alignedAllocator<vobj> alloc;
|
|
||||||
|
|
||||||
dealloc();
|
|
||||||
|
|
||||||
this->_odata_size = size;
|
|
||||||
if ( size )
|
|
||||||
this->_odata = alloc.allocate(this->_odata_size);
|
|
||||||
else
|
|
||||||
this->_odata = nullptr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
public:
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Can use to make accelerator dirty without copy from host ; useful for temporaries "dont care" prev contents
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
void SetViewMode(ViewMode mode) {
|
|
||||||
LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
|
|
||||||
accessor.ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper function to print the state of this object in the AccCache
|
|
||||||
void PrintCacheState(void)
|
|
||||||
{
|
|
||||||
MemoryManager::PrintState(this->_odata);
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Return a view object that may be dereferenced in site loops.
|
|
||||||
// The view is trivially copy constructible and may be copied to an accelerator device
|
|
||||||
// in device lambdas
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
LatticeView<vobj> View (ViewMode mode) const
|
|
||||||
{
|
|
||||||
LatticeView<vobj> accessor(*( (LatticeAccelerator<vobj> *) this),mode);
|
|
||||||
return accessor;
|
|
||||||
}
|
|
||||||
|
|
||||||
~Lattice() {
|
|
||||||
if ( this->_odata_size ) {
|
|
||||||
dealloc();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Expression Template closure support
|
|
||||||
////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template <typename Op, typename T1> inline Lattice<vobj> & operator=(const LatticeUnaryExpression<Op,T1> &expr)
|
|
||||||
{
|
|
||||||
GRID_TRACE("ExpressionTemplateEval");
|
|
||||||
GridBase *egrid(nullptr);
|
|
||||||
GridFromExpression(egrid,expr);
|
|
||||||
GRID_ASSERT(egrid!=nullptr);
|
|
||||||
conformable(this->_grid,egrid);
|
|
||||||
|
|
||||||
int cb=-1;
|
|
||||||
CBFromExpression(cb,expr);
|
|
||||||
GRID_ASSERT( (cb==Odd) || (cb==Even));
|
|
||||||
this->checkerboard=cb;
|
|
||||||
|
|
||||||
auto exprCopy = expr;
|
|
||||||
ExpressionViewOpen(exprCopy);
|
|
||||||
auto me = View(AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
|
||||||
auto tmp = eval(ss,exprCopy);
|
|
||||||
coalescedWrite(me[ss],tmp);
|
|
||||||
});
|
|
||||||
me.ViewClose();
|
|
||||||
ExpressionViewClose(exprCopy);
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1,typename T2> inline Lattice<vobj> & operator=(const LatticeBinaryExpression<Op,T1,T2> &expr)
|
|
||||||
{
|
|
||||||
GRID_TRACE("ExpressionTemplateEval");
|
|
||||||
GridBase *egrid(nullptr);
|
|
||||||
GridFromExpression(egrid,expr);
|
|
||||||
GRID_ASSERT(egrid!=nullptr);
|
|
||||||
conformable(this->_grid,egrid);
|
|
||||||
|
|
||||||
int cb=-1;
|
|
||||||
CBFromExpression(cb,expr);
|
|
||||||
GRID_ASSERT( (cb==Odd) || (cb==Even));
|
|
||||||
this->checkerboard=cb;
|
|
||||||
|
|
||||||
auto exprCopy = expr;
|
|
||||||
ExpressionViewOpen(exprCopy);
|
|
||||||
auto me = View(AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
|
||||||
auto tmp = eval(ss,exprCopy);
|
|
||||||
coalescedWrite(me[ss],tmp);
|
|
||||||
});
|
|
||||||
me.ViewClose();
|
|
||||||
ExpressionViewClose(exprCopy);
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
template <typename Op, typename T1,typename T2,typename T3> inline Lattice<vobj> & operator=(const LatticeTrinaryExpression<Op,T1,T2,T3> &expr)
|
|
||||||
{
|
|
||||||
GRID_TRACE("ExpressionTemplateEval");
|
|
||||||
GridBase *egrid(nullptr);
|
|
||||||
GridFromExpression(egrid,expr);
|
|
||||||
GRID_ASSERT(egrid!=nullptr);
|
|
||||||
conformable(this->_grid,egrid);
|
|
||||||
|
|
||||||
int cb=-1;
|
|
||||||
CBFromExpression(cb,expr);
|
|
||||||
GRID_ASSERT( (cb==Odd) || (cb==Even));
|
|
||||||
this->checkerboard=cb;
|
|
||||||
auto exprCopy = expr;
|
|
||||||
ExpressionViewOpen(exprCopy);
|
|
||||||
auto me = View(AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
|
||||||
auto tmp = eval(ss,exprCopy);
|
|
||||||
coalescedWrite(me[ss],tmp);
|
|
||||||
});
|
|
||||||
me.ViewClose();
|
|
||||||
ExpressionViewClose(exprCopy);
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
//GridFromExpression is tricky to do
|
|
||||||
template<class Op,class T1>
|
|
||||||
Lattice(const LatticeUnaryExpression<Op,T1> & expr) {
|
|
||||||
this->_grid = nullptr;
|
|
||||||
GridFromExpression(this->_grid,expr);
|
|
||||||
GRID_ASSERT(this->_grid!=nullptr);
|
|
||||||
|
|
||||||
int cb=-1;
|
|
||||||
CBFromExpression(cb,expr);
|
|
||||||
GRID_ASSERT( (cb==Odd) || (cb==Even));
|
|
||||||
this->checkerboard=cb;
|
|
||||||
|
|
||||||
resize(this->_grid->oSites());
|
|
||||||
|
|
||||||
*this = expr;
|
|
||||||
}
|
|
||||||
template<class Op,class T1, class T2>
|
|
||||||
Lattice(const LatticeBinaryExpression<Op,T1,T2> & expr) {
|
|
||||||
this->_grid = nullptr;
|
|
||||||
GridFromExpression(this->_grid,expr);
|
|
||||||
GRID_ASSERT(this->_grid!=nullptr);
|
|
||||||
|
|
||||||
int cb=-1;
|
|
||||||
CBFromExpression(cb,expr);
|
|
||||||
GRID_ASSERT( (cb==Odd) || (cb==Even));
|
|
||||||
this->checkerboard=cb;
|
|
||||||
|
|
||||||
resize(this->_grid->oSites());
|
|
||||||
|
|
||||||
*this = expr;
|
|
||||||
}
|
|
||||||
template<class Op,class T1, class T2, class T3>
|
|
||||||
Lattice(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr) {
|
|
||||||
this->_grid = nullptr;
|
|
||||||
GridFromExpression(this->_grid,expr);
|
|
||||||
GRID_ASSERT(this->_grid!=nullptr);
|
|
||||||
|
|
||||||
int cb=-1;
|
|
||||||
CBFromExpression(cb,expr);
|
|
||||||
GRID_ASSERT( (cb==Odd) || (cb==Even));
|
|
||||||
this->checkerboard=cb;
|
|
||||||
|
|
||||||
resize(this->_grid->oSites());
|
|
||||||
|
|
||||||
*this = expr;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class sobj> inline Lattice<vobj> & operator = (const sobj & r){
|
|
||||||
vobj vtmp;
|
|
||||||
vtmp = r;
|
|
||||||
#if 1
|
|
||||||
deviceVector<vobj> vvtmp(1);
|
|
||||||
acceleratorPut(vvtmp[0],vtmp);
|
|
||||||
vobj *vvtmp_p = & vvtmp[0];
|
|
||||||
auto me = View(AcceleratorWrite);
|
|
||||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
|
||||||
auto stmp=coalescedRead(*vvtmp_p);
|
|
||||||
coalescedWrite(me[ss],stmp);
|
|
||||||
});
|
|
||||||
#else
|
|
||||||
auto me = View(CpuWrite);
|
|
||||||
thread_for(ss,me.size(),{
|
|
||||||
me[ss]= r;
|
|
||||||
});
|
|
||||||
#endif
|
|
||||||
me.ViewClose();
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////
|
|
||||||
// Follow rule of five, with Constructor requires "grid" passed
|
|
||||||
// to user defined constructor
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// user defined constructor
|
|
||||||
///////////////////////////////////////////
|
|
||||||
Lattice(GridBase *grid,ViewMode mode=AcceleratorWriteDiscard) {
|
|
||||||
this->_grid = grid;
|
|
||||||
resize(this->_grid->oSites());
|
|
||||||
GRID_ASSERT((((uint64_t)&this->_odata[0])&0xF) ==0);
|
|
||||||
this->checkerboard=0;
|
|
||||||
SetViewMode(mode);
|
|
||||||
}
|
|
||||||
|
|
||||||
// virtual ~Lattice(void) = default;
|
|
||||||
|
|
||||||
void reset(GridBase* grid) {
|
|
||||||
if (this->_grid != grid) {
|
|
||||||
this->_grid = grid;
|
|
||||||
this->resize(grid->oSites());
|
|
||||||
this->checkerboard = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// copy constructor
|
|
||||||
///////////////////////////////////////////
|
|
||||||
Lattice(const Lattice& r){
|
|
||||||
this->_grid = r.Grid();
|
|
||||||
resize(this->_grid->oSites());
|
|
||||||
*this = r;
|
|
||||||
}
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// move constructor
|
|
||||||
///////////////////////////////////////////
|
|
||||||
Lattice(Lattice && r){
|
|
||||||
this->_grid = r.Grid();
|
|
||||||
this->_odata = r._odata;
|
|
||||||
this->_odata_size = r._odata_size;
|
|
||||||
this->checkerboard= r.Checkerboard();
|
|
||||||
r._odata = nullptr;
|
|
||||||
r._odata_size = 0;
|
|
||||||
}
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// assignment template
|
|
||||||
///////////////////////////////////////////
|
|
||||||
template<class robj> inline Lattice<vobj> & operator = (const Lattice<robj> & r){
|
|
||||||
typename std::enable_if<!std::is_same<robj,vobj>::value,int>::type i=0;
|
|
||||||
conformable(*this,r);
|
|
||||||
this->checkerboard = r.Checkerboard();
|
|
||||||
auto him= r.View(AcceleratorRead);
|
|
||||||
auto me = View(AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
|
||||||
coalescedWrite(me[ss],him(ss));
|
|
||||||
});
|
|
||||||
me.ViewClose(); him.ViewClose();
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// Copy assignment
|
|
||||||
///////////////////////////////////////////
|
|
||||||
inline Lattice<vobj> & operator = (const Lattice<vobj> & r){
|
|
||||||
this->checkerboard = r.Checkerboard();
|
|
||||||
conformable(*this,r);
|
|
||||||
auto him= r.View(AcceleratorRead);
|
|
||||||
auto me = View(AcceleratorWriteDiscard);
|
|
||||||
accelerator_for(ss,me.size(),vobj::Nsimd(),{
|
|
||||||
coalescedWrite(me[ss],him(ss));
|
|
||||||
});
|
|
||||||
me.ViewClose(); him.ViewClose();
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
///////////////////////////////////////////
|
|
||||||
// Move assignment possible if same type
|
|
||||||
///////////////////////////////////////////
|
|
||||||
inline Lattice<vobj> & operator = (Lattice<vobj> && r){
|
|
||||||
|
|
||||||
resize(0); // deletes if appropriate
|
|
||||||
this->_grid = r.Grid();
|
|
||||||
this->_odata = r._odata;
|
|
||||||
this->_odata_size = r._odata_size;
|
|
||||||
this->checkerboard= r.Checkerboard();
|
|
||||||
|
|
||||||
r._odata = nullptr;
|
|
||||||
r._odata_size = 0;
|
|
||||||
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////
|
|
||||||
// *=,+=,-= operators inherit behvour from correspond */+/- operation
|
|
||||||
/////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class T> inline Lattice<vobj> &operator *=(const T &r) {
|
|
||||||
*this = (*this)*r;
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class T> inline Lattice<vobj> &operator -=(const T &r) {
|
|
||||||
*this = (*this)-r;
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
template<class T> inline Lattice<vobj> &operator +=(const T &r) {
|
|
||||||
*this = (*this)+r;
|
|
||||||
return *this;
|
|
||||||
}
|
|
||||||
|
|
||||||
friend inline void swap(Lattice &l, Lattice &r) {
|
|
||||||
conformable(l,r);
|
|
||||||
LatticeAccelerator<vobj> tmp;
|
|
||||||
LatticeAccelerator<vobj> *lp = (LatticeAccelerator<vobj> *)&l;
|
|
||||||
LatticeAccelerator<vobj> *rp = (LatticeAccelerator<vobj> *)&r;
|
|
||||||
tmp = *lp; *lp=*rp; *rp=tmp;
|
|
||||||
}
|
|
||||||
|
|
||||||
}; // class Lattice
|
|
||||||
|
|
||||||
template<class vobj> std::ostream& operator<< (std::ostream& stream, const Lattice<vobj> &o){
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
for(int64_t g=0;g<o.Grid()->_gsites;g++){
|
|
||||||
|
|
||||||
Coordinate gcoor;
|
|
||||||
o.Grid()->GlobalIndexToGlobalCoor(g,gcoor);
|
|
||||||
|
|
||||||
sobj ss;
|
|
||||||
peekSite(ss,o,gcoor);
|
|
||||||
stream<<"[";
|
|
||||||
for(int d=0;d<gcoor.size();d++){
|
|
||||||
stream<<gcoor[d];
|
|
||||||
if(d!=gcoor.size()-1) stream<<",";
|
|
||||||
}
|
|
||||||
stream<<"]\t";
|
|
||||||
stream<<ss<<std::endl;
|
|
||||||
}
|
|
||||||
return stream;
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,235 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_basis.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christoph Lehner <christoph@lhnr.de>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution
|
|
||||||
directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
void basisOrthogonalize(std::vector<Field> &basis,Field &w,int k)
|
|
||||||
{
|
|
||||||
// If assume basis[j] are already orthonormal,
|
|
||||||
// can take all inner products in parallel saving 2x bandwidth
|
|
||||||
// Save 3x bandwidth on the second line of loop.
|
|
||||||
// perhaps 2.5x speed up.
|
|
||||||
// 2x overall in Multigrid Lanczos
|
|
||||||
for(int j=0; j<k; ++j){
|
|
||||||
auto ip = innerProduct(basis[j],w);
|
|
||||||
w = w - ip*basis[j];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class VField, class Matrix>
|
|
||||||
void basisRotate(VField &basis,Matrix& Qt,int j0, int j1, int k0,int k1,int Nm)
|
|
||||||
{
|
|
||||||
typedef decltype(basis[0]) Field;
|
|
||||||
typedef decltype(basis[0].View(AcceleratorRead)) View;
|
|
||||||
|
|
||||||
hostVector<View> h_basis_v(basis.size());
|
|
||||||
deviceVector<View> d_basis_v(basis.size());
|
|
||||||
typedef typename std::remove_reference<decltype(h_basis_v[0][0])>::type vobj;
|
|
||||||
typedef typename std::remove_reference<decltype(Qt(0,0))>::type Coeff_t;
|
|
||||||
|
|
||||||
GridBase* grid = basis[0].Grid();
|
|
||||||
|
|
||||||
for(int k=0;k<basis.size();k++){
|
|
||||||
h_basis_v[k] = basis[k].View(AcceleratorWrite);
|
|
||||||
acceleratorPut(d_basis_v[k],h_basis_v[k]);
|
|
||||||
}
|
|
||||||
|
|
||||||
View *basis_vp = &d_basis_v[0];
|
|
||||||
|
|
||||||
int nrot = j1-j0;
|
|
||||||
if (!nrot) // edge case not handled gracefully by Cuda
|
|
||||||
return;
|
|
||||||
|
|
||||||
uint64_t oSites =grid->oSites();
|
|
||||||
uint64_t siteBlock=(grid->oSites()+nrot-1)/nrot; // Maximum 1 additional vector overhead
|
|
||||||
|
|
||||||
deviceVector <vobj> Bt(siteBlock * nrot);
|
|
||||||
auto Bp=&Bt[0];
|
|
||||||
|
|
||||||
// GPU readable copy of matrix
|
|
||||||
hostVector<Coeff_t> h_Qt_jv(Nm*Nm);
|
|
||||||
deviceVector<Coeff_t> Qt_jv(Nm*Nm);
|
|
||||||
Coeff_t *Qt_p = & Qt_jv[0];
|
|
||||||
thread_for(i,Nm*Nm,{
|
|
||||||
int j = i/Nm;
|
|
||||||
int k = i%Nm;
|
|
||||||
h_Qt_jv[i]=Qt(j,k);
|
|
||||||
});
|
|
||||||
acceleratorCopyToDevice(&h_Qt_jv[0],Qt_p,Nm*Nm*sizeof(Coeff_t));
|
|
||||||
|
|
||||||
// Block the loop to keep storage footprint down
|
|
||||||
for(uint64_t s=0;s<oSites;s+=siteBlock){
|
|
||||||
|
|
||||||
// remaining work in this block
|
|
||||||
int ssites=MIN(siteBlock,oSites-s);
|
|
||||||
|
|
||||||
// zero out the accumulators
|
|
||||||
accelerator_for(ss,siteBlock*nrot,vobj::Nsimd(),{
|
|
||||||
decltype(coalescedRead(Bp[ss])) z;
|
|
||||||
z=Zero();
|
|
||||||
coalescedWrite(Bp[ss],z);
|
|
||||||
});
|
|
||||||
|
|
||||||
accelerator_for(sj,ssites*nrot,vobj::Nsimd(),{
|
|
||||||
|
|
||||||
int j =sj%nrot;
|
|
||||||
int jj =j0+j;
|
|
||||||
int ss =sj/nrot;
|
|
||||||
int sss=ss+s;
|
|
||||||
|
|
||||||
for(int k=k0; k<k1; ++k){
|
|
||||||
auto tmp = coalescedRead(Bp[ss*nrot+j]);
|
|
||||||
coalescedWrite(Bp[ss*nrot+j],tmp+ Qt_p[jj*Nm+k] * coalescedRead(basis_vp[k][sss]));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
accelerator_for(sj,ssites*nrot,vobj::Nsimd(),{
|
|
||||||
int j =sj%nrot;
|
|
||||||
int jj =j0+j;
|
|
||||||
int ss =sj/nrot;
|
|
||||||
int sss=ss+s;
|
|
||||||
coalescedWrite(basis_vp[jj][sss],coalescedRead(Bp[ss*nrot+j]));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int k=0;k<basis.size();k++) h_basis_v[k].ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract a single rotated vector
|
|
||||||
template<class Field>
|
|
||||||
void basisRotateJ(Field &result,std::vector<Field> &basis,Eigen::MatrixXd& Qt,int j, int k0,int k1,int Nm)
|
|
||||||
{
|
|
||||||
typedef decltype(basis[0].View(AcceleratorRead)) View;
|
|
||||||
typedef typename Field::vector_object vobj;
|
|
||||||
GridBase* grid = basis[0].Grid();
|
|
||||||
|
|
||||||
result.Checkerboard() = basis[0].Checkerboard();
|
|
||||||
|
|
||||||
hostVector<View> h_basis_v(basis.size());
|
|
||||||
deviceVector<View> d_basis_v(basis.size());
|
|
||||||
for(int k=0;k<basis.size();k++){
|
|
||||||
h_basis_v[k]=basis[k].View(AcceleratorRead);
|
|
||||||
acceleratorPut(d_basis_v[k],h_basis_v[k]);
|
|
||||||
}
|
|
||||||
|
|
||||||
vobj zz=Zero();
|
|
||||||
deviceVector<double> Qt_jv(Nm);
|
|
||||||
double * Qt_j = & Qt_jv[0];
|
|
||||||
for(int k=0;k<Nm;++k) acceleratorPut(Qt_j[k],Qt(j,k));
|
|
||||||
|
|
||||||
auto basis_vp=& d_basis_v[0];
|
|
||||||
autoView(result_v,result,AcceleratorWrite);
|
|
||||||
accelerator_for(ss, grid->oSites(),vobj::Nsimd(),{
|
|
||||||
vobj zzz=Zero();
|
|
||||||
auto B=coalescedRead(zzz);
|
|
||||||
for(int k=k0; k<k1; ++k){
|
|
||||||
B +=Qt_j[k] * coalescedRead(basis_vp[k][ss]);
|
|
||||||
}
|
|
||||||
coalescedWrite(result_v[ss], B);
|
|
||||||
});
|
|
||||||
for(int k=0;k<basis.size();k++) h_basis_v[k].ViewClose();
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
void basisReorderInPlace(std::vector<Field> &_v,std::vector<RealD>& sort_vals, std::vector<int>& idx)
|
|
||||||
{
|
|
||||||
int vlen = idx.size();
|
|
||||||
|
|
||||||
GRID_ASSERT(vlen>=1);
|
|
||||||
GRID_ASSERT(vlen<=sort_vals.size());
|
|
||||||
GRID_ASSERT(vlen<=_v.size());
|
|
||||||
|
|
||||||
for (size_t i=0;i<vlen;i++) {
|
|
||||||
|
|
||||||
if (idx[i] != i) {
|
|
||||||
|
|
||||||
//////////////////////////////////////
|
|
||||||
// idx[i] is a table of desired sources giving a permutation.
|
|
||||||
// Swap v[i] with v[idx[i]].
|
|
||||||
// Find j>i for which _vnew[j] = _vold[i],
|
|
||||||
// track the move idx[j] => idx[i]
|
|
||||||
// track the move idx[i] => i
|
|
||||||
//////////////////////////////////////
|
|
||||||
size_t j;
|
|
||||||
for (j=i;j<idx.size();j++)
|
|
||||||
if (idx[j]==i)
|
|
||||||
break;
|
|
||||||
|
|
||||||
GRID_ASSERT(idx[i] > i); GRID_ASSERT(j!=idx.size()); GRID_ASSERT(idx[j]==i);
|
|
||||||
|
|
||||||
swap(_v[i],_v[idx[i]]); // should use vector move constructor, no data copy
|
|
||||||
std::swap(sort_vals[i],sort_vals[idx[i]]);
|
|
||||||
|
|
||||||
idx[j] = idx[i];
|
|
||||||
idx[i] = i;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
inline std::vector<int> basisSortGetIndex(std::vector<RealD>& sort_vals)
|
|
||||||
{
|
|
||||||
std::vector<int> idx(sort_vals.size());
|
|
||||||
std::iota(idx.begin(), idx.end(), 0);
|
|
||||||
|
|
||||||
// sort indexes based on comparing values in v
|
|
||||||
std::sort(idx.begin(), idx.end(), [&sort_vals](int i1, int i2) {
|
|
||||||
return ::fabs(sort_vals[i1]) < ::fabs(sort_vals[i2]);
|
|
||||||
});
|
|
||||||
return idx;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Field>
|
|
||||||
void basisSortInPlace(std::vector<Field> & _v,std::vector<RealD>& sort_vals, bool reverse)
|
|
||||||
{
|
|
||||||
std::vector<int> idx = basisSortGetIndex(sort_vals);
|
|
||||||
if (reverse)
|
|
||||||
std::reverse(idx.begin(), idx.end());
|
|
||||||
|
|
||||||
basisReorderInPlace(_v,sort_vals,idx);
|
|
||||||
}
|
|
||||||
|
|
||||||
// PAB: faster to compute the inner products first then fuse loops.
|
|
||||||
// If performance critical can improve.
|
|
||||||
template<class Field>
|
|
||||||
void basisDeflate(const std::vector<Field> &_v,const std::vector<RealD>& eval,const Field& src_orig,Field& result) {
|
|
||||||
result = Zero();
|
|
||||||
GRID_ASSERT(_v.size()==eval.size());
|
|
||||||
int N = (int)_v.size();
|
|
||||||
for (int i=0;i<N;i++) {
|
|
||||||
Field& tmp = _v[i];
|
|
||||||
axpy(result,TensorRemove(innerProduct(tmp,src_orig)) / eval[i],tmp,result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
@@ -1,179 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_comparison.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_COMPARISON_H
|
|
||||||
#define GRID_LATTICE_COMPARISON_H
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// relational operators
|
|
||||||
//
|
|
||||||
// Support <,>,<=,>=,==,!=
|
|
||||||
//
|
|
||||||
//Query supporting bitwise &, |, ^, !
|
|
||||||
//Query supporting logical &&, ||,
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
typedef iScalar<vInteger> vPredicate ;
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// compare lattice to lattice
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class vfunctor,class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> LLComparison(vfunctor op,const Lattice<lobj> &lhs,const Lattice<robj> &rhs)
|
|
||||||
{
|
|
||||||
Lattice<vPredicate> ret(rhs.Grid());
|
|
||||||
autoView( lhs_v, lhs, CpuRead);
|
|
||||||
autoView( rhs_v, rhs, CpuRead);
|
|
||||||
autoView( ret_v, ret, CpuWrite);
|
|
||||||
thread_for( ss, rhs_v.size(), {
|
|
||||||
ret_v[ss]=op(lhs_v[ss],rhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// compare lattice to scalar
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class vfunctor,class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> LSComparison(vfunctor op,const Lattice<lobj> &lhs,const robj &rhs)
|
|
||||||
{
|
|
||||||
Lattice<vPredicate> ret(lhs.Grid());
|
|
||||||
autoView( lhs_v, lhs, CpuRead);
|
|
||||||
autoView( ret_v, ret, CpuWrite);
|
|
||||||
thread_for( ss, lhs_v.size(), {
|
|
||||||
ret_v[ss]=op(lhs_v[ss],rhs);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// compare scalar to lattice
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class vfunctor,class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> SLComparison(vfunctor op,const lobj &lhs,const Lattice<robj> &rhs)
|
|
||||||
{
|
|
||||||
Lattice<vPredicate> ret(rhs.Grid());
|
|
||||||
autoView( rhs_v, rhs, CpuRead);
|
|
||||||
autoView( ret_v, ret, CpuWrite);
|
|
||||||
thread_for( ss, rhs_v.size(), {
|
|
||||||
ret_v[ss]=op(lhs,rhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// Map to functors
|
|
||||||
//////////////////////////////////////////////////////////////////////////
|
|
||||||
// Less than
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator < (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return LLComparison(vlt<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator < (const Lattice<lobj> & lhs, const robj & rhs) {
|
|
||||||
return LSComparison(vlt<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator < (const lobj & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return SLComparison(vlt<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Less than equal
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator <= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return LLComparison(vle<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator <= (const Lattice<lobj> & lhs, const robj & rhs) {
|
|
||||||
return LSComparison(vle<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator <= (const lobj & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return SLComparison(vle<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Greater than
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator > (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return LLComparison(vgt<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator > (const Lattice<lobj> & lhs, const robj & rhs) {
|
|
||||||
return LSComparison(vgt<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator > (const lobj & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return SLComparison(vgt<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Greater than equal
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator >= (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return LLComparison(vge<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator >= (const Lattice<lobj> & lhs, const robj & rhs) {
|
|
||||||
return LSComparison(vge<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator >= (const lobj & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return SLComparison(vge<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
|
|
||||||
// equal
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator == (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return LLComparison(veq<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator == (const Lattice<lobj> & lhs, const robj & rhs) {
|
|
||||||
return LSComparison(veq<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator == (const lobj & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return SLComparison(veq<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// not equal
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator != (const Lattice<lobj> & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return LLComparison(vne<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator != (const Lattice<lobj> & lhs, const robj & rhs) {
|
|
||||||
return LSComparison(vne<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
template<class lobj,class robj>
|
|
||||||
inline Lattice<vPredicate> operator != (const lobj & lhs, const Lattice<robj> & rhs) {
|
|
||||||
return SLComparison(vne<lobj,robj>(),lhs,rhs);
|
|
||||||
}
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_coordinate.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class iobj> inline void LatticeCoordinate(Lattice<iobj> &l,int mu)
|
|
||||||
{
|
|
||||||
typedef typename iobj::scalar_type scalar_type;
|
|
||||||
typedef typename iobj::vector_type vector_type;
|
|
||||||
|
|
||||||
GridBase *grid = l.Grid();
|
|
||||||
int Nsimd = grid->iSites();
|
|
||||||
|
|
||||||
autoView(l_v, l, CpuWrite);
|
|
||||||
thread_for( o, grid->oSites(), {
|
|
||||||
vector_type vI;
|
|
||||||
Coordinate gcoor;
|
|
||||||
ExtractBuffer<scalar_type> mergebuf(Nsimd);
|
|
||||||
for(int i=0;i<grid->iSites();i++){
|
|
||||||
grid->RankIndexToGlobalCoor(grid->ThisRank(),o,i,gcoor);
|
|
||||||
mergebuf[i]=(Integer)gcoor[mu];
|
|
||||||
}
|
|
||||||
merge<vector_type,scalar_type>(vI,mergebuf);
|
|
||||||
l_v[o]=vI;
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_crc.h
|
|
||||||
|
|
||||||
Copyright (C) 2021
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class vobj> void DumpSliceNorm(std::string s,const Lattice<vobj> &f,int mu=-1)
|
|
||||||
{
|
|
||||||
auto ff = localNorm2(f);
|
|
||||||
if ( mu==-1 ) mu = f.Grid()->Nd()-1;
|
|
||||||
typedef typename vobj::tensor_reduced normtype;
|
|
||||||
typedef typename normtype::scalar_object scalar;
|
|
||||||
std::vector<scalar> sff;
|
|
||||||
sliceSum(ff,sff,mu);
|
|
||||||
for(int t=0;t<sff.size();t++){
|
|
||||||
std::cout << s<<" "<<t<<" "<<sff[t]<<std::endl;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> uint32_t crc(const Lattice<vobj> & buf)
|
|
||||||
{
|
|
||||||
autoView( buf_v , buf, CpuRead);
|
|
||||||
return ::crc32(0L,(unsigned char *)&buf_v[0],(size_t)sizeof(vobj)*buf.oSites());
|
|
||||||
}
|
|
||||||
|
|
||||||
#define CRC(U) std::cerr << "FingerPrint "<<__FILE__ <<" "<< __LINE__ <<" "<< #U <<" "<<crc(U)<<std::endl;
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_local.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_LOCALREDUCTION_H
|
|
||||||
#define GRID_LATTICE_LOCALREDUCTION_H
|
|
||||||
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
// localInner, localNorm, outerProduct
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
// Non site, reduced locally reduced routines
|
|
||||||
/////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// localNorm2,
|
|
||||||
template<class vobj>
|
|
||||||
inline auto localNorm2 (const Lattice<vobj> &rhs)-> Lattice<typename vobj::tensor_reduced>
|
|
||||||
{
|
|
||||||
Lattice<typename vobj::tensor_reduced> ret(rhs.Grid());
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
accelerator_for(ss,rhs_v.size(),vobj::Nsimd(),{
|
|
||||||
coalescedWrite(ret_v[ss],innerProduct(rhs_v(ss),rhs_v(ss)));
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
// localInnerProduct
|
|
||||||
template<class vobj>
|
|
||||||
inline auto localInnerProduct (const Lattice<vobj> &lhs,const Lattice<vobj> &rhs) -> Lattice<typename vobj::tensor_reduced>
|
|
||||||
{
|
|
||||||
Lattice<typename vobj::tensor_reduced> ret(rhs.Grid());
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
accelerator_for(ss,rhs_v.size(),vobj::Nsimd(),{
|
|
||||||
coalescedWrite(ret_v[ss],innerProduct(lhs_v(ss),rhs_v(ss)));
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
// outerProduct Scalar x Scalar -> Scalar
|
|
||||||
// Vector x Vector -> Matrix
|
|
||||||
template<class ll,class rr>
|
|
||||||
inline auto outerProduct (const Lattice<ll> &lhs,const Lattice<rr> &rhs) -> Lattice<decltype(outerProduct(ll(),rr()))>
|
|
||||||
{
|
|
||||||
typedef decltype(coalescedRead(ll())) sll;
|
|
||||||
typedef decltype(coalescedRead(rr())) srr;
|
|
||||||
Lattice<decltype(outerProduct(ll(),rr()))> ret(rhs.Grid());
|
|
||||||
autoView( lhs_v , lhs, AcceleratorRead);
|
|
||||||
autoView( rhs_v , rhs, AcceleratorRead);
|
|
||||||
autoView( ret_v , ret, AcceleratorWrite);
|
|
||||||
accelerator_for(ss,rhs_v.size(),1,{
|
|
||||||
// FIXME had issues with scalar version of outer
|
|
||||||
// Use vector [] operator and don't read coalesce this loop
|
|
||||||
ret_v[ss]=outerProduct(lhs_v[ss],rhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
@@ -1,199 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
Source file: ./lib/lattice/Lattice_reduction.h
|
|
||||||
Copyright (C) 2015
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
#include <Grid/Grid_Eigen_Dense.h>
|
|
||||||
|
|
||||||
#ifdef GRID_WARN_SUBOPTIMAL
|
|
||||||
#warning "Optimisation alert all these reduction loops are NOT threaded "
|
|
||||||
#endif
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,const Lattice<vobj> &Y,int Orthog,RealD scale=1.0)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
int Nblock = X.Grid()->GlobalDimensions()[Orthog];
|
|
||||||
|
|
||||||
GridBase *FullGrid = X.Grid();
|
|
||||||
// GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
|
|
||||||
|
|
||||||
// Lattice<vobj> Xslice(SliceGrid);
|
|
||||||
// Lattice<vobj> Rslice(SliceGrid);
|
|
||||||
|
|
||||||
GRID_ASSERT( FullGrid->_simd_layout[Orthog]==1);
|
|
||||||
|
|
||||||
//FIXME package in a convenient iterator
|
|
||||||
//Should loop over a plane orthogonal to direction "Orthog"
|
|
||||||
int stride=FullGrid->_slice_stride[Orthog];
|
|
||||||
int block =FullGrid->_slice_block [Orthog];
|
|
||||||
int nblock=FullGrid->_slice_nblock[Orthog];
|
|
||||||
int ostride=FullGrid->_ostride[Orthog];
|
|
||||||
autoView( X_v , X, CpuRead);
|
|
||||||
autoView( Y_v , Y, CpuRead);
|
|
||||||
autoView( R_v , R, CpuWrite);
|
|
||||||
thread_region
|
|
||||||
{
|
|
||||||
std::vector<vobj> s_x(Nblock);
|
|
||||||
|
|
||||||
thread_loop_collapse2( (int n=0;n<nblock;n++),{
|
|
||||||
for(int b=0;b<block;b++){
|
|
||||||
int o = n*stride + b;
|
|
||||||
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
s_x[i] = X_v[o+i*ostride];
|
|
||||||
}
|
|
||||||
|
|
||||||
vobj dot;
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
dot = Y_v[o+i*ostride];
|
|
||||||
for(int j=0;j<Nblock;j++){
|
|
||||||
dot = dot + s_x[j]*(scale*aa(j,i));
|
|
||||||
}
|
|
||||||
R_v[o+i*ostride]=dot;
|
|
||||||
}
|
|
||||||
}});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceMulMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,int Orthog,RealD scale=1.0)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
int Nblock = X.Grid()->GlobalDimensions()[Orthog];
|
|
||||||
|
|
||||||
GridBase *FullGrid = X.Grid();
|
|
||||||
GRID_ASSERT( FullGrid->_simd_layout[Orthog]==1);
|
|
||||||
|
|
||||||
//FIXME package in a convenient iterator
|
|
||||||
//Should loop over a plane orthogonal to direction "Orthog"
|
|
||||||
int stride=FullGrid->_slice_stride[Orthog];
|
|
||||||
int block =FullGrid->_slice_block [Orthog];
|
|
||||||
int nblock=FullGrid->_slice_nblock[Orthog];
|
|
||||||
int ostride=FullGrid->_ostride[Orthog];
|
|
||||||
|
|
||||||
autoView( X_v , X, CpuRead);
|
|
||||||
autoView( R_v , R, CpuWrite);
|
|
||||||
|
|
||||||
thread_region
|
|
||||||
{
|
|
||||||
std::vector<vobj> s_x(Nblock);
|
|
||||||
|
|
||||||
thread_loop_collapse2( (int n=0;n<nblock;n++),{
|
|
||||||
for(int b=0;b<block;b++){
|
|
||||||
int o = n*stride + b;
|
|
||||||
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
s_x[i] = X_v[o+i*ostride];
|
|
||||||
}
|
|
||||||
|
|
||||||
vobj dot;
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
dot = s_x[0]*(scale*aa(0,i));
|
|
||||||
for(int j=1;j<Nblock;j++){
|
|
||||||
dot = dot + s_x[j]*(scale*aa(j,i));
|
|
||||||
}
|
|
||||||
R_v[o+i*ostride]=dot;
|
|
||||||
}
|
|
||||||
}});
|
|
||||||
}
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceInnerProductMatrix( Eigen::MatrixXcd &mat, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
GridBase *FullGrid = lhs.Grid();
|
|
||||||
// GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
|
|
||||||
|
|
||||||
int Nblock = FullGrid->GlobalDimensions()[Orthog];
|
|
||||||
|
|
||||||
// Lattice<vobj> Lslice(SliceGrid);
|
|
||||||
// Lattice<vobj> Rslice(SliceGrid);
|
|
||||||
|
|
||||||
mat = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
|
||||||
|
|
||||||
GRID_ASSERT( FullGrid->_simd_layout[Orthog]==1);
|
|
||||||
// int nh = FullGrid->_ndimension;
|
|
||||||
// int nl = SliceGrid->_ndimension;
|
|
||||||
// int nl = nh-1;
|
|
||||||
|
|
||||||
//FIXME package in a convenient iterator
|
|
||||||
//Should loop over a plane orthogonal to direction "Orthog"
|
|
||||||
int stride=FullGrid->_slice_stride[Orthog];
|
|
||||||
int block =FullGrid->_slice_block [Orthog];
|
|
||||||
int nblock=FullGrid->_slice_nblock[Orthog];
|
|
||||||
int ostride=FullGrid->_ostride[Orthog];
|
|
||||||
|
|
||||||
typedef typename vobj::vector_typeD vector_typeD;
|
|
||||||
autoView( lhs_v , lhs, CpuRead);
|
|
||||||
autoView( rhs_v , rhs, CpuRead);
|
|
||||||
thread_region {
|
|
||||||
std::vector<vobj> Left(Nblock);
|
|
||||||
std::vector<vobj> Right(Nblock);
|
|
||||||
Eigen::MatrixXcd mat_thread = Eigen::MatrixXcd::Zero(Nblock,Nblock);
|
|
||||||
|
|
||||||
thread_loop_collapse2((int n=0;n<nblock;n++),{
|
|
||||||
for(int b=0;b<block;b++){
|
|
||||||
|
|
||||||
int o = n*stride + b;
|
|
||||||
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
Left [i] = lhs_v[o+i*ostride];
|
|
||||||
Right[i] = rhs_v[o+i*ostride];
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
for(int j=0;j<Nblock;j++){
|
|
||||||
auto tmp = innerProduct(Left[i],Right[j]);
|
|
||||||
auto rtmp = TensorRemove(tmp);
|
|
||||||
ComplexD z = Reduce(rtmp);
|
|
||||||
mat_thread(i,j) += std::complex<double>(real(z),imag(z));
|
|
||||||
}}
|
|
||||||
}});
|
|
||||||
thread_critical {
|
|
||||||
mat += mat_thread;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for(int i=0;i<Nblock;i++){
|
|
||||||
for(int j=0;j<Nblock;j++){
|
|
||||||
ComplexD sum = mat(i,j);
|
|
||||||
FullGrid->GlobalSum(sum);
|
|
||||||
mat(i,j)=sum;
|
|
||||||
}}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,231 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_peekpoke.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_PEEK_H
|
|
||||||
#define GRID_LATTICE_PEEK_H
|
|
||||||
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
// Peeking and poking around
|
|
||||||
///////////////////////////////////////////////
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
// FIXME accelerator_loop and accelerator_inline these
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Peek internal indices of a Lattice object
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<int Index,class vobj>
|
|
||||||
auto PeekIndex(const Lattice<vobj> &lhs,int i) -> Lattice<decltype(peekIndex<Index>(vobj(),i))>
|
|
||||||
{
|
|
||||||
Lattice<decltype(peekIndex<Index>(vobj(),i))> ret(lhs.Grid());
|
|
||||||
ret.Checkerboard()=lhs.Checkerboard();
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] = peekIndex<Index>(lhs_v[ss],i);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
template<int Index,class vobj>
|
|
||||||
auto PeekIndex(const Lattice<vobj> &lhs,int i,int j) -> Lattice<decltype(peekIndex<Index>(vobj(),i,j))>
|
|
||||||
{
|
|
||||||
Lattice<decltype(peekIndex<Index>(vobj(),i,j))> ret(lhs.Grid());
|
|
||||||
ret.Checkerboard()=lhs.Checkerboard();
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] = peekIndex<Index>(lhs_v[ss],i,j);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Poke internal indices of a Lattice object
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<int Index,class vobj>
|
|
||||||
void PokeIndex(Lattice<vobj> &lhs,const Lattice<decltype(peekIndex<Index>(vobj(),0))> & rhs,int i)
|
|
||||||
{
|
|
||||||
autoView( rhs_v, rhs, AcceleratorRead);
|
|
||||||
autoView( lhs_v, lhs, AcceleratorWrite);
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
pokeIndex<Index>(lhs_v[ss],rhs_v[ss],i);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
template<int Index,class vobj>
|
|
||||||
void PokeIndex(Lattice<vobj> &lhs,const Lattice<decltype(peekIndex<Index>(vobj(),0,0))> & rhs,int i,int j)
|
|
||||||
{
|
|
||||||
autoView( rhs_v, rhs, AcceleratorRead);
|
|
||||||
autoView( lhs_v, lhs, AcceleratorWrite);
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
pokeIndex<Index>(lhs_v[ss],rhs_v[ss],i,j);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// Poke a scalar object into the SIMD array
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
template<class vobj,class sobj>
|
|
||||||
void pokeSite(const sobj &s,Lattice<vobj> &l,const Coordinate &site){
|
|
||||||
|
|
||||||
GridBase *grid=l.Grid();
|
|
||||||
|
|
||||||
int Nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
GRID_ASSERT( l.Checkerboard()== l.Grid()->CheckerBoard(site));
|
|
||||||
GRID_ASSERT( sizeof(sobj)*Nsimd == sizeof(vobj));
|
|
||||||
|
|
||||||
int rank,odx,idx;
|
|
||||||
// Optional to broadcast from node 0.
|
|
||||||
grid->GlobalCoorToRankIndex(rank,odx,idx,site);
|
|
||||||
grid->Broadcast(grid->BossRank(),s);
|
|
||||||
|
|
||||||
// extract-modify-merge cycle is easiest way and this is not perf critical
|
|
||||||
ExtractBuffer<sobj> buf(Nsimd);
|
|
||||||
autoView( l_v , l, CpuWrite);
|
|
||||||
if ( rank == grid->ThisRank() ) {
|
|
||||||
extract(l_v[odx],buf);
|
|
||||||
buf[idx] = s;
|
|
||||||
merge(l_v[odx],buf);
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
// Peek a scalar object from the SIMD array
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
template<class vobj>
|
|
||||||
typename vobj::scalar_object peekSite(const Lattice<vobj> &l,const Coordinate &site){
|
|
||||||
typename vobj::scalar_object s;
|
|
||||||
peekSite(s,l,site);
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
template<class vobj,class sobj>
|
|
||||||
void peekSite(sobj &s,const Lattice<vobj> &l,const Coordinate &site){
|
|
||||||
|
|
||||||
GridBase *grid=l.Grid();
|
|
||||||
|
|
||||||
int Nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
GRID_ASSERT( l.Checkerboard() == l.Grid()->CheckerBoard(site));
|
|
||||||
|
|
||||||
int rank,odx,idx;
|
|
||||||
grid->GlobalCoorToRankIndex(rank,odx,idx,site);
|
|
||||||
|
|
||||||
ExtractBuffer<sobj> buf(Nsimd);
|
|
||||||
autoView( l_v , l, CpuWrite);
|
|
||||||
extract(l_v[odx],buf);
|
|
||||||
|
|
||||||
s = buf[idx];
|
|
||||||
|
|
||||||
grid->Broadcast(rank,s);
|
|
||||||
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
// Peek a scalar object from the SIMD array
|
|
||||||
//////////////////////////////////////////////////////////
|
|
||||||
// Must be CPU read view
|
|
||||||
template<class vobj,class sobj>
|
|
||||||
inline void peekLocalSite(sobj &s,const LatticeView<vobj> &l,Coordinate &site)
|
|
||||||
{
|
|
||||||
GridBase *grid = l.getGrid();
|
|
||||||
GRID_ASSERT(l.mode==CpuRead);
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
int Nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
// GRID_ASSERT( l.Checkerboard()== grid->CheckerBoard(site));
|
|
||||||
GRID_ASSERT( sizeof(sobj)*Nsimd == sizeof(vobj));
|
|
||||||
|
|
||||||
static const int words=sizeof(vobj)/sizeof(vector_type);
|
|
||||||
int odx,idx;
|
|
||||||
idx= grid->iIndex(site);
|
|
||||||
odx= grid->oIndex(site);
|
|
||||||
|
|
||||||
const vector_type *vp = (const vector_type *) &l[odx];
|
|
||||||
scalar_type * pt = (scalar_type *)&s;
|
|
||||||
|
|
||||||
for(int w=0;w<words;w++){
|
|
||||||
pt[w] = getlane(vp[w],idx);
|
|
||||||
}
|
|
||||||
// std::cout << "peekLocalSite "<<site<<" "<<odx<<","<<idx<<" "<<s<<std::endl;
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
template<class vobj,class sobj>
|
|
||||||
inline void peekLocalSite(sobj &s,const Lattice<vobj> &l,Coordinate &site)
|
|
||||||
{
|
|
||||||
autoView(lv,l,CpuRead);
|
|
||||||
peekLocalSite(s,lv,site);
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Must be CPU write view
|
|
||||||
template<class vobj,class sobj>
|
|
||||||
inline void pokeLocalSite(const sobj &s,LatticeView<vobj> &l,Coordinate &site)
|
|
||||||
{
|
|
||||||
GridBase *grid=l.getGrid();
|
|
||||||
GRID_ASSERT(l.mode==CpuWrite);
|
|
||||||
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
int Nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
// GRID_ASSERT( l.Checkerboard()== grid->CheckerBoard(site));
|
|
||||||
GRID_ASSERT( sizeof(sobj)*Nsimd == sizeof(vobj));
|
|
||||||
|
|
||||||
static const int words=sizeof(vobj)/sizeof(vector_type);
|
|
||||||
int odx,idx;
|
|
||||||
idx= grid->iIndex(site);
|
|
||||||
odx= grid->oIndex(site);
|
|
||||||
|
|
||||||
vector_type * vp = (vector_type *)&l[odx];
|
|
||||||
scalar_type * pt = (scalar_type *)&s;
|
|
||||||
for(int w=0;w<words;w++){
|
|
||||||
putlane(vp[w],pt[w],idx);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class vobj,class sobj>
|
|
||||||
inline void pokeLocalSite(const sobj &s, Lattice<vobj> &l,Coordinate &site)
|
|
||||||
{
|
|
||||||
autoView(lv,l,CpuWrite);
|
|
||||||
pokeLocalSite(s,lv,site);
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_reality.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: neo <cossu@post.kek.jp>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_REAL_IMAG_H
|
|
||||||
#define GRID_LATTICE_REAL_IMAG_H
|
|
||||||
|
|
||||||
|
|
||||||
// FIXME .. this is the sector of the code
|
|
||||||
// I am most worried about the directions
|
|
||||||
// The choice of burying complex in the SIMD
|
|
||||||
// is making the use of "real" and "imag" very cumbersome
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class vobj> inline Lattice<vobj> real(const Lattice<vobj> &lhs){
|
|
||||||
Lattice<vobj> ret(lhs.Grid());
|
|
||||||
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
|
|
||||||
ret.Checkerboard()=lhs.Checkerboard();
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] =real(lhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
template<class vobj> inline Lattice<vobj> imag(const Lattice<vobj> &lhs){
|
|
||||||
Lattice<vobj> ret(lhs.Grid());
|
|
||||||
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
|
|
||||||
ret.Checkerboard()=lhs.Checkerboard();
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] =imag(lhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
|
||||||
auto real(const Expression &expr) -> decltype(real(closure(expr)))
|
|
||||||
{
|
|
||||||
return real(closure(expr));
|
|
||||||
}
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
|
||||||
auto imag(const Expression &expr) -> decltype(imag(closure(expr)))
|
|
||||||
{
|
|
||||||
return imag(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
|
|
||||||
Source file: ./lib/lattice/Lattice_reality.h
|
|
||||||
|
|
||||||
Copyright (C) 2015
|
|
||||||
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: neo <cossu@post.kek.jp>
|
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#ifndef GRID_LATTICE_REALITY_H
|
|
||||||
#define GRID_LATTICE_REALITY_H
|
|
||||||
|
|
||||||
|
|
||||||
// FIXME .. this is the sector of the code
|
|
||||||
// I am most worried about the directions
|
|
||||||
// The choice of burying complex in the SIMD
|
|
||||||
// is making the use of "real" and "imag" very cumbersome
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
template<class vobj> inline Lattice<vobj> adj(const Lattice<vobj> &lhs){
|
|
||||||
Lattice<vobj> ret(lhs.Grid());
|
|
||||||
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
|
|
||||||
ret.Checkerboard()=lhs.Checkerboard();
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] = adj(lhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class vobj> inline Lattice<vobj> conjugate(const Lattice<vobj> &lhs){
|
|
||||||
Lattice<vobj> ret(lhs.Grid());
|
|
||||||
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
accelerator_for( ss, lhs_v.size(), vobj::Nsimd(), {
|
|
||||||
coalescedWrite( ret_v[ss] , conjugate(lhs_v(ss)));
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class vobj> inline Lattice<typename vobj::Complexified> toComplex(const Lattice<vobj> &lhs){
|
|
||||||
Lattice<typename vobj::Complexified> ret(lhs.Grid());
|
|
||||||
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] = toComplex(lhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
template<class vobj> inline Lattice<typename vobj::Realified> toReal(const Lattice<vobj> &lhs){
|
|
||||||
Lattice<typename vobj::Realified> ret(lhs.Grid());
|
|
||||||
|
|
||||||
autoView( lhs_v, lhs, AcceleratorRead);
|
|
||||||
autoView( ret_v, ret, AcceleratorWrite);
|
|
||||||
|
|
||||||
ret.Checkerboard() = lhs.Checkerboard();
|
|
||||||
accelerator_for( ss, lhs_v.size(), 1, {
|
|
||||||
ret_v[ss] = toReal(lhs_v[ss]);
|
|
||||||
});
|
|
||||||
return ret;
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
|
||||||
auto toComplex(const Expression &expr) -> decltype(closure(expr))
|
|
||||||
{
|
|
||||||
return toComplex(closure(expr));
|
|
||||||
}
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
|
||||||
auto toReal(const Expression &expr) -> decltype(closure(expr))
|
|
||||||
{
|
|
||||||
return toReal(closure(expr));
|
|
||||||
}
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
|
||||||
auto adj(const Expression &expr) -> decltype(closure(expr))
|
|
||||||
{
|
|
||||||
return adj(closure(expr));
|
|
||||||
}
|
|
||||||
template<class Expression,typename std::enable_if<is_lattice_expr<Expression>::value,void>::type * = nullptr>
|
|
||||||
auto conjugate(const Expression &expr) -> decltype(closure(expr))
|
|
||||||
{
|
|
||||||
return conjugate(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@@ -1,800 +0,0 @@
|
|||||||
/*************************************************************************************
|
|
||||||
Grid physics library, www.github.com/paboyle/Grid
|
|
||||||
Source file: ./lib/lattice/Lattice_reduction.h
|
|
||||||
Copyright (C) 2015
|
|
||||||
Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
|
|
||||||
Author: Peter Boyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: paboyle <paboyle@ph.ed.ac.uk>
|
|
||||||
Author: Christoph Lehner <christoph@lhnr.de>
|
|
||||||
This program is free software; you can redistribute it and/or modify
|
|
||||||
it under the terms of the GNU General Public License as published by
|
|
||||||
the Free Software Foundation; either version 2 of the License, or
|
|
||||||
(at your option) any later version.
|
|
||||||
This program is distributed in the hope that it will be useful,
|
|
||||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
GNU General Public License for more details.
|
|
||||||
You should have received a copy of the GNU General Public License along
|
|
||||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
|
||||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
||||||
See the full license in the file "LICENSE" in the top level distribution directory
|
|
||||||
*************************************************************************************/
|
|
||||||
/* END LEGAL */
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <Grid/Grid_Eigen_Dense.h>
|
|
||||||
|
|
||||||
|
|
||||||
#if defined(GRID_CUDA)||defined(GRID_HIP)
|
|
||||||
#include <Grid/lattice/Lattice_reduction_gpu.h>
|
|
||||||
#endif
|
|
||||||
#if defined(GRID_SYCL)
|
|
||||||
#include <Grid/lattice/Lattice_reduction_sycl.h>
|
|
||||||
#endif
|
|
||||||
#include <Grid/lattice/Lattice_slicesum_core.h>
|
|
||||||
|
|
||||||
NAMESPACE_BEGIN(Grid);
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
// FIXME this should promote to double and accumulate
|
|
||||||
//////////////////////////////////////////////////////
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_object sum_cpu(const vobj *arg, Integer osites)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
|
|
||||||
// const int Nsimd = vobj::Nsimd();
|
|
||||||
const int nthread = GridThread::GetThreads();
|
|
||||||
|
|
||||||
std::vector<sobj> sumarray(nthread);
|
|
||||||
for(int i=0;i<nthread;i++){
|
|
||||||
sumarray[i]=Zero();
|
|
||||||
}
|
|
||||||
|
|
||||||
thread_for(thr,nthread, {
|
|
||||||
int nwork, mywork, myoff;
|
|
||||||
nwork = osites;
|
|
||||||
GridThread::GetWork(nwork,thr,mywork,myoff);
|
|
||||||
vobj vvsum=Zero();
|
|
||||||
for(int ss=myoff;ss<mywork+myoff; ss++){
|
|
||||||
vvsum = vvsum + arg[ss];
|
|
||||||
}
|
|
||||||
sumarray[thr]=Reduce(vvsum);
|
|
||||||
});
|
|
||||||
|
|
||||||
sobj ssum=Zero(); // sum across threads
|
|
||||||
for(int i=0;i<nthread;i++){
|
|
||||||
ssum = ssum+sumarray[i];
|
|
||||||
}
|
|
||||||
return ssum;
|
|
||||||
}
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_objectD sumD_cpu(const vobj *arg, Integer osites)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_objectD sobj;
|
|
||||||
|
|
||||||
const int nthread = GridThread::GetThreads();
|
|
||||||
|
|
||||||
std::vector<sobj> sumarray(nthread);
|
|
||||||
for(int i=0;i<nthread;i++){
|
|
||||||
sumarray[i]=Zero();
|
|
||||||
}
|
|
||||||
|
|
||||||
thread_for(thr,nthread, {
|
|
||||||
int nwork, mywork, myoff;
|
|
||||||
nwork = osites;
|
|
||||||
GridThread::GetWork(nwork,thr,mywork,myoff);
|
|
||||||
vobj vvsum=Zero();
|
|
||||||
for(int ss=myoff;ss<mywork+myoff; ss++){
|
|
||||||
vvsum = vvsum + arg[ss];
|
|
||||||
}
|
|
||||||
sumarray[thr]=Reduce(vvsum);
|
|
||||||
});
|
|
||||||
|
|
||||||
sobj ssum=Zero(); // sum across threads
|
|
||||||
for(int i=0;i<nthread;i++){
|
|
||||||
ssum = ssum+sumarray[i];
|
|
||||||
}
|
|
||||||
return ssum;
|
|
||||||
}
|
|
||||||
/*
|
|
||||||
Threaded max, don't use for now
|
|
||||||
template<class Double>
|
|
||||||
inline Double max(const Double *arg, Integer osites)
|
|
||||||
{
|
|
||||||
// const int Nsimd = vobj::Nsimd();
|
|
||||||
const int nthread = GridThread::GetThreads();
|
|
||||||
|
|
||||||
std::vector<Double> maxarray(nthread);
|
|
||||||
|
|
||||||
thread_for(thr,nthread, {
|
|
||||||
int nwork, mywork, myoff;
|
|
||||||
nwork = osites;
|
|
||||||
GridThread::GetWork(nwork,thr,mywork,myoff);
|
|
||||||
Double max=arg[0];
|
|
||||||
for(int ss=myoff;ss<mywork+myoff; ss++){
|
|
||||||
if( arg[ss] > max ) max = arg[ss];
|
|
||||||
}
|
|
||||||
maxarray[thr]=max;
|
|
||||||
});
|
|
||||||
|
|
||||||
Double tmax=maxarray[0];
|
|
||||||
for(int i=0;i<nthread;i++){
|
|
||||||
if (maxarray[i]>tmax) tmax = maxarray[i];
|
|
||||||
}
|
|
||||||
return tmax;
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_object sum(const vobj *arg, Integer osites)
|
|
||||||
{
|
|
||||||
#if defined(GRID_CUDA)||defined(GRID_HIP)||defined(GRID_SYCL)
|
|
||||||
return sum_gpu(arg,osites);
|
|
||||||
#else
|
|
||||||
return sum_cpu(arg,osites);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_objectD sumD(const vobj *arg, Integer osites)
|
|
||||||
{
|
|
||||||
#if defined(GRID_CUDA)||defined(GRID_HIP)||defined(GRID_SYCL)
|
|
||||||
return sumD_gpu(arg,osites);
|
|
||||||
#else
|
|
||||||
return sumD_cpu(arg,osites);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_objectD sumD_large(const vobj *arg, Integer osites)
|
|
||||||
{
|
|
||||||
#if defined(GRID_CUDA)||defined(GRID_HIP)||defined(GRID_SYCL)
|
|
||||||
return sumD_gpu_large(arg,osites);
|
|
||||||
#else
|
|
||||||
return sumD_cpu(arg,osites);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_object rankSum(const Lattice<vobj> &arg)
|
|
||||||
{
|
|
||||||
Integer osites = arg.Grid()->oSites();
|
|
||||||
#if defined(GRID_CUDA)||defined(GRID_HIP)||defined(GRID_SYCL)
|
|
||||||
autoView( arg_v, arg, AcceleratorRead);
|
|
||||||
return sum_gpu(&arg_v[0],osites);
|
|
||||||
#else
|
|
||||||
autoView(arg_v, arg, CpuRead);
|
|
||||||
return sum_cpu(&arg_v[0],osites);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_object sum(const Lattice<vobj> &arg)
|
|
||||||
{
|
|
||||||
auto ssum = rankSum(arg);
|
|
||||||
arg.Grid()->GlobalSum(ssum);
|
|
||||||
return ssum;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_object rankSumLarge(const Lattice<vobj> &arg)
|
|
||||||
{
|
|
||||||
#if defined(GRID_CUDA)||defined(GRID_HIP)||defined(GRID_SYCL)
|
|
||||||
autoView( arg_v, arg, AcceleratorRead);
|
|
||||||
Integer osites = arg.Grid()->oSites();
|
|
||||||
return sum_gpu_large(&arg_v[0],osites);
|
|
||||||
#else
|
|
||||||
autoView(arg_v, arg, CpuRead);
|
|
||||||
Integer osites = arg.Grid()->oSites();
|
|
||||||
return sum_cpu(&arg_v[0],osites);
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
inline typename vobj::scalar_object sum_large(const Lattice<vobj> &arg)
|
|
||||||
{
|
|
||||||
auto ssum = rankSumLarge(arg);
|
|
||||||
arg.Grid()->GlobalSum(ssum);
|
|
||||||
return ssum;
|
|
||||||
}
|
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// Deterministic Reduction operations
|
|
||||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
template<class vobj> inline RealD norm2(const Lattice<vobj> &arg){
|
|
||||||
ComplexD nrm = innerProduct(arg,arg);
|
|
||||||
return real(nrm);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
template<class Op,class T1>
|
|
||||||
inline auto norm2(const LatticeUnaryExpression<Op,T1> & expr) ->RealD
|
|
||||||
{
|
|
||||||
return norm2(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Op,class T1,class T2>
|
|
||||||
inline auto norm2(const LatticeBinaryExpression<Op,T1,T2> & expr) ->RealD
|
|
||||||
{
|
|
||||||
return norm2(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
template<class Op,class T1,class T2,class T3>
|
|
||||||
inline auto norm2(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr) ->RealD
|
|
||||||
{
|
|
||||||
return norm2(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
//The global maximum of the site norm2
|
|
||||||
template<class vobj> inline RealD maxLocalNorm2(const Lattice<vobj> &arg)
|
|
||||||
{
|
|
||||||
typedef typename vobj::tensor_reduced vscalar; //iScalar<iScalar<.... <vPODtype> > >
|
|
||||||
typedef typename vscalar::scalar_object scalar; //iScalar<iScalar<.... <PODtype> > >
|
|
||||||
|
|
||||||
Lattice<vscalar> inner = localNorm2(arg);
|
|
||||||
|
|
||||||
auto grid = arg.Grid();
|
|
||||||
|
|
||||||
RealD max;
|
|
||||||
for(int l=0;l<grid->lSites();l++){
|
|
||||||
Coordinate coor;
|
|
||||||
scalar val;
|
|
||||||
RealD r;
|
|
||||||
grid->LocalIndexToLocalCoor(l,coor);
|
|
||||||
peekLocalSite(val,inner,coor);
|
|
||||||
r=real(TensorRemove(val));
|
|
||||||
if( (l==0) || (r>max)){
|
|
||||||
max=r;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
grid->GlobalMax(max);
|
|
||||||
return max;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Double inner product
|
|
||||||
template<class vobj>
|
|
||||||
inline ComplexD rankInnerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right)
|
|
||||||
{
|
|
||||||
typedef typename vobj::vector_typeD vector_type;
|
|
||||||
ComplexD nrm;
|
|
||||||
|
|
||||||
GridBase *grid = left.Grid();
|
|
||||||
|
|
||||||
const uint64_t nsimd = grid->Nsimd();
|
|
||||||
const uint64_t sites = grid->oSites();
|
|
||||||
|
|
||||||
// Might make all code paths go this way.
|
|
||||||
typedef decltype(innerProduct(vobj(),vobj())) inner_t;
|
|
||||||
deviceVector<inner_t> inner_tmp(sites);
|
|
||||||
auto inner_tmp_v = &inner_tmp[0];
|
|
||||||
|
|
||||||
{
|
|
||||||
autoView( left_v , left, AcceleratorRead);
|
|
||||||
autoView( right_v,right, AcceleratorRead);
|
|
||||||
|
|
||||||
// GPU - SIMT lane compliance...
|
|
||||||
accelerator_for( ss, sites, nsimd,{
|
|
||||||
auto x_l = left_v(ss);
|
|
||||||
auto y_l = right_v(ss);
|
|
||||||
coalescedWrite(inner_tmp_v[ss],innerProduct(x_l,y_l));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// This is in single precision and fails some tests
|
|
||||||
auto anrm = sumD(inner_tmp_v,sites);
|
|
||||||
nrm = anrm;
|
|
||||||
return nrm;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
inline ComplexD innerProduct(const Lattice<vobj> &left,const Lattice<vobj> &right) {
|
|
||||||
GridBase *grid = left.Grid();
|
|
||||||
|
|
||||||
bool ok;
|
|
||||||
#ifdef GRID_SYCL
|
|
||||||
// uint64_t csum=0;
|
|
||||||
// uint64_t csum2=0;
|
|
||||||
// if ( FlightRecorder::LoggingMode != FlightRecorder::LoggingModeNone)
|
|
||||||
// {
|
|
||||||
// Hack
|
|
||||||
// Fast integer xor checksum. Can also be used in comms now.
|
|
||||||
// autoView(l_v,left,AcceleratorRead);
|
|
||||||
// Integer words = left.Grid()->oSites()*sizeof(vobj)/sizeof(uint64_t);
|
|
||||||
// uint64_t *base= (uint64_t *)&l_v[0];
|
|
||||||
// csum=svm_xor(base,words);
|
|
||||||
// ok = FlightRecorder::CsumLog(csum);
|
|
||||||
// if ( !ok ) {
|
|
||||||
// csum2=svm_xor(base,words);
|
|
||||||
// std::cerr<< " Bad CSUM " << std::hex<< csum << " recomputed as "<<csum2<<std::dec<<std::endl;
|
|
||||||
// } else {
|
|
||||||
// csum2=svm_xor(base,words);
|
|
||||||
// std::cerr<< " ok CSUM " << std::hex<< csum << " recomputed as "<<csum2<<std::dec<<std::endl;
|
|
||||||
// }
|
|
||||||
// GRID_ASSERT(ok);
|
|
||||||
// }
|
|
||||||
#endif
|
|
||||||
FlightRecorder::StepLog("rank inner product");
|
|
||||||
ComplexD nrm = rankInnerProduct(left,right);
|
|
||||||
// ComplexD nrmck=nrm;
|
|
||||||
RealD local = real(nrm);
|
|
||||||
ok = FlightRecorder::NormLog(real(nrm));
|
|
||||||
if ( !ok ) {
|
|
||||||
ComplexD nrm2 = rankInnerProduct(left,right);
|
|
||||||
RealD local2 = real(nrm2);
|
|
||||||
std::cerr<< " Bad NORM " << local << " recomputed as "<<local2<<std::endl;
|
|
||||||
GRID_ASSERT(ok);
|
|
||||||
}
|
|
||||||
FlightRecorder::StepLog("Start global sum");
|
|
||||||
grid->GlobalSumP2P(nrm);
|
|
||||||
// grid->GlobalSum(nrm);
|
|
||||||
FlightRecorder::StepLog("Finished global sum");
|
|
||||||
// std::cout << " norm "<< nrm << " p2p norm "<<nrmck<<std::endl;
|
|
||||||
FlightRecorder::ReductionLog(local,real(nrm));
|
|
||||||
return nrm;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/////////////////////////
|
|
||||||
// Fast axpby_norm
|
|
||||||
// z = a x + b y
|
|
||||||
// return norm z
|
|
||||||
/////////////////////////
|
|
||||||
template<class sobj,class vobj> strong_inline RealD
|
|
||||||
axpy_norm_fast(Lattice<vobj> &z,sobj a,const Lattice<vobj> &x,const Lattice<vobj> &y)
|
|
||||||
{
|
|
||||||
sobj one(1.0);
|
|
||||||
return axpby_norm_fast(z,a,one,x,y);
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class sobj,class vobj> strong_inline RealD
|
|
||||||
axpby_norm_fast(Lattice<vobj> &z,sobj a,sobj b,const Lattice<vobj> &x,const Lattice<vobj> &y)
|
|
||||||
{
|
|
||||||
z.Checkerboard() = x.Checkerboard();
|
|
||||||
conformable(z,x);
|
|
||||||
conformable(x,y);
|
|
||||||
|
|
||||||
// typedef typename vobj::vector_typeD vector_type;
|
|
||||||
RealD nrm;
|
|
||||||
|
|
||||||
GridBase *grid = x.Grid();
|
|
||||||
|
|
||||||
const uint64_t nsimd = grid->Nsimd();
|
|
||||||
const uint64_t sites = grid->oSites();
|
|
||||||
|
|
||||||
// GPU
|
|
||||||
autoView( x_v, x, AcceleratorRead);
|
|
||||||
autoView( y_v, y, AcceleratorRead);
|
|
||||||
autoView( z_v, z, AcceleratorWrite);
|
|
||||||
typedef decltype(innerProduct(x_v[0],y_v[0])) inner_t;
|
|
||||||
deviceVector<inner_t> inner_tmp;
|
|
||||||
inner_tmp.resize(sites);
|
|
||||||
auto inner_tmp_v = &inner_tmp[0];
|
|
||||||
|
|
||||||
accelerator_for( ss, sites, nsimd,{
|
|
||||||
auto tmp = a*x_v(ss)+b*y_v(ss);
|
|
||||||
coalescedWrite(inner_tmp_v[ss],innerProduct(tmp,tmp));
|
|
||||||
coalescedWrite(z_v[ss],tmp);
|
|
||||||
});
|
|
||||||
bool ok;
|
|
||||||
nrm = real(TensorRemove(sumD(inner_tmp_v,sites)));
|
|
||||||
ok = FlightRecorder::NormLog(real(nrm));
|
|
||||||
GRID_ASSERT(ok);
|
|
||||||
RealD local = real(nrm);
|
|
||||||
grid->GlobalSum(nrm);
|
|
||||||
FlightRecorder::ReductionLog(local,real(nrm));
|
|
||||||
return nrm;
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj> strong_inline void
|
|
||||||
innerProductNorm(ComplexD& ip, RealD &nrm, const Lattice<vobj> &left,const Lattice<vobj> &right)
|
|
||||||
{
|
|
||||||
conformable(left,right);
|
|
||||||
|
|
||||||
typedef typename vobj::vector_typeD vector_type;
|
|
||||||
std::vector<ComplexD> tmp(2);
|
|
||||||
|
|
||||||
GridBase *grid = left.Grid();
|
|
||||||
|
|
||||||
const uint64_t nsimd = grid->Nsimd();
|
|
||||||
const uint64_t sites = grid->oSites();
|
|
||||||
|
|
||||||
// GPU
|
|
||||||
typedef decltype(innerProductD(vobj(),vobj())) inner_t;
|
|
||||||
typedef decltype(innerProductD(vobj(),vobj())) norm_t;
|
|
||||||
deviceVector<inner_t> inner_tmp(sites);
|
|
||||||
deviceVector<norm_t> norm_tmp(sites);
|
|
||||||
auto inner_tmp_v = &inner_tmp[0];
|
|
||||||
auto norm_tmp_v = &norm_tmp[0];
|
|
||||||
{
|
|
||||||
autoView(left_v,left, AcceleratorRead);
|
|
||||||
autoView(right_v,right,AcceleratorRead);
|
|
||||||
accelerator_for( ss, sites, 1,{
|
|
||||||
auto left_tmp = left_v[ss];
|
|
||||||
inner_tmp_v[ss]=innerProductD(left_tmp,right_v[ss]);
|
|
||||||
norm_tmp_v [ss]=innerProductD(left_tmp,left_tmp);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
tmp[0] = TensorRemove(sum(inner_tmp_v,sites));
|
|
||||||
tmp[1] = TensorRemove(sum(norm_tmp_v,sites));
|
|
||||||
|
|
||||||
grid->GlobalSumVector(&tmp[0],2); // keep norm Complex -> can use GlobalSumVector
|
|
||||||
ip = tmp[0];
|
|
||||||
nrm = real(tmp[1]);
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Op,class T1>
|
|
||||||
inline auto sum(const LatticeUnaryExpression<Op,T1> & expr)
|
|
||||||
->typename decltype(expr.op.func(eval(0,expr.arg1)))::scalar_object
|
|
||||||
{
|
|
||||||
return sum(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class Op,class T1,class T2>
|
|
||||||
inline auto sum(const LatticeBinaryExpression<Op,T1,T2> & expr)
|
|
||||||
->typename decltype(expr.op.func(eval(0,expr.arg1),eval(0,expr.arg2)))::scalar_object
|
|
||||||
{
|
|
||||||
return sum(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
template<class Op,class T1,class T2,class T3>
|
|
||||||
inline auto sum(const LatticeTrinaryExpression<Op,T1,T2,T3> & expr)
|
|
||||||
->typename decltype(expr.op.func(eval(0,expr.arg1),
|
|
||||||
eval(0,expr.arg2),
|
|
||||||
eval(0,expr.arg3)
|
|
||||||
))::scalar_object
|
|
||||||
{
|
|
||||||
return sum(closure(expr));
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
// sliceSum, sliceInnerProduct, sliceAxpy, sliceNorm etc...
|
|
||||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
template<class vobj> inline void sliceSum(const Lattice<vobj> &Data,
|
|
||||||
std::vector<typename vobj::scalar_object> &result,
|
|
||||||
int orthogdim)
|
|
||||||
{
|
|
||||||
///////////////////////////////////////////////////////
|
|
||||||
// FIXME precision promoted summation
|
|
||||||
// may be important for correlation functions
|
|
||||||
// But easily avoided by using double precision fields
|
|
||||||
///////////////////////////////////////////////////////
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::scalar_object::scalar_type scalar_type;
|
|
||||||
GridBase *grid = Data.Grid();
|
|
||||||
GRID_ASSERT(grid!=NULL);
|
|
||||||
|
|
||||||
const int Nd = grid->_ndimension;
|
|
||||||
const int Nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
GRID_ASSERT(orthogdim >= 0);
|
|
||||||
GRID_ASSERT(orthogdim < Nd);
|
|
||||||
|
|
||||||
int fd=grid->_fdimensions[orthogdim];
|
|
||||||
int ld=grid->_ldimensions[orthogdim];
|
|
||||||
int rd=grid->_rdimensions[orthogdim];
|
|
||||||
|
|
||||||
std::vector<vobj> lvSum(rd); // will locally sum vectors first
|
|
||||||
std::vector<sobj> lsSum(ld,Zero()); // sum across these down to scalars
|
|
||||||
ExtractBuffer<sobj> extracted(Nsimd); // splitting the SIMD
|
|
||||||
|
|
||||||
result.resize(fd); // And then global sum to return the same vector to every node
|
|
||||||
for(int r=0;r<rd;r++){
|
|
||||||
lvSum[r]=Zero();
|
|
||||||
}
|
|
||||||
|
|
||||||
int e1= grid->_slice_nblock[orthogdim];
|
|
||||||
int e2= grid->_slice_block [orthogdim];
|
|
||||||
int stride=grid->_slice_stride[orthogdim];
|
|
||||||
int ostride=grid->_ostride[orthogdim];
|
|
||||||
|
|
||||||
//Reduce Data down to lvSum
|
|
||||||
sliceSumReduction(Data,lvSum,rd, e1,e2,stride,ostride,Nsimd);
|
|
||||||
|
|
||||||
// Sum across simd lanes in the plane, breaking out orthog dir.
|
|
||||||
Coordinate icoor(Nd);
|
|
||||||
|
|
||||||
for(int rt=0;rt<rd;rt++){
|
|
||||||
|
|
||||||
extract(lvSum[rt],extracted);
|
|
||||||
|
|
||||||
for(int idx=0;idx<Nsimd;idx++){
|
|
||||||
|
|
||||||
grid->iCoorFromIindex(icoor,idx);
|
|
||||||
|
|
||||||
int ldx =rt+icoor[orthogdim]*rd;
|
|
||||||
|
|
||||||
lsSum[ldx]=lsSum[ldx]+extracted[idx];
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum over nodes.
|
|
||||||
for(int t=0;t<fd;t++){
|
|
||||||
int pt = t/ld; // processor plane
|
|
||||||
int lt = t%ld;
|
|
||||||
if ( pt == grid->_processor_coor[orthogdim] ) {
|
|
||||||
result[t]=lsSum[lt];
|
|
||||||
} else {
|
|
||||||
result[t]=Zero();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
scalar_type * ptr = (scalar_type *) &result[0];
|
|
||||||
int words = fd*sizeof(sobj)/sizeof(scalar_type);
|
|
||||||
grid->GlobalSumVector(ptr, words);
|
|
||||||
// std::cout << GridLogMessage << " sliceSum local"<<t_sum<<" us, host+mpi "<<t_rest<<std::endl;
|
|
||||||
|
|
||||||
}
|
|
||||||
template<class vobj> inline
|
|
||||||
std::vector<typename vobj::scalar_object>
|
|
||||||
sliceSum(const Lattice<vobj> &Data,int orthogdim)
|
|
||||||
{
|
|
||||||
std::vector<typename vobj::scalar_object> result;
|
|
||||||
sliceSum(Data,result,orthogdim);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
Reimplement
|
|
||||||
|
|
||||||
1)
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,const Lattice<vobj> &Y,int Orthog,RealD scale=1.0)
|
|
||||||
|
|
||||||
2)
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceInnerProductMatrix( Eigen::MatrixXcd &mat, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog)
|
|
||||||
|
|
||||||
3)
|
|
||||||
-- Make Slice Mul Matrix call sliceMaddMatrix
|
|
||||||
*/
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceInnerProductVector( std::vector<ComplexD> & result, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int orthogdim)
|
|
||||||
{
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
GridBase *grid = lhs.Grid();
|
|
||||||
GRID_ASSERT(grid!=NULL);
|
|
||||||
conformable(grid,rhs.Grid());
|
|
||||||
|
|
||||||
const int Nd = grid->_ndimension;
|
|
||||||
const int Nsimd = grid->Nsimd();
|
|
||||||
|
|
||||||
GRID_ASSERT(orthogdim >= 0);
|
|
||||||
GRID_ASSERT(orthogdim < Nd);
|
|
||||||
|
|
||||||
int fd=grid->_fdimensions[orthogdim];
|
|
||||||
int ld=grid->_ldimensions[orthogdim];
|
|
||||||
int rd=grid->_rdimensions[orthogdim];
|
|
||||||
|
|
||||||
std::vector<vector_type> lvSum(rd); // will locally sum vectors first
|
|
||||||
std::vector<scalar_type > lsSum(ld,scalar_type(0.0)); // sum across these down to scalars
|
|
||||||
ExtractBuffer<iScalar<scalar_type> > extracted(Nsimd); // splitting the SIMD
|
|
||||||
|
|
||||||
result.resize(fd); // And then global sum to return the same vector to every node for IO to file
|
|
||||||
for(int r=0;r<rd;r++){
|
|
||||||
lvSum[r]=Zero();
|
|
||||||
}
|
|
||||||
|
|
||||||
int e1= grid->_slice_nblock[orthogdim];
|
|
||||||
int e2= grid->_slice_block [orthogdim];
|
|
||||||
int stride=grid->_slice_stride[orthogdim];
|
|
||||||
|
|
||||||
autoView( lhv, lhs, CpuRead);
|
|
||||||
autoView( rhv, rhs, CpuRead);
|
|
||||||
thread_for( r,rd,{
|
|
||||||
|
|
||||||
int so=r*grid->_ostride[orthogdim]; // base offset for start of plane
|
|
||||||
|
|
||||||
for(int n=0;n<e1;n++){
|
|
||||||
for(int b=0;b<e2;b++){
|
|
||||||
int ss= so+n*stride+b;
|
|
||||||
vector_type vv = TensorRemove(innerProduct(lhv[ss],rhv[ss]));
|
|
||||||
lvSum[r]=lvSum[r]+vv;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Sum across simd lanes in the plane, breaking out orthog dir.
|
|
||||||
Coordinate icoor(Nd);
|
|
||||||
for(int rt=0;rt<rd;rt++){
|
|
||||||
|
|
||||||
iScalar<vector_type> temp;
|
|
||||||
temp._internal = lvSum[rt];
|
|
||||||
extract(temp,extracted);
|
|
||||||
|
|
||||||
for(int idx=0;idx<Nsimd;idx++){
|
|
||||||
|
|
||||||
grid->iCoorFromIindex(icoor,idx);
|
|
||||||
|
|
||||||
int ldx =rt+icoor[orthogdim]*rd;
|
|
||||||
|
|
||||||
lsSum[ldx]=lsSum[ldx]+extracted[idx]._internal;
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum over nodes.
|
|
||||||
scalar_type gsum;
|
|
||||||
for(int t=0;t<fd;t++){
|
|
||||||
int pt = t/ld; // processor plane
|
|
||||||
int lt = t%ld;
|
|
||||||
if ( pt == grid->_processor_coor[orthogdim] ) {
|
|
||||||
gsum=lsSum[lt];
|
|
||||||
} else {
|
|
||||||
gsum=scalar_type(0.0);
|
|
||||||
}
|
|
||||||
|
|
||||||
grid->GlobalSum(gsum);
|
|
||||||
|
|
||||||
result[t]=gsum;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceNorm (std::vector<RealD> &sn,const Lattice<vobj> &rhs,int Orthog)
|
|
||||||
{
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
|
|
||||||
int Nblock = rhs.Grid()->GlobalDimensions()[Orthog];
|
|
||||||
std::vector<ComplexD> ip(Nblock);
|
|
||||||
sn.resize(Nblock);
|
|
||||||
|
|
||||||
sliceInnerProductVector(ip,rhs,rhs,Orthog);
|
|
||||||
for(int ss=0;ss<Nblock;ss++){
|
|
||||||
sn[ss] = real(ip[ss]);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceMaddVector(Lattice<vobj> &R,std::vector<RealD> &a,const Lattice<vobj> &X,const Lattice<vobj> &Y,
|
|
||||||
int orthogdim,RealD scale=1.0)
|
|
||||||
{
|
|
||||||
// perhaps easier to just promote A to a field and use regular madd
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::scalar_type scalar_type;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
typedef typename vobj::tensor_reduced tensor_reduced;
|
|
||||||
|
|
||||||
scalar_type zscale(scale);
|
|
||||||
|
|
||||||
GridBase *grid = X.Grid();
|
|
||||||
|
|
||||||
int Nsimd =grid->Nsimd();
|
|
||||||
int Nblock =grid->GlobalDimensions()[orthogdim];
|
|
||||||
|
|
||||||
int fd =grid->_fdimensions[orthogdim];
|
|
||||||
int ld =grid->_ldimensions[orthogdim];
|
|
||||||
int rd =grid->_rdimensions[orthogdim];
|
|
||||||
|
|
||||||
int e1 =grid->_slice_nblock[orthogdim];
|
|
||||||
int e2 =grid->_slice_block [orthogdim];
|
|
||||||
int stride =grid->_slice_stride[orthogdim];
|
|
||||||
|
|
||||||
Coordinate icoor;
|
|
||||||
for(int r=0;r<rd;r++){
|
|
||||||
|
|
||||||
int so=r*grid->_ostride[orthogdim]; // base offset for start of plane
|
|
||||||
|
|
||||||
vector_type av;
|
|
||||||
|
|
||||||
for(int l=0;l<Nsimd;l++){
|
|
||||||
grid->iCoorFromIindex(icoor,l);
|
|
||||||
int ldx =r+icoor[orthogdim]*rd;
|
|
||||||
av.putlane(scalar_type(a[ldx])*zscale,l);
|
|
||||||
}
|
|
||||||
|
|
||||||
tensor_reduced at; at=av;
|
|
||||||
|
|
||||||
autoView( Rv, R, CpuWrite);
|
|
||||||
autoView( Xv, X, CpuRead);
|
|
||||||
autoView( Yv, Y, CpuRead);
|
|
||||||
thread_for2d( n, e1, b,e2, {
|
|
||||||
int ss= so+n*stride+b;
|
|
||||||
Rv[ss] = at*Xv[ss]+Yv[ss];
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
inline GridBase *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Orthog)
|
|
||||||
{
|
|
||||||
int NN = BlockSolverGrid->_ndimension;
|
|
||||||
int nsimd = BlockSolverGrid->Nsimd();
|
|
||||||
|
|
||||||
std::vector<int> latt_phys(NN-1);
|
|
||||||
Coordinate simd_phys;
|
|
||||||
std::vector<int> mpi_phys(NN-1);
|
|
||||||
Coordinate checker_dim_mask(NN-1);
|
|
||||||
int checker_dim=-1;
|
|
||||||
|
|
||||||
int dd;
|
|
||||||
for(int d=0;d<NN;d++){
|
|
||||||
if( d!=Orthog ) {
|
|
||||||
latt_phys[dd]=BlockSolverGrid->_fdimensions[d];
|
|
||||||
mpi_phys[dd] =BlockSolverGrid->_processors[d];
|
|
||||||
checker_dim_mask[dd] = BlockSolverGrid->_checker_dim_mask[d];
|
|
||||||
if ( d == BlockSolverGrid->_checker_dim ) checker_dim = dd;
|
|
||||||
dd++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
simd_phys=GridDefaultSimd(latt_phys.size(),nsimd);
|
|
||||||
GridCartesian *tmp = new GridCartesian(latt_phys,simd_phys,mpi_phys);
|
|
||||||
if(BlockSolverGrid->_isCheckerBoarded) {
|
|
||||||
GridRedBlackCartesian *ret = new GridRedBlackCartesian(tmp,checker_dim_mask,checker_dim);
|
|
||||||
delete tmp;
|
|
||||||
return (GridBase *) ret;
|
|
||||||
} else {
|
|
||||||
return (GridBase *) tmp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,const Lattice<vobj> &Y,int Orthog,RealD scale=1.0)
|
|
||||||
{
|
|
||||||
GridBase *FullGrid = X.Grid();
|
|
||||||
GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
|
|
||||||
|
|
||||||
Lattice<vobj> Ys(SliceGrid);
|
|
||||||
Lattice<vobj> Rs(SliceGrid);
|
|
||||||
Lattice<vobj> Xs(SliceGrid);
|
|
||||||
Lattice<vobj> RR(FullGrid);
|
|
||||||
|
|
||||||
RR = R; // Copies checkerboard for insert
|
|
||||||
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
int Nslice = X.Grid()->GlobalDimensions()[Orthog];
|
|
||||||
for(int i=0;i<Nslice;i++){
|
|
||||||
ExtractSlice(Ys,Y,i,Orthog);
|
|
||||||
ExtractSlice(Rs,R,i,Orthog);
|
|
||||||
Rs=Ys;
|
|
||||||
for(int j=0;j<Nslice;j++){
|
|
||||||
ExtractSlice(Xs,X,j,Orthog);
|
|
||||||
Rs = Rs + Xs*(scale*aa(j,i));
|
|
||||||
}
|
|
||||||
InsertSlice(Rs,RR,i,Orthog);
|
|
||||||
}
|
|
||||||
R=RR; // Copy back handles arguments aliasing case
|
|
||||||
delete SliceGrid;
|
|
||||||
};
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceMulMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,int Orthog,RealD scale=1.0)
|
|
||||||
{
|
|
||||||
R=Zero();
|
|
||||||
sliceMaddMatrix(R,aa,X,R,Orthog,scale);
|
|
||||||
};
|
|
||||||
|
|
||||||
|
|
||||||
template<class vobj>
|
|
||||||
static void sliceInnerProductMatrix( Eigen::MatrixXcd &mat, const Lattice<vobj> &lhs,const Lattice<vobj> &rhs,int Orthog)
|
|
||||||
{
|
|
||||||
GridBase *SliceGrid = makeSubSliceGrid(lhs.Grid(),Orthog);
|
|
||||||
|
|
||||||
Lattice<vobj> ls(SliceGrid);
|
|
||||||
Lattice<vobj> rs(SliceGrid);
|
|
||||||
|
|
||||||
typedef typename vobj::scalar_object sobj;
|
|
||||||
typedef typename vobj::vector_type vector_type;
|
|
||||||
int Nslice = lhs.Grid()->GlobalDimensions()[Orthog];
|
|
||||||
mat = Eigen::MatrixXcd::Zero(Nslice,Nslice);
|
|
||||||
for(int s=0;s<Nslice;s++){
|
|
||||||
ExtractSlice(ls,lhs,s,Orthog);
|
|
||||||
for(int ss=0;ss<Nslice;ss++){
|
|
||||||
ExtractSlice(rs,rhs,ss,Orthog);
|
|
||||||
mat(s,ss) = innerProduct(ls,rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
delete SliceGrid;
|
|
||||||
}
|
|
||||||
|
|
||||||
NAMESPACE_END(Grid);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user